course_ratings.rb 3.11 KB
Newer Older
Zac Wood's avatar
Zac Wood committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
require 'httparty'
require 'nokogiri'

headers = {
  'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0',
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
  'Accept-Encoding': 'gzip, deflate, br',
  'Referer': 'https://crserating.gmu.edu/ReportPaper/',
  'Content-Type': 'application/x-www-form-urlencoded',
  'Connection': 'keep-alive',
  'Cookie': 'ga=GA1.2.1878572417.1543773996; __unam=74bec77-167701a5090-6fe3a784-10; __insp_slim=1549744302281; CFID=1938650; CFTOKEN=b1990d0cd24bb3f4-19FA09EB-5056-9470-35EEF241BAFA23D5; CFGLOBALS=urltoken%3DCFID%23%3D1938650%26CFTOKEN%23%3Db1990d0cd24bb3f4%2D19FA09EB%2D5056%2D9470%2D35EEF241BAFA23D5%23lastvisit%3D%7Bts%20%272019%2D01%2D29%2019%3A40%3A10%27%7D%23timecreated%3D%7Bts%20%272019%2D01%2D29%2019%3A39%3A51%27%7D%23hitcount%3D7%23cftoken%3Db1990d0cd24bb3f4%2D19FA09EB%2D5056%2D9470%2D35EEF241BAFA23D5%23cfid%3D1938650%23; __insp_wid=1435896333; __insp_nv=true; __insp_ref=aHR0cHM6Ly9pcnIyLmdtdS5lZHUvTmV3L05fRW5yb2xsT2ZmL0VucmxTdHMuY2Zt; __insp_targlpu=https%3A%2F%2Firr2.gmu.edu%2FNew%2FN_EnrollOff%2FEnrlSts.cfm; __insp_targlpt=Office%20of%20Institutional%20Research%20and%20Effectiveness; BIGipServer~web.gmu.edu~goose.gmu.edu_p80=1242087434.20480.0000',
  'Upgrade-Insecure-Requests': '1',
  'Cache-Control': 'max-age=0'
}

resp = HTTParty.post('https://crserating.gmu.edu/ReportPaper/InstructorList.cfm',
                     body: "SearchType=instructor&semester=f17&ctitle=&iname=&divsname=&deptname=&disc=&ckey=&orig=off&SearchTypeHid=instructor",
                     headers: headers).body

document = Nokogiri::HTML(resp)
values = document.css('select option').map { |e| e['value'] }
i = values.index('--Select an Instructor--')
values = values[i+1..-1]
all = {}
c = values.count
counter = 1
values.each do |v|
  puts "#{counter}/#{c} getting data for #{v}..."
  counter += 1
  resp = HTTParty.post('https://crserating.gmu.edu/ReportPaper/InstructorList.cfm',
                       body: "SearchType=instructor&semester=f17&ctitle=&iname=#{v}&divsname=&deptname=&disc=&ckey=&orig=off&SearchTypeHid=instructor",
                       headers: headers)

  document = Nokogiri::HTML(resp)
  rows = document.css('tr')
  i = rows.index { |r| r.css('td').first&.text == 'Course' }
  rows[i+1..-3].each do |s_tr|
    tds = s_tr.css('td')
    
    id = tds.first.css('font a').first['href'].match(/[0-9]+/)[0]
    section, instr = tds.first&.text, tds[2]&.text
    
    resp = HTTParty.post("https://crserating.gmu.edu/ReportPaper/MeansSummary16.cfm?rat_num=#{id}&caller=InstructorList",
                         body: "semester=f17&formseq=62&orig=off&iname=#{instr}",
                         headers: headers)
    document = Nokogiri::HTML(resp)
    rows = document.css('tr')
    qs = [9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39].map do |i|
      datas = rows[i].css('td')
      { q: datas[0].text.match(/[A-Z].*/)[0], resp: datas[1].text.strip, instr_mean: datas[2].text.strip, dept_mean: datas[3].text.strip }
    end
    all[section] = qs
  end
  puts '------------------------------'
end

File.write('f17.json', all.to_json)