Commit dc751ea1 authored by Zac Wood's avatar Zac Wood
Browse files

monthly linting

parent eb721652
Pipeline #2825 passed with stage
in 2 minutes and 15 seconds
...@@ -52,3 +52,5 @@ gem 'icalendar' ...@@ -52,3 +52,5 @@ gem 'icalendar'
gem 'nokogiri' gem 'nokogiri'
gem 'rack-cors', require: 'rack/cors' gem 'rack-cors', require: 'rack/cors'
gem 'rubyXL' gem 'rubyXL'
gem "rubocop", "~> 0.58.2"
...@@ -41,6 +41,7 @@ GEM ...@@ -41,6 +41,7 @@ GEM
addressable (2.5.2) addressable (2.5.2)
public_suffix (>= 2.0.2, < 4.0) public_suffix (>= 2.0.2, < 4.0)
arel (8.0.0) arel (8.0.0)
ast (2.4.0)
bindex (0.5.0) bindex (0.5.0)
builder (3.2.3) builder (3.2.3)
byebug (10.0.2) byebug (10.0.2)
...@@ -65,6 +66,7 @@ GEM ...@@ -65,6 +66,7 @@ GEM
i18n (1.0.1) i18n (1.0.1)
concurrent-ruby (~> 1.0) concurrent-ruby (~> 1.0)
icalendar (2.4.1) icalendar (2.4.1)
jaro_winkler (1.5.1)
jbuilder (2.7.0) jbuilder (2.7.0)
activesupport (>= 4.2.0) activesupport (>= 4.2.0)
multi_json (>= 1.2) multi_json (>= 1.2)
...@@ -86,6 +88,10 @@ GEM ...@@ -86,6 +88,10 @@ GEM
nio4r (2.3.1) nio4r (2.3.1)
nokogiri (1.8.3) nokogiri (1.8.3)
mini_portile2 (~> 2.3.0) mini_portile2 (~> 2.3.0)
parallel (1.12.1)
parser (2.5.1.2)
ast (~> 2.4.0)
powerpack (0.1.2)
pry (0.11.3) pry (0.11.3)
coderay (~> 1.1.0) coderay (~> 1.1.0)
method_source (~> 0.9.0) method_source (~> 0.9.0)
...@@ -121,10 +127,20 @@ GEM ...@@ -121,10 +127,20 @@ GEM
method_source method_source
rake (>= 0.8.7) rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0) thor (>= 0.18.1, < 2.0)
rainbow (3.0.0)
rake (12.3.1) rake (12.3.1)
rb-fsevent (0.10.3) rb-fsevent (0.10.3)
rb-inotify (0.9.10) rb-inotify (0.9.10)
ffi (>= 0.5.0, < 2) ffi (>= 0.5.0, < 2)
rubocop (0.58.2)
jaro_winkler (~> 1.5.1)
parallel (~> 1.10)
parser (>= 2.5, != 2.5.1.1)
powerpack (~> 0.1)
rainbow (>= 2.2.2, < 4.0)
ruby-progressbar (~> 1.7)
unicode-display_width (~> 1.0, >= 1.0.1)
ruby-progressbar (1.10.0)
rubyXL (3.3.29) rubyXL (3.3.29)
nokogiri (>= 1.4.4) nokogiri (>= 1.4.4)
rubyzip (>= 1.1.6) rubyzip (>= 1.1.6)
...@@ -153,6 +169,7 @@ GEM ...@@ -153,6 +169,7 @@ GEM
turbolinks-source (5.1.0) turbolinks-source (5.1.0)
tzinfo (1.2.5) tzinfo (1.2.5)
thread_safe (~> 0.1) thread_safe (~> 0.1)
unicode-display_width (1.4.0)
web-console (3.6.2) web-console (3.6.2)
actionview (>= 5.0) actionview (>= 5.0)
activemodel (>= 5.0) activemodel (>= 5.0)
...@@ -181,6 +198,7 @@ DEPENDENCIES ...@@ -181,6 +198,7 @@ DEPENDENCIES
puma (~> 3.7) puma (~> 3.7)
rack-cors rack-cors
rails (~> 5.1.6) rails (~> 5.1.6)
rubocop (~> 0.58.2)
rubyXL rubyXL
selenium-webdriver selenium-webdriver
spring spring
...@@ -191,4 +209,4 @@ DEPENDENCIES ...@@ -191,4 +209,4 @@ DEPENDENCIES
web-console (>= 3.3.0) web-console (>= 3.3.0)
BUNDLED WITH BUNDLED WITH
1.16.2 1.16.3
...@@ -7,7 +7,7 @@ class CourseSectionsController < ApplicationController ...@@ -7,7 +7,7 @@ class CourseSectionsController < ApplicationController
@sections = @sections.where(course_id: params[:course_id]) if params.key?(:course_id) @sections = @sections.where(course_id: params[:course_id]) if params.key?(:course_id)
@sections = @sections.where(crn: params[:crn]) if params.key?(:crn) @sections = @sections.where(crn: params[:crn]) if params.key?(:crn)
render json: @sections render json: @sections
end end
end end
...@@ -3,7 +3,7 @@ require 'time' ...@@ -3,7 +3,7 @@ require 'time'
# Contains functionality for generating schedules. # Contains functionality for generating schedules.
class SchedulesController < ApplicationController class SchedulesController < ApplicationController
# Render an iCal file containing the schedules of all the # Render an iCal file containing the schedules of all the
# course sections with the given CRNs. # course sections with the given CRNs.
def index def index
crns = params["crns"].split ',' crns = params["crns"].split ','
......
...@@ -5,7 +5,7 @@ class Schedule ...@@ -5,7 +5,7 @@ class Schedule
def initialize(crns) def initialize(crns)
@cal = Icalendar::Calendar.new @cal = Icalendar::Calendar.new
@cal.x_wr_calname = 'GMU Fall 2018' @cal.x_wr_calname = 'GMU Fall 2018'
@course_sections = crns.map do |crn| @course_sections = crns.map do |crn|
CourseSection.find_by crn: crn CourseSection.find_by crn: crn
end end
...@@ -23,7 +23,7 @@ class Schedule ...@@ -23,7 +23,7 @@ class Schedule
def load_events def load_events
@course_sections.each do |section| @course_sections.each do |section|
unless section.start_time == "TBA" || section.end_time == "TBA" unless section.start_time == "TBA" || section.end_time == "TBA"
event = generate_event_from_section(section) event = generate_event_from_section(section)
@cal.add_event(event) @cal.add_event(event)
end end
...@@ -89,7 +89,7 @@ class Schedule ...@@ -89,7 +89,7 @@ class Schedule
# @return [Array] # @return [Array]
def exdates_for_section(section) def exdates_for_section(section)
# Generate exdates for all closures in a semester # Generate exdates for all closures in a semester
exdates = Closure.where(semester: section.course.semester).map { |closure| exdates = Closure.where(semester: section.course.semester).map { |closure|
generate_exdate(closure.date.to_formatted_s(:number), section.start_time) generate_exdate(closure.date.to_formatted_s(:number), section.start_time)
} }
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
# A +Semester+ is a simple model that consists of a +year+ and a +season+, e.g. "Fall 2018". # A +Semester+ is a simple model that consists of a +year+ and a +season+, e.g. "Fall 2018".
class Semester < ApplicationRecord class Semester < ApplicationRecord
has_many :courses has_many :courses
# Ensure necessary fields are present. # Ensure necessary fields are present.
validates :year, presence: true validates :year, presence: true
validates :season, presence: true validates :season, presence: true
......
...@@ -19,7 +19,7 @@ module Schedules ...@@ -19,7 +19,7 @@ module Schedules
config.middleware.insert_before 0, Rack::Cors do config.middleware.insert_before 0, Rack::Cors do
allow do allow do
origins '*' origins '*'
resource '*', :headers => :any, :methods => [:get, :post, :options] resource '*', headers: :any, methods: [:get, :post, :options]
end end
end end
end end
......
...@@ -27,7 +27,7 @@ class ExcelLoader ...@@ -27,7 +27,7 @@ class ExcelLoader
end end
private private
# create closures for the days there will be no classes # create closures for the days there will be no classes
# see: https://registrar.gmu.edu/calendars/fall-2018/ # see: https://registrar.gmu.edu/calendars/fall-2018/
def load_closures def load_closures
...@@ -36,7 +36,7 @@ class ExcelLoader ...@@ -36,7 +36,7 @@ class ExcelLoader
(21..25).each { |n| Closure.create! date: Date.new(2018, 11, n), semester: @semester } (21..25).each { |n| Closure.create! date: Date.new(2018, 11, n), semester: @semester }
(10..19).each { |n| Closure.create! date: Date.new(2018, 12, n), semester: @semester } (10..19).each { |n| Closure.create! date: Date.new(2018, 12, n), semester: @semester }
end end
# Prints the failure, deletes all data added during loading, and raises the failure error. # Prints the failure, deletes all data added during loading, and raises the failure error.
def fail(error) def fail(error)
logger.fatal error.message logger.fatal error.message
...@@ -67,42 +67,42 @@ class ExcelLoader ...@@ -67,42 +67,42 @@ class ExcelLoader
# Tries to create a section from a given row. # Tries to create a section from a given row.
def configure_section?(row) def configure_section?(row)
section_name = row.cells[2]&.value section_name = row.cells[2]&.value
# If there is no valid section name, just continue to the next row # If there is no valid section name, just continue to the next row
unless section_name.blank? || section_name == 'Total' return nil if section_name.blank? || section_name == 'Total'
# The time field in the spreadsheet uses the format "start_time - end_time" i.e. "12:00 PM - 1:15 PM".
# So, split the times string by the - character # The time field in the spreadsheet uses the format "start_time - end_time" i.e. "12:00 PM - 1:15 PM".
times = row.cells[23]&.value # So, split the times string by the - character
time_strs = times.split('-') times = row.cells[23]&.value
time_strs = times.split('-')
instructor_val = row.cells[16]
instructor = if instructor_val.nil? || instructor_val.value == "'-" instructor_val = row.cells[16]
"TBA" instructor = if instructor_val.nil? || instructor_val.value == "'-"
else
instructor_val.value
end
location_cell = row.cells[25]
location = if location_cell.nil? || location_cell.value.include?("'-")
"TBA" "TBA"
else else
location_cell.value instructor_val.value
end end
section = CourseSection.create name: section_name, location_cell = row.cells[25]
course: @current_course, location = if location_cell.nil? || location_cell.value.include?("'-")
crn: row.cells[6]&.value, "TBA"
section_type: row.cells[8]&.value, else
title: row.cells[11]&.value, location_cell.value
instructor: instructor, end
start_date: row.cells[18]&.value,
end_date: row.cells[21]&.value, section = CourseSection.create name: section_name,
days: row.cells[22]&.value, course: @current_course,
start_time: time_strs[0].strip, crn: row.cells[6]&.value,
end_time: time_strs[1].strip, section_type: row.cells[8]&.value,
location: location title: row.cells[11]&.value,
instructor: instructor,
section start_date: row.cells[18]&.value,
end end_date: row.cells[21]&.value,
days: row.cells[22]&.value,
start_time: time_strs[0].strip,
end_time: time_strs[1].strip,
location: location
section
end end
end end
# frozen_string_literal: true
require 'thwait'
require 'httparty'
require 'nokogiri'
require 'json'
#
# USAGE:
#
# Just run it and it dynamically dumps the latest semester. There's a bit to do it for all of the ones in history commented out below but it'll thrash your RAM and probably piss off PatriotWeb. Also note this script could be trivially modified to correlate human readable names to semester IDs since they're just the .text attribute of the option node.
#
# There's a few minor issues like multiple spaces in teacher names and we could be scraping out email addresses but no major ones.
#
# DISCLAIMER/WARNING:
#
# This opens a number of connections pretty transparently from a script to PatriotWeb. I am not liable if you run this a million times and somehow kill over PatriotWeb. It's a scraper, not a DoS utility.
#
# Credit stackoverflow
class String
def alpha?
!!match(/^[[:alpha:]]+$/)
end
end
def get_details(data, titledetails, titledata)
crn = titledetails[1].strip
data[crn] = {} unless data[titledetails[1]]
crsinfo = { 'name': titledetails[0].strip }
uniquedata = { 'sect': titledetails[3].strip, 'crn': titledetails[1].strip }
general = { 'subj': titledata[0].strip, 'code': titledata[1].strip }
data[crn] = general.merge(uniquedata.merge(crsinfo))
data[crn][:code] = titledetails[2].split(' ')[1]
[data, data[crn]]
end
def sort_item(item, currentobj, data)
if item.name == 'th'
if item.to_html.include? '-'
titletxt = item.text
if item.text.include? ' - Honors'
titletxt = titletxt.gsub(' - Honors', ' (Honors)')
end
titledetails = titletxt.split(' - ')
if titledetails.count > 4
titledetails = ["#{titledetails[0]} #{titledetails[1]}", titledetails[2], titledetails[3], titledetails[4]]
end
titledata = titledetails[2].split(' ')
begin
data = get_details(data, titledetails, titledata)[0]
currentobj = get_details(data, titledetails, titledata)[1]
rescue StandardError => e
puts item
puts e
exit(1)
end
currentobj[:fields] = []
end
elsif item.is_a? Nokogiri::XML::Element
item.css('th').each do |field|
currentobj[:fields].push(field.text.downcase.tr(' ', '_'))
end
iter = 0
if currentobj
if currentobj[:fields]
upper = currentobj[:fields].count - 1
while iter <= upper
assign = item.css('td')[iter].text
currentobj[currentobj[:fields][iter]] = assign
iter += 1
end
end
end
end
currentobj
end
def feed_course_info(searcher)
table = searcher.css('html body div.pagebodydiv table.datadisplaytable')
data = {}
currentobj = nil
table.css('table.datadisplaytable').first.children.each do |row|
next unless row.name == 'tr'
row.children.each do |item|
currentobj = sort_item(item, currentobj, data)
end
end
data
end
def get_crn(title, code, section)
puts "TODO #{title} #{code} #{section}"
end
def full_major(major)
resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_get_crse_unsec',
body: "term_in=201870&sel_subj=dummy&sel_day=dummy&sel_schd=dummy&sel_insm=dummy&sel_camp=dummy&sel_levl=dummy&sel_sess=dummy&sel_instr=dummy&sel_ptrm=dummy&sel_attr=dummy&sel_subj=#{major}&sel_crse=&sel_title=&sel_schd=%25&sel_from_cred=&sel_to_cred=&sel_camp=%25&sel_levl=%25&sel_ptrm=%25&sel_instr=%25&begin_hh=0&begin_mi=0&begin_ap=x&end_hh=0&end_mi=0&end_ap=x",
headers: {
'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8'
})
searcher = Nokogiri::HTML(resp)
data = feed_course_info(searcher)
end
def initialize_req(subj, num)
base_url = 'https://patriotweb.gmu.edu/pls/prod/bwckctlg.p_disp_listcrse?term_in=201870'
stub = "subj_in=#{subj}&crse_in=#{num}&schd_in=%25"
resp = HTTParty.get("#{base_url}&#{stub}")
searcher = Nokogiri::HTML(resp)
data = feed_course_info(searcher)
end
def getSemesters
semesters = []
resp = HTTParty.get('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_disp_dyn_sched')
searcher = Nokogiri::HTML(resp)
searcher.css('option').each do |opt|
if opt.attr('value').start_with? '20'
semesters.push(opt.attr('value'))
end
end
semesters
end
def getCourses(semester)
semesters = []
resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckgens.p_proc_term_date',
body: "p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=#{semester}&p_by_date=Y&p_from_date=&p_to_date=",
headers: {
'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8'
})
searcher = Nokogiri::HTML(resp)
# puts searcher.inspect
searcher.xpath('//*[@id="subj_id"]/option').each do |opt|
if opt.attr('value').strip.alpha?
semesters.push(opt.attr('value'))
end
end
semesters
end
# end
# total.each { |subject|
# puts subject.first
# subject[1].each { |section|
# puts section
# }
# }
def load_data
# Initialize threads to be waited on array
threads = []
total = {}
# below will get you literally all semesters which is wildly overkill
# getSemesters.each do |semester|
semester = getSemesters.first
getCourses(semester).each do |course|
threads << Thread.new {
total[course] = full_major(course)
}
end
ThreadsWait.all_waits(*threads)
Semester.delete_all
Course.delete_all
Section.delete_all
semester = Semester.create! season: 'Fall', year: '2018'
semester.save!
total.each { |subject|
subject[1].each { |crn|
section = crn[1]
course = Course.find_or_create_by(subject: section[:subj],
course_number: section[:code])
course.semester = semester
course.save!
section_name = "#{section[:subj]} #{section[:code]} #{section[:sect]}"
Section.create!(name: section_name,
crn: section[:crn],
title: section[:name],
course: course)
puts "#{section[:subj]} #{section[:code]} #{section[:sect]} #{section[:name]}"
}
}
end
...@@ -14,10 +14,10 @@ module PatriotWeb ...@@ -14,10 +14,10 @@ module PatriotWeb
def initialize def initialize
@networker = PatriotWeb::Networker.new @networker = PatriotWeb::Networker.new
end end
# Parses all semesters avaliable on Patriot Web # Parses all semesters avaliable on Patriot Web
def parse_semesters def parse_semesters
response = @networker.fetch_page_containing_semester_data response = @networker.fetch_page_containing_semester_data
document = Nokogiri::HTML(response) # parse the document from the HTTP response document = Nokogiri::HTML(response) # parse the document from the HTTP response
get_semesters_from_option_values(document).compact get_semesters_from_option_values(document).compact
...@@ -40,7 +40,7 @@ module PatriotWeb ...@@ -40,7 +40,7 @@ module PatriotWeb
end end
private private
# Parse the values of all different options on the Patriot Web # Parse the values of all different options on the Patriot Web
# semester select page # semester select page
# @param document [Nokogiri::HTML::Document] # @param document [Nokogiri::HTML::Document]
...@@ -63,10 +63,10 @@ module PatriotWeb ...@@ -63,10 +63,10 @@ module PatriotWeb
end end
end end
# Parse all courses from the subject search page # Parse all courses from the subject search page
# @param document [Nokogiri::HTML::Document] # @param document [Nokogiri::HTML::Document]
# @return [Array] courses # @return [Array] courses
def get_courses(document, subject) def get_courses(document, _subject)
table = document.css('html body div.pagebodydiv table.datadisplaytable') table = document.css('html body div.pagebodydiv table.datadisplaytable')
rows = table.css('tr') rows = table.css('tr')
# rows[100..110].each_with_index do |row, i| # rows[100..110].each_with_index do |row, i|
...@@ -78,11 +78,10 @@ module PatriotWeb ...@@ -78,11 +78,10 @@ module PatriotWeb
def data_from(rows) def data_from(rows)
i = 0 i = 0
title_index = 0
result = [] result = []
while i < rows.length while i < rows.length
if is_title(rows[i].text) # check if the row is a title if title?(rows[i].text) # check if the row is a title
data = {} data = {}
title_elements = rows[i].text.split(' - ') title_elements = rows[i].text.split(' - ')
...@@ -94,15 +93,15 @@ module PatriotWeb ...@@ -94,15 +93,15 @@ module PatriotWeb
data[:course_number] = full_name[1] data[:course_number] = full_name[1]
data[:section] = title_elements[3].strip data[:section] = title_elements[3].strip
details = rows[i+2].css('td table tr td') details = rows[i + 2].css('td table tr td')
unless details.length > 0 unless !details.empty?
puts "#{full_name.join(' ')} is fake news" puts "#{full_name.join(' ')} is fake news"
i += 1 i += 1
next next
end end
times = details[1].text.split(' - ') times = details[1].text.split(' - ')
if (times.length == 1) if times.length == 1
data[:start_time] = 'TBA' data[:start_time] = 'TBA'
data[:end_time] = 'TBA' data[:end_time] = 'TBA'
else else
...@@ -112,11 +111,11 @@ module PatriotWeb ...@@ -112,11 +111,11 @@ module PatriotWeb
data[:days] = details[2].text.strip