Commit 3cb72415 authored by Zac Wood's avatar Zac Wood

TAKE THAT PATRIOT WEB

parent 8bcc6f48
......@@ -67,20 +67,74 @@ module PatriotWeb
# @param document [Nokogiri::HTML::Document]
# @return [Array] courses
def get_courses(document)
table = document.css('html body div.pagebodydiv table.datadisplaytable').first
File.write('help', table)
rows = table.children.drop 2 # first two elements are junk
# puts rows[0].text
# puts rows[2].css('table.datadisplaytable td').children.drop(1).map { |c| c.text }
# puts rows[3]
# puts rows[4]
table = document.css('html body div.pagebodydiv table.datadisplaytable')
rows = table.css('tr')
(0..(rows.length/6-1)).map do |i|
start = i*5
data = {}
title = rows[start].text
# the title looks this: Survey of Accounting - 71117 - ACCT 203 - 001
# so split it by ' - ' and extract
title_elements = title.split(' - ')
next unless title_elements.length == 4
data[:title] = title_elements[0].strip
data[:crn] = title_elements[1]
full_name = title_elements[2].split(' ')
next unless full_name.length == 2
data[:subj] = title_elements[2].split(' ')[0]
data[:course_number] = title_elements[2].split(' ')[1]
data[:section] = title_elements[3].strip
# rows 1 to 3 contain info about registration and drop dates.
# for now we're gonna ignore them and skip to row 4, which contains details
details = rows[start+2].css('td table tr td')
next unless details.length > 0 # if there are no details, skip this item
# details = detail_rows.last.text.split("\n").compact.reject(&:empty?) # skip empty strings
times = details[1].text.split(' - ')
if (times.length == 1)
data[:start_time] = 'TBA'
data[:end_time] = 'TBA'
else
data[:start_time] = times[0]
data[:end_time] = times[1]
end
(0..(rows.length/4 - 1)).map do |i|
start = i*4
puts rows[start].text
puts rows[start+2].css('table.datadisplaytable td').children.drop(1).map { |c| c.text }
data[:days] = details[2].text.strip
data[:location] = details[3].text.strip
dates = details[4].text.split(' - ')
data[:start_date] = dates[0]
data[:end_date] = dates[1]
data[:type] = details[5].text
data[:instructor] = details[6].text
data
end
# puts rows[0].text
# puts rows[2].css('td table tr td')
# puts rows[5].text
# puts rows[7].css('td table tr td')
# puts rows[10].text
# puts rows[12].css('td table tr td')
# (0..(rows.length/3-1)).each do |i|
# start = i*3
# puts rows[start].search('th').first.text
# section_data = rows[start+2].css('td table.datadisplaytable').search('td')
# puts section_data[1].text
# end
# puts rows[3]
# puts rows[3].search('th').first.text
# section_data = rows[5].css('td table.datadisplaytable').search('td')
# puts section_data[1].text
# end
# each section is represented by 6 rows in the table
# (0..(rows.length/6 - 1)).map do |i|
# start = i*6
......
# This file should contain all the record creation needed to seed the database with its default values.
# The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup).
require_relative 'patriot_web_parser'
require 'thwait'
require 'httparty'
require 'nokogiri'
require 'json'
threads = []
total = {}
parser = PatriotWeb::Parser.new
# get the first semester only
semester = parser.parse_semesters.first
puts "DDOSing Patriot Web, buckle up kids"
# # parse all subjects and their courses in the semester
# parser.parse_subjects(semester).each do |subject|
# puts "Getting courses for #{subject}"
# threads << Thread.new {
# total[subject] = parser.parse_courses_in_subject(subject)
# }
# end
# For testing, only get first subject
subject = parser.parse_subjects(semester).first
total[subject] = parser.parse_courses_in_subject(subject)
# wait for all the threads to finish
# ThreadsWait.all_waits(*threads)
# delete everything in the current database
Closure.delete_all
CourseSection.delete_all
Course.delete_all
Semester.delete_all
# create a semester for the next semester
semester = Semester.create! season: 'Fall', year: 2018
semester.save!
total.each do |subject, sections|
puts "Adding courses for #{subject}..."
sections.each do |section|
next if section.nil? || !section.key?(:subj) || !section.key?(:course_number)
# Find or create a course and set its semester
# TODO: this breaks when you try to do more than one semester,
# since just the subject + course_number do not uniquely identify a course
# Check the semester as well
course = Course.find_or_create_by(subject: section[:subj],
course_number: section[:course_number])
course.semester = semester
course.save!
section_name = "#{section[:subj]} #{section[:course_number]} #{section[:section]}"
puts "Adding #{section_name}..."
CourseSection.create!(name: section_name,
crn: section[:crn],
section_type: section[:type],
title: section[:title],
instructor: section[:instructor],
start_date: section[:start_date],
end_date: section[:end_date],
days: section[:days],
start_time: section[:start_time],
end_time: section[:end_time],
location: section[:location],
course: course)
end
end
# create closures for the days there will be no classes
# see: https://registrar.gmu.edu/calendars/fall-2018/
Closure.create! date: Date.new(2018, 9, 3), semester: semester
Closure.create! date: Date.new(2018, 10, 8), semester: semester
(21..25).each { |n| Closure.create! date: Date.new(2018, 11, n), semester: semester }
(10..19).each { |n| Closure.create! date: Date.new(2018, 12, n), semester: semester }
require_relative 'excel_loader'
# Deletes all records from the database.
Closure.delete_all
CourseSection.delete_all
Course.delete_all
Semester.delete_all
loader = ExcelLoader.new 'db/data/fall2018.xlsx'
loader.load_data
require_relative 'excel_loader'
# This file should contain all the record creation needed to seed the database with its default values.
# The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup).
# Deletes all records from the database.
require_relative 'patriot_web_parser'
require 'thwait'
require 'httparty'
require 'nokogiri'
require 'json'
threads = []
total = {}
parser = PatriotWeb::Parser.new
# get the first semester only
semester = parser.parse_semesters.first
puts "DDOSing Patriot Web, buckle up kids"
# parse all subjects and their courses in the semester
parser.parse_subjects(semester).each do |subject|
puts "Getting courses for #{subject}"
threads << Thread.new {
total[subject] = parser.parse_courses_in_subject(subject)
}
end
# For testing, only get first subject
# subject = parser.parse_subjects(semester).first
# total[subject] = parser.parse_courses_in_subject(subject)
# wait for all the threads to finish
ThreadsWait.all_waits(*threads)
# delete everything in the current database
Closure.delete_all
CourseSection.delete_all
Course.delete_all
Semester.delete_all
loader = ExcelLoader.new 'db/data/fall2018.xlsx'
loader.load_data
# create a semester for the next semester
semester = Semester.create! season: 'Fall', year: 2018
semester.save!
total.each do |subject, sections|
puts "Adding courses for #{subject}..."
sections.each do |section|
next if section.nil? || !section.key?(:subj) || !section.key?(:course_number)
# Find or create a course and set its semester
# TODO: this breaks when you try to do more than one semester,
# since just the subject + course_number do not uniquely identify a course
# Check the semester as well
course = Course.find_or_create_by(subject: section[:subj],
course_number: section[:course_number])
course.semester = semester
course.save!
section_name = "#{section[:subj]} #{section[:course_number]} #{section[:section]}"
# puts "Adding #{section_name}..."
CourseSection.create!(name: section_name,
crn: section[:crn],
section_type: section[:type],
title: section[:title],
instructor: section[:instructor],
start_date: section[:start_date],
end_date: section[:end_date],
days: section[:days],
start_time: section[:start_time],
end_time: section[:end_time],
location: section[:location],
course: course)
end
end
# create closures for the days there will be no classes
# see: https://registrar.gmu.edu/calendars/fall-2018/
Closure.create! date: Date.new(2018, 9, 3), semester: semester
Closure.create! date: Date.new(2018, 10, 8), semester: semester
(21..25).each { |n| Closure.create! date: Date.new(2018, 11, n), semester: semester }
(10..19).each { |n| Closure.create! date: Date.new(2018, 12, n), semester: semester }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment