Commit 8bcc6f48 authored by Zac Wood's avatar Zac Wood
Browse files

Fixed behavior around Columbus Day.

Also switched temporarily back to seeds.rb using Excel instead of Patriot Web
while Patriot Web parser is rip
parent 69f6e0f7
Pipeline #2778 passed with stage
in 2 minutes and 15 seconds
...@@ -21,5 +21,3 @@ ...@@ -21,5 +21,3 @@
/yarn-error.log /yarn-error.log
.byebug_history .byebug_history
*.xlsx
\ No newline at end of file
...@@ -15,6 +15,11 @@ class CalendarGeneratorController < ApplicationController ...@@ -15,6 +15,11 @@ class CalendarGeneratorController < ApplicationController
event = generate_event_from_section(section) event = generate_event_from_section(section)
cal.add_event(event) cal.add_event(event)
end end
if section.days.start_with? "M"
col_day_makeup = generate_event_after_columbus_day(section)
cal.add_event(col_day_makeup)
end
end end
render plain: cal.to_ical # render a plaintext iCal file render plain: cal.to_ical # render a plaintext iCal file
...@@ -90,6 +95,14 @@ class CalendarGeneratorController < ApplicationController ...@@ -90,6 +95,14 @@ class CalendarGeneratorController < ApplicationController
) )
end end
# If the section meets on Tuesdays, add an exdate for the day after columbus day
if section.days.start_with? "T"
exdates << generate_exdate(
Date.new(2018, 10, 9).to_formatted_s(:number),
section.start_time
)
end
exdates exdates
end end
...@@ -102,4 +115,20 @@ class CalendarGeneratorController < ApplicationController ...@@ -102,4 +115,20 @@ class CalendarGeneratorController < ApplicationController
formatted_time = Time.parse(time).strftime("%H%M%S") formatted_time = Time.parse(time).strftime("%H%M%S")
Icalendar::Values::DateTime.new("#{date}T#{formatted_time}") Icalendar::Values::DateTime.new("#{date}T#{formatted_time}")
end end
# Configures a calendar event for the day after columbus day
# @param section [CourseSection]
def generate_event_after_columbus_day(section)
event = Icalendar::Event.new
event.summary = section.name + " (Columbus Day makeup)"
event.description = section.title + " (Columbus Day makeup)"
event.location = section.location
after_columbus_day = Date.new 2018, 10, 9
event.dtstart = Icalendar::Values::DateTime.new(formatted_datetime_str(after_columbus_day, section.start_time))
event.dtend = Icalendar::Values::DateTime.new(formatted_datetime_str(after_columbus_day, section.end_time))
event
end
end end
...@@ -22,25 +22,28 @@ class ExcelLoader ...@@ -22,25 +22,28 @@ class ExcelLoader
end end
configure_section?(row)&.save! # If this row contained a section, save it configure_section?(row)&.save! # If this row contained a section, save it
end end
load_closures
end end
private private
# create closures for the days there will be no classes
# see: https://registrar.gmu.edu/calendars/fall-2018/
def load_closures
Closure.create! date: Date.new(2018, 9, 3), semester: @semester
Closure.create! date: Date.new(2018, 10, 8), semester: @semester
(21..25).each { |n| Closure.create! date: Date.new(2018, 11, n), semester: @semester }
(10..19).each { |n| Closure.create! date: Date.new(2018, 12, n), semester: @semester }
end
# Prints the failure, deletes all data added during loading, and raises the failure error. # Prints the failure, deletes all data added during loading, and raises the failure error.
def fail(error) def fail(error)
logger.fatal error.message logger.fatal error.message
logger.fatal error.backtrace logger.fatal error.backtrace
delete_all_records
raise error raise error
end end
# Deletes all records from the database.
def delete_all_records
Semester.delete_all
Course.delete_all
CourseSection.delete_all
end
# Tries to create a course from a given row. # Tries to create a course from a given row.
def configure_course?(row) def configure_course?(row)
course_name = row.cells[1]&.value course_name = row.cells[1]&.value
...@@ -64,27 +67,42 @@ class ExcelLoader ...@@ -64,27 +67,42 @@ class ExcelLoader
# Tries to create a section from a given row. # Tries to create a section from a given row.
def configure_section?(row) def configure_section?(row)
section_name = row.cells[2]&.value section_name = row.cells[2]&.value
# If there is no valid section name, just continue to the next row # If there is no valid section name, just continue to the next row
unless section_name.blank? || section_name == 'Total' unless section_name.blank? || section_name == 'Total'
# The time field in the spreadsheet uses the format "start_time - end_time" i.e. "12:00 PM - 1:15 PM". # The time field in the spreadsheet uses the format "start_time - end_time" i.e. "12:00 PM - 1:15 PM".
# So, split the times string by the - character # So, split the times string by the - character
times = row.cells[23]&.value times = row.cells[23]&.value
time_strs = times.split('-') time_strs = times.split('-')
instructor_val = row.cells[16]
instructor = if instructor_val.nil? || instructor_val.value == "'-"
"TBA"
else
instructor_val.value
end
location_cell = row.cells[25]
location = if location_cell.nil? || location_cell.value.include?("'-")
"TBA"
else
location_cell.value
end
section = CourseSection.create name: section_name, section = CourseSection.create name: section_name,
course: @current_course, course: @current_course,
crn: row.cells[6]&.value, crn: row.cells[6]&.value,
section_type: row.cells[8]&.value, section_type: row.cells[8]&.value,
title: row.cells[11]&.value, title: row.cells[11]&.value,
instructor: row.cells[16]&.value, instructor: instructor,
start_date: row.cells[18]&.value, start_date: row.cells[18]&.value,
end_date: row.cells[21]&.value, end_date: row.cells[21]&.value,
days: row.cells[22]&.value, days: row.cells[22]&.value,
start_time: time_strs[0].strip, start_time: time_strs[0].strip,
end_time: time_strs[1].strip, end_time: time_strs[1].strip,
location: row.cells[25]&.value location: location
# TODO: Add campus, notes, and size limit fields section
end end
section
end end
end end
...@@ -68,52 +68,64 @@ module PatriotWeb ...@@ -68,52 +68,64 @@ module PatriotWeb
# @return [Array] courses # @return [Array] courses
def get_courses(document) def get_courses(document)
table = document.css('html body div.pagebodydiv table.datadisplaytable').first table = document.css('html body div.pagebodydiv table.datadisplaytable').first
File.write('help', table)
rows = table.children.drop 2 # first two elements are junk rows = table.children.drop 2 # first two elements are junk
# puts rows[0].text
# puts rows[2].css('table.datadisplaytable td').children.drop(1).map { |c| c.text }
# puts rows[3]
# puts rows[4]
(0..(rows.length/4 - 1)).map do |i|
start = i*4
puts rows[start].text
puts rows[start+2].css('table.datadisplaytable td').children.drop(1).map { |c| c.text }
end
# each section is represented by 6 rows in the table # each section is represented by 6 rows in the table
(0..(rows.length/6 - 1)).map do |i| # (0..(rows.length/6 - 1)).map do |i|
start = i*6 # start = i*6
data = {} # data = {}
title = rows[start].text # title = rows[start].text
# the title looks this: Survey of Accounting - 71117 - ACCT 203 - 001 # # the title looks this: Survey of Accounting - 71117 - ACCT 203 - 001
# so split it by ' - ' and extract # # so split it by ' - ' and extract
title_elements = title.split(' - ') # title_elements = title.split(' - ')
data[:title] = title_elements[0].strip # next unless title_elements.length == 4
data[:crn] = title_elements[1] # data[:title] = title_elements[0].strip
# data[:crn] = title_elements[1]
full_name = title_elements[2].split(' ') # full_name = title_elements[2].split(' ')
next unless full_name.length == 2 # next unless full_name.length == 2
data[:subj] = title_elements[2].split(' ')[0] # data[:subj] = title_elements[2].split(' ')[0]
data[:course_number] = title_elements[2].split(' ')[1] # data[:course_number] = title_elements[2].split(' ')[1]
data[:section] = title_elements[3].strip # data[:section] = title_elements[3].strip
# rows 1 to 3 contain info about registration and drop dates. # # rows 1 to 3 contain info about registration and drop dates.
# for now we're gonna ignore them and skip to row 4, which contains details # # for now we're gonna ignore them and skip to row 4, which contains details
detail_rows = rows[start+4].css('tr') # detail_rows = rows[start+4].css('tr')
next unless detail_rows.length > 0 # if there are no details, skip this item # next unless detail_rows.length > 0 # if there are no details, skip this item
details = detail_rows.last.text.split("\n").compact.reject(&:empty?) # skip empty strings # details = detail_rows.last.text.split("\n").compact.reject(&:empty?) # skip empty strings
times = details[1].split(' - ') # times = details[1].split(' - ')
if (times.length == 1) # if (times.length == 1)
data[:start_time] = 'TBA' # data[:start_time] = 'TBA'
data[:end_time] = 'TBA' # data[:end_time] = 'TBA'
else # else
data[:start_time] = times[0] # data[:start_time] = times[0]
data[:end_time] = times[1] # data[:end_time] = times[1]
end # end
data[:days] = details[2].strip # data[:days] = details[2].strip
data[:location] = details[3].strip # data[:location] = details[3].strip
dates = details[4].split(' - ') # dates = details[4].split(' - ')
data[:start_date] = dates[0] # data[:start_date] = dates[0]
data[:end_date] = dates[1] # data[:end_date] = dates[1]
data[:type] = details[5] # data[:type] = details[5]
data[:instructor] = details[6] # data[:instructor] = details[6]
data # data
end # end
end end
end end
end end
# This file should contain all the record creation needed to seed the database with its default values.
# The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup).
require_relative 'patriot_web_parser'
require 'thwait'
require 'httparty'
require 'nokogiri'
require 'json'
threads = []
total = {}
parser = PatriotWeb::Parser.new
# get the first semester only
semester = parser.parse_semesters.first
puts "DDOSing Patriot Web, buckle up kids"
# # parse all subjects and their courses in the semester
# parser.parse_subjects(semester).each do |subject|
# puts "Getting courses for #{subject}"
# threads << Thread.new {
# total[subject] = parser.parse_courses_in_subject(subject)
# }
# end
# For testing, only get first subject
subject = parser.parse_subjects(semester).first
total[subject] = parser.parse_courses_in_subject(subject)
# wait for all the threads to finish
# ThreadsWait.all_waits(*threads)
# delete everything in the current database
Closure.delete_all
CourseSection.delete_all
Course.delete_all
Semester.delete_all
# create a semester for the next semester
semester = Semester.create! season: 'Fall', year: 2018
semester.save!
total.each do |subject, sections|
puts "Adding courses for #{subject}..."
sections.each do |section|
next if section.nil? || !section.key?(:subj) || !section.key?(:course_number)
# Find or create a course and set its semester
# TODO: this breaks when you try to do more than one semester,
# since just the subject + course_number do not uniquely identify a course
# Check the semester as well
course = Course.find_or_create_by(subject: section[:subj],
course_number: section[:course_number])
course.semester = semester
course.save!
section_name = "#{section[:subj]} #{section[:course_number]} #{section[:section]}"
puts "Adding #{section_name}..."
CourseSection.create!(name: section_name,
crn: section[:crn],
section_type: section[:type],
title: section[:title],
instructor: section[:instructor],
start_date: section[:start_date],
end_date: section[:end_date],
days: section[:days],
start_time: section[:start_time],
end_time: section[:end_time],
location: section[:location],
course: course)
end
end
# create closures for the days there will be no classes
# see: https://registrar.gmu.edu/calendars/fall-2018/
Closure.create! date: Date.new(2018, 9, 3), semester: semester
Closure.create! date: Date.new(2018, 10, 8), semester: semester
(21..25).each { |n| Closure.create! date: Date.new(2018, 11, n), semester: semester }
(10..19).each { |n| Closure.create! date: Date.new(2018, 12, n), semester: semester }
# This file should contain all the record creation needed to seed the database with its default values. require_relative 'excel_loader'
# The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup).
require_relative 'patriot_web_parser' # Deletes all records from the database.
require 'thwait'
require 'httparty'
require 'nokogiri'
require 'json'
threads = []
total = {}
parser = PatriotWeb::Parser.new
# get the first semester only
semester = parser.parse_semesters.first
puts "DDOSing Patriot Web, buckle up kids"
# parse all subjects and their courses in the semester
parser.parse_subjects(semester).each do |subject|
puts "Getting courses for #{subject}"
threads << Thread.new {
total[subject] = parser.parse_courses_in_subject(subject)
}
end
# For testing, only get first subject
# subject = parser.parse_subjects(semester).first
# total[subject] = parser.parse_courses_in_subject(subject)
# wait for all the threads to finish
ThreadsWait.all_waits(*threads)
# delete everything in the current database
Closure.delete_all Closure.delete_all
CourseSection.delete_all CourseSection.delete_all
Course.delete_all Course.delete_all
Semester.delete_all Semester.delete_all
loader = ExcelLoader.new 'db/data/fall2018.xlsx'
loader.load_data
# create a semester for the next semester
semester = Semester.create! season: 'Fall', year: 2018
semester.save!
total.each do |subject, sections|
puts "Adding courses for #{subject}..."
sections.each do |section|
next if section.nil? || !section.key?(:subj) || !section.key?(:course_number)
# Find or create a course and set its semester
# TODO: this breaks when you try to do more than one semester,
# since just the subject + course_number do not uniquely identify a course
# Check the semester as well
course = Course.find_or_create_by(subject: section[:subj],
course_number: section[:course_number])
course.semester = semester
course.save!
section_name = "#{section[:subj]} #{section[:course_number]} #{section[:section]}"
puts "Adding #{section_name}..."
CourseSection.create!(name: section_name,
crn: section[:crn],
section_type: section[:type],
title: section[:title],
instructor: section[:instructor],
start_date: section[:start_date],
end_date: section[:end_date],
days: section[:days],
start_time: section[:start_time],
end_time: section[:end_time],
location: section[:location],
course: course)
end
end
# create closures for the days there will be no classes
# see: https://registrar.gmu.edu/calendars/fall-2018/
Closure.create! date: Date.new(2018, 9, 3), semester: semester
Closure.create! date: Date.new(2018, 10, 8), semester: semester
(21..25).each { |n| Closure.create! date: Date.new(2018, 11, n), semester: semester }
(10..19).each { |n| Closure.create! date: Date.new(2018, 12, n), semester: semester }
...@@ -13,6 +13,13 @@ EXDATE:20180530T120000 ...@@ -13,6 +13,13 @@ EXDATE:20180530T120000
EXDATE:20180531T120000 EXDATE:20180531T120000
END:VEVENT END:VEVENT
BEGIN:VEVENT BEGIN:VEVENT
DTSTART:20181009T120000
DTEND:20181009T130000
DESCRIPTION:MyString (Columbus Day makeup)
LOCATION:MyString
SUMMARY:MyString (Columbus Day makeup)
END:VEVENT
BEGIN:VEVENT
DTSTART:20180521T110000 DTSTART:20180521T110000
DTEND:20180521T140000 DTEND:20180521T140000
DESCRIPTION:MyString DESCRIPTION:MyString
...@@ -22,5 +29,6 @@ RRULE:FREQ=WEEKLY;UNTIL=20180604T140000;BYDAY=TU,TH ...@@ -22,5 +29,6 @@ RRULE:FREQ=WEEKLY;UNTIL=20180604T140000;BYDAY=TU,TH
EXDATE:20180530T110000 EXDATE:20180530T110000
EXDATE:20180531T110000 EXDATE:20180531T110000
EXDATE:20180521T110000 EXDATE:20180521T110000
EXDATE:20181009T110000
END:VEVENT END:VEVENT
END:VCALENDAR END:VCALENDAR
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment