Commit 459ba299 authored by Zac Wood's avatar Zac Wood

Merge branch '4-ical-gen' into 'master'

Resolve "Add iCal generation"

Closes #4

See merge request !7
parents f0e4a549 e26728c2
.vscode
\ No newline at end of file
......@@ -21,3 +21,5 @@
/yarn-error.log
.byebug_history
*.xlsx
\ No newline at end of file
......@@ -35,6 +35,8 @@ group :development, :test do
gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
# Adds support for Capybara system testing and selenium driver
gem 'capybara', '~> 2.13'
gem 'pry'
gem 'pry-doc'
gem 'selenium-webdriver'
end
......@@ -50,4 +52,8 @@ end
# Windows does not include zoneinfo files, so bundle the tzinfo-data gem
gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
gem 'httparty'
gem 'nokogiri'
gem 'rubyXL'
gem 'icalendar'
......@@ -53,6 +53,7 @@ GEM
xpath (>= 2.0, < 4.0)
childprocess (0.9.0)
ffi (~> 1.0, >= 1.0.11)
coderay (1.1.2)
concurrent-ruby (1.0.5)
crass (1.0.3)
erubi (1.7.1)
......@@ -60,8 +61,11 @@ GEM
ffi (1.9.23)
globalid (0.4.1)
activesupport (>= 4.2.0)
httparty (0.16.2)
multi_xml (>= 0.5.2)
i18n (1.0.0)
concurrent-ruby (~> 1.0)
icalendar (2.4.1)
jbuilder (2.7.0)
activesupport (>= 4.2.0)
multi_json (>= 1.2)
......@@ -79,9 +83,16 @@ GEM
mini_portile2 (2.3.0)
minitest (5.11.3)
multi_json (1.13.1)
multi_xml (0.6.0)
nio4r (2.3.0)
nokogiri (1.8.2)
mini_portile2 (~> 2.3.0)
pry (0.11.3)
coderay (~> 1.1.0)
method_source (~> 0.9.0)
pry-doc (0.13.4)
pry (~> 0.11)
yard (~> 0.9.11)
public_suffix (3.0.2)
puma (3.11.3)
rack (2.0.4)
......@@ -166,6 +177,7 @@ GEM
websocket-extensions (0.1.3)
xpath (3.0.0)
nokogiri (~> 1.8)
yard (0.9.12)
PLATFORMS
ruby
......@@ -173,8 +185,13 @@ PLATFORMS
DEPENDENCIES
byebug
capybara (~> 2.13)
httparty
icalendar
jbuilder (~> 2.5)
listen (>= 3.0.5, < 3.2)
nokogiri
pry
pry-doc
puma (~> 3.7)
rails (~> 5.1.6)
rubyXL
......
// Place all the behaviors and hooks related to the matching controller here.
// All this logic will automatically be available in application.js.
// Place all the styles related to the CalendarGenerator controller here.
// They will automatically be included in application.css.
// You can use Sass (SCSS) here: http://sass-lang.com/
# Configures the application.
class ApplicationController < ActionController::Base
protect_from_forgery with: :exception
protect_from_forgery with: :null_session
end
require 'icalendar'
require 'time'
class CalendarGeneratorController < ApplicationController
def generate
cal = Icalendar::Calendar.new
posted_crns = JSON.parse(request.body.read)
posted_crns.each do |crn|
section = Section.find_by_crn(crn)
event = generate_event_from_section(section)
cal.add_event(event)
end
render plain: cal.to_ical
end
private
def generate_event_from_section(section)
event = Icalendar::Event.new
event.summary = section.name
event.description = section.title
event.location = section.location
event.dtstart = Icalendar::Values::DateTime.new(formatted_datetime_str(section.start_date, section.start_time))
event.dtend = Icalendar::Values::DateTime.new(formatted_datetime_str(section.start_date, section.end_time))
event.rrule = Icalendar::Values::Recur.new(recurrence_rule_str(section))
event.exdate = exdates_for_section(section)
event
end
def formatted_datetime_str(date, time)
formatted_date = date.to_s.tr('-', '')
formatted_time = Time.parse(time).strftime("%H%M%S")
"#{formatted_date}T#{formatted_time}"
end
DAYS = {
"M" => "MO",
"T" => "TU",
"W" => "WE",
"R" => "TH",
"F" => "FR",
"S" => "SA",
"U" => "SU"
}.freeze
def recurrence_rule_str(section)
days = section.days.split("").map do |day|
DAYS[day]
end
"FREQ=WEEKLY;UNTIL=#{formatted_datetime_str(section.end_date, section.end_time)};BYDAY=#{days.join(',')}"
end
def exdates_for_section(section)
exdates = Closure.where(semester: section.course.semester).map { |closure|
generate_exdate(closure.date.to_formatted_s(:number), section.start_time)
}
# Every section's start_date is the first Monday of the semester.
# So we need to add an exclusion for that day unless the class is held on Mondays
unless section.days.start_with? "M"
exdates << generate_exdate(
section.start_date.to_formatted_s(:number),
section.start_time
)
end
exdates
end
def generate_exdate(date, time)
formatted_time = Time.parse(time).strftime("%H%M%S")
Icalendar::Values::DateTime.new("#{date}T#{formatted_time}")
end
end
class Closure < ApplicationRecord
belongs_to :semester
validates :date, presence: true
end
......@@ -11,8 +11,8 @@ class Section < ApplicationRecord
# Unsure if necessary
# validates :section_type, presence: true
validates :title, presence: true
validates :start_date, presence: true
validates :end_date, presence: true
validates :days, presence: true
# validates :start_date, presence: true
# validates :end_date, presence: true
# validates :days, presence: true
validates :course_id, presence: true
end
<h1>CalendarGenerator#generate</h1>
<p>Find me in app/views/calendar_generator/generate.html.erb</p>
<h1>Schedules</h1>
<input type="text" id="search" title="search_text" placeholder="Enter CRN..."/>
<button title="search" onclick="search()">Search</button>
<!-- <h2>Search results</h2>
<table id="searchTable">
<tr>
<th>Course</th>
<th>Section Name</th>
<th>CRN</th>
<th>Professor</th>
<th>Location</th>
<th>Days</th>
<th>Times</th>
</tr>
</table>
-->
<!-- <br><br> -->
<input type="text" title="search_text" placeholder="Enter CRN..."/>
<button title="search">Search</button>
<h2>Your classes</h2>
<table id="scheduleTable">
<table>
<tr>
<th>Course</th>
<th>Section Name</th>
......
......@@ -6,6 +6,7 @@ Rails.application.routes.draw do
end
get 'search', controller: 'search', action: 'index'
post 'generate', controller: 'calendar_generator', action: 'generate'
end
root 'courses#index' # Set the root to be the courses API endpoint
......
class CreateClosures < ActiveRecord::Migration[5.1]
def change
create_table :closures do |t|
t.date :date
t.references :semester, foreign_key: true
t.timestamps
end
end
end
# frozen_string_literal: true
require 'thwait'
require 'httparty'
require 'nokogiri'
require 'json'
#
# USAGE:
#
# Just run it and it dynamically dumps the latest semester. There's a bit to do it for all of the ones in history commented out below but it'll thrash your RAM and probably piss off PatriotWeb. Also note this script could be trivially modified to correlate human readable names to semester IDs since they're just the .text attribute of the option node.
#
# There's a few minor issues like multiple spaces in teacher names and we could be scraping out email addresses but no major ones.
#
# DISCLAIMER/WARNING:
#
# This opens a number of connections pretty transparently from a script to PatriotWeb. I am not liable if you run this a million times and somehow kill over PatriotWeb. It's a scraper, not a DoS utility.
#
# Credit stackoverflow
class String
def alpha?
!!match(/^[[:alpha:]]+$/)
end
end
def get_details(data, titledetails, titledata)
crn = titledetails[1].strip
data[crn] = {} unless data[titledetails[1]]
crsinfo = { 'name': titledetails[0].strip }
uniquedata = { 'sect': titledetails[3].strip, 'crn': titledetails[1].strip }
general = { 'subj': titledata[0].strip, 'code': titledata[1].strip }
data[crn] = general.merge(uniquedata.merge(crsinfo))
data[crn][:code] = titledetails[2].split(' ')[1]
[data, data[crn]]
end
def sort_item(item, currentobj, data)
if item.name == 'th'
if item.to_html.include? '-'
titletxt = item.text
if item.text.include? ' - Honors'
titletxt = titletxt.gsub(' - Honors', ' (Honors)')
end
titledetails = titletxt.split(' - ')
if titledetails.count > 4
titledetails = ["#{titledetails[0]} #{titledetails[1]}", titledetails[2], titledetails[3], titledetails[4]]
end
titledata = titledetails[2].split(' ')
begin
data = get_details(data, titledetails, titledata)[0]
currentobj = get_details(data, titledetails, titledata)[1]
rescue StandardError => e
puts item
puts e
exit(1)
end
currentobj[:fields] = []
end
elsif item.is_a? Nokogiri::XML::Element
item.css('th').each do |field|
currentobj[:fields].push(field.text.downcase.tr(' ', '_'))
end
iter = 0
if currentobj
if currentobj[:fields]
upper = currentobj[:fields].count - 1
while iter <= upper
assign = item.css('td')[iter].text
currentobj[currentobj[:fields][iter]] = assign
iter += 1
end
end
end
end
currentobj
end
def feed_course_info(searcher)
table = searcher.css('html body div.pagebodydiv table.datadisplaytable')
data = {}
currentobj = nil
table.css('table.datadisplaytable').first.children.each do |row|
next unless row.name == 'tr'
row.children.each do |item|
currentobj = sort_item(item, currentobj, data)
end
end
data
end
def get_crn(title, code, section)
puts "TODO #{title} #{code} #{section}"
end
def full_major(major)
resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_get_crse_unsec',
body: "term_in=201870&sel_subj=dummy&sel_day=dummy&sel_schd=dummy&sel_insm=dummy&sel_camp=dummy&sel_levl=dummy&sel_sess=dummy&sel_instr=dummy&sel_ptrm=dummy&sel_attr=dummy&sel_subj=#{major}&sel_crse=&sel_title=&sel_schd=%25&sel_from_cred=&sel_to_cred=&sel_camp=%25&sel_levl=%25&sel_ptrm=%25&sel_instr=%25&begin_hh=0&begin_mi=0&begin_ap=x&end_hh=0&end_mi=0&end_ap=x",
headers: {
'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8'
})
searcher = Nokogiri::HTML(resp)
data = feed_course_info(searcher)
end
def initialize_req(subj, num)
base_url = 'https://patriotweb.gmu.edu/pls/prod/bwckctlg.p_disp_listcrse?term_in=201870'
stub = "subj_in=#{subj}&crse_in=#{num}&schd_in=%25"
resp = HTTParty.get("#{base_url}&#{stub}")
searcher = Nokogiri::HTML(resp)
data = feed_course_info(searcher)
end
def getSemesters
semesters = []
resp = HTTParty.get('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_disp_dyn_sched')
searcher = Nokogiri::HTML(resp)
searcher.css('option').each do |opt|
if opt.attr('value').start_with? '20'
semesters.push(opt.attr('value'))
end
end
semesters
end
def getCourses(semester)
semesters = []
resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckgens.p_proc_term_date',
body: "p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=#{semester}&p_by_date=Y&p_from_date=&p_to_date=",
headers: {
'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8'
})
searcher = Nokogiri::HTML(resp)
# puts searcher.inspect
searcher.xpath('//*[@id="subj_id"]/option').each do |opt|
if opt.attr('value').strip.alpha?
semesters.push(opt.attr('value'))
end
end
semesters
end
# end
# total.each { |subject|
# puts subject.first
# subject[1].each { |section|
# puts section
# }
# }
def load_data
# Initialize threads to be waited on array
threads = []
total = {}
# below will get you literally all semesters which is wildly overkill
# getSemesters.each do |semester|
semester = getSemesters.first
getCourses(semester).each do |course|
threads << Thread.new {
total[course] = full_major(course)
}
end
ThreadsWait.all_waits(*threads)
Semester.delete_all
Course.delete_all
Section.delete_all
semester = Semester.create! season: 'Fall', year: '2018'
semester.save!
total.each { |subject|
subject[1].each { |crn|
section = crn[1]
course = Course.find_or_create_by(subject: section[:subj],
course_number: section[:code])
course.semester = semester
course.save!
section_name = "#{section[:subj]} #{section[:code]} #{section[:sect]}"
Section.create!(name: section_name,
crn: section[:crn],
title: section[:name],
course: course)
puts "#{section[:subj]} #{section[:code]} #{section[:sect]} #{section[:name]}"
}
}
end
require 'httparty'
module PatriotWeb
class Networker
def fetch_page_containing_semester_data
HTTParty.get('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_disp_dyn_sched')
end
def fetch_subjects(semester_id)
HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckgens.p_proc_term_date',
body: "p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=#{semester_id}&p_by_date=Y&p_from_date=&p_to_date=",
headers: {
'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8'
})
end
def fetch_courses_in_subject(subject)
HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_get_crse_unsec',
body: "term_in=201870&sel_subj=dummy&sel_day=dummy&sel_schd=dummy&sel_insm=dummy&sel_camp=dummy&sel_levl=dummy&sel_sess=dummy&sel_instr=dummy&sel_ptrm=dummy&sel_attr=dummy&sel_subj=#{subject}&sel_crse=&sel_title=&sel_schd=%25&sel_from_cred=&sel_to_cred=&sel_camp=%25&sel_levl=%25&sel_ptrm=%25&sel_instr=%25&begin_hh=0&begin_mi=0&begin_ap=x&end_hh=0&end_mi=0&end_ap=x",
headers: {
'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8'
})
end
end
end
require_relative 'patriot_web_networker'
require 'nokogiri'
class String
def alpha?
!!match(/^[[:alpha:]]+$/)
end
end
module PatriotWeb
class Parser
def initialize
@networker = PatriotWeb::Networker.new
end
def parse_semesters
response = @networker.fetch_page_containing_semester_data
searcher = Nokogiri::HTML(response)
get_semesters_from_option_values(searcher).compact
end
def parse_subjects(semester_id)
response = @networker.fetch_subjects(semester_id)
searcher = Nokogiri::HTML(response)
get_alpha_option_values(searcher)
end
def parse_courses_in_subject(subject)
resp = @networker.fetch_courses_in_subject(subject)
searcher = Nokogiri::HTML(resp)
feed_course_info(searcher)
end
private
def get_alpha_option_values(searcher)
searcher.xpath('//*[@id="subj_id"]/option').map do |opt|
if opt.attr('value').strip.alpha?
opt.attr('value')
end
end
end
def get_semesters_from_option_values(searcher)
searcher.css('option').map do |opt|
if opt.attr('value').start_with? '20'
opt.attr('value')
end
end
end
def feed_course_info(searcher)
table = searcher.css('html body div.pagebodydiv table.datadisplaytable')
data = {}
currentobj = nil
table.css('table.datadisplaytable').first.children.each do |row|
next unless row.name == 'tr'
row.children.each do |item|
currentobj = sort_item(item, currentobj, data)
end
end
data
end
def sort_item(item, currentobj, data)
if item.name == 'th'
if item.to_html.include? '-'
titletxt = item.text
if item.text.include? ' - Honors'
titletxt = titletxt.gsub(' - Honors', ' (Honors)')
end
titledetails = titletxt.split(' - ')
if titledetails.count > 4
titledetails = ["#{titledetails[0]} #{titledetails[1]}", titledetails[2], titledetails[3], titledetails[4]]
end
titledata = titledetails[2].split(' ')
begin
data = get_details(data, titledetails, titledata)[0]
currentobj = get_details(data, titledetails, titledata)[1]
rescue StandardError => e
puts item
puts e
exit(1)
end
currentobj[:fields] = []
end
elsif item.is_a? Nokogiri::XML::Element
item.css('th').each do |field|
currentobj[:fields].push(field.text.downcase.tr(' ', '_'))
end
iter = 0
if currentobj
if currentobj[:fields]
upper = currentobj[:fields].count - 1
while iter <= upper
assign = item.css('td')[iter].text
currentobj[currentobj[:fields][iter]] = assign
iter += 1
end
end
end
end
currentobj
end
def get_details(data, titledetails, titledata)
crn = titledetails[1].strip
data[crn] = {} unless data[titledetails[1]]
crsinfo = { 'name': titledetails[0].strip }
uniquedata = { 'sect': titledetails[3].strip, 'crn': titledetails[1].strip }
general = { 'subj': titledata[0].strip, 'code': titledata[1].strip }
data[crn] = general.merge(uniquedata.merge(crsinfo))
data[crn][:code] = titledetails[2].split(' ')[1]
[data, data[crn]]
end
end
end
......@@ -10,7 +10,15 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20180407190750) do
ActiveRecord::Schema.define(version: 20180505195736) do
create_table "closures", force: :cascade do |t|
t.date "date"
t.integer "semester_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["semester_id"], name: "index_closures_on_semester_id"
end
create_table "courses", force: :cascade do |t|
t.string "subject"
......
......@@ -5,13 +5,78 @@
#
# movies = Movie.create([{ name: 'Star Wars' }, { name: 'Lord of the Rings' }])
# Character.create(name: 'Luke', movie: movies.first)
require 'rubyXL'
require_relative 'excel_loader'
loader = if Rails.env.test?
ExcelLoader.new 'db/data/testdata.xlsx'
else
ExcelLoader.new 'db/data/allsections.xlsx'
end
require_relative 'patriot_web_parser'
require 'thwait'
require 'httparty'
require 'nokogiri'
require 'json'
loader.load_data
threads = []
total = []
parser = PatriotWeb::Parser.new
semester = parser.parse_semesters.first
parser.parse_subjects(semester).each do |subject|
threads << Thread.new {
total << parser.parse_courses_in_subject(subject)
}
end
# For testing, only get first subject
# subject = parser.parse_subjects(semester).first
# total << parser.parse_courses_in_subject(subject)
ThreadsWait.all_waits(*threads)
Closure.delete_all
Section.delete_all
Course.delete_all
Semester.delete_all
semester = Semester.create! season: 'Fall', year: 2018
semester.save!
total.each do |subject|
subject.each_value do |section|
next unless (section.key? "date_range") && (section.key? "instructors") && (section.key? "days")
course = Course.find_or_create_by(subject: section[:subj],
course_number: section[:code])
course.semester = semester
course.save!
section_name = "#{section[:subj]} #{section[:code]} #{section[:sect]}"
puts "Adding #{section_name}..."
start_time = if section.key? "time"
section["time"].split(' - ').first
else
"N/A"
end
end_time = if section.key? "time"
section["time"].split(' - ').last
else
"N/A"
end
Section.create!(name: section_name,
crn: section[:crn],
title: section[:name],
location: section["where"],
days: section["days"],
start_date: section["date_range"].split(' - ').first,
end_date: section["date_range"].split(' - ').last,
start_time: start_time,
end_time: end_time,