TAKE THAT PATRIOT WEB

schedules_api/db/patriot_web_parser.rb

@@ -67,20 +67,74 @@ module PatriotWeb
     # @param document [Nokogiri::HTML::Document]
     # @return [Array] courses
     def get_courses(document)
-      table = document.css('html body div.pagebodydiv table.datadisplaytable').first
-      File.write('help', table)
-      rows = table.children.drop 2 # first two elements are junk
-      # puts rows[0].text
-      # puts rows[2].css('table.datadisplaytable td').children.drop(1).map { |c| c.text }
-      # puts rows[3]
-      # puts rows[4]
+      table = document.css('html body div.pagebodydiv table.datadisplaytable')
+      rows = table.css('tr')
+
+      (0..(rows.length/6-1)).map do |i|
+        start = i*5
+        data = {}
+        title = rows[start].text
+        # the title looks this: Survey of Accounting - 71117 - ACCT 203 - 001
+        # so split it by ' - ' and extract
+        title_elements = title.split(' - ')
+        next unless title_elements.length == 4
+        data[:title] = title_elements[0].strip
+        data[:crn] = title_elements[1]
+        
+        full_name = title_elements[2].split(' ')
+        next unless full_name.length == 2
+        data[:subj] = title_elements[2].split(' ')[0]
+        data[:course_number] = title_elements[2].split(' ')[1]
+        
+        data[:section] = title_elements[3].strip
+
+        # rows 1 to 3 contain info about registration and drop dates.
+        # for now we're gonna ignore them and skip to row 4, which contains details
+        details = rows[start+2].css('td table tr td')
+
+        next unless details.length > 0 # if there are no details, skip this item
+        # details = detail_rows.last.text.split("\n").compact.reject(&:empty?) # skip empty strings
+        
+        times = details[1].text.split(' - ')
+        if (times.length == 1)
+          data[:start_time] = 'TBA'
+          data[:end_time] = 'TBA'
+        else
+          data[:start_time] = times[0]
+          data[:end_time] = times[1]
+        end
 
-      (0..(rows.length/4 - 1)).map do |i|
-        start = i*4
-        puts rows[start].text
-        puts rows[start+2].css('table.datadisplaytable td').children.drop(1).map { |c| c.text }
+        data[:days] = details[2].text.strip
+        data[:location] = details[3].text.strip
+        
+        dates = details[4].text.split(' - ')
+        data[:start_date] = dates[0]
+        data[:end_date] = dates[1]
+        
+        data[:type] = details[5].text
+        data[:instructor] = details[6].text
+        data
       end
+      # puts rows[0].text
+      # puts rows[2].css('td table tr td')
+      # puts rows[5].text
+      # puts rows[7].css('td table tr td')
       
+      # puts rows[10].text
+      # puts rows[12].css('td table tr td')
+      # (0..(rows.length/3-1)).each do |i|
+      #   start = i*3
+        
+      #   puts rows[start].search('th').first.text
+      #   section_data = rows[start+2].css('td table.datadisplaytable').search('td')
+      #   puts section_data[1].text
+      # end
+      # puts rows[3]
+      # puts rows[3].search('th').first.text
+      # section_data = rows[5].css('td table.datadisplaytable').search('td')
+      # puts section_data[1].text
+
+      # end
       # each section is represented by 6 rows in the table
       # (0..(rows.length/6 - 1)).map do |i|
       #   start = i*6

schedules_api/db/seeds-broken.rb

-# This file should contain all the record creation needed to seed the database with its default values.
-# The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup).
-
-require_relative 'patriot_web_parser'
-require 'thwait'
-require 'httparty'
-require 'nokogiri'
-require 'json'
-
-threads = []
-total = {}
-parser = PatriotWeb::Parser.new
-
-# get the first semester only
-semester = parser.parse_semesters.first
-
-puts "DDOSing Patriot Web, buckle up kids"
-
-# # parse all subjects and their courses in the semester
-# parser.parse_subjects(semester).each do |subject|
-#   puts "Getting courses for #{subject}"
-#   threads << Thread.new {
-#     total[subject] = parser.parse_courses_in_subject(subject)    
-#   }
-# end
-
-# For testing, only get first subject
-subject = parser.parse_subjects(semester).first
-total[subject] = parser.parse_courses_in_subject(subject)
-
-# wait for all the threads to finish
-# ThreadsWait.all_waits(*threads)
-
-# delete everything in the current database
-Closure.delete_all
-CourseSection.delete_all
-Course.delete_all
-Semester.delete_all
-
-# create a semester for the next semester
-semester = Semester.create! season: 'Fall', year: 2018
-semester.save!
-
-total.each do |subject, sections|
-  puts "Adding courses for #{subject}..."
-  sections.each do |section|
-    next if section.nil? || !section.key?(:subj) || !section.key?(:course_number)
-    
-    # Find or create a course and set its semester
-    # TODO: this breaks when you try to do more than one semester,
-    # since just the subject + course_number do not uniquely identify a course
-    # Check the semester as well
-    course = Course.find_or_create_by(subject: section[:subj],
-                                      course_number: section[:course_number])
-    course.semester = semester
-    course.save!
-
-    section_name = "#{section[:subj]} #{section[:course_number]} #{section[:section]}"
-    
-    puts "Adding #{section_name}..."
-
-    CourseSection.create!(name: section_name,
-                    crn: section[:crn],
-                    section_type: section[:type],
-                    title: section[:title],
-                    instructor: section[:instructor],
-                    start_date: section[:start_date],
-                    end_date: section[:end_date],
-                    days: section[:days],
-                    start_time: section[:start_time],
-                    end_time: section[:end_time],
-                    location: section[:location],
-                    course: course)
-                   
-  end
-end
-
-# create closures for the days there will be no classes
-# see: https://registrar.gmu.edu/calendars/fall-2018/
-Closure.create! date: Date.new(2018, 9, 3), semester: semester
-Closure.create! date: Date.new(2018, 10, 8), semester: semester
-(21..25).each { |n| Closure.create! date: Date.new(2018, 11, n), semester: semester }
-(10..19).each { |n| Closure.create! date: Date.new(2018, 12, n), semester: semester }

schedules_api/db/seeds-excel.rb

+require_relative 'excel_loader'
+
+# Deletes all records from the database.
+Closure.delete_all
+CourseSection.delete_all
+Course.delete_all
+Semester.delete_all
+  
+loader = ExcelLoader.new 'db/data/fall2018.xlsx'
+         
+loader.load_data
+
+

schedules_api/db/seeds.rb

-require_relative 'excel_loader'
+# This file should contain all the record creation needed to seed the database with its default values.
+# The data can then be loaded with the rails db:seed command (or created alongside the database with db:setup).
 
-# Deletes all records from the database.
+require_relative 'patriot_web_parser'
+require 'thwait'
+require 'httparty'
+require 'nokogiri'
+require 'json'
+
+threads = []
+total = {}
+parser = PatriotWeb::Parser.new
+
+# get the first semester only
+semester = parser.parse_semesters.first
+
+puts "DDOSing Patriot Web, buckle up kids"
+
+# parse all subjects and their courses in the semester
+parser.parse_subjects(semester).each do |subject|
+  puts "Getting courses for #{subject}"
+  threads << Thread.new {
+    total[subject] = parser.parse_courses_in_subject(subject)    
+  }
+end
+
+# For testing, only get first subject
+# subject = parser.parse_subjects(semester).first
+# total[subject] = parser.parse_courses_in_subject(subject)
+
+# wait for all the threads to finish
+ThreadsWait.all_waits(*threads)
+
+# delete everything in the current database
 Closure.delete_all
 CourseSection.delete_all
 Course.delete_all
 Semester.delete_all
-  
-loader = ExcelLoader.new 'db/data/fall2018.xlsx'
-         
-loader.load_data
 
+# create a semester for the next semester
+semester = Semester.create! season: 'Fall', year: 2018
+semester.save!
+
+total.each do |subject, sections|
+  puts "Adding courses for #{subject}..."
+  sections.each do |section|
+    next if section.nil? || !section.key?(:subj) || !section.key?(:course_number)
+    
+    # Find or create a course and set its semester
+    # TODO: this breaks when you try to do more than one semester,
+    # since just the subject + course_number do not uniquely identify a course
+    # Check the semester as well
+    course = Course.find_or_create_by(subject: section[:subj],
+                                      course_number: section[:course_number])
+    course.semester = semester
+    course.save!
+
+    section_name = "#{section[:subj]} #{section[:course_number]} #{section[:section]}"
+    
+    # puts "Adding #{section_name}..."
+
+    CourseSection.create!(name: section_name,
+                    crn: section[:crn],
+                    section_type: section[:type],
+                    title: section[:title],
+                    instructor: section[:instructor],
+                    start_date: section[:start_date],
+                    end_date: section[:end_date],
+                    days: section[:days],
+                    start_time: section[:start_time],
+                    end_time: section[:end_time],
+                    location: section[:location],
+                    course: course)
+                   
+  end
+end
 
+# create closures for the days there will be no classes
+# see: https://registrar.gmu.edu/calendars/fall-2018/
+Closure.create! date: Date.new(2018, 9, 3), semester: semester
+Closure.create! date: Date.new(2018, 10, 8), semester: semester
+(21..25).each { |n| Closure.create! date: Date.new(2018, 11, n), semester: semester }
+(10..19).each { |n| Closure.create! date: Date.new(2018, 12, n), semester: semester }