Commit 512d3abf authored by Zac Wood's avatar Zac Wood
Browse files

Merge branch '15-bulk-insertion' into 'master'

Resolve "Change seeds.rb script to use bulk insertion"

Closes #15

See merge request !22
parents b0d18849 d0a3a0e2
Pipeline #2879 passed with stage
in 2 minutes and 13 seconds
...@@ -11,3 +11,5 @@ Style/FrozenStringLiteralComment: ...@@ -11,3 +11,5 @@ Style/FrozenStringLiteralComment:
Style/SymbolArray: Style/SymbolArray:
EnforcedStyle: brackets # [:symbol1, :symbol2] EnforcedStyle: brackets # [:symbol1, :symbol2]
Metrics/BlockLength:
Enabled: false
...@@ -7,7 +7,7 @@ require 'httparty' ...@@ -7,7 +7,7 @@ require 'httparty'
require 'nokogiri' require 'nokogiri'
require 'json' require 'json'
# threads = [] threads = []
total = {} total = {}
parser = PatriotWeb::Parser.new parser = PatriotWeb::Parser.new
...@@ -19,9 +19,9 @@ puts "DDOSing Patriot Web, buckle up kids" ...@@ -19,9 +19,9 @@ puts "DDOSing Patriot Web, buckle up kids"
# parse all subjects and their courses in the semester # parse all subjects and their courses in the semester
parser.parse_subjects(semester).each do |subject| parser.parse_subjects(semester).each do |subject|
puts "Getting courses for #{subject}" puts "Getting courses for #{subject}"
# threads << Thread.new { threads << Thread.new {
total[subject] = parser.parse_courses_in_subject(subject) total[subject] = parser.parse_courses_in_subject(subject)
# } }
end end
# For testing, only get first subject # For testing, only get first subject
...@@ -29,7 +29,7 @@ end ...@@ -29,7 +29,7 @@ end
# total[subject] = parser.parse_courses_in_subject(subject) # total[subject] = parser.parse_courses_in_subject(subject)
# wait for all the threads to finish # wait for all the threads to finish
# ThreadsWait.all_waits(*threads) ThreadsWait.all_waits(*threads)
# delete everything in the current database # delete everything in the current database
Closure.delete_all Closure.delete_all
...@@ -41,40 +41,53 @@ Semester.delete_all ...@@ -41,40 +41,53 @@ Semester.delete_all
semester = Semester.create! season: 'Fall', year: 2018 semester = Semester.create! season: 'Fall', year: 2018
semester.save! semester.save!
# Taking a course and a list of courses, checks if the course is already in that
# list. If it isn't, create an active record and store it in the list for that
# course. If it is, grab the pre-existing database entry.
def get_course(course, all_courses)
all_courses.each do |c|
if c[:course_number] == course[:course_number]
return c[:db_object]
end
end
course[:db_object] = Course.create!(course)
all_courses.push(course)
course[:db_object]
end
total.each do |subject, sections| total.each do |subject, sections|
puts "Adding courses for #{subject}..." puts "Adding courses for #{subject}..."
all_sections = []
all_courses = []
sections.each do |section| sections.each do |section|
if section.nil? || !section.key?(:subj) || !section.key?(:course_number) if section.nil? || !section.key?(:subj) || !section.key?(:course_number)
puts "#{subject} failed section: #{section.class}" puts "#{subject} failed section: #{section.class}"
next next
end end
# Find or create a course and set its semester course = get_course({ subject: section[:subj],
# TODO: this breaks when you try to do more than one semester, course_number: section[:course_number],
# since just the subject + course_number do not uniquely identify a course semester: semester }, all_courses)
# Check the semester as well
course = Course.find_or_create_by(subject: section[:subj],
course_number: section[:course_number])
course.semester = semester
course.save!
section_name = "#{section[:subj]} #{section[:course_number]} #{section[:section]}" section_name = "#{section[:subj]} #{section[:course_number]} #{section[:section]}"
# puts "Adding #{section_name}..." all_sections.push(name: section_name,
crn: section[:crn],
CourseSection.create!(name: section_name, section_type: section[:type],
crn: section[:crn], title: section[:title],
section_type: section[:type], instructor: section[:instructor],
title: section[:title], start_date: section[:start_date],
instructor: section[:instructor], end_date: section[:end_date],
start_date: section[:start_date], days: section[:days],
end_date: section[:end_date], start_time: section[:start_time],
days: section[:days], end_time: section[:end_time],
start_time: section[:start_time], location: section[:location],
end_time: section[:end_time], course: course)
location: section[:location],
course: course)
end end
CourseSection.create!(all_sections)
end end
# create closures for the days there will be no classes # create closures for the days there will be no classes
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment