Commit 512d3abf authored by Zac Wood's avatar Zac Wood

Merge branch '15-bulk-insertion' into 'master'

Resolve "Change seeds.rb script to use bulk insertion"

Closes #15

See merge request !22
parents b0d18849 d0a3a0e2
Pipeline #2879 passed with stage
in 2 minutes and 13 seconds
......@@ -11,3 +11,5 @@ Style/FrozenStringLiteralComment:
Style/SymbolArray:
EnforcedStyle: brackets # [:symbol1, :symbol2]
Metrics/BlockLength:
Enabled: false
......@@ -7,7 +7,7 @@ require 'httparty'
require 'nokogiri'
require 'json'
# threads = []
threads = []
total = {}
parser = PatriotWeb::Parser.new
......@@ -19,9 +19,9 @@ puts "DDOSing Patriot Web, buckle up kids"
# parse all subjects and their courses in the semester
parser.parse_subjects(semester).each do |subject|
puts "Getting courses for #{subject}"
# threads << Thread.new {
total[subject] = parser.parse_courses_in_subject(subject)
# }
threads << Thread.new {
total[subject] = parser.parse_courses_in_subject(subject)
}
end
# For testing, only get first subject
......@@ -29,7 +29,7 @@ end
# total[subject] = parser.parse_courses_in_subject(subject)
# wait for all the threads to finish
# ThreadsWait.all_waits(*threads)
ThreadsWait.all_waits(*threads)
# delete everything in the current database
Closure.delete_all
......@@ -41,40 +41,53 @@ Semester.delete_all
semester = Semester.create! season: 'Fall', year: 2018
semester.save!
# Taking a course and a list of courses, checks if the course is already in that
# list. If it isn't, create an active record and store it in the list for that
# course. If it is, grab the pre-existing database entry.
def get_course(course, all_courses)
all_courses.each do |c|
if c[:course_number] == course[:course_number]
return c[:db_object]
end
end
course[:db_object] = Course.create!(course)
all_courses.push(course)
course[:db_object]
end
total.each do |subject, sections|
puts "Adding courses for #{subject}..."
all_sections = []
all_courses = []
sections.each do |section|
if section.nil? || !section.key?(:subj) || !section.key?(:course_number)
puts "#{subject} failed section: #{section.class}"
next
end
# Find or create a course and set its semester
# TODO: this breaks when you try to do more than one semester,
# since just the subject + course_number do not uniquely identify a course
# Check the semester as well
course = Course.find_or_create_by(subject: section[:subj],
course_number: section[:course_number])
course.semester = semester
course.save!
course = get_course({ subject: section[:subj],
course_number: section[:course_number],
semester: semester }, all_courses)
section_name = "#{section[:subj]} #{section[:course_number]} #{section[:section]}"
# puts "Adding #{section_name}..."
CourseSection.create!(name: section_name,
crn: section[:crn],
section_type: section[:type],
title: section[:title],
instructor: section[:instructor],
start_date: section[:start_date],
end_date: section[:end_date],
days: section[:days],
start_time: section[:start_time],
end_time: section[:end_time],
location: section[:location],
course: course)
all_sections.push(name: section_name,
crn: section[:crn],
section_type: section[:type],
title: section[:title],
instructor: section[:instructor],
start_date: section[:start_date],
end_date: section[:end_date],
days: section[:days],
start_time: section[:start_time],
end_time: section[:end_time],
location: section[:location],
course: course)
end
CourseSection.create!(all_sections)
end
# create closures for the days there will be no classes
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment