Commit 90141ab1 authored by Mark Stenglein's avatar Mark Stenglein

FINALLY HAVE DATA IN THE DATABASE!!!!!!!

- Still has some issues though...
    - Look into why it stops at the 500th record for sections....I'm
    guessing that I need to switch over to a real database instead of
    sqlite for development.
    - Some information, such as the relation back to semesters are not
    being added to the sections.
    - Professors are not being added correctly to the classes it would
    appear. check that code pls tyty
parent 0466dfb5
Pipeline #686 failed with stage
in 42 seconds
......@@ -10,7 +10,7 @@
* contains inside of it an array of each `Semester` object, which in turn has
* all of the thousands of `Section` objects.
*
* Sequelize (Schedules' ORM) takes care of converting the nested objects into
* Sequelize (Schedules's ORM) takes care of converting the nested objects into
* Has-Many relationships (forming a splitting tree of data from the base
* university objects).
*/
......@@ -23,168 +23,166 @@ var schools = config.schools
var sequelize = db.sequelize
var models = db.sequelize.models
console.log('day')
// Synchronize the database tables with their models (leave `{force: true}` to
// empty the database at each run)
// TODO: create a config switch that decides locally if the database is to be
// completely wiped each and every run of the application or not.
sequelize.sync({ force: true })
// Loop through each school stored in the configuration and build the
// university object for it. Everything else happens _insider_ this loop.
for (var i = 0; i < schools.length; i++) {
var school = schools[i]
// Barebone university object pulls data from config and leaves an empty
// array for any `Semesters` to be stored.
var university = {
'slug': school.slug,
'name': school.name,
'website': school.website,
'Semesters': []
}
/*
* Look through each of the `university`'s semesters and build a `semester`
* object for them. Again, most everything else happens in here.
*
* NOTE: This entire loop runs independently for _each_ different university
* configured in the `config.yaml` file.
*/
for (var j = 0; j < school.semesters.length; j++) {
// pull the raw semester data to work with (really to shorten further uses)
var rawSemester = school.semesters[j]
// Fetch the large JSON object file that stores all of the `Section` data
// for this particular semester.
var rawSections = require(path.join(__dirname, 'dataFiles', rawSemester.dataFile))
// Build a basic semester object according to the ORM. Again, leaves an
// empty array to be populated in the next loop.
/*
* Loop through each school stored in the configuration and build the
* university object for it. Once this is built, it is added to the database.
*
* Once the university object's database operation completes, the semesters
* are built for the university while the function moves onto the next
* school at the same time. (yay promises!)
*
*/
var syncUniversities = function() {
schools.forEach(function (school) {
var university = {
'slug': school.slug,
'name': school.name,
'website': school.website
}
models.University.create(university).then(function () {
syncSemesters(school)
})
})
}
/*
* Loop through each semester for a given school stored in the configuration
* and build the semester object for it. Once built, it is added to the
* database and the sections are built immediately after each semester's
* database operation completes.
*
* The function moves onto the next semester at the same time (yay promises!)
*/
var syncSemesters = function(school) {
school.semesters.forEach(function (rawSemester) {
var semester = {
'slug': rawSemester.slug,
'name': rawSemester.name,
'Sections': []
'university': school.slug
}
models.Semester.create(semester).then(function () {
syncSections(rawSemester)
})
})
}
/*
* Loop through each section of a given semester stored in the data files and
* build the section object for it. Once built, it is added to the section
* table. After the last section is loaded, all data will be finished being
* added to the system.
*/
var syncSections = function(semester) {
// Fetch the large JSON object file that stores all of the `Section` data for
// this particular semester.
var rawSections = require(path.join(__dirname, 'dataFiles', semester.dataFile))
for (var z = 0; z < rawSections.classes.length; z++) {
var rawSection = rawSections.classes[z]
var sectionDateRange = rawSection.session_templates[0].date_range
var section = {
'crn': rawSection.crn,
'name': rawSection.name,
'title': rawSection.title,
'section': rawSection.section,
'instructors': '', // build this from all total instructors
'campus': rawSection.campus,
'location': '', // taken from only the first session_template
'class_type': '', // taken from only the first session_template
'startDate': sectionDateRange.substr(0, 12),
'endDate': sectionDateRange.substr(15, 12),
'Msession': false,
'MtimeStart': null,
'MtimeEnd': null,
'Mlocation': null,
'MclassType': null,
'Tsession': false,
'TtimeStart': null,
'TtimeEnd': null,
'Tlocation': null,
'TclassType': null,
'Wsession': false,
'WtimeStart': null,
'WtimeEnd': null,
'Wlocation': null,
'WclassType': null,
'Rsession': false,
'RtimeStart': null,
'RtimeEnd': null,
'Rlocation': null,
'RclassType': null,
'Fsession': false,
'FtimeStart': null,
'FtimeEnd': null,
'Flocation': null,
'FclassType': null
}
// Pull generic session information (location and type) from first
// session template
section.location = rawSection.session_templates[0].location
section.class_type = rawSection.session_templates[0].class_type
/*
* Loop through all of the hundreds of classes for the current university's
* semester being processed currently. Builds the base `section` object and
* does some processing to get the raw data into the ORM's needed format.
*
* NOTE: This is done independently for each `Semester` in each
* `University`. Keep in mind that this loop is run a stupid amount
* before adding some crazy expensive processing here.
* Loop through each session template for each section/class. This loop
* is what allows different days of the week to meet on different times
* and or locations.
*/
for (var z = 0; z < rawSections.classes.length; z++) {
var rawSection = rawSections.classes[z]
var sectionDateRange = rawSection.session_templates[0].date_range
var section = {
'crn': rawSection.crn,
'name': rawSection.name,
'title': rawSection.title,
'section': rawSection.section,
'instructors': '', // build this from all total instructors
'campus': rawSection.campus,
'location': '', // taken from only the first session_template
'class_type': '', // taken from only the first session_template
'startDate': sectionDateRange.substr(0, 12),
'endDate': sectionDateRange.substr(15, 12),
'Msession': false,
'MtimeStart': null,
'MtimeEnd': null,
'Mlocation': null,
'MclassType': null,
'Tsession': false,
'TtimeStart': null,
'TtimeEnd': null,
'Tlocation': null,
'TclassType': null,
'Wsession': false,
'WtimeStart': null,
'WtimeEnd': null,
'Wlocation': null,
'WclassType': null,
'Rsession': false,
'RtimeStart': null,
'RtimeEnd': null,
'Rlocation': null,
'RclassType': null,
'Fsession': false,
'FtimeStart': null,
'FtimeEnd': null,
'Flocation': null,
'FclassType': null
}
// Pull generic session information (location and type) from first
// session template
section.location = rawSection.session_templates[0].location
section.class_type = rawSection.session_templates[0].class_type
/*
* Loop through each session template for each section/class. This loop
* is what allows different days of the week to meet on different times
* and or locations.
*/
for (var c = 0; c < rawSection.session_templates.length; c++) {
var session_template = rawSection.session_templates[c]
// raw value of the start/stop times (to be processed to time values)
var sessionTime = session_template.time;
(['M', 'T', 'W', 'R', 'F']).forEach(function(day) {
if (session_template.days.indexOf(day) > -1) {
// if here then there is a session for the day
section[day + 'session'] = true
// set initial length for start time
var startLength = 8
// test if the time has 2 digits in hours (adjust length if yes)
if (sessionTime[1] === ':') {
startLength = 7
}
// set the start time
section[day + 'timeStart'] = sessionTime.substr(0, startLength)
// set initial length for end time
var endLength = 8
// test if the time has 2 digits in hours (adjust length if yes)
if (sessionTime[sessionTime.length - 8] === ' ') {
endLength = 7
}
var endOffset = sessionTime.length - endLength
// set the end time
section[day + 'timeEnd'] = sessionTime.substr(endOffset, endLength)
// Note down location and type information
section[day + 'location'] = session_template.location
section[day + 'classType'] = session_template.class_type
// If the instructor is not yet in the entire listing, add it.
// TODO: Make this tolerant of multiple instructors defined for
// one section. (ie. currently if you have 2 instructors in
// one section it'll search for all of them as one term and
// consequently add all of them to the full list.) Should
if (section.instructors.search(session_template.instructors) < 0) {
section.instructors += session_template.instructors
}
for (var c = 0; c < rawSection.session_templates.length; c++) {
var session_template = rawSection.session_templates[c]
// raw value of the start/stop times (to be processed to time values)
var sessionTime = session_template.time;
(['M', 'T', 'W', 'R', 'F']).forEach(function(day) {
if (session_template.days.indexOf(day) > -1) {
// if here then there is a session for the day
section[day + 'session'] = true
// set initial length for start time
var startLength = 8
// test if the time has 2 digits in hours (adjust length if yes)
if (sessionTime[1] === ':') {
startLength = 7
}
// set the start time
section[day + 'timeStart'] = sessionTime.substr(0, startLength)
// set initial length for end time
var endLength = 8
// test if the time has 2 digits in hours (adjust length if yes)
if (sessionTime[sessionTime.length - 8] === ' ') {
endLength = 7
}
var endOffset = sessionTime.length - endLength
// set the end time
section[day + 'timeEnd'] = sessionTime.substr(endOffset, endLength)
// Note down location and type information
section[day + 'location'] = session_template.location
section[day + 'classType'] = session_template.class_type
// If the instructor is not yet in the entire listing, add it.
// TODO: Make this tolerant of multiple instructors defined for
// one section. (ie. currently if you have 2 instructors in
// one section it'll search for all of them as one term and
// consequently add all of them to the full list.) Should
if (section.instructors.search(session_template.instructors) < 0) {
section.instructors += session_template.instructors
}
})
}
/* END JANKY WORKAROUND DESCRIBED IN ABOVE BLOCK */
console.log(section)
semester.Sections.push(section)
}
})
}
university.Semesters.push(semester)
/* END JANKY WORKAROUND DESCRIBED IN ABOVE BLOCK */
models.Section.create(section)
}
// Actually add the data to the database and sync it.
sequelize.sync().then(function () {
return models.University.create(university, {
include: [
models.Semester
]
})
})
}
// Synchronize the database tables with their models (leave `{force: true}` to
// empty the database at each run)
// TODO: create a config switch that decides locally if the database is to be
// completely wiped each and every run of the application or not.
sequelize.sync({ force: true, logging: console.log }).then(syncUniversities)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment