Skip to content

GitLab

schedules-legacy
schedules-legacy
Commit 90141ab1 authored by Mark Stenglein's avatar Mark Stenglein
Browse files
Options

FINALLY HAVE DATA IN THE DATABASE!!!!!!! 

- Still has some issues though...
    - Look into why it stops at the 500th record for sections....I'm
    guessing that I need to switch over to a real database instead of
    sqlite for development.
    - Some information, such as the relation back to semesters are not
    being added to the sections.
    - Professors are not being added correctly to the classes it would
    appear. check that code pls tyty
parent 0466dfb5
Pipeline #686 failed with stage
    in 42 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 152 additions and 154 deletions
    data/index.js
    View file @ 90141ab1
    ......@@ -10,7 +10,7 @@
    * contains inside of it an array of each `Semester` object, which in turn has
    * all of the thousands of `Section` objects.
    *
    * Sequelize (Schedules' ORM) takes care of converting the nested objects into
    * Sequelize (Schedules's ORM) takes care of converting the nested objects into
    * Has-Many relationships (forming a splitting tree of data from the base
    * university objects).
    */
    ......@@ -23,168 +23,166 @@ var schools = config.schools
    var sequelize = db.sequelize
    var models = db.sequelize.models
    console.log('day')
    // Synchronize the database tables with their models (leave `{force: true}` to
    // empty the database at each run)
    // TODO: create a config switch that decides locally if the database is to be
    // completely wiped each and every run of the application or not.
    sequelize.sync({ force: true })
    // Loop through each school stored in the configuration and build the
    // university object for it. Everything else happens _insider_ this loop.
    for (var i = 0; i < schools.length; i++) {
    var school = schools[i]
    // Barebone university object pulls data from config and leaves an empty
    // array for any `Semesters` to be stored.
    var university = {
    'slug': school.slug,
    'name': school.name,
    'website': school.website,
    'Semesters': []
    }
    /*
    * Look through each of the `university`'s semesters and build a `semester`
    * object for them. Again, most everything else happens in here.
    *
    * NOTE: This entire loop runs independently for _each_ different university
    * configured in the `config.yaml` file.
    */
    for (var j = 0; j < school.semesters.length; j++) {
    // pull the raw semester data to work with (really to shorten further uses)
    var rawSemester = school.semesters[j]
    // Fetch the large JSON object file that stores all of the `Section` data
    // for this particular semester.
    var rawSections = require(path.join(__dirname, 'dataFiles', rawSemester.dataFile))
    // Build a basic semester object according to the ORM. Again, leaves an
    // empty array to be populated in the next loop.
    /*
    * Loop through each school stored in the configuration and build the
    * university object for it. Once this is built, it is added to the database.
    *
    * Once the university object's database operation completes, the semesters
    * are built for the university while the function moves onto the next
    * school at the same time. (yay promises!)
    *
    */
    var syncUniversities = function() {
    schools.forEach(function (school) {
    var university = {
    'slug': school.slug,
    'name': school.name,
    'website': school.website
    }
    models.University.create(university).then(function () {
    syncSemesters(school)
    })
    })
    }
    /*
    * Loop through each semester for a given school stored in the configuration
    * and build the semester object for it. Once built, it is added to the
    * database and the sections are built immediately after each semester's
    * database operation completes.
    *
    * The function moves onto the next semester at the same time (yay promises!)
    */
    var syncSemesters = function(school) {
    school.semesters.forEach(function (rawSemester) {
    var semester = {
    'slug': rawSemester.slug,
    'name': rawSemester.name,
    'Sections': []
    'university': school.slug
    }
    models.Semester.create(semester).then(function () {
    syncSections(rawSemester)
    })
    })
    }
    /*
    * Loop through each section of a given semester stored in the data files and
    * build the section object for it. Once built, it is added to the section
    * table. After the last section is loaded, all data will be finished being
    * added to the system.
    */
    var syncSections = function(semester) {
    // Fetch the large JSON object file that stores all of the `Section` data for
    // this particular semester.
    var rawSections = require(path.join(__dirname, 'dataFiles', semester.dataFile))
    for (var z = 0; z < rawSections.classes.length; z++) {
    var rawSection = rawSections.classes[z]
    var sectionDateRange = rawSection.session_templates[0].date_range
    var section = {
    'crn': rawSection.crn,
    'name': rawSection.name,
    'title': rawSection.title,
    'section': rawSection.section,
    'instructors': '', // build this from all total instructors
    'campus': rawSection.campus,
    'location': '', // taken from only the first session_template
    'class_type': '', // taken from only the first session_template
    'startDate': sectionDateRange.substr(0, 12),
    'endDate': sectionDateRange.substr(15, 12),
    'Msession': false,
    'MtimeStart': null,
    'MtimeEnd': null,
    'Mlocation': null,
    'MclassType': null,
    'Tsession': false,
    'TtimeStart': null,
    'TtimeEnd': null,
    'Tlocation': null,
    'TclassType': null,
    'Wsession': false,
    'WtimeStart': null,
    'WtimeEnd': null,
    'Wlocation': null,
    'WclassType': null,
    'Rsession': false,
    'RtimeStart': null,
    'RtimeEnd': null,
    'Rlocation': null,
    'RclassType': null,
    'Fsession': false,
    'FtimeStart': null,
    'FtimeEnd': null,
    'Flocation': null,
    'FclassType': null
    }
    // Pull generic session information (location and type) from first
    // session template
    section.location = rawSection.session_templates[0].location
    section.class_type = rawSection.session_templates[0].class_type
    /*
    * Loop through all of the hundreds of classes for the current university's
    * semester being processed currently. Builds the base `section` object and
    * does some processing to get the raw data into the ORM's needed format.
    *
    * NOTE: This is done independently for each `Semester` in each
    * `University`. Keep in mind that this loop is run a stupid amount
    * before adding some crazy expensive processing here.
    * Loop through each session template for each section/class. This loop
    * is what allows different days of the week to meet on different times
    * and or locations.
    */
    for (var z = 0; z < rawSections.classes.length; z++) {
    var rawSection = rawSections.classes[z]
    var sectionDateRange = rawSection.session_templates[0].date_range
    var section = {
    'crn': rawSection.crn,
    'name': rawSection.name,
    'title': rawSection.title,
    'section': rawSection.section,
    'instructors': '', // build this from all total instructors
    'campus': rawSection.campus,
    'location': '', // taken from only the first session_template
    'class_type': '', // taken from only the first session_template
    'startDate': sectionDateRange.substr(0, 12),
    'endDate': sectionDateRange.substr(15, 12),
    'Msession': false,
    'MtimeStart': null,
    'MtimeEnd': null,
    'Mlocation': null,
    'MclassType': null,
    'Tsession': false,
    'TtimeStart': null,
    'TtimeEnd': null,
    'Tlocation': null,
    'TclassType': null,
    'Wsession': false,
    'WtimeStart': null,
    'WtimeEnd': null,
    'Wlocation': null,
    'WclassType': null,
    'Rsession': false,
    'RtimeStart': null,
    'RtimeEnd': null,
    'Rlocation': null,
    'RclassType': null,
    'Fsession': false,
    'FtimeStart': null,
    'FtimeEnd': null,
    'Flocation': null,
    'FclassType': null
    }
    // Pull generic session information (location and type) from first
    // session template
    section.location = rawSection.session_templates[0].location
    section.class_type = rawSection.session_templates[0].class_type
    /*
    * Loop through each session template for each section/class. This loop
    * is what allows different days of the week to meet on different times
    * and or locations.
    */
    for (var c = 0; c < rawSection.session_templates.length; c++) {
    var session_template = rawSection.session_templates[c]
    // raw value of the start/stop times (to be processed to time values)
    var sessionTime = session_template.time;
    (['M', 'T', 'W', 'R', 'F']).forEach(function(day) {
    if (session_template.days.indexOf(day) > -1) {
    // if here then there is a session for the day
    section[day + 'session'] = true
    // set initial length for start time
    var startLength = 8
    // test if the time has 2 digits in hours (adjust length if yes)
    if (sessionTime[1] === ':') {
    startLength = 7
    }
    // set the start time
    section[day + 'timeStart'] = sessionTime.substr(0, startLength)
    // set initial length for end time
    var endLength = 8
    // test if the time has 2 digits in hours (adjust length if yes)
    if (sessionTime[sessionTime.length - 8] === ' ') {
    endLength = 7
    }
    var endOffset = sessionTime.length - endLength
    // set the end time
    section[day + 'timeEnd'] = sessionTime.substr(endOffset, endLength)
    // Note down location and type information
    section[day + 'location'] = session_template.location
    section[day + 'classType'] = session_template.class_type
    // If the instructor is not yet in the entire listing, add it.
    // TODO: Make this tolerant of multiple instructors defined for
    // one section. (ie. currently if you have 2 instructors in
    // one section it'll search for all of them as one term and
    // consequently add all of them to the full list.) Should
    if (section.instructors.search(session_template.instructors) < 0) {
    section.instructors += session_template.instructors
    }
    for (var c = 0; c < rawSection.session_templates.length; c++) {
    var session_template = rawSection.session_templates[c]
    // raw value of the start/stop times (to be processed to time values)
    var sessionTime = session_template.time;
    (['M', 'T', 'W', 'R', 'F']).forEach(function(day) {
    if (session_template.days.indexOf(day) > -1) {
    // if here then there is a session for the day
    section[day + 'session'] = true
    // set initial length for start time
    var startLength = 8
    // test if the time has 2 digits in hours (adjust length if yes)
    if (sessionTime[1] === ':') {
    startLength = 7
    }
    // set the start time
    section[day + 'timeStart'] = sessionTime.substr(0, startLength)
    // set initial length for end time
    var endLength = 8
    // test if the time has 2 digits in hours (adjust length if yes)
    if (sessionTime[sessionTime.length - 8] === ' ') {
    endLength = 7
    }
    var endOffset = sessionTime.length - endLength
    // set the end time
    section[day + 'timeEnd'] = sessionTime.substr(endOffset, endLength)
    // Note down location and type information
    section[day + 'location'] = session_template.location
    section[day + 'classType'] = session_template.class_type
    // If the instructor is not yet in the entire listing, add it.
    // TODO: Make this tolerant of multiple instructors defined for
    // one section. (ie. currently if you have 2 instructors in
    // one section it'll search for all of them as one term and
    // consequently add all of them to the full list.) Should
    if (section.instructors.search(session_template.instructors) < 0) {
    section.instructors += session_template.instructors
    }
    })
    }
    /* END JANKY WORKAROUND DESCRIBED IN ABOVE BLOCK */
    console.log(section)
    semester.Sections.push(section)
    }
    })
    }
    university.Semesters.push(semester)
    /* END JANKY WORKAROUND DESCRIBED IN ABOVE BLOCK */
    models.Section.create(section)
    }
    // Actually add the data to the database and sync it.
    sequelize.sync().then(function () {
    return models.University.create(university, {
    include: [
    models.Semester
    ]
    })
    })
    }
    // Synchronize the database tables with their models (leave `{force: true}` to
    // empty the database at each run)
    // TODO: create a config switch that decides locally if the database is to be
    // completely wiped each and every run of the application or not.
    sequelize.sync({ force: true, logging: console.log }).then(syncUniversities)
    Markdown is supported
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment