Commit 683bbe4d authored by Zac Wood's avatar Zac Wood
Browse files

SQLification

parent e3a17955
inherit_from:
- .rubocop_autogen.yml # Some auto-generated Rails code we shouldn't worry about
- .rubocop_relaxed.yml
AllCops:
TargetRubyVersion: 2.5.0
Style/FrozenStringLiteralComment:
Enabled: false
Style/SymbolArray:
EnforcedStyle: brackets # [:symbol1, :symbol2]
# This configuration was generated by
# `rubocop --auto-gen-config`
# on 2018-04-09 23:19:33 -0400 using RuboCop version 0.54.0.
# The point is for the user to remove these configuration records
# one by one as the offenses are removed from the code base.
# Note that changes in the inspected code, or installation of new
# versions of RuboCop, may require this file to be generated again.
# Offense count: 2
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
# SupportedStyles: empty_lines, no_empty_lines
Layout/EmptyLinesAroundBlockBody:
Exclude:
- 'db/schema.rb'
# Offense count: 2
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBrackets.
# SupportedStyles: space, no_space, compact
# SupportedStylesForEmptyBrackets: space, no_space
Layout/SpaceInsideArrayLiteralBrackets:
Exclude:
- 'config/environments/production.rb'
# Offense count: 1
# Configuration parameters: CountComments, ExcludedMethods.
# Metrics/BlockLength:
# Max: 35
Metrics/MethodLength:
Exclude:
- 'db/migrate/*'
Metrics/BlockLength:
Exclude:
- 'db/schema.rb'
# Offense count: 1
# Cop supports --auto-correct.
# Configuration parameters: AutoCorrect, EnforcedStyle.
# SupportedStyles: nested, compact
Style/ClassAndModuleChildren:
Exclude:
- 'test/test_helper.rb'
# Offense count: 12
Style/Documentation:
Exclude:
- 'spec/**/*'
- 'test/**/*'
- 'app/helpers/*'
- 'app/mailers/application_mailer.rb'
- 'app/models/application_record.rb'
- 'config/application.rb'
- 'db/migrate/20180407190422_create_semesters.rb'
- 'db/migrate/20180407190502_create_courses.rb'
- 'db/migrate/20180407190750_create_sections.rb'
# Offense count: 4
# Cop supports --auto-correct.
Style/ExpandPathArguments:
Exclude:
- 'bin/bundle'
- 'bin/setup'
- 'bin/update'
- 'test/test_helper.rb'
# Offense count: 2
Style/MixinUsage:
Exclude:
- 'bin/setup'
- 'bin/update'
# Offense count: 1
# Cop supports --auto-correct.
# Configuration parameters: Strict.
Style/NumericLiterals:
MinDigits: 15
# Offense count: 1
# Cop supports --auto-correct.
# Configuration parameters: PreferredDelimiters.
Style/PercentLiteralDelimiters:
Exclude:
- 'config/spring.rb'
# Offense count: 2
# Cop supports --auto-correct.
Style/StderrPuts:
Exclude:
- 'bin/yarn'
# Offense count: 47
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
# SupportedStyles: single_quotes, double_quotes
Style/StringLiterals:
Exclude:
- 'Gemfile'
- 'bin/yarn'
- 'config/environments/production.rb'
- 'config/puma.rb'
- 'db/schema.rb'
- 'test/application_system_test_case.rb'
Rails/FilePath:
Exclude:
- "config/environments/development.rb" # auto generated file
# Relaxed.Ruby.Style
## Version 2.2
Style/Alias:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylealias
Style/AsciiComments:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleasciicomments
Style/BeginBlock:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylebeginblock
Style/BlockDelimiters:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleblockdelimiters
Style/CommentAnnotation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylecommentannotation
Style/Documentation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styledocumentation
Layout/DotPosition:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutdotposition
Style/DoubleNegation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styledoublenegation
Style/EndBlock:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleendblock
Style/FormatString:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleformatstring
Style/IfUnlessModifier:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleifunlessmodifier
Style/Lambda:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylelambda
Style/ModuleFunction:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylemodulefunction
Style/MultilineBlockChain:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylemultilineblockchain
Style/NegatedIf:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylenegatedif
Style/NegatedWhile:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylenegatedwhile
Style/ParallelAssignment:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleparallelassignment
Style/PercentLiteralDelimiters:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylepercentliteraldelimiters
Style/PerlBackrefs:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleperlbackrefs
Style/Semicolon:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesemicolon
Style/SignalException:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesignalexception
Style/SingleLineBlockParams:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesinglelineblockparams
Style/SingleLineMethods:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesinglelinemethods
Layout/SpaceBeforeBlockBraces:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutspacebeforeblockbraces
Layout/SpaceInsideParens:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutspaceinsideparens
Style/SpecialGlobalVars:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylespecialglobalvars
Style/StringLiterals:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylestringliterals
Style/TrailingCommaInArguments:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainarguments
Style/TrailingCommaInArrayLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainarrayliteral
Style/TrailingCommaInHashLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainhashliteral
Style/WhileUntilModifier:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylewhileuntilmodifier
Style/WordArray:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylewordarray
Lint/AmbiguousRegexpLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#lintambiguousregexpliteral
Lint/AssignmentInCondition:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#lintassignmentincondition
Metrics/AbcSize:
Enabled: false
Metrics/BlockNesting:
Enabled: false
Metrics/ClassLength:
Enabled: false
Metrics/ModuleLength:
Enabled: false
Metrics/CyclomaticComplexity:
Enabled: false
Metrics/LineLength:
Enabled: false
Metrics/MethodLength:
Enabled: false
Metrics/ParameterLists:
Enabled: false
Metrics/PerceivedComplexity:
Enabled: false
Style/Alias:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylealias
Style/AsciiComments:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleasciicomments
Style/BeginBlock:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylebeginblock
Style/BlockDelimiters:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleblockdelimiters
Style/CommentAnnotation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylecommentannotation
Style/Documentation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styledocumentation
Layout/DotPosition:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutdotposition
Style/DoubleNegation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styledoublenegation
Style/EndBlock:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleendblock
Style/FormatString:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleformatstring
Style/IfUnlessModifier:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleifunlessmodifier
Style/Lambda:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylelambda
Style/ModuleFunction:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylemodulefunction
Style/MultilineBlockChain:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylemultilineblockchain
Style/NegatedIf:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylenegatedif
Style/NegatedWhile:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylenegatedwhile
Style/ParallelAssignment:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleparallelassignment
Style/PercentLiteralDelimiters:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylepercentliteraldelimiters
Style/PerlBackrefs:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleperlbackrefs
Style/Semicolon:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesemicolon
Style/SignalException:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesignalexception
Style/SingleLineBlockParams:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesinglelineblockparams
Style/SingleLineMethods:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesinglelinemethods
Layout/SpaceBeforeBlockBraces:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutspacebeforeblockbraces
Layout/SpaceInsideParens:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutspaceinsideparens
Style/SpecialGlobalVars:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylespecialglobalvars
Style/StringLiterals:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylestringliterals
Style/TrailingCommaInArguments:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainarguments
Style/TrailingCommaInArrayLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainarrayliteral
Style/TrailingCommaInHashLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainhashliteral
Style/WhileUntilModifier:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylewhileuntilmodifier
Style/WordArray:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylewordarray
Lint/AmbiguousRegexpLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#lintambiguousregexpliteral
Lint/AssignmentInCondition:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#lintassignmentincondition
Metrics/AbcSize:
Enabled: false
Metrics/BlockNesting:
Enabled: false
Metrics/ClassLength:
Enabled: false
Metrics/ModuleLength:
Enabled: false
Metrics/CyclomaticComplexity:
Enabled: false
Metrics/LineLength:
Enabled: false
Metrics/MethodLength:
Enabled: false
Metrics/ParameterLists:
Enabled: false
Metrics/PerceivedComplexity:
Enabled: false
source 'https://rubygems.org' source 'https://rubygems.org'
gem 'nokogiri'
gem 'httparty' gem 'httparty'
gem 'nokogiri'
GEM
remote: https://rubygems.org/
specs:
httparty (0.16.2)
multi_xml (>= 0.5.2)
mini_portile2 (2.3.0)
multi_xml (0.6.0)
nokogiri (1.8.2)
mini_portile2 (~> 2.3.0)
PLATFORMS
ruby
DEPENDENCIES
httparty
nokogiri
BUNDLED WITH
1.16.1
# frozen_string_literal: true # frozen_string_literal: true
require 'thread'
require 'thwait' require 'thwait'
require 'httparty' require 'httparty'
require 'nokogiri' require 'nokogiri'
require 'json' require 'json'
=begin #
# USAGE:
USAGE: #
# Just run it and it dynamically dumps the latest semester. There's a bit to do it for all of the ones in history commented out below but it'll thrash your RAM and probably piss off PatriotWeb. Also note this script could be trivially modified to correlate human readable names to semester IDs since they're just the .text attribute of the option node.
Just run it and it dynamically dumps the latest semester. There's a bit to do it for all of the ones in history commented out below but it'll thrash your RAM and probably piss off PatriotWeb. Also note this script could be trivially modified to correlate human readable names to semester IDs since they're just the .text attribute of the option node. #
# There's a few minor issues like multiple spaces in teacher names and we could be scraping out email addresses but no major ones.
There's a few minor issues like multiple spaces in teacher names and we could be scraping out email addresses but no major ones. #
# DISCLAIMER/WARNING:
DISCLAIMER/WARNING: #
# This opens a number of connections pretty transparently from a script to PatriotWeb. I am not liable if you run this a million times and somehow kill over PatriotWeb. It's a scraper, not a DoS utility.
This opens a number of connections pretty transparently from a script to PatriotWeb. I am not liable if you run this a million times and somehow kill over PatriotWeb. It's a scraper, not a DoS utility. #
=end
# Credit stackoverflow # Credit stackoverflow
class String class String
...@@ -38,23 +35,22 @@ def get_details(data, titledetails, titledata) ...@@ -38,23 +35,22 @@ def get_details(data, titledetails, titledata)
[data, data[crn]] [data, data[crn]]
end end
def sort_item(item, currentobj, data) def sort_item(item, currentobj, data)
if item.name == 'th' if item.name == 'th'
if item.to_html.include? '-' if item.to_html.include? '-'
titletxt = item.text titletxt = item.text
if item.text.include? ' - Honors' if item.text.include? ' - Honors'
titletxt = titletxt.gsub(' - Honors',' (Honors)') titletxt = titletxt.gsub(' - Honors', ' (Honors)')
end end
titledetails = titletxt.split(' - ') titledetails = titletxt.split(' - ')
if titledetails.count > 4 if titledetails.count > 4
titledetails = ["#{titledetails[0]} #{titledetails[1]}",titledetails[2],titledetails[3],titledetails[4]] titledetails = ["#{titledetails[0]} #{titledetails[1]}", titledetails[2], titledetails[3], titledetails[4]]
end end
titledata = titledetails[2].split(' ') titledata = titledetails[2].split(' ')
begin begin
data = get_details(data, titledetails, titledata)[0] data = get_details(data, titledetails, titledata)[0]
currentobj = get_details(data, titledetails, titledata)[1] currentobj = get_details(data, titledetails, titledata)[1]
rescue => e rescue StandardError => e
puts item puts item
puts e puts e
exit(1) exit(1)
...@@ -98,13 +94,12 @@ def get_crn(title, code, section) ...@@ -98,13 +94,12 @@ def get_crn(title, code, section)
end end
def full_major(major) def full_major(major)
resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_get_crse_unsec', { resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_get_crse_unsec',
body: "term_in=201870&sel_subj=dummy&sel_day=dummy&sel_schd=dummy&sel_insm=dummy&sel_camp=dummy&sel_levl=dummy&sel_sess=dummy&sel_instr=dummy&sel_ptrm=dummy&sel_attr=dummy&sel_subj=#{major}&sel_crse=&sel_title=&sel_schd=%25&sel_from_cred=&sel_to_cred=&sel_camp=%25&sel_levl=%25&sel_ptrm=%25&sel_instr=%25&begin_hh=0&begin_mi=0&begin_ap=x&end_hh=0&end_mi=0&end_ap=x", body: "term_in=201870&sel_subj=dummy&sel_day=dummy&sel_schd=dummy&sel_insm=dummy&sel_camp=dummy&sel_levl=dummy&sel_sess=dummy&sel_instr=dummy&sel_ptrm=dummy&sel_attr=dummy&sel_subj=#{major}&sel_crse=&sel_title=&sel_schd=%25&sel_from_cred=&sel_to_cred=&sel_camp=%25&sel_levl=%25&sel_ptrm=%25&sel_instr=%25&begin_hh=0&begin_mi=0&begin_ap=x&end_hh=0&end_mi=0&end_ap=x",
headers: { headers: {
'Content-Type' => 'application/x-www-form-urlencoded', 'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8' 'charset' => 'utf-8'
} })
})
searcher = Nokogiri::HTML(resp) searcher = Nokogiri::HTML(resp)
data = feed_course_info(searcher) data = feed_course_info(searcher)
end end
...@@ -117,7 +112,6 @@ def initialize_req(subj, num) ...@@ -117,7 +112,6 @@ def initialize_req(subj, num)
data = feed_course_info(searcher) data = feed_course_info(searcher)
end end
def getSemesters def getSemesters
semesters = [] semesters = []
resp = HTTParty.get('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_disp_dyn_sched') resp = HTTParty.get('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_disp_dyn_sched')
...@@ -130,17 +124,16 @@ def getSemesters ...@@ -130,17 +124,16 @@ def getSemesters
semesters semesters
end end
def getCourses(semester) def getCourses(semester)
semesters = [] semesters = []
resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckgens.p_proc_term_date', { resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckgens.p_proc_term_date',
body: "p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=#{semester}&p_by_date=Y&p_from_date=&p_to_date=", body: "p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=#{semester}&p_by_date=Y&p_from_date=&p_to_date=",
headers: { headers: {
'Content-Type' => 'application/x-www-form-urlencoded', 'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8' 'charset' => 'utf-8'
}}) })
searcher = Nokogiri::HTML(resp) searcher = Nokogiri::HTML(resp)
# puts searcher.inspect # puts searcher.inspect
searcher.xpath('//*[@id="subj_id"]/option').each do |opt| searcher.xpath('//*[@id="subj_id"]/option').each do |opt|
if opt.attr('value').strip.alpha? if opt.attr('value').strip.alpha?
semesters.push(opt.attr('value')) semesters.push(opt.attr('value'))
...@@ -149,22 +142,33 @@ def getCourses(semester) ...@@ -149,22 +142,33 @@ def getCourses(semester)
semesters semesters
end end
#Initialize threads to be waited on array # Initialize threads to be waited on array