Commit 683bbe4d authored by Zac Wood's avatar Zac Wood

SQLification

parent e3a17955
inherit_from:
- .rubocop_autogen.yml # Some auto-generated Rails code we shouldn't worry about
- .rubocop_relaxed.yml
AllCops:
TargetRubyVersion: 2.5.0
Style/FrozenStringLiteralComment:
Enabled: false
Style/SymbolArray:
EnforcedStyle: brackets # [:symbol1, :symbol2]
# This configuration was generated by
# `rubocop --auto-gen-config`
# on 2018-04-09 23:19:33 -0400 using RuboCop version 0.54.0.
# The point is for the user to remove these configuration records
# one by one as the offenses are removed from the code base.
# Note that changes in the inspected code, or installation of new
# versions of RuboCop, may require this file to be generated again.
# Offense count: 2
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle.
# SupportedStyles: empty_lines, no_empty_lines
Layout/EmptyLinesAroundBlockBody:
Exclude:
- 'db/schema.rb'
# Offense count: 2
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBrackets.
# SupportedStyles: space, no_space, compact
# SupportedStylesForEmptyBrackets: space, no_space
Layout/SpaceInsideArrayLiteralBrackets:
Exclude:
- 'config/environments/production.rb'
# Offense count: 1
# Configuration parameters: CountComments, ExcludedMethods.
# Metrics/BlockLength:
# Max: 35
Metrics/MethodLength:
Exclude:
- 'db/migrate/*'
Metrics/BlockLength:
Exclude:
- 'db/schema.rb'
# Offense count: 1
# Cop supports --auto-correct.
# Configuration parameters: AutoCorrect, EnforcedStyle.
# SupportedStyles: nested, compact
Style/ClassAndModuleChildren:
Exclude:
- 'test/test_helper.rb'
# Offense count: 12
Style/Documentation:
Exclude:
- 'spec/**/*'
- 'test/**/*'
- 'app/helpers/*'
- 'app/mailers/application_mailer.rb'
- 'app/models/application_record.rb'
- 'config/application.rb'
- 'db/migrate/20180407190422_create_semesters.rb'
- 'db/migrate/20180407190502_create_courses.rb'
- 'db/migrate/20180407190750_create_sections.rb'
# Offense count: 4
# Cop supports --auto-correct.
Style/ExpandPathArguments:
Exclude:
- 'bin/bundle'
- 'bin/setup'
- 'bin/update'
- 'test/test_helper.rb'
# Offense count: 2
Style/MixinUsage:
Exclude:
- 'bin/setup'
- 'bin/update'
# Offense count: 1
# Cop supports --auto-correct.
# Configuration parameters: Strict.
Style/NumericLiterals:
MinDigits: 15
# Offense count: 1
# Cop supports --auto-correct.
# Configuration parameters: PreferredDelimiters.
Style/PercentLiteralDelimiters:
Exclude:
- 'config/spring.rb'
# Offense count: 2
# Cop supports --auto-correct.
Style/StderrPuts:
Exclude:
- 'bin/yarn'
# Offense count: 47
# Cop supports --auto-correct.
# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
# SupportedStyles: single_quotes, double_quotes
Style/StringLiterals:
Exclude:
- 'Gemfile'
- 'bin/yarn'
- 'config/environments/production.rb'
- 'config/puma.rb'
- 'db/schema.rb'
- 'test/application_system_test_case.rb'
Rails/FilePath:
Exclude:
- "config/environments/development.rb" # auto generated file
# Relaxed.Ruby.Style
## Version 2.2
Style/Alias:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylealias
Style/AsciiComments:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleasciicomments
Style/BeginBlock:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylebeginblock
Style/BlockDelimiters:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleblockdelimiters
Style/CommentAnnotation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylecommentannotation
Style/Documentation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styledocumentation
Layout/DotPosition:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutdotposition
Style/DoubleNegation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styledoublenegation
Style/EndBlock:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleendblock
Style/FormatString:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleformatstring
Style/IfUnlessModifier:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleifunlessmodifier
Style/Lambda:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylelambda
Style/ModuleFunction:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylemodulefunction
Style/MultilineBlockChain:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylemultilineblockchain
Style/NegatedIf:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylenegatedif
Style/NegatedWhile:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylenegatedwhile
Style/ParallelAssignment:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleparallelassignment
Style/PercentLiteralDelimiters:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylepercentliteraldelimiters
Style/PerlBackrefs:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleperlbackrefs
Style/Semicolon:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesemicolon
Style/SignalException:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesignalexception
Style/SingleLineBlockParams:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesinglelineblockparams
Style/SingleLineMethods:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesinglelinemethods
Layout/SpaceBeforeBlockBraces:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutspacebeforeblockbraces
Layout/SpaceInsideParens:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutspaceinsideparens
Style/SpecialGlobalVars:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylespecialglobalvars
Style/StringLiterals:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylestringliterals
Style/TrailingCommaInArguments:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainarguments
Style/TrailingCommaInArrayLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainarrayliteral
Style/TrailingCommaInHashLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainhashliteral
Style/WhileUntilModifier:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylewhileuntilmodifier
Style/WordArray:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylewordarray
Lint/AmbiguousRegexpLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#lintambiguousregexpliteral
Lint/AssignmentInCondition:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#lintassignmentincondition
Metrics/AbcSize:
Enabled: false
Metrics/BlockNesting:
Enabled: false
Metrics/ClassLength:
Enabled: false
Metrics/ModuleLength:
Enabled: false
Metrics/CyclomaticComplexity:
Enabled: false
Metrics/LineLength:
Enabled: false
Metrics/MethodLength:
Enabled: false
Metrics/ParameterLists:
Enabled: false
Metrics/PerceivedComplexity:
Enabled: false
Style/Alias:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylealias
Style/AsciiComments:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleasciicomments
Style/BeginBlock:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylebeginblock
Style/BlockDelimiters:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleblockdelimiters
Style/CommentAnnotation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylecommentannotation
Style/Documentation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styledocumentation
Layout/DotPosition:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutdotposition
Style/DoubleNegation:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styledoublenegation
Style/EndBlock:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleendblock
Style/FormatString:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleformatstring
Style/IfUnlessModifier:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleifunlessmodifier
Style/Lambda:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylelambda
Style/ModuleFunction:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylemodulefunction
Style/MultilineBlockChain:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylemultilineblockchain
Style/NegatedIf:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylenegatedif
Style/NegatedWhile:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylenegatedwhile
Style/ParallelAssignment:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleparallelassignment
Style/PercentLiteralDelimiters:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylepercentliteraldelimiters
Style/PerlBackrefs:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styleperlbackrefs
Style/Semicolon:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesemicolon
Style/SignalException:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesignalexception
Style/SingleLineBlockParams:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesinglelineblockparams
Style/SingleLineMethods:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylesinglelinemethods
Layout/SpaceBeforeBlockBraces:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutspacebeforeblockbraces
Layout/SpaceInsideParens:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#layoutspaceinsideparens
Style/SpecialGlobalVars:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylespecialglobalvars
Style/StringLiterals:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylestringliterals
Style/TrailingCommaInArguments:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainarguments
Style/TrailingCommaInArrayLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainarrayliteral
Style/TrailingCommaInHashLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#styletrailingcommainhashliteral
Style/WhileUntilModifier:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylewhileuntilmodifier
Style/WordArray:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#stylewordarray
Lint/AmbiguousRegexpLiteral:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#lintambiguousregexpliteral
Lint/AssignmentInCondition:
Enabled: false
StyleGuide: http://relaxed.ruby.style/#lintassignmentincondition
Metrics/AbcSize:
Enabled: false
Metrics/BlockNesting:
Enabled: false
Metrics/ClassLength:
Enabled: false
Metrics/ModuleLength:
Enabled: false
Metrics/CyclomaticComplexity:
Enabled: false
Metrics/LineLength:
Enabled: false
Metrics/MethodLength:
Enabled: false
Metrics/ParameterLists:
Enabled: false
Metrics/PerceivedComplexity:
Enabled: false
source 'https://rubygems.org'
gem 'nokogiri'
gem 'httparty'
gem 'nokogiri'
GEM
remote: https://rubygems.org/
specs:
httparty (0.16.2)
multi_xml (>= 0.5.2)
mini_portile2 (2.3.0)
multi_xml (0.6.0)
nokogiri (1.8.2)
mini_portile2 (~> 2.3.0)
PLATFORMS
ruby
DEPENDENCIES
httparty
nokogiri
BUNDLED WITH
1.16.1
# frozen_string_literal: true
require 'thread'
require 'thwait'
require 'httparty'
require 'nokogiri'
require 'json'
=begin
USAGE:
Just run it and it dynamically dumps the latest semester. There's a bit to do it for all of the ones in history commented out below but it'll thrash your RAM and probably piss off PatriotWeb. Also note this script could be trivially modified to correlate human readable names to semester IDs since they're just the .text attribute of the option node.
There's a few minor issues like multiple spaces in teacher names and we could be scraping out email addresses but no major ones.
DISCLAIMER/WARNING:
This opens a number of connections pretty transparently from a script to PatriotWeb. I am not liable if you run this a million times and somehow kill over PatriotWeb. It's a scraper, not a DoS utility.
=end
#
# USAGE:
#
# Just run it and it dynamically dumps the latest semester. There's a bit to do it for all of the ones in history commented out below but it'll thrash your RAM and probably piss off PatriotWeb. Also note this script could be trivially modified to correlate human readable names to semester IDs since they're just the .text attribute of the option node.
#
# There's a few minor issues like multiple spaces in teacher names and we could be scraping out email addresses but no major ones.
#
# DISCLAIMER/WARNING:
#
# This opens a number of connections pretty transparently from a script to PatriotWeb. I am not liable if you run this a million times and somehow kill over PatriotWeb. It's a scraper, not a DoS utility.
#
# Credit stackoverflow
class String
......@@ -38,23 +35,22 @@ def get_details(data, titledetails, titledata)
[data, data[crn]]
end
def sort_item(item, currentobj, data)
if item.name == 'th'
if item.to_html.include? '-'
titletxt = item.text
if item.text.include? ' - Honors'
titletxt = titletxt.gsub(' - Honors',' (Honors)')
titletxt = titletxt.gsub(' - Honors', ' (Honors)')
end
titledetails = titletxt.split(' - ')
if titledetails.count > 4
titledetails = ["#{titledetails[0]} #{titledetails[1]}",titledetails[2],titledetails[3],titledetails[4]]
titledetails = ["#{titledetails[0]} #{titledetails[1]}", titledetails[2], titledetails[3], titledetails[4]]
end
titledata = titledetails[2].split(' ')
begin
data = get_details(data, titledetails, titledata)[0]
currentobj = get_details(data, titledetails, titledata)[1]
rescue => e
rescue StandardError => e
puts item
puts e
exit(1)
......@@ -98,13 +94,12 @@ def get_crn(title, code, section)
end
def full_major(major)
resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_get_crse_unsec', {
body: "term_in=201870&sel_subj=dummy&sel_day=dummy&sel_schd=dummy&sel_insm=dummy&sel_camp=dummy&sel_levl=dummy&sel_sess=dummy&sel_instr=dummy&sel_ptrm=dummy&sel_attr=dummy&sel_subj=#{major}&sel_crse=&sel_title=&sel_schd=%25&sel_from_cred=&sel_to_cred=&sel_camp=%25&sel_levl=%25&sel_ptrm=%25&sel_instr=%25&begin_hh=0&begin_mi=0&begin_ap=x&end_hh=0&end_mi=0&end_ap=x",
headers: {
'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8'
}
})
resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_get_crse_unsec',
body: "term_in=201870&sel_subj=dummy&sel_day=dummy&sel_schd=dummy&sel_insm=dummy&sel_camp=dummy&sel_levl=dummy&sel_sess=dummy&sel_instr=dummy&sel_ptrm=dummy&sel_attr=dummy&sel_subj=#{major}&sel_crse=&sel_title=&sel_schd=%25&sel_from_cred=&sel_to_cred=&sel_camp=%25&sel_levl=%25&sel_ptrm=%25&sel_instr=%25&begin_hh=0&begin_mi=0&begin_ap=x&end_hh=0&end_mi=0&end_ap=x",
headers: {
'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8'
})
searcher = Nokogiri::HTML(resp)
data = feed_course_info(searcher)
end
......@@ -117,7 +112,6 @@ def initialize_req(subj, num)
data = feed_course_info(searcher)
end
def getSemesters
semesters = []
resp = HTTParty.get('https://patriotweb.gmu.edu/pls/prod/bwckschd.p_disp_dyn_sched')
......@@ -130,17 +124,16 @@ def getSemesters
semesters
end
def getCourses(semester)
semesters = []
resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckgens.p_proc_term_date', {
body: "p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=#{semester}&p_by_date=Y&p_from_date=&p_to_date=",
headers: {
'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8'
}})
resp = HTTParty.post('https://patriotweb.gmu.edu/pls/prod/bwckgens.p_proc_term_date',
body: "p_calling_proc=bwckschd.p_disp_dyn_sched&p_term=#{semester}&p_by_date=Y&p_from_date=&p_to_date=",
headers: {
'Content-Type' => 'application/x-www-form-urlencoded',
'charset' => 'utf-8'
})
searcher = Nokogiri::HTML(resp)
# puts searcher.inspect
# puts searcher.inspect
searcher.xpath('//*[@id="subj_id"]/option').each do |opt|
if opt.attr('value').strip.alpha?
semesters.push(opt.attr('value'))
......@@ -149,22 +142,33 @@ def getCourses(semester)
semesters
end
#Initialize threads to be waited on array
# Initialize threads to be waited on array
threads = []
total = {}
#below will get you literally all semesters which is wildly overkill
#getSemesters.each do |semester|
semester = getSemesters.first
getCourses(semester).each do |course|
threads << Thread.new {
total[course] = full_major(course)
}
end
#end
# below will get you literally all semesters which is wildly overkill
# getSemesters.each do |semester|
semester = getSemesters.first
getCourses(semester).each do |course|
threads << Thread.new {
total[course] = full_major(course)
}
end
# end
ThreadsWait.all_waits(*threads)
puts total.to_json
# total.each { |subject|
# puts subject.first
# subject[1].each { |section|
# puts section
# }
# }
total.each { |subject|
subject[1].each { |crn|
section = crn[1]
puts section
# puts "#{section[:subj]} #{section[:code]} #{section[:sect]} #{section[:name]}"
}
}
......@@ -50,4 +50,6 @@ end
# Windows does not include zoneinfo files, so bundle the tzinfo-data gem
gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
gem 'httparty'
gem 'nokogiri'
gem 'rubyXL'
......@@ -60,6 +60,8 @@ GEM
ffi (1.9.23)
globalid (0.4.1)
activesupport (>= 4.2.0)
httparty (0.16.2)
multi_xml (>= 0.5.2)
i18n (1.0.0)
concurrent-ruby (~> 1.0)
jbuilder (2.7.0)
......@@ -79,6 +81,7 @@ GEM
mini_portile2 (2.3.0)
minitest (5.11.3)
multi_json (1.13.1)
multi_xml (0.6.0)
nio4r (2.3.0)