Commit 0f69a429 authored by Landon DeCoito's avatar Landon DeCoito

Merge branch '4-get-redis' into 'master'

Resolve "Get redis"

Closes #4

See merge request !33
parents cf1112d3 72cfd150
...@@ -17,3 +17,7 @@ Step 3 - Create a web page that list out all the events and change what are pres ...@@ -17,3 +17,7 @@ Step 3 - Create a web page that list out all the events and change what are pres
1) Create a `virtualenv` if you know what that is 1) Create a `virtualenv` if you know what that is
2) Install the project's dependencies by running `pip install -r requirements.txt` 2) Install the project's dependencies by running `pip install -r requirements.txt`
3) Start the Flask development server by running `./start.sh` 3) Start the Flask development server by running `./start.sh`
# Documentation for the packages we use
* Schedule https://pypi.org/project/schedule/
* redis-py http://redis-py.readthedocs.io/en/latest/
\ No newline at end of file
...@@ -4,12 +4,18 @@ from flask import Response ...@@ -4,12 +4,18 @@ from flask import Response
from flask import render_template from flask import render_template
# app imports # app imports
from parscript import load_data from appmethods import update_both_dbs, run_schedule_loop
from getconnectedscript import load_getconn_data from redisactions import redisdb
# python imports # python imports
import json import json
import thread
# other imports
import redis
# setting up flask instance
app = Flask(__name__) app = Flask(__name__)
...@@ -21,15 +27,31 @@ def display_default(): ...@@ -21,15 +27,31 @@ def display_default():
@app.route("/api/25live") @app.route("/api/25live")
def display_data(): def display_data():
resp = Response(json.dumps(load_data(), ensure_ascii=False) resp = Response(redisdb.get("livedict")) # .encode('utf-8'))
.encode('utf-8'))
resp.headers['Content-Type'] = 'application/json; charset=utf-8' resp.headers['Content-Type'] = 'application/json; charset=utf-8'
return resp return resp
@app.route("/api/getconnected") @app.route("/api/getconnected")
def display_GC_data(): def display_GC_data():
resp = Response(json.dumps(load_getconn_data(), ensure_ascii=False) resp = Response(redisdb.get("gcdict")) # .encode('utf-8'))
.encode('utf-8'))
resp.headers['Content-Type'] = 'application/json; charset=utf-8' resp.headers['Content-Type'] = 'application/json; charset=utf-8'
return resp return resp
@app.route("/api/lastupdate")
def get_last_update():
resp = Response(redisdb.lindex("dbupdatelog", 0).replace("\n", "</br>"))
return resp
# make sure the cacheing is set up on init
update_both_dbs()
try:
thread.start_new_thread(run_schedule_loop, ())
print "started thread!"
except:
print "===================================================" \
+ "Unable to start scheduling thread" \
+ "==================================================="
# app imports
from parscript import load_data
from getconnectedscript import load_gc_data
import redisactions as f
# python imports
import json
import datetime
import time
# other imports
import redis
import schedule
# attempts to update both dbs and logs the result
def update_both_dbs():
livesuccess = f.live_db_fill(json.dumps(load_data(), ensure_ascii=False))
gcsuccess = f.gc_db_fill(json.dumps(load_gc_data(), ensure_ascii=False))
successLog = str(datetime.datetime.now()) \
+ "\n\nAttempted to update cache." \
+ "\n25Live: " + str(livesuccess) \
+ "\nGC: " + str(gcsuccess)
f.append_to_update_log(successLog)
return successLog
# setting up cacheing
# this must be done after the update_both_dbs def
schedule.every().day.at("02:00").do(update_both_dbs)
def run_schedule_loop():
runScheduler = True
while runScheduler:
schedule.run_pending()
time.sleep(1800)
...@@ -25,7 +25,7 @@ def splitAndConvertTime(strin): ...@@ -25,7 +25,7 @@ def splitAndConvertTime(strin):
return returnlist return returnlist
def load_getconn_data(): def load_gc_data():
feedtext = requests.get( feedtext = requests.get(
"https://getconnected.gmu.edu/events/events.rss").text "https://getconnected.gmu.edu/events/events.rss").text
feedtext = cleanup(feedtext) feedtext = cleanup(feedtext)
......
# python imports
import datetime
# other imports
import redis
# I'm thinking we store a couple things
# first: a key-value where the value is the dictlist
# second: a k-v for a list of errored events
# use rpush(key, value) to append a dblist (rpushx() to check if it exists)
# use del(key) to remove a k-v
# so everytime we run parscript or gcscript we want to run a dbfill()
# function. and every time we find an error we want to run a dberrorfill()
# function.
# setting up redis database
redisdb = redis.from_url("redis://localhost:6379/0", db=0)
# this will update the live dictlist and the cachedate
# returns true if the dictlist is not empty, false otherwise
def gc_db_fill(dictlist):
success = redisdb.set("gcdict", dictlist)
log = str(datetime.datetime.now())
redisdb.set("gccachedate", log)
success = redisdb.get("gccachedate") == log and success
return redisdb.get("gcdict") is not None and success
# saves new dictlist in place of previous 25Live dictlist
# returns true if the dictlist is not empty, false otherwise
def live_db_fill(dictlist):
success = redisdb.set("livedict", dictlist)
log = str(datetime.datetime.now())
redisdb.set("livecachedate", log)
success = redisdb.get("livecachedate") == log and success
return redisdb.get("livedict") is not None and success
# appends the log string to the head of our update long
# returns true if the head is the newest update
def append_to_update_log(logstring):
redisdb.lpush("dbupdatelog", logstring)
return redisdb.lindex("dbupdatelog", 0) == logstring
...@@ -10,8 +10,11 @@ itsdangerous==0.24 ...@@ -10,8 +10,11 @@ itsdangerous==0.24
Jinja2==2.10 Jinja2==2.10
lxml==4.1.1 lxml==4.1.1
MarkupSafe==1.0 MarkupSafe==1.0
pycodestyle==2.4.0
python-dateutil==2.6.1 python-dateutil==2.6.1
redis==2.10.6
requests==2.18.4 requests==2.18.4
schedule==0.5.0
six==1.11.0 six==1.11.0
urllib3==1.22 urllib3==1.22
Werkzeug==0.14.1 Werkzeug==0.14.1
#!/bin/sh #!/bin/sh
export FLASK_APP=mason-today/__init__.py export FLASK_APP=mason-today/__init__.py
export FLASK_DEBUG=1 export FLASK_DEBUG=0
flask run flask run
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment