Merge branch '4-get-redis' into 'master'

Resolve "Get redis"

Closes #4

See merge request !33

README.md

@@ -17,3 +17,7 @@ Step 3 - Create a web page that list out all the events and change what are pres
 1) Create a `virtualenv` if you know what that is  
 2) Install the project's dependencies by running `pip install -r requirements.txt`  
 3) Start the Flask development server by running `./start.sh`  
+
+# Documentation for the packages we use
+* Schedule https://pypi.org/project/schedule/
+* redis-py http://redis-py.readthedocs.io/en/latest/
\ No newline at end of file

mason-today/__init__.py

@@ -4,12 +4,18 @@ from flask import Response
 from flask import render_template
 
 # app imports
-from parscript import load_data
-from getconnectedscript import load_getconn_data
+from appmethods import update_both_dbs, run_schedule_loop
+from redisactions import redisdb
 
 # python imports
 import json
+import thread
 
+# other imports
+import redis
+
+
+# setting up flask instance
 app = Flask(__name__)
 
 
@@ -21,15 +27,31 @@ def display_default():
 
 @app.route("/api/25live")
 def display_data():
-    resp = Response(json.dumps(load_data(), ensure_ascii=False)
-                    .encode('utf-8'))
+    resp = Response(redisdb.get("livedict"))  # .encode('utf-8'))
     resp.headers['Content-Type'] = 'application/json; charset=utf-8'
     return resp
 
 
 @app.route("/api/getconnected")
 def display_GC_data():
-    resp = Response(json.dumps(load_getconn_data(), ensure_ascii=False)
-                    .encode('utf-8'))
+    resp = Response(redisdb.get("gcdict"))  # .encode('utf-8'))
     resp.headers['Content-Type'] = 'application/json; charset=utf-8'
     return resp
+
+
+@app.route("/api/lastupdate")
+def get_last_update():
+    resp = Response(redisdb.lindex("dbupdatelog", 0).replace("\n", "</br>"))
+    return resp
+
+
+# make sure the cacheing is set up on init
+update_both_dbs()
+
+try:
+    thread.start_new_thread(run_schedule_loop, ())
+    print "started thread!"
+except:
+    print "===================================================" \
+        + "Unable to start scheduling thread" \
+        + "==================================================="

mason-today/appmethods.py

+# app imports
+from parscript import load_data
+from getconnectedscript import load_gc_data
+import redisactions as f
+
+# python imports
+import json
+import datetime
+import time
+
+# other imports
+import redis
+import schedule
+
+
+# attempts to update both dbs and logs the result
+def update_both_dbs():
+    livesuccess = f.live_db_fill(json.dumps(load_data(), ensure_ascii=False))
+    gcsuccess = f.gc_db_fill(json.dumps(load_gc_data(), ensure_ascii=False))
+    successLog = str(datetime.datetime.now()) \
+        + "\n\nAttempted to update cache." \
+        + "\n25Live: " + str(livesuccess) \
+        + "\nGC: " + str(gcsuccess)
+
+    f.append_to_update_log(successLog)
+    return successLog
+
+
+# setting up cacheing
+# this must be done after the update_both_dbs def
+schedule.every().day.at("02:00").do(update_both_dbs)
+
+
+def run_schedule_loop():
+    runScheduler = True
+    while runScheduler:
+        schedule.run_pending()
+        time.sleep(1800)

mason-today/getconnectedscript.py

@@ -25,7 +25,7 @@ def splitAndConvertTime(strin):
     return returnlist
 
 
-def load_getconn_data():
+def load_gc_data():
     feedtext = requests.get(
             "https://getconnected.gmu.edu/events/events.rss").text
     feedtext = cleanup(feedtext)

mason-today/redisactions.py

+# python imports
+import datetime
+
+# other imports
+import redis
+
+# I'm thinking we store a couple things
+# first: a key-value where the value is the dictlist
+# second: a k-v for a list of errored events
+
+# use rpush(key, value) to append a dblist (rpushx() to check if it exists)
+# use del(key) to remove a k-v
+
+# so everytime we run parscript or gcscript we want to run a dbfill()
+# function. and every time we find an error we want to run a dberrorfill()
+# function.
+
+
+# setting up redis database
+redisdb = redis.from_url("redis://localhost:6379/0", db=0)
+
+
+# this will update the live dictlist and the cachedate
+# returns true if the dictlist is not empty, false otherwise
+def gc_db_fill(dictlist):
+    success = redisdb.set("gcdict", dictlist)
+
+    log = str(datetime.datetime.now())
+    redisdb.set("gccachedate", log)
+    success = redisdb.get("gccachedate") == log and success
+
+    return redisdb.get("gcdict") is not None and success
+
+
+# saves new dictlist in place of previous 25Live dictlist
+# returns true if the dictlist is not empty, false otherwise
+def live_db_fill(dictlist):
+    success = redisdb.set("livedict", dictlist)
+
+    log = str(datetime.datetime.now())
+    redisdb.set("livecachedate", log)
+    success = redisdb.get("livecachedate") == log and success
+
+    return redisdb.get("livedict") is not None and success
+
+
+# appends the log string to the head of our update long
+# returns true if the head is the newest update
+def append_to_update_log(logstring):
+    redisdb.lpush("dbupdatelog", logstring)
+
+    return redisdb.lindex("dbupdatelog", 0) == logstring

requirements.txt

@@ -10,8 +10,11 @@ itsdangerous==0.24
 Jinja2==2.10
 lxml==4.1.1
 MarkupSafe==1.0
+pycodestyle==2.4.0
 python-dateutil==2.6.1
+redis==2.10.6
 requests==2.18.4
+schedule==0.5.0
 six==1.11.0
 urllib3==1.22
 Werkzeug==0.14.1

start.sh

 #!/bin/sh
 export FLASK_APP=mason-today/__init__.py
-export FLASK_DEBUG=1
+export FLASK_DEBUG=0
 flask run