Commit cf1112d3 authored by Landon DeCoito's avatar Landon DeCoito

Merge branch 'feature/filter-data-into-days' into 'master'

Feature/filter data into days

See merge request !32
parents 6a72b33e ad1828d8
...@@ -2,8 +2,25 @@ ...@@ -2,8 +2,25 @@
# third party imports # third party imports
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from datetime import datetime
import requests import requests
import collections
_MONTH_DICT = {
"January": 1,
"Febuary": 2,
"March": 3,
"April": 4,
"May": 5,
"June": 6,
"July": 7,
"August": 8,
"September": 9,
"October": 10,
"November": 11,
"December": 12
}
# this function cleans up some of the useless html leftovers to characters we can actually use # this function cleans up some of the useless html leftovers to characters we can actually use
def cleanup(dirtystring): def cleanup(dirtystring):
...@@ -26,7 +43,7 @@ def cleanup(dirtystring): ...@@ -26,7 +43,7 @@ def cleanup(dirtystring):
dirtystring = dirtystring.replace(replacement[0], replacement[1]) dirtystring = dirtystring.replace(replacement[0], replacement[1])
return dirtystring[:-1] return dirtystring[:-1]
# Simple event quality test # Simple event quality test
def qualityTest(desc): def qualityTest(desc):
# none, bad, okay, good # none, bad, okay, good
...@@ -42,7 +59,7 @@ def qualityTest(desc): ...@@ -42,7 +59,7 @@ def qualityTest(desc):
return "good" return "good"
elif length < 100: elif length < 100:
return "verygood" return "verygood"
else: else:
return "excellent" return "excellent"
# convertTime accepts strings in the form of "" # convertTime accepts strings in the form of ""
...@@ -80,6 +97,27 @@ def convertTime(stri): # this function is used for splicing the event times. ...@@ -80,6 +97,27 @@ def convertTime(stri): # this function is used for splicing the event times.
raise Exception("Issue with time dilation. Input string: " + stri) raise Exception("Issue with time dilation. Input string: " + stri)
def filter_data_into_days(dictlist):
day_dict = collections.OrderedDict()
for event in dictlist:
if "error" in event:
continue
event_date = "{}/{}/{}".format(event["dayofmonth"],
_MONTH_DICT[event["month"]],
event["year"])
if event_date in day_dict:
day_dict[event_date]["data"].append(event)
else:
timestamp = datetime(
int(event["year"]), _MONTH_DICT[event["month"]], int(event["dayofmonth"]))
day_dict[event_date] = {
"date": event_date, "datetime": timestamp.isoformat(' '), "year": event["year"], "month": event["month"], "dayofmonth": event["dayofmonth"], "data": [event]}
day_list = [
day_dict[day]
for day in day_dict
]
return day_list
def load_data(): def load_data():
""" """
Parses the XML from Mason and mines 2 BTC. Parses the XML from Mason and mines 2 BTC.
...@@ -257,23 +295,25 @@ def load_data(): ...@@ -257,23 +295,25 @@ def load_data():
except Exception as e: except Exception as e:
error.append("Time Dilation Error: " + str(e)) error.append("Time Dilation Error: " + str(e))
'''print "-----------------------------------------------------------------------------" # print "-----------------------------------------------------------------------------"
print location # print location
print day # print day
print month # print month
print monthday # print monthday
print year # print year
print timestart # print timestart
print timestop # print timestop
print description # print description
print "----------------------------------------------------------------------------" # print "----------------------------------------------------------------------------"
'''
if (error == []): if (error == []):
quality = qualityTest(description) quality = qualityTest(description)
dictlist.append({"id": uniqueid, "quality": quality, "title": entry_title, "dayofweek": day, "dayofmonth": monthday, "month": month, dictlist.append({"id": uniqueid, "quality": quality, "title": entry_title, "dayofweek": day, "dayofmonth": monthday, "month": month,
"year": year, "timestart": timestart, "timestop": timestop, "location": location, "description": description}) "year": year, "timestart": timestart, "timestop": timestop, "location": location, "description": description})
else: else:
dictlist.append({"id": uniqueid, "error": error}) dictlist.append({"id": uniqueid, "error": error})
return dictlist
return filter_data_into_days(dictlist)
# everything in the house is fuzzy, stupid dogs were acting like pollinators, if that's how you even spell it # everything in the house is fuzzy, stupid dogs were acting like pollinators, if that's how you even spell it
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment