Commit fb1926cd authored by Josh Roesslein's avatar Josh Roesslein
Browse files

Implemented file-based cache.

parent a518381f
......@@ -14,6 +14,7 @@ Features:
Supports most of the twitter api endpoints.
Python 3 support. (See py3k branch)
Streaming API
Cache system (memory and file based)
Getting started:
Check out the tutorial folder to get started with Tweepy.
......
......@@ -3,6 +3,7 @@ Stuff that needs to be done...
- finish unit tests
- search API
- caching system
+ in-memory cache done. now just need a file-based cache (pickle?)
- memcache? database?
+ memory and file caches implemented
- needs docs, tutors, examples, etc
- commandline client
......@@ -4,6 +4,10 @@
import time
import threading
import os
import hashlib
import fcntl
import cPickle as pickle
"""Cache interface"""
class Cache(object):
......@@ -71,10 +75,7 @@ class MemoryCache(Cache):
# use provided timeout in arguments if provided
# otherwise use the one provided during init.
if timeout is None:
_timeout = self.timeout
else:
_timeout = timeout
_timeout = self.timeout if timeout is None else timeout
# make sure entry is not expired
if self._is_expired(entry, _timeout):
......@@ -95,3 +96,89 @@ class MemoryCache(Cache):
with self.lock:
self._entries.clear()
"""File-based cache"""
class FileCache(Cache):
def __init__(self, cache_dir, timeout=60):
Cache.__init__(self, timeout)
if os.path.exists(cache_dir) is False:
os.mkdir(cache_dir)
self.cache_dir = cache_dir
self.lock = threading.Lock()
def _get_path(self, key):
md5 = hashlib.md5()
md5.update(key)
return os.path.join(self.cache_dir, md5.hexdigest())
def _lock_file(self, path, exclusive=True):
lock_path = path + '.lock'
if exclusive is True:
f_lock = open(lock_path, 'w')
fcntl.lockf(f_lock, fcntl.LOCK_EX)
else:
f_lock = open(lock_path, 'r')
fcntl.lockf(f_lock, fcntl.LOCK_SH)
if os.path.exists(lock_path) is False:
f_lock.close()
return None
return f_lock
def _delete_file(self, path):
os.remove(path)
os.remove(path + '.lock')
def store(self, key, value):
path = self._get_path(key)
with self.lock:
# acquire lock and open file
f_lock = self._lock_file(path)
datafile = open(path, 'wb')
# write data
pickle.dump((time.time(), value), datafile)
# close and unlock file
datafile.close()
f_lock.close()
def get(self, key, timeout=None):
return self._get(self._get_path(key), timeout)
def _get(self, path, timeout):
if os.path.exists(path) is False:
# no record
return None
while self.lock:
# acquire lock and open
f_lock = self._lock_file(path, False)
if f_lock is None:
# does not exist
return None
datafile = open(path, 'rb')
# read pickled object
created_time, value = pickle.load(datafile)
datafile.close()
# check if value is expired
_timeout = self.timeout if timeout is None else timeout
if _timeout > 0 and (time.time() - created_time) >= _timeout:
# expired! delete from cache
value = None
self._delete_file(path)
# unlock and return result
f_lock.close()
return value
def cleanup(self):
for entry in os.listdir(self.cache_dir):
if entry.endswith('.lock'): continue
self._get(os.path.join(self.cache_dir, entry), None)
def flush(self):
for entry in os.listdir(self.cache_dir):
if entry.endswith('.lock'): continue
self._delete_file(os.path.join(self.cache_dir, entry))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment