175 lines
5.1 KiB
Python
175 lines
5.1 KiB
Python
from datetime import datetime
|
|
import fcntl
|
|
from functools import wraps
|
|
import os
|
|
from osclib.cache_manager import CacheManager
|
|
import shelve
|
|
import pickle
|
|
|
|
# Where the cache files are stored
|
|
CACHEDIR = CacheManager.directory('memoize')
|
|
|
|
|
|
def memoize(ttl=None, session=False, add_invalidate=False):
|
|
"""Decorator function to implement a persistent cache.
|
|
|
|
>>> @memoize()
|
|
... def test_func(a):
|
|
... return a
|
|
|
|
Internally, the memoized function has a cache:
|
|
|
|
>>> cache = [c.cell_contents for c in test_func.func_closure if 'sync' in dir(c.cell_contents)][0]
|
|
>>> 'sync' in dir(cache)
|
|
True
|
|
|
|
There is a limit of the size of the cache
|
|
|
|
>>> for k in cache:
|
|
... del cache[k]
|
|
>>> len(cache)
|
|
0
|
|
|
|
>>> for i in range(4095):
|
|
... test_func(i)
|
|
... len(cache)
|
|
4095
|
|
|
|
>>> test_func(0)
|
|
0
|
|
|
|
>>> len(cache)
|
|
4095
|
|
|
|
>>> test_func(4095)
|
|
4095
|
|
|
|
>>> len(cache)
|
|
3072
|
|
|
|
>>> test_func(0)
|
|
0
|
|
|
|
>>> len(cache)
|
|
3073
|
|
|
|
>>> from datetime import timedelta
|
|
>>> k = [k for k in cache if cPickle.loads(k) == ((0,), {})][0]
|
|
>>> t, v = cache[k]
|
|
>>> t = t - timedelta(days=10)
|
|
>>> cache[k] = (t, v)
|
|
>>> test_func(0)
|
|
0
|
|
>>> t2, v = cache[k]
|
|
>>> t != t2
|
|
True
|
|
|
|
"""
|
|
|
|
# Configuration variables
|
|
SLOTS = 4096 # Number of slots in the cache file
|
|
NCLEAN = 1024 # Number of slots to remove when limit reached
|
|
TIMEOUT = 60 * 60 * 2 # Time to live for every cache slot (seconds)
|
|
memoize.session_functions = []
|
|
|
|
def _memoize(fn):
|
|
# Implement a POSIX lock / unlock extension for shelves. Inspired
|
|
# on ActiveState Code recipe #576591
|
|
def _lock(filename):
|
|
lckfile = open(filename + '.lck', 'w')
|
|
fcntl.flock(lckfile.fileno(), fcntl.LOCK_EX)
|
|
return lckfile
|
|
|
|
def _unlock(lckfile):
|
|
fcntl.flock(lckfile.fileno(), fcntl.LOCK_UN)
|
|
lckfile.close()
|
|
|
|
def _open_cache(cache_name):
|
|
if not session:
|
|
lckfile = _lock(cache_name)
|
|
cache = shelve.open(cache_name, protocol=-1)
|
|
# Store a reference to the lckfile to avoid to be
|
|
# closed by gc
|
|
cache.lckfile = lckfile
|
|
else:
|
|
if not hasattr(fn, '_memoize_session_cache'):
|
|
fn._memoize_session_cache = {}
|
|
memoize.session_functions.append(fn)
|
|
cache = fn._memoize_session_cache
|
|
return cache
|
|
|
|
def _close_cache(cache):
|
|
if not session:
|
|
cache.close()
|
|
_unlock(cache.lckfile)
|
|
|
|
def _clean_cache(cache):
|
|
len_cache = len(cache)
|
|
if len_cache >= SLOTS:
|
|
nclean = NCLEAN + len_cache - SLOTS
|
|
keys_to_delete = sorted(cache, key=lambda k: cache[k][0])[:nclean]
|
|
for key in keys_to_delete:
|
|
del cache[key]
|
|
|
|
def _key(obj):
|
|
# Pickle doesn't guarantee that there is a single
|
|
# representation for every serialization. We can try to
|
|
# picke / depickle twice to have a canonical
|
|
# representation.
|
|
key = pickle.dumps(obj, protocol=-1)
|
|
key = pickle.dumps(pickle.loads(key), protocol=-1)
|
|
return key
|
|
|
|
def _invalidate(*args, **kwargs):
|
|
key = _key((args, kwargs))
|
|
cache = _open_cache(cache_name)
|
|
if key in cache:
|
|
del cache[key]
|
|
|
|
def _invalidate_all():
|
|
cache = _open_cache(cache_name)
|
|
cache.clear()
|
|
|
|
def _add_invalidate_method(_self):
|
|
name = '_invalidate_%s' % fn.__name__
|
|
if not hasattr(_self, name):
|
|
setattr(_self, name, _invalidate)
|
|
|
|
name = '_invalidate_all'
|
|
if not hasattr(_self, name):
|
|
setattr(_self, name, _invalidate_all)
|
|
|
|
@wraps(fn)
|
|
def _fn(*args, **kwargs):
|
|
def total_seconds(td):
|
|
return (td.microseconds + (td.seconds + td.days * 24 * 3600.) * 10**6) / 10**6
|
|
now = datetime.now()
|
|
if add_invalidate:
|
|
_self = args[0]
|
|
_add_invalidate_method(_self)
|
|
first = str(args[0]) if isinstance(args[0], object) else args[0]
|
|
key = _key((first, args[1:], kwargs))
|
|
updated = False
|
|
cache = _open_cache(cache_name)
|
|
if key in cache:
|
|
timestamp, value = cache[key]
|
|
updated = True if total_seconds(now - timestamp) < ttl else False
|
|
if not updated:
|
|
value = fn(*args, **kwargs)
|
|
cache[key] = (now, value)
|
|
_clean_cache(cache)
|
|
_close_cache(cache)
|
|
return value
|
|
|
|
cache_name = os.path.join(CACHEDIR, fn.__name__)
|
|
return _fn
|
|
|
|
ttl = ttl if ttl else TIMEOUT
|
|
return _memoize
|
|
|
|
|
|
def memoize_session_reset():
|
|
"""Reset all session caches."""
|
|
for i, _ in enumerate(memoize.session_functions):
|
|
memoize.session_functions[i]._memoize_session_cache = {}
|