yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve clean_cache method. Add a test.

Browse Source
This commit is contained in:
Alberto Planas 2014-10-14 13:06:32 +02:00
parent 816414949c
commit 4af48f4c96
4 changed files with 121 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
osclib/checkrepo.py
View File

@ -119,13 +119,13 @@ class Request(object):
class CheckRepo(object): class CheckRepo(object):
def __init__(self, apiurl, opensuse='Factory', readonly=False, debug=False): def __init__(self, apiurl, opensuse='Factory', readonly=False, force_clean=False, debug=False):
"""CheckRepo constructor.""" """CheckRepo constructor."""
self.apiurl = apiurl self.apiurl = apiurl
self.opensuse = opensuse self.opensuse = opensuse
self.staging = StagingAPI(apiurl, opensuse) self.staging = StagingAPI(apiurl, opensuse)
self.pkgcache = PkgCache(BINCACHE) self.pkgcache = PkgCache(BINCACHE, force_clean=force_clean)
# grouped = { id: staging, } # grouped = { id: staging, }
self.grouped = {} self.grouped = {}

73
osclib/pkgcache.py
View File

@ -24,6 +24,7 @@ except:
import pickle import pickle
import shelve import shelve
import shutil import shutil
import time
from UserDict import DictMixin from UserDict import DictMixin
@ -41,6 +42,8 @@ class PkgCache(DictMixin):
if not os.path.exists(self.cachedir): if not os.path.exists(self.cachedir):
os.makedirs(self.cachedir) os.makedirs(self.cachedir)
self._clean_cache()
def _lock(self, filename): def _lock(self, filename):
"""Get a lock for the index file.""" """Get a lock for the index file."""
lckfile = open(filename + '.lck', 'w') lckfile = open(filename + '.lck', 'w')
@ -65,22 +68,33 @@ class PkgCache(DictMixin):
index.close() index.close()
self._unlock(index.lckfile) self._unlock(index.lckfile)
# def _clean_cache(self, index=None): def _clean_cache(self, ttl=14*24*60*60, index=None):
# """Remove elements in the cache that thare the same prefix of the key """Remove elements in the cache that share the same prefix of the key
# (all except the mtime), and keep the latest one. (all except the mtime), and keep the latest one. Also remove
old entries based on the TTL.
# """ """
# _i = self._open_index() if not index else index _i = self._open_index() if index is None else index
# keys = sorted(_i)
# last = [] # Ugly hack to assure that the key is serialized always with
# for key in keys: # the same pickle format. Pickle do not guarantee that the
# if last[:-1] == key[:-1]: # same object is serialized always in the same string.
# self.__delitem__(key, _i) skeys = {pickle.loads(key): key for key in _i}
# last = key keys = sorted(skeys)
# if not index: now = int(time.time())
# self._close_index(_i) last = None
for key in keys:
if last and last[:-1] == key[:-1]:
self.__delitem__(key=skeys[last], skey=True, index=_i)
last = key
elif now - key[-1] >= ttl:
self.__delitem__(key=skeys[key], skey=True, index=_i)
else:
last = key
if index is None:
self._close_index(_i)
def __getitem__(self, key, index=None): def __getitem__(self, key, index=None):
"""Get a element in the cache. """Get a element in the cache.
@ -89,11 +103,14 @@ class PkgCache(DictMixin):
(project, repository, arch, package, filename, mtime) (project, repository, arch, package, filename, mtime)
""" """
_i = self._open_index() if not index else index _i = self._open_index() if index is None else index
key = pickle.dumps(key, protocol=-1) key = pickle.dumps(key, protocol=-1)
value = pickle.loads(_i[key]) value = pickle.loads(_i[key])
if not index:
if index is None:
self._close_index(_i) self._close_index(_i)
return value return value
def __setitem__(self, key, value, index=None): def __setitem__(self, key, value, index=None):
@ -101,7 +118,8 @@ class PkgCache(DictMixin):
path of file. path of file.
""" """
_i = self._open_index() if not index else index _i = self._open_index() if index is None else index
key = pickle.dumps(key, protocol=-1) key = pickle.dumps(key, protocol=-1)
original_value = value original_value = value
@ -128,13 +146,14 @@ class PkgCache(DictMixin):
os.makedirs(dirname) os.makedirs(dirname)
os.link(original_value, cache_fn) os.link(original_value, cache_fn)
if not index: if index is None:
self._close_index(_i) self._close_index(_i)
def __delitem__(self, key, index=None): def __delitem__(self, key, skey=False, index=None):
"""Remove a file from the cache.""" """Remove a file from the cache."""
_i = self._open_index() if not index else index _i = self._open_index() if index is None else index
key = pickle.dumps(key, protocol=-1)
key = pickle.dumps(key, protocol=-1) if not skey else key
value = pickle.loads(_i[key]) value = pickle.loads(_i[key])
md5, _ = value md5, _ = value
@ -154,19 +173,22 @@ class PkgCache(DictMixin):
del _i[key] del _i[key]
if not index: if index is None:
self._close_index(_i) self._close_index(_i)
def keys(self, index=None): def keys(self, index=None):
_i = self._open_index() if not index else index _i = self._open_index() if index is None else index
keys = [pickle.loads(key) for key in _i] keys = [pickle.loads(key) for key in _i]
if not index:
if index is None:
self._close_index(_i) self._close_index(_i)
return keys return keys
def linkto(self, key, target, index=None): def linkto(self, key, target, index=None):
"""Create a link between the cached object and the target""" """Create a link between the cached object and the target"""
_i = self._open_index() if not index else index _i = self._open_index() if index is None else index
md5, filename = self.__getitem__(key, index=_i) md5, filename = self.__getitem__(key, index=_i)
if filename != target: if filename != target:
@ -175,6 +197,5 @@ class PkgCache(DictMixin):
cache_fn = os.path.join(self.cachedir, md5[:2], md5[2:]) cache_fn = os.path.join(self.cachedir, md5[:2], md5[2:])
os.link(cache_fn, target) os.link(cache_fn, target)
if not index: if index is None:
self._close_index(_i) self._close_index(_i)

2
tests/checkrepo_tests.py
View File

@ -31,7 +31,7 @@ class TestCheckRepoCalls(unittest.TestCase):
"""Initialize the configuration.""" """Initialize the configuration."""
self.obs = OBS() self.obs = OBS()
self.checkrepo = CheckRepo(APIURL) self.checkrepo = CheckRepo(APIURL, force_clean=True)
# Des-memoize some functions # Des-memoize some functions
self.checkrepo.build = self.checkrepo._build self.checkrepo.build = self.checkrepo._build
self.checkrepo.last_build_success = self.checkrepo._last_build_success self.checkrepo.last_build_success = self.checkrepo._last_build_success

124
tests/pkgcache_tests.py
View File

@ -8,96 +8,114 @@ import os
import shutil import shutil
import unittest import unittest
from mock import MagicMock
import osclib.pkgcache
from osclib.pkgcache import PkgCache from osclib.pkgcache import PkgCache
class TestPkgCache(unittest.TestCase): class TestPkgCache(unittest.TestCase):
def setUp(self): def setUp(self):
"""Initialize the environment""" """Initialize the environment."""
self.cache = PkgCache('cache', force_clean=True) self.cache = PkgCache('/tmp/cache', force_clean=True)
for fn in ('file_a', 'file_b', 'file_c'): for fn in ('file_a', 'file_b', 'file_c'):
with open(fn, 'w') as f: with open(os.path.join('/tmp', fn), 'w') as f:
print >>f, fn print >>f, fn
def tearDown(self): def tearDown(self):
"""Clean the environment""" """Clean the environment."""
shutil.rmtree('cache') shutil.rmtree('/tmp/cache')
for fn in ('file_a', 'file_b', 'file_c'): for fn in ('/tmp/file_a', '/tmp/file_b', '/tmp/file_c'):
os.unlink(fn) os.unlink(fn)
def test_insertion(self): def test_insertion(self):
self.cache[('file_a', 1)] = 'file_a' self.cache[('file_a', 1)] = '/tmp/file_a'
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertEqual(open('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a').read(), 'file_a\n') self.assertEqual(open('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a').read(), 'file_a\n')
self.cache[('file_b', 1)] = 'file_b' self.cache[('file_b', 1)] = '/tmp/file_b'
self.assertTrue(os.path.exists('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
self.assertEqual(open('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3').read(), 'file_b\n') self.assertEqual(open('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3').read(), 'file_b\n')
self.cache[('file_c', 1)] = 'file_c' self.cache[('file_c', 1)] = '/tmp/file_c'
self.assertTrue(os.path.exists('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
self.assertEqual(open('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f').read(), 'file_c\n') self.assertEqual(open('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f').read(), 'file_c\n')
def test_index(self): def test_index(self):
self.cache[('file_a', 1)] = 'file_a' self.cache[('file_a', 1)] = '/tmp/file_a'
self.assertEqual(self.cache[('file_a', 1)], ('c7f33375edf32d8fb62d4b505c74519a', 'file_a')) self.assertEqual(self.cache[('file_a', 1)], ('c7f33375edf32d8fb62d4b505c74519a', 'file_a'))
self.cache[('file_b', 1)] = 'file_b' self.cache[('file_b', 1)] = '/tmp/file_b'
self.assertEqual(self.cache[('file_b', 1)], ('a7004efbb89078ebcc8f21d55354e2f3', 'file_b')) self.assertEqual(self.cache[('file_b', 1)], ('a7004efbb89078ebcc8f21d55354e2f3', 'file_b'))
self.cache[('file_c', 1)] = 'file_c' self.cache[('file_c', 1)] = '/tmp/file_c'
self.assertEqual(self.cache[('file_c', 1)], ('22ee05516c08f3672cb25e03ce7f045f', 'file_c')) self.assertEqual(self.cache[('file_c', 1)], ('22ee05516c08f3672cb25e03ce7f045f', 'file_c'))
self.assertEqual(set(self.cache.keys()), set((('file_a', 1), ('file_b', 1), ('file_c', 1)))) self.assertEqual(set(self.cache.keys()), set((('file_a', 1), ('file_b', 1), ('file_c', 1))))
def test_delete(self): def test_delete(self):
self.cache[('file_a', 1)] = 'file_a' self.cache[('file_a', 1)] = '/tmp/file_a'
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
del self.cache[('file_a', 1)] del self.cache[('file_a', 1)]
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertFalse(os.path.exists('cache/pkgcache/c7')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7'))
self.assertTrue(os.path.exists('cache/pkgcache')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache'))
self.cache[('file_b', 1)] = 'file_b' self.cache[('file_b', 1)] = '/tmp/file_b'
self.assertTrue(os.path.exists('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
del self.cache[('file_b', 1)] del self.cache[('file_b', 1)]
self.assertFalse(os.path.exists('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
self.assertFalse(os.path.exists('cache/pkgcache/a7')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7'))
self.assertTrue(os.path.exists('cache/pkgcache')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache'))
self.cache[('file_c', 1)] = 'file_c' self.cache[('file_c', 1)] = '/tmp/file_c'
self.assertTrue(os.path.exists('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
del self.cache[('file_c', 1)] del self.cache[('file_c', 1)]
self.assertFalse(os.path.exists('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
self.assertFalse(os.path.exists('cache/pkgcache/22')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22'))
self.assertTrue(os.path.exists('cache/pkgcache')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache'))
def test_collision(self): def test_collision(self):
self.cache[('file_a', 1)] = 'file_a' self.cache[('file_a', 1)] = '/tmp/file_a'
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.cache[('file_a', 2)] = 'file_a' self.cache[('file_a', 2)] = '/tmp/file_a'
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.cache[('file_a', 3)] = 'file_a' self.cache[('file_a', 3)] = '/tmp/file_a'
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
del self.cache[('file_a', 2)] del self.cache[('file_a', 2)]
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
del self.cache[('file_a', 1)] del self.cache[('file_a', 1)]
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
del self.cache[('file_a', 3)] del self.cache[('file_a', 3)]
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
def test_linkto(self): def test_linkto(self):
self.cache[('file_a', 1)] = 'file_a' self.cache[('file_a', 1)] = '/tmp/file_a'
self.cache.linkto(('file_a', 1), 'file_a-1') self.cache.linkto(('file_a', 1), '/tmp/file_a_')
self.assertEqual(open('file_a-1').read(), 'file_a\n') self.assertEqual(open('/tmp/file_a_').read(), 'file_a\n')
os.unlink('file_a-1') os.unlink('/tmp/file_a_')
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
def test_clean(self):
self.cache[('file_a', 1)] = '/tmp/file_a'
self.cache[('file_a', 2)] = '/tmp/file_a'
self.cache[('file_a', 3)] = '/tmp/file_a'
self.cache[('file_b', 1)] = '/tmp/file_b'
self.cache[('file_c', 1)] = '/tmp/file_c'
osclib.pkgcache.time.time = MagicMock(return_value=3)
self.cache._clean_cache(ttl=2)
self.assertFalse(('file_a', 1) in self.cache)
self.assertFalse(('file_a', 2) in self.cache)
self.assertTrue(('file_a', 3) in self.cache)
self.assertFalse(('file_b', 1) in self.cache)
self.assertFalse(('file_c', 1) in self.cache)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()