diff --git a/osc-check_repo.py b/osc-check_repo.py index df37374d..22eea562 100644 --- a/osc-check_repo.py +++ b/osc-check_repo.py @@ -401,7 +401,11 @@ def do_check_repo(self, subcmd, opts, *args): # Makes sure to remove the place where are the downloaded files # (is not the cache) - shutil.rmtree(DOWNLOADS) + try: + shutil.rmtree(DOWNLOADS) + except: + pass + os.makedirs(DOWNLOADS) self.checkrepo = CheckRepo(self.get_api_url(), opts.project, readonly=opts.dry, debug=opts.verbose) diff --git a/osclib/checkrepo.py b/osclib/checkrepo.py index 22583299..46abaa89 100644 --- a/osclib/checkrepo.py +++ b/osclib/checkrepo.py @@ -119,13 +119,13 @@ class Request(object): class CheckRepo(object): - def __init__(self, apiurl, opensuse='Factory', readonly=False, debug=False): + def __init__(self, apiurl, opensuse='Factory', readonly=False, force_clean=False, debug=False): """CheckRepo constructor.""" self.apiurl = apiurl self.opensuse = opensuse self.staging = StagingAPI(apiurl, opensuse) - self.pkgcache = PkgCache(BINCACHE) + self.pkgcache = PkgCache(BINCACHE, force_clean=force_clean) # grouped = { id: staging, } self.grouped = {} diff --git a/osclib/pkgcache.py b/osclib/pkgcache.py index d9cafbff..2c0bbbe3 100644 --- a/osclib/pkgcache.py +++ b/osclib/pkgcache.py @@ -24,6 +24,7 @@ except: import pickle import shelve import shutil +import time from UserDict import DictMixin @@ -41,6 +42,8 @@ class PkgCache(DictMixin): if not os.path.exists(self.cachedir): os.makedirs(self.cachedir) + self._clean_cache() + def _lock(self, filename): """Get a lock for the index file.""" lckfile = open(filename + '.lck', 'w') @@ -65,22 +68,33 @@ class PkgCache(DictMixin): index.close() self._unlock(index.lckfile) - # def _clean_cache(self, index=None): - # """Remove elements in the cache that thare the same prefix of the key - # (all except the mtime), and keep the latest one. + def _clean_cache(self, ttl=14*24*60*60, index=None): + """Remove elements in the cache that share the same prefix of the key + (all except the mtime), and keep the latest one. Also remove + old entries based on the TTL. - # """ - # _i = self._open_index() if not index else index - # keys = sorted(_i) + """ + _i = self._open_index() if index is None else index - # last = [] - # for key in keys: - # if last[:-1] == key[:-1]: - # self.__delitem__(key, _i) - # last = key + # Ugly hack to assure that the key is serialized always with + # the same pickle format. Pickle do not guarantee that the + # same object is serialized always in the same string. + skeys = {pickle.loads(key): key for key in _i} + keys = sorted(skeys) - # if not index: - # self._close_index(_i) + now = int(time.time()) + last = None + for key in keys: + if last and last[:-1] == key[:-1]: + self.__delitem__(key=skeys[last], skey=True, index=_i) + last = key + elif now - key[-1] >= ttl: + self.__delitem__(key=skeys[key], skey=True, index=_i) + else: + last = key + + if index is None: + self._close_index(_i) def __getitem__(self, key, index=None): """Get a element in the cache. @@ -89,11 +103,14 @@ class PkgCache(DictMixin): (project, repository, arch, package, filename, mtime) """ - _i = self._open_index() if not index else index + _i = self._open_index() if index is None else index + key = pickle.dumps(key, protocol=-1) value = pickle.loads(_i[key]) - if not index: + + if index is None: self._close_index(_i) + return value def __setitem__(self, key, value, index=None): @@ -101,7 +118,8 @@ class PkgCache(DictMixin): path of file. """ - _i = self._open_index() if not index else index + _i = self._open_index() if index is None else index + key = pickle.dumps(key, protocol=-1) original_value = value @@ -128,13 +146,14 @@ class PkgCache(DictMixin): os.makedirs(dirname) os.link(original_value, cache_fn) - if not index: + if index is None: self._close_index(_i) - def __delitem__(self, key, index=None): + def __delitem__(self, key, skey=False, index=None): """Remove a file from the cache.""" - _i = self._open_index() if not index else index - key = pickle.dumps(key, protocol=-1) + _i = self._open_index() if index is None else index + + key = pickle.dumps(key, protocol=-1) if not skey else key value = pickle.loads(_i[key]) md5, _ = value @@ -154,19 +173,22 @@ class PkgCache(DictMixin): del _i[key] - if not index: + if index is None: self._close_index(_i) def keys(self, index=None): - _i = self._open_index() if not index else index + _i = self._open_index() if index is None else index + keys = [pickle.loads(key) for key in _i] - if not index: + + if index is None: self._close_index(_i) + return keys def linkto(self, key, target, index=None): """Create a link between the cached object and the target""" - _i = self._open_index() if not index else index + _i = self._open_index() if index is None else index md5, filename = self.__getitem__(key, index=_i) if filename != target: @@ -175,6 +197,5 @@ class PkgCache(DictMixin): cache_fn = os.path.join(self.cachedir, md5[:2], md5[2:]) os.link(cache_fn, target) - if not index: + if index is None: self._close_index(_i) - diff --git a/tests/checkrepo_tests.py b/tests/checkrepo_tests.py index f0bad77f..363f8fad 100644 --- a/tests/checkrepo_tests.py +++ b/tests/checkrepo_tests.py @@ -31,7 +31,7 @@ class TestCheckRepoCalls(unittest.TestCase): """Initialize the configuration.""" self.obs = OBS() - self.checkrepo = CheckRepo(APIURL) + self.checkrepo = CheckRepo(APIURL, force_clean=True) # Des-memoize some functions self.checkrepo.build = self.checkrepo._build self.checkrepo.last_build_success = self.checkrepo._last_build_success diff --git a/tests/pkgcache_tests.py b/tests/pkgcache_tests.py index 9cc268fe..891290d6 100644 --- a/tests/pkgcache_tests.py +++ b/tests/pkgcache_tests.py @@ -8,96 +8,114 @@ import os import shutil import unittest +from mock import MagicMock + +import osclib.pkgcache from osclib.pkgcache import PkgCache class TestPkgCache(unittest.TestCase): def setUp(self): - """Initialize the environment""" - self.cache = PkgCache('cache', force_clean=True) + """Initialize the environment.""" + self.cache = PkgCache('/tmp/cache', force_clean=True) for fn in ('file_a', 'file_b', 'file_c'): - with open(fn, 'w') as f: + with open(os.path.join('/tmp', fn), 'w') as f: print >>f, fn def tearDown(self): - """Clean the environment""" - shutil.rmtree('cache') - for fn in ('file_a', 'file_b', 'file_c'): + """Clean the environment.""" + shutil.rmtree('/tmp/cache') + for fn in ('/tmp/file_a', '/tmp/file_b', '/tmp/file_c'): os.unlink(fn) def test_insertion(self): - self.cache[('file_a', 1)] = 'file_a' - self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.assertEqual(open('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a').read(), 'file_a\n') - self.cache[('file_b', 1)] = 'file_b' - self.assertTrue(os.path.exists('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) - self.assertEqual(open('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3').read(), 'file_b\n') - self.cache[('file_c', 1)] = 'file_c' - self.assertTrue(os.path.exists('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) - self.assertEqual(open('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f').read(), 'file_c\n') + self.cache[('file_a', 1)] = '/tmp/file_a' + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) + self.assertEqual(open('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a').read(), 'file_a\n') + self.cache[('file_b', 1)] = '/tmp/file_b' + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) + self.assertEqual(open('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3').read(), 'file_b\n') + self.cache[('file_c', 1)] = '/tmp/file_c' + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) + self.assertEqual(open('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f').read(), 'file_c\n') def test_index(self): - self.cache[('file_a', 1)] = 'file_a' + self.cache[('file_a', 1)] = '/tmp/file_a' self.assertEqual(self.cache[('file_a', 1)], ('c7f33375edf32d8fb62d4b505c74519a', 'file_a')) - self.cache[('file_b', 1)] = 'file_b' + self.cache[('file_b', 1)] = '/tmp/file_b' self.assertEqual(self.cache[('file_b', 1)], ('a7004efbb89078ebcc8f21d55354e2f3', 'file_b')) - self.cache[('file_c', 1)] = 'file_c' + self.cache[('file_c', 1)] = '/tmp/file_c' self.assertEqual(self.cache[('file_c', 1)], ('22ee05516c08f3672cb25e03ce7f045f', 'file_c')) self.assertEqual(set(self.cache.keys()), set((('file_a', 1), ('file_b', 1), ('file_c', 1)))) def test_delete(self): - self.cache[('file_a', 1)] = 'file_a' - self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) + self.cache[('file_a', 1)] = '/tmp/file_a' + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) del self.cache[('file_a', 1)] - self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.assertFalse(os.path.exists('cache/pkgcache/c7')) - self.assertTrue(os.path.exists('cache/pkgcache')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7')) + self.assertTrue(os.path.exists('/tmp/cache/pkgcache')) - self.cache[('file_b', 1)] = 'file_b' - self.assertTrue(os.path.exists('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) + self.cache[('file_b', 1)] = '/tmp/file_b' + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) del self.cache[('file_b', 1)] - self.assertFalse(os.path.exists('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) - self.assertFalse(os.path.exists('cache/pkgcache/a7')) - self.assertTrue(os.path.exists('cache/pkgcache')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7')) + self.assertTrue(os.path.exists('/tmp/cache/pkgcache')) - self.cache[('file_c', 1)] = 'file_c' - self.assertTrue(os.path.exists('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) + self.cache[('file_c', 1)] = '/tmp/file_c' + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) del self.cache[('file_c', 1)] - self.assertFalse(os.path.exists('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) - self.assertFalse(os.path.exists('cache/pkgcache/22')) - self.assertTrue(os.path.exists('cache/pkgcache')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22')) + self.assertTrue(os.path.exists('/tmp/cache/pkgcache')) def test_collision(self): - self.cache[('file_a', 1)] = 'file_a' - self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.cache[('file_a', 2)] = 'file_a' - self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) - self.cache[('file_a', 3)] = 'file_a' - self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) + self.cache[('file_a', 1)] = '/tmp/file_a' + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) + self.cache[('file_a', 2)] = '/tmp/file_a' + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) + self.cache[('file_a', 3)] = '/tmp/file_a' + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) del self.cache[('file_a', 2)] - self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) - self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) del self.cache[('file_a', 1)] - self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) - self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) del self.cache[('file_a', 3)] - self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) - self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) + self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) def test_linkto(self): - self.cache[('file_a', 1)] = 'file_a' - self.cache.linkto(('file_a', 1), 'file_a-1') - self.assertEqual(open('file_a-1').read(), 'file_a\n') + self.cache[('file_a', 1)] = '/tmp/file_a' + self.cache.linkto(('file_a', 1), '/tmp/file_a_') + self.assertEqual(open('/tmp/file_a_').read(), 'file_a\n') - os.unlink('file_a-1') - self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) + os.unlink('/tmp/file_a_') + self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) + def test_clean(self): + self.cache[('file_a', 1)] = '/tmp/file_a' + self.cache[('file_a', 2)] = '/tmp/file_a' + self.cache[('file_a', 3)] = '/tmp/file_a' + + self.cache[('file_b', 1)] = '/tmp/file_b' + self.cache[('file_c', 1)] = '/tmp/file_c' + + osclib.pkgcache.time.time = MagicMock(return_value=3) + self.cache._clean_cache(ttl=2) + self.assertFalse(('file_a', 1) in self.cache) + self.assertFalse(('file_a', 2) in self.cache) + self.assertTrue(('file_a', 3) in self.cache) + self.assertFalse(('file_b', 1) in self.cache) + self.assertFalse(('file_c', 1) in self.cache) if __name__ == '__main__': unittest.main()