yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve clean_cache method. Add a test.

Browse Source
This commit is contained in:
Alberto Planas 2014-10-14 13:06:32 +02:00
parent 816414949c
commit 4af48f4c96
4 changed files with 121 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
osclib/checkrepo.py
View File

@ -119,13 +119,13 @@ class Request(object):
class CheckRepo(object):
def __init__(self, apiurl, opensuse='Factory', readonly=False, debug=False):
def __init__(self, apiurl, opensuse='Factory', readonly=False, force_clean=False, debug=False):
"""CheckRepo constructor."""
self.apiurl = apiurl
self.opensuse = opensuse
self.staging = StagingAPI(apiurl, opensuse)
self.pkgcache = PkgCache(BINCACHE)
self.pkgcache = PkgCache(BINCACHE, force_clean=force_clean)
# grouped = { id: staging, }
self.grouped = {}

73
osclib/pkgcache.py
View File

@ -24,6 +24,7 @@ except:
import pickle
import shelve
import shutil
import time
from UserDict import DictMixin
@ -41,6 +42,8 @@ class PkgCache(DictMixin):
if not os.path.exists(self.cachedir):
os.makedirs(self.cachedir)
self._clean_cache()
def _lock(self, filename):
"""Get a lock for the index file."""
lckfile = open(filename + '.lck', 'w')
@ -65,22 +68,33 @@ class PkgCache(DictMixin):
index.close()
self._unlock(index.lckfile)
# def _clean_cache(self, index=None):
# """Remove elements in the cache that thare the same prefix of the key
# (all except the mtime), and keep the latest one.
def _clean_cache(self, ttl=14*24*60*60, index=None):
"""Remove elements in the cache that share the same prefix of the key
(all except the mtime), and keep the latest one. Also remove
old entries based on the TTL.
# """
# _i = self._open_index() if not index else index
# keys = sorted(_i)
"""
_i = self._open_index() if index is None else index
# last = []
# for key in keys:
# if last[:-1] == key[:-1]:
# self.__delitem__(key, _i)
# last = key
# Ugly hack to assure that the key is serialized always with
# the same pickle format. Pickle do not guarantee that the
# same object is serialized always in the same string.
skeys = {pickle.loads(key): key for key in _i}
keys = sorted(skeys)
# if not index:
# self._close_index(_i)
now = int(time.time())
last = None
for key in keys:
if last and last[:-1] == key[:-1]:
self.__delitem__(key=skeys[last], skey=True, index=_i)
last = key
elif now - key[-1] >= ttl:
self.__delitem__(key=skeys[key], skey=True, index=_i)
else:
last = key
if index is None:
self._close_index(_i)
def __getitem__(self, key, index=None):
"""Get a element in the cache.
@ -89,11 +103,14 @@ class PkgCache(DictMixin):
(project, repository, arch, package, filename, mtime)
"""
_i = self._open_index() if not index else index
_i = self._open_index() if index is None else index
key = pickle.dumps(key, protocol=-1)
value = pickle.loads(_i[key])
if not index:
if index is None:
self._close_index(_i)
return value
def __setitem__(self, key, value, index=None):
@ -101,7 +118,8 @@ class PkgCache(DictMixin):
path of file.
"""
_i = self._open_index() if not index else index
_i = self._open_index() if index is None else index
key = pickle.dumps(key, protocol=-1)
original_value = value
@ -128,13 +146,14 @@ class PkgCache(DictMixin):
os.makedirs(dirname)
os.link(original_value, cache_fn)
if not index:
if index is None:
self._close_index(_i)
def __delitem__(self, key, index=None):
def __delitem__(self, key, skey=False, index=None):
"""Remove a file from the cache."""
_i = self._open_index() if not index else index
key = pickle.dumps(key, protocol=-1)
_i = self._open_index() if index is None else index
key = pickle.dumps(key, protocol=-1) if not skey else key
value = pickle.loads(_i[key])
md5, _ = value
@ -154,19 +173,22 @@ class PkgCache(DictMixin):
del _i[key]
if not index:
if index is None:
self._close_index(_i)
def keys(self, index=None):
_i = self._open_index() if not index else index
_i = self._open_index() if index is None else index
keys = [pickle.loads(key) for key in _i]
if not index:
if index is None:
self._close_index(_i)
return keys
def linkto(self, key, target, index=None):
"""Create a link between the cached object and the target"""
_i = self._open_index() if not index else index
_i = self._open_index() if index is None else index
md5, filename = self.__getitem__(key, index=_i)
if filename != target:
@ -175,6 +197,5 @@ class PkgCache(DictMixin):
cache_fn = os.path.join(self.cachedir, md5[:2], md5[2:])
os.link(cache_fn, target)
if not index:
if index is None:
self._close_index(_i)

2
tests/checkrepo_tests.py
View File

@ -31,7 +31,7 @@ class TestCheckRepoCalls(unittest.TestCase):
"""Initialize the configuration."""
self.obs = OBS()
self.checkrepo = CheckRepo(APIURL)
self.checkrepo = CheckRepo(APIURL, force_clean=True)
# Des-memoize some functions
self.checkrepo.build = self.checkrepo._build
self.checkrepo.last_build_success = self.checkrepo._last_build_success

124
tests/pkgcache_tests.py
View File

@ -8,96 +8,114 @@ import os
import shutil
import unittest
from mock import MagicMock
import osclib.pkgcache
from osclib.pkgcache import PkgCache
class TestPkgCache(unittest.TestCase):
def setUp(self):
"""Initialize the environment"""
self.cache = PkgCache('cache', force_clean=True)
"""Initialize the environment."""
self.cache = PkgCache('/tmp/cache', force_clean=True)
for fn in ('file_a', 'file_b', 'file_c'):
with open(fn, 'w') as f:
with open(os.path.join('/tmp', fn), 'w') as f:
print >>f, fn
def tearDown(self):
"""Clean the environment"""
shutil.rmtree('cache')
for fn in ('file_a', 'file_b', 'file_c'):
"""Clean the environment."""
shutil.rmtree('/tmp/cache')
for fn in ('/tmp/file_a', '/tmp/file_b', '/tmp/file_c'):
os.unlink(fn)
def test_insertion(self):
self.cache[('file_a', 1)] = 'file_a'
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertEqual(open('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a').read(), 'file_a\n')
self.cache[('file_b', 1)] = 'file_b'
self.assertTrue(os.path.exists('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
self.assertEqual(open('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3').read(), 'file_b\n')
self.cache[('file_c', 1)] = 'file_c'
self.assertTrue(os.path.exists('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
self.assertEqual(open('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f').read(), 'file_c\n')
self.cache[('file_a', 1)] = '/tmp/file_a'
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertEqual(open('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a').read(), 'file_a\n')
self.cache[('file_b', 1)] = '/tmp/file_b'
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
self.assertEqual(open('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3').read(), 'file_b\n')
self.cache[('file_c', 1)] = '/tmp/file_c'
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
self.assertEqual(open('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f').read(), 'file_c\n')
def test_index(self):
self.cache[('file_a', 1)] = 'file_a'
self.cache[('file_a', 1)] = '/tmp/file_a'
self.assertEqual(self.cache[('file_a', 1)], ('c7f33375edf32d8fb62d4b505c74519a', 'file_a'))
self.cache[('file_b', 1)] = 'file_b'
self.cache[('file_b', 1)] = '/tmp/file_b'
self.assertEqual(self.cache[('file_b', 1)], ('a7004efbb89078ebcc8f21d55354e2f3', 'file_b'))
self.cache[('file_c', 1)] = 'file_c'
self.cache[('file_c', 1)] = '/tmp/file_c'
self.assertEqual(self.cache[('file_c', 1)], ('22ee05516c08f3672cb25e03ce7f045f', 'file_c'))
self.assertEqual(set(self.cache.keys()), set((('file_a', 1), ('file_b', 1), ('file_c', 1))))
def test_delete(self):
self.cache[('file_a', 1)] = 'file_a'
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.cache[('file_a', 1)] = '/tmp/file_a'
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
del self.cache[('file_a', 1)]
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertFalse(os.path.exists('cache/pkgcache/c7'))
self.assertTrue(os.path.exists('cache/pkgcache'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7'))
self.assertTrue(os.path.exists('/tmp/cache/pkgcache'))
self.cache[('file_b', 1)] = 'file_b'
self.assertTrue(os.path.exists('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
self.cache[('file_b', 1)] = '/tmp/file_b'
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
del self.cache[('file_b', 1)]
self.assertFalse(os.path.exists('cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
self.assertFalse(os.path.exists('cache/pkgcache/a7'))
self.assertTrue(os.path.exists('cache/pkgcache'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7'))
self.assertTrue(os.path.exists('/tmp/cache/pkgcache'))
self.cache[('file_c', 1)] = 'file_c'
self.assertTrue(os.path.exists('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
self.cache[('file_c', 1)] = '/tmp/file_c'
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
del self.cache[('file_c', 1)]
self.assertFalse(os.path.exists('cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
self.assertFalse(os.path.exists('cache/pkgcache/22'))
self.assertTrue(os.path.exists('cache/pkgcache'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22'))
self.assertTrue(os.path.exists('/tmp/cache/pkgcache'))
def test_collision(self):
self.cache[('file_a', 1)] = 'file_a'
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.cache[('file_a', 2)] = 'file_a'
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.cache[('file_a', 3)] = 'file_a'
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
self.cache[('file_a', 1)] = '/tmp/file_a'
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.cache[('file_a', 2)] = '/tmp/file_a'
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.cache[('file_a', 3)] = '/tmp/file_a'
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
del self.cache[('file_a', 2)]
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
del self.cache[('file_a', 1)]
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
del self.cache[('file_a', 3)]
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.assertFalse(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
def test_linkto(self):
self.cache[('file_a', 1)] = 'file_a'
self.cache.linkto(('file_a', 1), 'file_a-1')
self.assertEqual(open('file_a-1').read(), 'file_a\n')
self.cache[('file_a', 1)] = '/tmp/file_a'
self.cache.linkto(('file_a', 1), '/tmp/file_a_')
self.assertEqual(open('/tmp/file_a_').read(), 'file_a\n')
os.unlink('file_a-1')
self.assertTrue(os.path.exists('cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
os.unlink('/tmp/file_a_')
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
def test_clean(self):
self.cache[('file_a', 1)] = '/tmp/file_a'
self.cache[('file_a', 2)] = '/tmp/file_a'
self.cache[('file_a', 3)] = '/tmp/file_a'
self.cache[('file_b', 1)] = '/tmp/file_b'
self.cache[('file_c', 1)] = '/tmp/file_c'
osclib.pkgcache.time.time = MagicMock(return_value=3)
self.cache._clean_cache(ttl=2)
self.assertFalse(('file_a', 1) in self.cache)
self.assertFalse(('file_a', 2) in self.cache)
self.assertTrue(('file_a', 3) in self.cache)
self.assertFalse(('file_b', 1) in self.cache)
self.assertFalse(('file_c', 1) in self.cache)
if __name__ == '__main__':
unittest.main()