diff --git a/abichecker/abichecker.py b/abichecker/abichecker.py index 3bb22e0c..878a703e 100755 --- a/abichecker/abichecker.py +++ b/abichecker/abichecker.py @@ -32,7 +32,6 @@ except ImportError: import rpm from collections import namedtuple -from osclib.pkgcache import PkgCache from osclib.comments import CommentAPI from abichecker_common import CACHEDIR @@ -68,11 +67,7 @@ ARCH_WHITELIST = { } # Directory where download binary packages. -# TODO: move to CACHEDIR, just here for consistency with repochecker -BINCACHE = os.path.expanduser('~/co') -DOWNLOADS = os.path.join(BINCACHE, 'downloads') - -# Where the cache files are stored +DOWNLOADS = os.path.join(CACHEDIR, 'downloads') UNPACKDIR = os.path.join(CACHEDIR, 'unpacked') so_re = re.compile(r'^(?:/usr)?/lib(?:64)?/lib([^/]+)\.so(?:\.[^/]+)?') @@ -181,8 +176,6 @@ class ABIChecker(ReviewBot.ReviewBot): self.ts = rpm.TransactionSet() self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES) - self.pkgcache = PkgCache(BINCACHE) - # reports of source submission self.reports = [] # textual report summary for use in accept/decline message @@ -711,6 +704,7 @@ class ABIChecker(ReviewBot.ReviewBot): if r != 0: raise FetchError("failed to extract %s!"%fn) tmpfd.close() + os.unlink(downloaded[fn]) cpio = CpioRead(tmpfile) cpio.read() for ch in cpio: @@ -752,22 +746,12 @@ class ABIChecker(ReviewBot.ReviewBot): downloaded[fn] = t return downloaded - # XXX: from repochecker def _get_binary_file(self, project, repository, arch, package, filename, target, mtime): """Get a binary file from OBS.""" - # Check if the file is already there. - key = (project, repository, arch, package, filename, mtime) - if key in self.pkgcache: - try: - os.unlink(target) - except: - pass - self.pkgcache.linkto(key, target) - else: - osc.core.get_binary_file(self.apiurl, project, repository, arch, - filename, package=package, - target_filename=target) - self.pkgcache[key] = target + # Used to cache, but dropped as part of python3 migration. + osc.core.get_binary_file(self.apiurl, project, repository, arch, + filename, package=package, + target_filename=target) def readRpmHeaderFD(self, fd): h = None diff --git a/osclib/pkgcache.py b/osclib/pkgcache.py deleted file mode 100644 index 6769a8cc..00000000 --- a/osclib/pkgcache.py +++ /dev/null @@ -1,185 +0,0 @@ -import fcntl -import glob -import hashlib -import os.path -try: - import cPickle as pickle -except: - import pickle -import shelve -import shutil -import time -from UserDict import DictMixin - - -class PkgCache(DictMixin): - def __init__(self, basecachedir, force_clean=False): - self.cachedir = os.path.join(basecachedir, 'pkgcache') - self.index_fn = os.path.join(self.cachedir, 'index.db') - - if force_clean: - try: - shutil.rmtree(self.cachedir) - except OSError: - pass - - if not os.path.exists(self.cachedir): - os.makedirs(self.cachedir) - - self._clean_cache() - - def _lock(self, filename): - """Get a lock for the index file.""" - lckfile = open(filename + '.lck', 'w') - fcntl.flock(lckfile.fileno(), fcntl.LOCK_EX) - return lckfile - - def _unlock(self, lckfile): - """Release the lock for the index file.""" - fcntl.flock(lckfile.fileno(), fcntl.LOCK_UN) - lckfile.close() - - def _open_index(self): - """Open the index file for the cache / container.""" - lckfile = self._lock(self.index_fn) - index = shelve.open(self.index_fn, protocol=-1) - # Store a reference to the lckfile to avoid to be closed by gc - index.lckfile = lckfile - return index - - def _close_index(self, index): - """Close the index file for the cache / container.""" - index.close() - self._unlock(index.lckfile) - - def _clean_cache(self, ttl=14*24*60*60, index=None): - """Remove elements in the cache that share the same prefix of the key - (all except the mtime), and keep the latest one. Also remove - old entries based on the TTL. - - """ - _i = self._open_index() if index is None else index - - # Ugly hack to assure that the key is serialized always with - # the same pickle format. Pickle do not guarantee that the - # same object is serialized always in the same string. - skeys = {pickle.loads(key): key for key in _i} - keys = sorted(skeys) - - now = int(time.time()) - last = None - for key in keys: - if last and last[:-1] == key[:-1]: - self.__delitem__(key=skeys[last], skey=True, index=_i) - last = key - elif now - int(key[-1]) >= ttl: - self.__delitem__(key=skeys[key], skey=True, index=_i) - else: - last = key - - if index is None: - self._close_index(_i) - - def __getitem__(self, key, index=None): - """Get a element in the cache. - - For the container perspective, the key is a tuple like this: - (project, repository, arch, package, filename, mtime) - - """ - _i = self._open_index() if index is None else index - - key = pickle.dumps(key, protocol=-1) - value = pickle.loads(_i[key]) - - if index is None: - self._close_index(_i) - - return value - - def __setitem__(self, key, value, index=None): - """Add a new file in the cache. 'value' is expected to contains the - path of file. - - """ - _i = self._open_index() if index is None else index - - key = pickle.dumps(key, protocol=-1) - - original_value = value - - md5 = hashlib.md5(open(value, 'rb').read()).hexdigest() - filename = os.path.basename(value) - value = (md5, filename) - value = pickle.dumps(value, protocol=-1) - _i[key] = value - - # Move the file into the container using a hard link - cache_fn = os.path.join(self.cachedir, md5[:2], md5[2:]) - if os.path.exists(cache_fn): - # Manage collisions using hard links and refcount - collisions = sorted(glob.glob(cache_fn + '-*')) - next_refcount = 1 - if collisions: - next_refcount = int(collisions[-1][-3:]) + 1 - next_cache_fn = cache_fn + '-%03d' % next_refcount - os.link(cache_fn, next_cache_fn) - else: - dirname = os.path.dirname(cache_fn) - if not os.path.exists(dirname): - os.makedirs(dirname) - os.link(original_value, cache_fn) - - if index is None: - self._close_index(_i) - - def __delitem__(self, key, skey=False, index=None): - """Remove a file from the cache.""" - _i = self._open_index() if index is None else index - - key = pickle.dumps(key, protocol=-1) if not skey else key - value = pickle.loads(_i[key]) - - md5, _ = value - - # Remove the file (taking care of collision) and the directory - # if it is empty - cache_fn = os.path.join(self.cachedir, md5[:2], md5[2:]) - collisions = sorted(glob.glob(cache_fn + '-*')) - if collisions: - os.unlink(collisions[-1]) - else: - os.unlink(cache_fn) - - dirname = os.path.dirname(cache_fn) - if not os.listdir(dirname): - os.rmdir(dirname) - - del _i[key] - - if index is None: - self._close_index(_i) - - def keys(self, index=None): - _i = self._open_index() if index is None else index - - keys = [pickle.loads(key) for key in _i] - - if index is None: - self._close_index(_i) - - return keys - - def linkto(self, key, target, index=None): - """Create a link between the cached object and the target""" - _i = self._open_index() if index is None else index - - md5, filename = self.__getitem__(key, index=_i) - if filename != target: - pass - # print 'Warning. The target name (%s) is different from the original name (%s)' % (target, filename) - cache_fn = os.path.join(self.cachedir, md5[:2], md5[2:]) - os.link(cache_fn, target) - - if index is None: - self._close_index(_i) diff --git a/tests/pkgcache_tests.py b/tests/pkgcache_tests.py deleted file mode 100644 index 9402f01d..00000000 --- a/tests/pkgcache_tests.py +++ /dev/null @@ -1,117 +0,0 @@ -from __future__ import print_function - -import os -import shutil -import unittest - -from mock import MagicMock - -import osclib.pkgcache -from osclib.pkgcache import PkgCache - - -class TestPkgCache(unittest.TestCase): - def setUp(self): - """Initialize the environment.""" - self.cache = PkgCache('/tmp/cache', force_clean=True) - for fn in ('file_a', 'file_b', 'file_c'): - with open(os.path.join('/tmp', fn), 'w') as f: - print(fn, file=f) - - def tearDown(self): - """Clean the environment.""" - shutil.rmtree('/tmp/cache') - for fn in ('/tmp/file_a', '/tmp/file_b', '/tmp/file_c'): - os.unlink(fn) - - def test_insertion(self): - self.cache[('file_a', 1)] = '/tmp/file_a' - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.assertEqual(open('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a').read(), 'file_a\n') - self.cache[('file_b', 1)] = '/tmp/file_b' - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) - self.assertEqual(open('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3').read(), 'file_b\n') - self.cache[('file_c', 1)] = '/tmp/file_c' - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) - self.assertEqual(open('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f').read(), 'file_c\n') - - def test_index(self): - self.cache[('file_a', 1)] = '/tmp/file_a' - self.assertEqual(self.cache[('file_a', 1)], ('c7f33375edf32d8fb62d4b505c74519a', 'file_a')) - self.cache[('file_b', 1)] = '/tmp/file_b' - self.assertEqual(self.cache[('file_b', 1)], ('a7004efbb89078ebcc8f21d55354e2f3', 'file_b')) - self.cache[('file_c', 1)] = '/tmp/file_c' - self.assertEqual(self.cache[('file_c', 1)], ('22ee05516c08f3672cb25e03ce7f045f', 'file_c')) - self.assertEqual(set(self.cache.keys()), set((('file_a', 1), ('file_b', 1), ('file_c', 1)))) - - def test_delete(self): - self.cache[('file_a', 1)] = '/tmp/file_a' - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - del self.cache[('file_a', 1)] - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7')) - self.assertTrue(os.path.exists('/tmp/cache/pkgcache')) - - self.cache[('file_b', 1)] = '/tmp/file_b' - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) - del self.cache[('file_b', 1)] - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7')) - self.assertTrue(os.path.exists('/tmp/cache/pkgcache')) - - self.cache[('file_c', 1)] = '/tmp/file_c' - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) - del self.cache[('file_c', 1)] - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22')) - self.assertTrue(os.path.exists('/tmp/cache/pkgcache')) - - def test_collision(self): - self.cache[('file_a', 1)] = '/tmp/file_a' - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.cache[('file_a', 2)] = '/tmp/file_a' - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) - self.cache[('file_a', 3)] = '/tmp/file_a' - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) - - del self.cache[('file_a', 2)] - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) - - del self.cache[('file_a', 1)] - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) - - del self.cache[('file_a', 3)] - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) - self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) - - def test_linkto(self): - self.cache[('file_a', 1)] = '/tmp/file_a' - self.cache.linkto(('file_a', 1), '/tmp/file_a_') - self.assertEqual(open('/tmp/file_a_').read(), 'file_a\n') - - os.unlink('/tmp/file_a_') - self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) - - def test_clean(self): - self.cache[('file_a', 1)] = '/tmp/file_a' - self.cache[('file_a', 2)] = '/tmp/file_a' - self.cache[('file_a', 3)] = '/tmp/file_a' - - self.cache[('file_b', 1)] = '/tmp/file_b' - self.cache[('file_c', 1)] = '/tmp/file_c' - - osclib.pkgcache.time.time = MagicMock(return_value=3) - self.cache._clean_cache(ttl=2) - self.assertFalse(('file_a', 1) in self.cache) - self.assertFalse(('file_a', 2) in self.cache) - self.assertTrue(('file_a', 3) in self.cache) - self.assertFalse(('file_b', 1) in self.cache) - self.assertFalse(('file_c', 1) in self.cache) - -if __name__ == '__main__': - unittest.main()