Merge pull request #1970 from jberry-suse/abichecker-drop-pkgcache
abichecker: remove PkgCache usage and drop PkgCache.
This commit is contained in:
Stephan Kulow
GitHub
commit
491142d2fa
@ -32,7 +32,6 @@ except ImportError:
|
||||
|
||||
import rpm
|
||||
from collections import namedtuple
|
||||
from osclib.pkgcache import PkgCache
|
||||
from osclib.comments import CommentAPI
|
||||
|
||||
from abichecker_common import CACHEDIR
|
||||
@ -68,11 +67,7 @@ ARCH_WHITELIST = {
|
||||
}
|
||||
|
||||
# Directory where download binary packages.
|
||||
# TODO: move to CACHEDIR, just here for consistency with repochecker
|
||||
BINCACHE = os.path.expanduser('~/co')
|
||||
DOWNLOADS = os.path.join(BINCACHE, 'downloads')
|
||||
|
||||
# Where the cache files are stored
|
||||
DOWNLOADS = os.path.join(CACHEDIR, 'downloads')
|
||||
UNPACKDIR = os.path.join(CACHEDIR, 'unpacked')
|
||||
|
||||
so_re = re.compile(r'^(?:/usr)?/lib(?:64)?/lib([^/]+)\.so(?:\.[^/]+)?')
|
||||
@ -181,8 +176,6 @@ class ABIChecker(ReviewBot.ReviewBot):
|
||||
self.ts = rpm.TransactionSet()
|
||||
self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
|
||||
|
||||
self.pkgcache = PkgCache(BINCACHE)
|
||||
|
||||
# reports of source submission
|
||||
self.reports = []
|
||||
# textual report summary for use in accept/decline message
|
||||
@ -711,6 +704,7 @@ class ABIChecker(ReviewBot.ReviewBot):
|
||||
if r != 0:
|
||||
raise FetchError("failed to extract %s!"%fn)
|
||||
tmpfd.close()
|
||||
os.unlink(downloaded[fn])
|
||||
cpio = CpioRead(tmpfile)
|
||||
cpio.read()
|
||||
for ch in cpio:
|
||||
@ -752,22 +746,12 @@ class ABIChecker(ReviewBot.ReviewBot):
|
||||
downloaded[fn] = t
|
||||
return downloaded
|
||||
|
||||
# XXX: from repochecker
|
||||
def _get_binary_file(self, project, repository, arch, package, filename, target, mtime):
|
||||
"""Get a binary file from OBS."""
|
||||
# Check if the file is already there.
|
||||
key = (project, repository, arch, package, filename, mtime)
|
||||
if key in self.pkgcache:
|
||||
try:
|
||||
os.unlink(target)
|
||||
except:
|
||||
pass
|
||||
self.pkgcache.linkto(key, target)
|
||||
else:
|
||||
osc.core.get_binary_file(self.apiurl, project, repository, arch,
|
||||
filename, package=package,
|
||||
target_filename=target)
|
||||
self.pkgcache[key] = target
|
||||
# Used to cache, but dropped as part of python3 migration.
|
||||
osc.core.get_binary_file(self.apiurl, project, repository, arch,
|
||||
filename, package=package,
|
||||
target_filename=target)
|
||||
|
||||
def readRpmHeaderFD(self, fd):
|
||||
h = None
|
||||
|
||||
@ -1,185 +0,0 @@
|
||||
import fcntl
|
||||
import glob
|
||||
import hashlib
|
||||
import os.path
|
||||
try:
|
||||
import cPickle as pickle
|
||||
except:
|
||||
import pickle
|
||||
import shelve
|
||||
import shutil
|
||||
import time
|
||||
from UserDict import DictMixin
|
||||
|
||||
|
||||
class PkgCache(DictMixin):
|
||||
def __init__(self, basecachedir, force_clean=False):
|
||||
self.cachedir = os.path.join(basecachedir, 'pkgcache')
|
||||
self.index_fn = os.path.join(self.cachedir, 'index.db')
|
||||
|
||||
if force_clean:
|
||||
try:
|
||||
shutil.rmtree(self.cachedir)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if not os.path.exists(self.cachedir):
|
||||
os.makedirs(self.cachedir)
|
||||
|
||||
self._clean_cache()
|
||||
|
||||
def _lock(self, filename):
|
||||
"""Get a lock for the index file."""
|
||||
lckfile = open(filename + '.lck', 'w')
|
||||
fcntl.flock(lckfile.fileno(), fcntl.LOCK_EX)
|
||||
return lckfile
|
||||
|
||||
def _unlock(self, lckfile):
|
||||
"""Release the lock for the index file."""
|
||||
fcntl.flock(lckfile.fileno(), fcntl.LOCK_UN)
|
||||
lckfile.close()
|
||||
|
||||
def _open_index(self):
|
||||
"""Open the index file for the cache / container."""
|
||||
lckfile = self._lock(self.index_fn)
|
||||
index = shelve.open(self.index_fn, protocol=-1)
|
||||
# Store a reference to the lckfile to avoid to be closed by gc
|
||||
index.lckfile = lckfile
|
||||
return index
|
||||
|
||||
def _close_index(self, index):
|
||||
"""Close the index file for the cache / container."""
|
||||
index.close()
|
||||
self._unlock(index.lckfile)
|
||||
|
||||
def _clean_cache(self, ttl=14*24*60*60, index=None):
|
||||
"""Remove elements in the cache that share the same prefix of the key
|
||||
(all except the mtime), and keep the latest one. Also remove
|
||||
old entries based on the TTL.
|
||||
|
||||
"""
|
||||
_i = self._open_index() if index is None else index
|
||||
|
||||
# Ugly hack to assure that the key is serialized always with
|
||||
# the same pickle format. Pickle do not guarantee that the
|
||||
# same object is serialized always in the same string.
|
||||
skeys = {pickle.loads(key): key for key in _i}
|
||||
keys = sorted(skeys)
|
||||
|
||||
now = int(time.time())
|
||||
last = None
|
||||
for key in keys:
|
||||
if last and last[:-1] == key[:-1]:
|
||||
self.__delitem__(key=skeys[last], skey=True, index=_i)
|
||||
last = key
|
||||
elif now - int(key[-1]) >= ttl:
|
||||
self.__delitem__(key=skeys[key], skey=True, index=_i)
|
||||
else:
|
||||
last = key
|
||||
|
||||
if index is None:
|
||||
self._close_index(_i)
|
||||
|
||||
def __getitem__(self, key, index=None):
|
||||
"""Get a element in the cache.
|
||||
|
||||
For the container perspective, the key is a tuple like this:
|
||||
(project, repository, arch, package, filename, mtime)
|
||||
|
||||
"""
|
||||
_i = self._open_index() if index is None else index
|
||||
|
||||
key = pickle.dumps(key, protocol=-1)
|
||||
value = pickle.loads(_i[key])
|
||||
|
||||
if index is None:
|
||||
self._close_index(_i)
|
||||
|
||||
return value
|
||||
|
||||
def __setitem__(self, key, value, index=None):
|
||||
"""Add a new file in the cache. 'value' is expected to contains the
|
||||
path of file.
|
||||
|
||||
"""
|
||||
_i = self._open_index() if index is None else index
|
||||
|
||||
key = pickle.dumps(key, protocol=-1)
|
||||
|
||||
original_value = value
|
||||
|
||||
md5 = hashlib.md5(open(value, 'rb').read()).hexdigest()
|
||||
filename = os.path.basename(value)
|
||||
value = (md5, filename)
|
||||
value = pickle.dumps(value, protocol=-1)
|
||||
_i[key] = value
|
||||
|
||||
# Move the file into the container using a hard link
|
||||
cache_fn = os.path.join(self.cachedir, md5[:2], md5[2:])
|
||||
if os.path.exists(cache_fn):
|
||||
# Manage collisions using hard links and refcount
|
||||
collisions = sorted(glob.glob(cache_fn + '-*'))
|
||||
next_refcount = 1
|
||||
if collisions:
|
||||
next_refcount = int(collisions[-1][-3:]) + 1
|
||||
next_cache_fn = cache_fn + '-%03d' % next_refcount
|
||||
os.link(cache_fn, next_cache_fn)
|
||||
else:
|
||||
dirname = os.path.dirname(cache_fn)
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
os.link(original_value, cache_fn)
|
||||
|
||||
if index is None:
|
||||
self._close_index(_i)
|
||||
|
||||
def __delitem__(self, key, skey=False, index=None):
|
||||
"""Remove a file from the cache."""
|
||||
_i = self._open_index() if index is None else index
|
||||
|
||||
key = pickle.dumps(key, protocol=-1) if not skey else key
|
||||
value = pickle.loads(_i[key])
|
||||
|
||||
md5, _ = value
|
||||
|
||||
# Remove the file (taking care of collision) and the directory
|
||||
# if it is empty
|
||||
cache_fn = os.path.join(self.cachedir, md5[:2], md5[2:])
|
||||
collisions = sorted(glob.glob(cache_fn + '-*'))
|
||||
if collisions:
|
||||
os.unlink(collisions[-1])
|
||||
else:
|
||||
os.unlink(cache_fn)
|
||||
|
||||
dirname = os.path.dirname(cache_fn)
|
||||
if not os.listdir(dirname):
|
||||
os.rmdir(dirname)
|
||||
|
||||
del _i[key]
|
||||
|
||||
if index is None:
|
||||
self._close_index(_i)
|
||||
|
||||
def keys(self, index=None):
|
||||
_i = self._open_index() if index is None else index
|
||||
|
||||
keys = [pickle.loads(key) for key in _i]
|
||||
|
||||
if index is None:
|
||||
self._close_index(_i)
|
||||
|
||||
return keys
|
||||
|
||||
def linkto(self, key, target, index=None):
|
||||
"""Create a link between the cached object and the target"""
|
||||
_i = self._open_index() if index is None else index
|
||||
|
||||
md5, filename = self.__getitem__(key, index=_i)
|
||||
if filename != target:
|
||||
pass
|
||||
# print 'Warning. The target name (%s) is different from the original name (%s)' % (target, filename)
|
||||
cache_fn = os.path.join(self.cachedir, md5[:2], md5[2:])
|
||||
os.link(cache_fn, target)
|
||||
|
||||
if index is None:
|
||||
self._close_index(_i)
|
||||
@ -1,117 +0,0 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import unittest
|
||||
|
||||
from mock import MagicMock
|
||||
|
||||
import osclib.pkgcache
|
||||
from osclib.pkgcache import PkgCache
|
||||
|
||||
|
||||
class TestPkgCache(unittest.TestCase):
|
||||
def setUp(self):
|
||||
"""Initialize the environment."""
|
||||
self.cache = PkgCache('/tmp/cache', force_clean=True)
|
||||
for fn in ('file_a', 'file_b', 'file_c'):
|
||||
with open(os.path.join('/tmp', fn), 'w') as f:
|
||||
print(fn, file=f)
|
||||
|
||||
def tearDown(self):
|
||||
"""Clean the environment."""
|
||||
shutil.rmtree('/tmp/cache')
|
||||
for fn in ('/tmp/file_a', '/tmp/file_b', '/tmp/file_c'):
|
||||
os.unlink(fn)
|
||||
|
||||
def test_insertion(self):
|
||||
self.cache[('file_a', 1)] = '/tmp/file_a'
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
|
||||
self.assertEqual(open('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a').read(), 'file_a\n')
|
||||
self.cache[('file_b', 1)] = '/tmp/file_b'
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
|
||||
self.assertEqual(open('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3').read(), 'file_b\n')
|
||||
self.cache[('file_c', 1)] = '/tmp/file_c'
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
|
||||
self.assertEqual(open('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f').read(), 'file_c\n')
|
||||
|
||||
def test_index(self):
|
||||
self.cache[('file_a', 1)] = '/tmp/file_a'
|
||||
self.assertEqual(self.cache[('file_a', 1)], ('c7f33375edf32d8fb62d4b505c74519a', 'file_a'))
|
||||
self.cache[('file_b', 1)] = '/tmp/file_b'
|
||||
self.assertEqual(self.cache[('file_b', 1)], ('a7004efbb89078ebcc8f21d55354e2f3', 'file_b'))
|
||||
self.cache[('file_c', 1)] = '/tmp/file_c'
|
||||
self.assertEqual(self.cache[('file_c', 1)], ('22ee05516c08f3672cb25e03ce7f045f', 'file_c'))
|
||||
self.assertEqual(set(self.cache.keys()), set((('file_a', 1), ('file_b', 1), ('file_c', 1))))
|
||||
|
||||
def test_delete(self):
|
||||
self.cache[('file_a', 1)] = '/tmp/file_a'
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
|
||||
del self.cache[('file_a', 1)]
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7'))
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache'))
|
||||
|
||||
self.cache[('file_b', 1)] = '/tmp/file_b'
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
|
||||
del self.cache[('file_b', 1)]
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3'))
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7'))
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache'))
|
||||
|
||||
self.cache[('file_c', 1)] = '/tmp/file_c'
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
|
||||
del self.cache[('file_c', 1)]
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f'))
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22'))
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache'))
|
||||
|
||||
def test_collision(self):
|
||||
self.cache[('file_a', 1)] = '/tmp/file_a'
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
|
||||
self.cache[('file_a', 2)] = '/tmp/file_a'
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
|
||||
self.cache[('file_a', 3)] = '/tmp/file_a'
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
|
||||
|
||||
del self.cache[('file_a', 2)]
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
|
||||
|
||||
del self.cache[('file_a', 1)]
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
|
||||
|
||||
del self.cache[('file_a', 3)]
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001'))
|
||||
self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002'))
|
||||
|
||||
def test_linkto(self):
|
||||
self.cache[('file_a', 1)] = '/tmp/file_a'
|
||||
self.cache.linkto(('file_a', 1), '/tmp/file_a_')
|
||||
self.assertEqual(open('/tmp/file_a_').read(), 'file_a\n')
|
||||
|
||||
os.unlink('/tmp/file_a_')
|
||||
self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a'))
|
||||
|
||||
def test_clean(self):
|
||||
self.cache[('file_a', 1)] = '/tmp/file_a'
|
||||
self.cache[('file_a', 2)] = '/tmp/file_a'
|
||||
self.cache[('file_a', 3)] = '/tmp/file_a'
|
||||
|
||||
self.cache[('file_b', 1)] = '/tmp/file_b'
|
||||
self.cache[('file_c', 1)] = '/tmp/file_c'
|
||||
|
||||
osclib.pkgcache.time.time = MagicMock(return_value=3)
|
||||
self.cache._clean_cache(ttl=2)
|
||||
self.assertFalse(('file_a', 1) in self.cache)
|
||||
self.assertFalse(('file_a', 2) in self.cache)
|
||||
self.assertTrue(('file_a', 3) in self.cache)
|
||||
self.assertFalse(('file_b', 1) in self.cache)
|
||||
self.assertFalse(('file_c', 1) in self.cache)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
x
Reference in New Issue
Block a user