From 482da28be67a7aaefbc1c6a1a4977b6b918b6b24 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Mon, 29 Sep 2014 14:04:38 +0200 Subject: [PATCH 1/2] Do not download requests without RPM. --- osclib/checkrepo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/osclib/checkrepo.py b/osclib/checkrepo.py index c169d359..15b67b75 100644 --- a/osclib/checkrepo.py +++ b/osclib/checkrepo.py @@ -595,6 +595,10 @@ class CheckRepo(object): request.downloads[(_project, _repo, disturl)].append(file_in_disturl) + # Some subpackage do not have any rpm (e.g. rpmlint) + if not last_disturldir: + return + for _project, _repo, arch, fn, mt in todownload_rest: repodir = os.path.join(DOWNLOADS, request.src_package, _project, _repo) if not os.path.exists(repodir): From 503ca7f6f2507f8d8f3cbbd2fc416f0a96ebb2d7 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Mon, 29 Sep 2014 16:28:11 +0200 Subject: [PATCH 2/2] Remove all cache logic... for now. --- osc-check_repo.py | 56 +++----------------------------- osclib/checkrepo.py | 78 ++------------------------------------------- 2 files changed, 7 insertions(+), 127 deletions(-) diff --git a/osc-check_repo.py b/osc-check_repo.py index bcffcc90..590da33b 100644 --- a/osc-check_repo.py +++ b/osc-check_repo.py @@ -20,7 +20,6 @@ from collections import defaultdict from collections import namedtuple import os -import re import shutil import subprocess import tempfile @@ -39,50 +38,9 @@ from osclib.cycle import CycleDetector from osclib.memoize import CACHEDIR -def _fix_local_cache(self, request, keys): - """Remove old package versions from the cache.""" - def _cmp(x, y): - x = [int(i) for i in x.split('.')] - y = [int(i) for i in y.split('.')] - return cmp(x, y) - - # Detect the most updated version - dirnames = set() - newest = (None, None, None, None) - for key in keys: - for full_filename in request.downloads[key]: - dirname = os.path.dirname(full_filename) - dirnames.add(dirname) - - filename = os.path.basename(full_filename) - result = re.match(r'.*-([^-]*)-([^-]*)\.[^-\.]+\.rpm', filename) - if result: - version, revision = result.groups() - _, cversion, crevision, _ = newest - if (not cversion - or _cmp(version, cversion) > 0 - or (not _cmp(version, cversion) and _cmp(revision, crevision) > 0)): - newest = (key, version, revision, dirname) - - # Remove the rest - most_updated_key, _, _, correct_dirname = newest - dirnames.remove(correct_dirname) - for dirname in dirnames: - shutil.rmtree(dirname) - - return [most_updated_key] - - def _check_repo_download(self, request): request.downloads = defaultdict(list) - # Found cached version for the request, but the cache can be - # partial. For example, in a rebuild we can have locally a - # working package. So the download list needs to be updated with - # the local copies. - if request.is_partially_cached: - request.downloads = self.checkrepo._get_downloads_from_local(request) - if request.build_excluded: return set() @@ -284,16 +242,7 @@ def _check_repo_group(self, id_, requests, debug=False): print 'DEBUG Keys', keys if keys: - # From time to time, a rebuild of a package can create - # another revision. In this case can happend that an - # old revision of the package with a still valid - # DISTURL is also in the cache. In this case - # `len(keys) > 1`, and one workaround is to take the - # newest package and remove the old one. - if len(keys) > 1: - keys = self._fix_local_cache(rq, keys) - if DEBUG_PLAN: - print 'DEBUG Cleaning the cache. New keys', keys + assert len(keys) == 1, 'Found more than one candidate for the plan' execution_plan[plan].append((rq, plan, rq.downloads[keys[0]])) if DEBUG_PLAN: @@ -531,3 +480,6 @@ def do_check_repo(self, subcmd, opts, *args): self._check_repo_group(id_, reqs, debug=opts.verbose) print print + + # Makes sure to remove the cache for now + shutil.rmtree(CACHEDIR) diff --git a/osclib/checkrepo.py b/osclib/checkrepo.py index 15b67b75..221166dc 100644 --- a/osclib/checkrepo.py +++ b/osclib/checkrepo.py @@ -14,7 +14,6 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from collections import defaultdict import os import re import subprocess @@ -64,7 +63,6 @@ class Request(object): self.updated = False self.error = None self.build_excluded = False - self.is_partially_cached = False self.action_type = 'submit' # assume default self.downloads = [] self.is_shadow_devel = False @@ -574,11 +572,7 @@ class CheckRepo(object): if not os.path.exists(repodir): os.makedirs(repodir) t = os.path.join(repodir, fn) - was_cached = self._get_binary_file(_project, _repo, arch, - request.src_package, - fn, t, mt) - if was_cached: - continue + self._get_binary_file(_project, _repo, arch, request.src_package, fn, t, mt) # Organize the files into DISTURL directories. disturl = self._md5_disturl(self._disturl(t)) @@ -604,11 +598,7 @@ class CheckRepo(object): if not os.path.exists(repodir): os.makedirs(repodir) t = os.path.join(repodir, fn) - was_cached = self._get_binary_file(_project, _repo, arch, - request.src_package, - fn, t, mt) - if was_cached: - continue + self._get_binary_file(_project, _repo, arch, request.src_package, fn, t, mt) file_in_disturl = os.path.join(last_disturldir, fn) if last_disturldir: @@ -690,58 +680,6 @@ class CheckRepo(object): return False - def is_request_cached(self, request): - """Search the request in the local cache.""" - result = False - - package_dir = os.path.join(DOWNLOADS, request.src_package) - rpm_packages = [] - for dirpath, dirnames, filenames in os.walk(package_dir): - rpm_packages.extend(os.path.join(dirpath, f) for f in filenames if f.endswith('.rpm')) - - result = any(self.check_disturl(request, filename=rpm) for rpm in rpm_packages) - - return result - - def _get_goodrepos_from_local(self, request): - """Calculate 'goodrepos' from local cache.""" - - # 'goodrepos' store the tuples (project, repos) - goodrepos = [] - - package_dir = os.path.join(DOWNLOADS, request.src_package) - projects = os.walk(package_dir).next()[1] - - # XXX TODO - The generated list can be false, we need to check - # if this is still a goodrepo. To do this we need to check - # verifymd5 or something like that, because the build status - # can be a different value from 'success'. - for project in projects: - project_dir = os.path.join(package_dir, project) - repos = os.walk(project_dir).next()[1] - - for repo in repos: - goodrepos.append((project, repo)) - - return goodrepos - - def _get_downloads_from_local(self, request): - """Calculate 'downloads' from local cache.""" - downloads = defaultdict(list) - - package_dir = os.path.join(DOWNLOADS, request.src_package) - - for project, repo in self._get_goodrepos_from_local(request): - repo_dir = os.path.join(package_dir, project, repo) - disturls = os.walk(repo_dir).next()[1] - - for disturl in disturls: - disturl_dir = os.path.join(DOWNLOADS, request.src_package, project, repo, disturl) - filenames = os.walk(disturl_dir).next()[2] - downloads[(project, repo, disturl)] = [os.path.join(disturl_dir, f) for f in filenames] - - return downloads - def get_missings(self, request): """Get the list of packages that are in missing status.""" @@ -775,14 +713,6 @@ class CheckRepo(object): """ - # Check if we have a local version of the package before - # checking it. If this is the case partially preload the - # 'goodrepos' and 'missings' fields. - if self.is_request_cached(request): - request.is_partially_cached = True - request.goodrepos = self._get_goodrepos_from_local(request) - request.missings = self.get_missings(request) - # If the request do not build properly in both Intel platforms, # return False. repos_to_check = self.repositories_to_check(request) @@ -851,9 +781,7 @@ class CheckRepo(object): if r_foundfailed: foundfailed = r_foundfailed - # If the request is partially cached, maybe there are some - # content in request.missings. - request.missings = sorted(set(request.missings) | missings) + request.missings = sorted(missings) if result: return True