From 9c92af9a7096a3d9e16681a6573ac0d71929ba55 Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Thu, 29 Aug 2013 11:27:32 +0200 Subject: [PATCH 1/3] Iterate over groups --- osc-check_repo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osc-check_repo.py b/osc-check_repo.py index 645e2a3d..a0c8e62b 100644 --- a/osc-check_repo.py +++ b/osc-check_repo.py @@ -857,7 +857,7 @@ def _check_repo_group(self, id_, reqs, opts): subpkgs = factory_graph.subpkgs - for p in reqs: + for p in packs: # Take the dependencies and subpackages pkg = self._get_builddepinfo(opts, p.sproject, p.goodrepo, arch, p.spackage) @@ -866,7 +866,7 @@ def _check_repo_group(self, id_, reqs, opts): if not pkg: continue - # Update the currect graph and see if we have different cycles + # Update the current graph and see if we have different cycles if pkg.pkg in current_graph: current_graph[pkg.pkg] = pkg current_graph.remove_edges_from(set((pkg.pkg, p_) for p_ in current_graph.edges(pkg.pkg))) From 5c0ad1d84a0f35dde102d6b34200c2ad87dad09a Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Thu, 29 Aug 2013 15:05:39 +0200 Subject: [PATCH 2/3] Open / close the cache only when is needed. --- osc-check_repo.py | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/osc-check_repo.py b/osc-check_repo.py index a0c8e62b..6e1954f0 100644 --- a/osc-check_repo.py +++ b/osc-check_repo.py @@ -261,7 +261,22 @@ def memoize(ttl=None): fcntl.flock(lckfile.fileno(), fcntl.LOCK_EX) return lckfile - def _clean_cache(): + def _unlock(lckfile): + fcntl.flock(lckfile.fileno(), fcntl.LOCK_UN) + lckfile.close() + + def _open_cache(cache_name): + lckfile = _lock(cache_name) + cache = shelve.open(cache_name, protocol=-1) + # Store a reference to the lckfile to avoid to be closed by gc + cache.lckfile = lckfile + return cache + + def _close_cache(cache): + cache.close() + _unlock(cache.lckfile) + + def _clean_cache(cache): len_cache = len(cache) if len_cache >= SLOTS: nclean = NCLEAN + len_cache - SLOTS @@ -276,23 +291,21 @@ def memoize(ttl=None): now = datetime.now() key = cPickle.dumps((args, kwargs), protocol=-1) updated = False + cache = _open_cache(cache_name) if key in cache: timestamp, value = cache[key] updated = True if total_seconds(now-timestamp) < ttl else False if not updated: value = f(*args, **kwargs) cache[key] = (now, value) - _clean_cache() + _clean_cache(cache) + _close_cache(cache) return value cache_dir = os.path.expanduser(TMPDIR) if not os.path.exists(cache_dir): os.makedirs(cache_dir) cache_name = os.path.join(cache_dir, f.__name__) - lckfile = _lock(cache_name) - cache = shelve.open(cache_name, protocol=-1) - # Store a reference to the lckfile to avoid to be closed by gc - cache.lckfile = lckfile return _f ttl = ttl if ttl else TIMEOUT @@ -855,26 +868,28 @@ def _check_repo_group(self, id_, reqs, opts): # This graph will be updated for every request current_graph = deepcopy(factory_graph) - subpkgs = factory_graph.subpkgs + subpkgs = current_graph.subpkgs - for p in packs: + # Recover all packages at once, ingoring some packages that + # can't be found in x86_64 architecture + all_packages = [self._get_builddepinfo(opts, p.sproject, p.goodrepo, arch, p.spackage) for p in packs] + all_packages = [pkg for pkg in all_packages if pkg] + + for pkg in all_packages: # Take the dependencies and subpackages pkg = self._get_builddepinfo(opts, p.sproject, p.goodrepo, arch, p.spackage) - # Some packages can't be found in x86_64 architecture. We - # can ignore them. - if not pkg: - continue - # Update the current graph and see if we have different cycles if pkg.pkg in current_graph: current_graph[pkg.pkg] = pkg - current_graph.remove_edges_from(set((pkg.pkg, p_) for p_ in current_graph.edges(pkg.pkg))) + current_graph.remove_edges_from(set((pkg.pkg, p) for p in current_graph.edges(pkg.pkg))) else: current_graph.add_node(pkg.pkg, pkg) - current_graph.add_edges_from((pkg.pkg, subpkgs[p_]) for p_ in pkg.deps if p_ in subpkgs) + current_graph.add_edges_from((pkg.pkg, subpkgs[p]) for p in pkg.deps if p in subpkgs) - subpkgs.update(dict((p_, pkg.pkg) for p_ in pkg.subs)) + print p, pkg + #print p, subpkgs[ + subpkgs.update(dict((p, pkg.pkg) for p in pkg.subs)) for cycle in current_graph.cycles(): if cycle not in factory_cycles: From 8b25cd3eedf4137ebcc9332bd065058eb71695ce Mon Sep 17 00:00:00 2001 From: Alberto Planas Date: Thu, 29 Aug 2013 15:09:31 +0200 Subject: [PATCH 3/3] Remove debug prints --- osc-check_repo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/osc-check_repo.py b/osc-check_repo.py index 6e1954f0..808907a5 100644 --- a/osc-check_repo.py +++ b/osc-check_repo.py @@ -887,8 +887,6 @@ def _check_repo_group(self, id_, reqs, opts): current_graph.add_node(pkg.pkg, pkg) current_graph.add_edges_from((pkg.pkg, subpkgs[p]) for p in pkg.deps if p in subpkgs) - print p, pkg - #print p, subpkgs[ subpkgs.update(dict((p, pkg.pkg) for p in pkg.subs)) for cycle in current_graph.cycles():