From c47bb4c902430b6894b2dc020c1340e6a4449b32 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Tue, 4 Sep 2018 15:03:03 -0500 Subject: [PATCH] Utilize CacheManager for all existing caches. --- .travis.yml | 2 +- docs/metrics.md | 2 +- legal-auto.py | 4 ++-- metrics.py | 3 +-- metrics/access/README.md | 2 +- metrics/access/aggregate.php | 2 +- obs_clone.py | 3 +-- osclib/cache.py | 10 ++++++++-- osclib/git.py | 5 +++-- osclib/memoize.py | 20 +++----------------- pkglistgen.py | 8 ++++---- rebuildpacs.pl | 3 ++- repo_checker.py | 3 ++- tests/obs.py | 6 +++--- 14 files changed, 33 insertions(+), 40 deletions(-) diff --git a/.travis.yml b/.travis.yml index c96fe940..8a2970a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,7 +85,7 @@ deploy: cache: directories: # obs_clone.py - - ~/.cache/openSUSE-release-tools-clone + - ~/.cache/openSUSE-release-tools/request/clone # distribution jobs `osc build` - .docker-tmp pip: true diff --git a/docs/metrics.md b/docs/metrics.md index 72852a94..1c21d08b 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -46,7 +46,7 @@ to unintentionally change the default project, annotation state, or time period by saving the dashboard with different defaults. Use the `--debug` option and inspect individual request XML dumps by looking in -`~/.cache/openSUSE-release-tools-metrics` or: +`~/.cache/openSUSE-release-tools/request/metrics` or: ``` osc api '/request/$reqid?withfullhistory=1' diff --git a/legal-auto.py b/legal-auto.py index 68d0d4b4..dc2b9798 100755 --- a/legal-auto.py +++ b/legal-auto.py @@ -26,6 +26,7 @@ except: import osc.conf import osc.core +from osclib.cache_manager import CacheManager import ReviewBot from osclib.comments import CommentAPI @@ -189,8 +190,7 @@ class LegalAuto(ReviewBot.ReviewBot): self.delete_from_db(req.reqid) def _pkl_path(self): - CACHE_DIR = os.path.expanduser('~/.cache/osc-plugin-factory') - return os.path.join(CACHE_DIR, 'legaldb') + return CacheManager.directory('legal-auto') def update_project(self, project): try: diff --git a/metrics.py b/metrics.py index cd872b29..3796af36 100755 --- a/metrics.py +++ b/metrics.py @@ -544,14 +544,13 @@ def main(args): # Use separate cache since it is persistent. _, package = project_pseudometa_package(apiurl, args.project) - Cache.CACHE_DIR = Cache.CACHE_DIR + '-metrics' if args.wipe_cache: Cache.delete_all() if args.heavy_cache: Cache.PATTERNS['/search/request'] = sys.maxint Cache.PATTERNS['/source/[^/]+/{}/_history'.format(package)] = sys.maxint Cache.PATTERNS['/source/[^/]+/{}/[^/]+\?rev=.*'.format(package)] = sys.maxint - Cache.init() + Cache.init('metrics') Config(apiurl, args.project) api = StagingAPI(apiurl, args.project) diff --git a/metrics/access/README.md b/metrics/access/README.md index 54264086..958f181a 100644 --- a/metrics/access/README.md +++ b/metrics/access/README.md @@ -22,7 +22,7 @@ A separate, minimal set of aggregation done for each IP protocol data. - `aggregate.php`: invoke to manage entire process including calling (`ingest.php`) - `ingest.php`: used to parse a single log file / stream and dump summary JSON to stdout -See `~/.cache/openSUSE-release-tools-access` for cache data separated by IP protocol. A single JSON file corresponds to a single access log file. +See `~/.cache/openSUSE-release-tools/metrics-access` for cache data separated by IP protocol. A single JSON file corresponds to a single access log file. ## Future product versions diff --git a/metrics/access/aggregate.php b/metrics/access/aggregate.php index 5da35f96..0c0f5de5 100755 --- a/metrics/access/aggregate.php +++ b/metrics/access/aggregate.php @@ -4,7 +4,7 @@ use InfluxDB\Point; use InfluxDB\Database; -$CACHE_DIR = $_SERVER['HOME'] . '/.cache/openSUSE-release-tools-access'; +$CACHE_DIR = $_SERVER['HOME'] . '/.cache/openSUSE-release-tools/metrics-access'; const PROTOCOLS = ['ipv4', 'ipv6']; const PONTIFEX = 'http://pontifex.infra.opensuse.org/logs'; const LANGLEY = 'http://langley.suse.de/pub/pontifex%s-opensuse.suse.de'; diff --git a/obs_clone.py b/obs_clone.py index b5ece818..dd230387 100755 --- a/obs_clone.py +++ b/obs_clone.py @@ -212,12 +212,11 @@ if __name__ == '__main__': if args.cache: from osclib.cache import Cache - Cache.CACHE_DIR = Cache.CACHE_DIR + '-clone' Cache.PATTERNS = {} # Prevent caching source information from local clone. Cache.PATTERNS['/source/[^/]+/[^/]+/[^/]+?rev'] = 0 Cache.PATTERNS['.*'] = Cache.TTL_LONG * 2 - Cache.init() + Cache.init('clone') osc.conf.config['debug'] = args.debug project_fence.project = args.project diff --git a/osclib/cache.py b/osclib/cache.py index acb92940..2e5f41df 100644 --- a/osclib/cache.py +++ b/osclib/cache.py @@ -12,6 +12,7 @@ import urllib from StringIO import StringIO from osc import conf from osc.core import urlopen +from osclib.cache_manager import CacheManager from time import time try: @@ -64,7 +65,7 @@ class Cache(object): cache, but obviously not for other contributors. """ - CACHE_DIR = os.path.expanduser('~/.cache/openSUSE-release-tools') + CACHE_DIR = None TTL_LONG = 12 * 60 * 60 TTL_MEDIUM = 30 * 60 TTL_SHORT = 5 * 60 @@ -101,7 +102,9 @@ class Cache(object): last_updated = {} @staticmethod - def init(): + def init(directory='main'): + Cache.CACHE_DIR = CacheManager.directory('request', directory) + Cache.patterns = [] for pattern in Cache.PATTERNS: Cache.patterns.append(re.compile(pattern)) @@ -249,6 +252,9 @@ class Cache(object): @staticmethod def path(url, project, include_file=False, makedirs=False): + if not Cache.CACHE_DIR: + raise Exception('Cache.init() must be called first') + parts = [Cache.CACHE_DIR] o = urlparse.urlsplit(url) diff --git a/osclib/git.py b/osclib/git.py index d80062fc..0a003b07 100644 --- a/osclib/git.py +++ b/osclib/git.py @@ -1,9 +1,10 @@ import os from os import path +from osclib.cache_manager import CacheManager import subprocess -from xdg.BaseDirectory import save_cache_path -CACHE_DIR = save_cache_path('openSUSE-release-tools', 'git') +# Git will not be happy if pruned, but not used enough to be worth excluding. +CACHE_DIR = CacheManager.directory('git') def clone(url, directory): return_code = subprocess.call(['git', 'clone', url, directory]) diff --git a/osclib/memoize.py b/osclib/memoize.py index f5261b34..3aef743c 100644 --- a/osclib/memoize.py +++ b/osclib/memoize.py @@ -2,26 +2,15 @@ from datetime import datetime import fcntl from functools import wraps import os +from osclib.cache_manager import CacheManager import shelve try: import cPickle as pickle except: import pickle - -try: - from xdg.BaseDirectory import save_cache_path -except ImportError: - from xdg.BaseDirectory import xdg_cache_home - - def save_cache_path(*name): - path = os.path.join(xdg_cache_home, *name) - if not os.path.isdir(path): - os.makedirs(path) - return path - # Where the cache files are stored -CACHEDIR = save_cache_path('opensuse-repo-checker') +CACHEDIR = CacheManager.directory('memoize') def memoize(ttl=None, session=False, add_invalidate=False): @@ -175,10 +164,7 @@ def memoize(ttl=None, session=False, add_invalidate=False): _close_cache(cache) return value - cache_dir = os.path.expanduser(CACHEDIR) - if not os.path.exists(cache_dir): - os.makedirs(cache_dir) - cache_name = os.path.join(cache_dir, fn.__name__) + cache_name = os.path.join(CACHEDIR, fn.__name__) return _fn ttl = ttl if ttl else TIMEOUT diff --git a/pkglistgen.py b/pkglistgen.py index 96490959..ec0dd931 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -19,12 +19,12 @@ from osc.core import Package from osc.core import show_results_meta from osc.core import undelete_package from osc import conf +from osclib.cache_manager import CacheManager from osclib.conf import Config, str2bool from osclib.core import repository_path_expand from osclib.stagingapi import StagingAPI from osclib.util import project_list_family from osclib.util import project_list_family_prior -from xdg.BaseDirectory import save_cache_path import glob import hashlib import io @@ -47,7 +47,7 @@ import string import ToolBase # share header cache with repochecker -from osclib.memoize import CACHEDIR +CACHEDIR = CacheManager.directory('repository-meta') logger = logging.getLogger() @@ -1235,7 +1235,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): # Cache dir specific to hostname and project. host = urlparse.urlparse(api.apiurl).hostname - cache_dir = save_cache_path('opensuse-packagelists', host, opts.project) + cache_dir = CacheManager.directory('pkglistgen', host, opts.project) if not opts.no_checkout: if os.path.exists(cache_dir): @@ -1298,7 +1298,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): if drop_list: # Ensure solv files from all releases in product family are updated. print('-> solv_cache_update') - cache_dir_solv = save_cache_path('opensuse-packagelists', 'solv') + cache_dir_solv = CacheManager.directory('pkglistgen', 'solv') family_last = target_config.get('pkglistgen-product-family-last') family_include = target_config.get('pkglistgen-product-family-include') solv_prior = self.solv_cache_update( diff --git a/rebuildpacs.pl b/rebuildpacs.pl index 17528af3..a4f4cf4c 100755 --- a/rebuildpacs.pl +++ b/rebuildpacs.pl @@ -175,7 +175,8 @@ sub mirror_repo($$$) { my $repo = shift; my $arch = shift; - my $repodir = ($ENV{XDG_CACHE_HOME}||$ENV{HOME}."/.cache")."/opensuse-repo-checker/repo-$project-$repo-$arch"; + # Old and new in single directory, but never deployed together. + my $repodir = ($ENV{XDG_CACHE_HOME}||$ENV{HOME}."/.cache")."/openSUSE-release-tools/repository-meta/repo-$project-$repo-$arch"; mkdir($repodir); system( diff --git a/repo_checker.py b/repo_checker.py index 64c58ee4..f1dfa5da 100755 --- a/repo_checker.py +++ b/repo_checker.py @@ -14,6 +14,7 @@ import subprocess import sys import tempfile +from osclib.cache_manager import CacheManager from osclib.conf import Config from osclib.conf import str2bool from osclib.core import BINARY_REGEX @@ -31,12 +32,12 @@ from osclib.core import repositories_states from osclib.core import repositories_published from osclib.core import target_archs from osclib.cycle import CycleDetector -from osclib.memoize import CACHEDIR from osclib.memoize import memoize from osclib.util import sha1_short import ReviewBot +CACHEDIR = CacheManager.directory('repository-meta') SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) CheckResult = namedtuple('CheckResult', ('success', 'comment')) INSTALL_REGEX = r"^(?:can't install (.*?)|found conflict of (.*?) with (.*?)):$" diff --git a/tests/obs.py b/tests/obs.py index 9fd086bc..f8000ca2 100644 --- a/tests/obs.py +++ b/tests/obs.py @@ -13,6 +13,7 @@ import xml.etree.cElementTree as ET import httpretty import osc from osclib.cache import Cache +from osclib.cache_manager import CacheManager APIURL = 'http://localhost' @@ -114,9 +115,8 @@ class OBS(object): """Instance constructor.""" self.fixtures = fixtures - if not hasattr(Cache, '_CACHE_DIR'): - Cache._CACHE_DIR = True - Cache.CACHE_DIR += '-test' + CacheManager.directory_test() + Cache.init() Cache.delete_all() httpretty.enable()