openSUSE-release-tools/osclib/cache_manager.py

from __future__ import print_function
import os
from osclib.common import NAME
import shutil
import sys
from time import time
from xdg.BaseDirectory import save_cache_path

# Provide general cache management in the form of directory location and pruned
# contents. Of the variety of caches utilized there will be content that will
# cease to be useful, but never get cleaned up since it will never be accessed.
# This manager ensures that the entire cache is pruned periodically to remove
# files that have not been accessed recently and avoid endless growth.


class CacheManager(object):
    PRUNE_FREQUENCY = 60 * 60 * 24 * 7
    PRUNE_TTL = 60 * 60 * 24 * 30

    pruned = False
    test = False

    @staticmethod
    def directory(*args):
        CacheManager.prune_all()
        if CacheManager.test:
            return save_cache_path(NAME, '.test', *args)
        return save_cache_path(NAME, *args)

    @staticmethod
    def prune_all():
        if CacheManager.pruned:
            return
        CacheManager.pruned = True

        prune_lock = os.path.join(CacheManager.directory(), '.prune')
        if not os.path.exists(prune_lock):
            CacheManager.migrate()
        elif time() - os.stat(prune_lock).st_mtime < CacheManager.PRUNE_FREQUENCY:
            return

        with open(prune_lock, 'a'):
            os.utime(prune_lock, None)

        print('> pruning cache', file=sys.stderr)

        accessed_prune = time() - CacheManager.PRUNE_TTL
        files_pruned = 0
        bytes_pruned = 0
        for directory, subdirectories, files in os.walk(CacheManager.directory()):
            files_pruned_directory = 0
            for filename in files:
                path = os.path.join(directory, filename)
                stat = os.stat(path)
                accessed = stat.st_atime
                if accessed < accessed_prune:
                    files_pruned_directory += 1
                    files_pruned += 1
                    bytes_pruned += stat.st_size
                    os.remove(path)

            if len(subdirectories) == 0 and len(files) - files_pruned_directory == 0:
                os.rmdir(directory)

        print('> pruned {:,} files comprised of {:,} bytes'.format(
            files_pruned, bytes_pruned), file=sys.stderr)

    # Migrate the variety of prior cache locations within a single parent.
    @staticmethod
    def migrate(first=True):
        # If a old path exists then perform migration.
        cache_root = save_cache_path('')
        for source, destination in CacheManager.migrate_paths():
            if not os.path.exists(source):
                continue

            if first:
                print('> migrating caches', file=sys.stderr)

                # Move existing dir out of the way in order to nest.
                cache_moved = CacheManager.directory() + '-main'
                if not os.path.exists(cache_moved):
                    os.rename(CacheManager.directory(), cache_moved)

                # Detected need to migrate, but may have already passed -main.
                CacheManager.migrate(False)
                return

            # If either incompatible format, explicit removal, or newer source
            # was already migrated remove the cache entirely.
            if destination and os.path.exists(destination):
                # Set to None to make clear in message.
                destination = None

            print(
                '> - {} -> {}'.format(
                    os.path.relpath(source, cache_root),
                    os.path.relpath(destination, cache_root) if destination else None),
                file=sys.stderr)

            if not destination:
                shutil.rmtree(source)
                continue

            # Ensure parent directory exists and then move within.
            destination_parent = os.path.dirname(destination)
            if not os.path.exists(destination_parent):
                os.makedirs(destination_parent)
            os.rename(source, destination)

    @staticmethod
    def migrate_paths():
        path_map = {
            '{}-access': 'metrics-access',
            '{}-clone': 'request/clone',
            '{}-metrics': 'request/metrics',
            '{}-main': 'request/main',
            '{}-test': None,
            'opensuse-packagelists': 'pkglistgen',
            'opensuse-repo-checker': 'repository-meta',
            'opensuse-repo-checker-http': None,
            'osc-plugin-factory': None,
        }
        bases = [NAME, 'osc-plugin-factory']
        cache_root = save_cache_path('')
        for base in bases:
            for source, destination in path_map.items():
                source = os.path.join(cache_root, source.format(base))
                if destination:
                    destination = os.path.join(CacheManager.directory(), destination)

                yield source, destination