osc-origin: provide initial origin osc subcommand.

osc-origin.py

@@ -0,0 +1,203 @@
+import logging
+import os
+import os.path
+from osc import cmdln
+from osc import core
+from osc import oscerr
+from osclib.cache import Cache
+from osclib.cache_manager import CacheManager
+from osclib.core import package_list_without_links
+from osclib.origin import config_load
+from osclib.origin import origin_find
+from osclib.origin import config_origin_list
+from osclib.util import mail_send
+from shutil import copyfile
+import time
+import yaml
+
+OSRT_ORIGIN_LOOKUP_TTL = 60 * 60 * 24 * 7
+
+@cmdln.option('--debug', action='store_true', help='output debug information')
+@cmdln.option('--diff', action='store_true', help='diff against previous report')
+@cmdln.option('--dry', action='store_true', help='perform a dry-run where applicable')
+@cmdln.option('--force-refresh', action='store_true', help='force refresh of data')
+@cmdln.option('--mail', action='store_true', help='mail report to <confg:mail-release-list>')
+@cmdln.option('--origins-only', action='store_true', help='list origins instead of expanded config')
+@cmdln.option('-p', '--project', help='project on which to operate (default is openSUSE:Factory)')
+def do_origin(self, subcmd, opts, *args):
+    """${cmd_name}: tools for working with origin information
+
+    ${cmd_option_list}
+
+    config: print expanded OSRT:OriginConfig
+    list: print all packages and their origin
+    package: print the origin of package
+    report: print origin summary report
+
+    Usage:
+        osc origin config [--origins-only]
+        osc origin list [--force-refresh]
+        osc origin package [--debug] PACKAGE
+        osc origin report [--diff] [--force-refresh] [--mail]
+    """
+
+    if len(args) == 0:
+        raise oscerr.WrongArgs('A command must be indicated.')
+    command = args[0]
+    if command not in ['config', 'list', 'package', 'report']:
+        raise oscerr.WrongArgs('Unknown command: {}'.format(command))
+    if command == 'package' and len(args) < 2:
+        raise oscerr.WrongArgs('A package must be indicated.')
+
+    level = logging.DEBUG if opts.debug else None
+    logging.basicConfig(level=level, format='[%(levelname).1s] %(message)s')
+
+    # Allow for determining project from osc store.
+    if not opts.project:
+        if core.is_project_dir('.'):
+            opts.project = core.store_read_project('.')
+        else:
+            opts.project = 'openSUSE:Factory'
+
+    Cache.init()
+    apiurl = self.get_api_url()
+    config = config_load(apiurl, opts.project)
+    if not config:
+        raise oscerr.WrongArgs('OSRT:OriginConfig attribute missing from {}'.format(opts.project))
+
+    function = 'osrt_origin_{}'.format(command)
+    globals()[function](apiurl, opts, *args[1:])
+
+def osrt_origin_config(apiurl, opts, *args):
+    config = config_load(apiurl, opts.project)
+
+    if opts.origins_only:
+        print('\n'.join(config_origin_list(config)))
+    else:
+        yaml.Dumper.ignore_aliases = lambda *args : True
+        print(yaml.dump(config))
+
+def osrt_origin_lookup_file(previous=False):
+    lookup_name = 'lookup.yaml' if not previous else 'lookup.previous.yaml'
+    cache_dir = CacheManager.directory('origin-manager')
+    return os.path.join(cache_dir, lookup_name)
+
+def osrt_origin_lookup(apiurl, project, force_refresh=False, previous=False):
+    lookup_path = osrt_origin_lookup_file(previous)
+    if not force_refresh and os.path.exists(lookup_path):
+        if not previous and time.time() - os.stat(lookup_path).st_mtime > OSRT_ORIGIN_LOOKUP_TTL:
+            return osrt_origin_lookup(apiurl, project, True)
+
+        with open(lookup_path, 'r') as lookup_stream:
+            lookup = yaml.safe_load(lookup_stream)
+    else:
+        if previous:
+            return None
+
+        packages = package_list_without_links(apiurl, project)
+        logging.debug('{} packages found in {}'.format(len(packages), project))
+
+        lookup = {}
+        for package in packages:
+            lookup[str(package)] = str(origin_find(apiurl, project, package))
+
+        if os.path.exists(lookup_path):
+            lookup_path_previous = osrt_origin_lookup_file(True)
+            copyfile(lookup_path, lookup_path_previous)
+
+        with open(lookup_path, 'w+') as lookup_stream:
+            yaml.dump(lookup, lookup_stream, default_flow_style=False)
+
+    return lookup
+
+def osrt_origin_max_key(dictionary, minimum):
+    return max(len(max(dictionary.keys(), key=len)), minimum)
+
+def osrt_origin_list(apiurl, opts, *args):
+    lookup = osrt_origin_lookup(apiurl, opts.project, opts.force_refresh)
+
+    line_format = '{:<' + str(osrt_origin_max_key(lookup, 7)) + '}  {}'
+    print(line_format.format('package', 'origin'))
+
+    for package, origin in lookup.items():
+        print(line_format.format(package, origin))
+
+def osrt_origin_package(apiurl, opts, *packages):
+    origin_info = origin_find(apiurl, opts.project, packages[0])
+    print(origin_info)
+
+def osrt_origin_report_count(lookup):
+    origin_count = {}
+    for package, origin in lookup.items():
+        origin_count.setdefault(origin, 0)
+        origin_count[origin] += 1
+
+    return origin_count
+
+def osrt_origin_report_count_diff(origin_count, origin_count_previous):
+    origin_count_change = {}
+    for origin, count in origin_count.items():
+        delta = count - origin_count_previous.get(origin, 0)
+        delta = '+' + str(delta) if delta > 0 else str(delta)
+        origin_count_change[origin] = delta
+
+    return origin_count_change
+
+def osrt_origin_report_diff(lookup, lookup_previous):
+    diff = {}
+    for package, origin in lookup.items():
+        origin_previous = lookup_previous.get(package)
+        if origin != origin_previous:
+            diff[package] = (origin, origin_previous)
+
+    return diff
+
+def osrt_origin_report(apiurl, opts, *args):
+    lookup = osrt_origin_lookup(apiurl, opts.project, opts.force_refresh)
+    origin_count = osrt_origin_report_count(lookup)
+
+    columns = ['origin', 'count', 'percent']
+    column_formats = [
+        '{:<' + str(osrt_origin_max_key(origin_count, 6)) + '}',
+        '{:>5}',
+        '{:>7}',
+    ]
+
+    if opts.diff:
+        columns.insert(2, 'change')
+        column_formats.insert(2, '{:>6}')
+
+        lookup_previous = osrt_origin_lookup(apiurl, opts.project, previous=True)
+        if lookup_previous is not None:
+            origin_count_previous = osrt_origin_report_count(lookup_previous)
+            origin_count_change = osrt_origin_report_count_diff(origin_count, origin_count_previous)
+            package_diff = osrt_origin_report_diff(lookup, lookup_previous)
+        else:
+            origin_count_change = {}
+            package_diff = []
+
+    line_format = '  '.join(column_formats)
+    report = [line_format.format(*columns)]
+
+    total = len(lookup)
+    for origin, count in sorted(origin_count.items(), key=lambda x : x[1], reverse=True):
+        values = [origin, count, round(float(count) / total * 100, 2)]
+        if opts.diff:
+            values.insert(2, origin_count_change.get(origin, 0))
+        report.append(line_format.format(*values))
+
+    if opts.diff and len(package_diff):
+        line_format = '{:<' + str(osrt_origin_max_key(package_diff, 7)) + '}  ' + \
+            '  '.join([column_formats[0]] * 2)
+        report.append('')
+        report.append(line_format.format('package', 'origin', 'origin previous'))
+        for package, origins in sorted(package_diff.items()):
+            report.append(line_format.format(package, *origins))
+
+
+    body = '\n'.join(report)
+    print(body)
+
+    if opts.mail:
+        mail_send(apiurl, opts.project, 'release-list', '{} origin report'.format(opts.project),
+                  body, None, dry=opts.dry)