diff --git a/osc-origin.py b/osc-origin.py new file mode 100644 index 00000000..da8bcdf4 --- /dev/null +++ b/osc-origin.py @@ -0,0 +1,203 @@ +import logging +import os +import os.path +from osc import cmdln +from osc import core +from osc import oscerr +from osclib.cache import Cache +from osclib.cache_manager import CacheManager +from osclib.core import package_list_without_links +from osclib.origin import config_load +from osclib.origin import origin_find +from osclib.origin import config_origin_list +from osclib.util import mail_send +from shutil import copyfile +import time +import yaml + +OSRT_ORIGIN_LOOKUP_TTL = 60 * 60 * 24 * 7 + +@cmdln.option('--debug', action='store_true', help='output debug information') +@cmdln.option('--diff', action='store_true', help='diff against previous report') +@cmdln.option('--dry', action='store_true', help='perform a dry-run where applicable') +@cmdln.option('--force-refresh', action='store_true', help='force refresh of data') +@cmdln.option('--mail', action='store_true', help='mail report to ') +@cmdln.option('--origins-only', action='store_true', help='list origins instead of expanded config') +@cmdln.option('-p', '--project', help='project on which to operate (default is openSUSE:Factory)') +def do_origin(self, subcmd, opts, *args): + """${cmd_name}: tools for working with origin information + + ${cmd_option_list} + + config: print expanded OSRT:OriginConfig + list: print all packages and their origin + package: print the origin of package + report: print origin summary report + + Usage: + osc origin config [--origins-only] + osc origin list [--force-refresh] + osc origin package [--debug] PACKAGE + osc origin report [--diff] [--force-refresh] [--mail] + """ + + if len(args) == 0: + raise oscerr.WrongArgs('A command must be indicated.') + command = args[0] + if command not in ['config', 'list', 'package', 'report']: + raise oscerr.WrongArgs('Unknown command: {}'.format(command)) + if command == 'package' and len(args) < 2: + raise oscerr.WrongArgs('A package must be indicated.') + + level = logging.DEBUG if opts.debug else None + logging.basicConfig(level=level, format='[%(levelname).1s] %(message)s') + + # Allow for determining project from osc store. + if not opts.project: + if core.is_project_dir('.'): + opts.project = core.store_read_project('.') + else: + opts.project = 'openSUSE:Factory' + + Cache.init() + apiurl = self.get_api_url() + config = config_load(apiurl, opts.project) + if not config: + raise oscerr.WrongArgs('OSRT:OriginConfig attribute missing from {}'.format(opts.project)) + + function = 'osrt_origin_{}'.format(command) + globals()[function](apiurl, opts, *args[1:]) + +def osrt_origin_config(apiurl, opts, *args): + config = config_load(apiurl, opts.project) + + if opts.origins_only: + print('\n'.join(config_origin_list(config))) + else: + yaml.Dumper.ignore_aliases = lambda *args : True + print(yaml.dump(config)) + +def osrt_origin_lookup_file(previous=False): + lookup_name = 'lookup.yaml' if not previous else 'lookup.previous.yaml' + cache_dir = CacheManager.directory('origin-manager') + return os.path.join(cache_dir, lookup_name) + +def osrt_origin_lookup(apiurl, project, force_refresh=False, previous=False): + lookup_path = osrt_origin_lookup_file(previous) + if not force_refresh and os.path.exists(lookup_path): + if not previous and time.time() - os.stat(lookup_path).st_mtime > OSRT_ORIGIN_LOOKUP_TTL: + return osrt_origin_lookup(apiurl, project, True) + + with open(lookup_path, 'r') as lookup_stream: + lookup = yaml.safe_load(lookup_stream) + else: + if previous: + return None + + packages = package_list_without_links(apiurl, project) + logging.debug('{} packages found in {}'.format(len(packages), project)) + + lookup = {} + for package in packages: + lookup[str(package)] = str(origin_find(apiurl, project, package)) + + if os.path.exists(lookup_path): + lookup_path_previous = osrt_origin_lookup_file(True) + copyfile(lookup_path, lookup_path_previous) + + with open(lookup_path, 'w+') as lookup_stream: + yaml.dump(lookup, lookup_stream, default_flow_style=False) + + return lookup + +def osrt_origin_max_key(dictionary, minimum): + return max(len(max(dictionary.keys(), key=len)), minimum) + +def osrt_origin_list(apiurl, opts, *args): + lookup = osrt_origin_lookup(apiurl, opts.project, opts.force_refresh) + + line_format = '{:<' + str(osrt_origin_max_key(lookup, 7)) + '} {}' + print(line_format.format('package', 'origin')) + + for package, origin in lookup.items(): + print(line_format.format(package, origin)) + +def osrt_origin_package(apiurl, opts, *packages): + origin_info = origin_find(apiurl, opts.project, packages[0]) + print(origin_info) + +def osrt_origin_report_count(lookup): + origin_count = {} + for package, origin in lookup.items(): + origin_count.setdefault(origin, 0) + origin_count[origin] += 1 + + return origin_count + +def osrt_origin_report_count_diff(origin_count, origin_count_previous): + origin_count_change = {} + for origin, count in origin_count.items(): + delta = count - origin_count_previous.get(origin, 0) + delta = '+' + str(delta) if delta > 0 else str(delta) + origin_count_change[origin] = delta + + return origin_count_change + +def osrt_origin_report_diff(lookup, lookup_previous): + diff = {} + for package, origin in lookup.items(): + origin_previous = lookup_previous.get(package) + if origin != origin_previous: + diff[package] = (origin, origin_previous) + + return diff + +def osrt_origin_report(apiurl, opts, *args): + lookup = osrt_origin_lookup(apiurl, opts.project, opts.force_refresh) + origin_count = osrt_origin_report_count(lookup) + + columns = ['origin', 'count', 'percent'] + column_formats = [ + '{:<' + str(osrt_origin_max_key(origin_count, 6)) + '}', + '{:>5}', + '{:>7}', + ] + + if opts.diff: + columns.insert(2, 'change') + column_formats.insert(2, '{:>6}') + + lookup_previous = osrt_origin_lookup(apiurl, opts.project, previous=True) + if lookup_previous is not None: + origin_count_previous = osrt_origin_report_count(lookup_previous) + origin_count_change = osrt_origin_report_count_diff(origin_count, origin_count_previous) + package_diff = osrt_origin_report_diff(lookup, lookup_previous) + else: + origin_count_change = {} + package_diff = [] + + line_format = ' '.join(column_formats) + report = [line_format.format(*columns)] + + total = len(lookup) + for origin, count in sorted(origin_count.items(), key=lambda x : x[1], reverse=True): + values = [origin, count, round(float(count) / total * 100, 2)] + if opts.diff: + values.insert(2, origin_count_change.get(origin, 0)) + report.append(line_format.format(*values)) + + if opts.diff and len(package_diff): + line_format = '{:<' + str(osrt_origin_max_key(package_diff, 7)) + '} ' + \ + ' '.join([column_formats[0]] * 2) + report.append('') + report.append(line_format.format('package', 'origin', 'origin previous')) + for package, origins in sorted(package_diff.items()): + report.append(line_format.format(package, *origins)) + + + body = '\n'.join(report) + print(body) + + if opts.mail: + mail_send(apiurl, opts.project, 'release-list', '{} origin report'.format(opts.project), + body, None, dry=opts.dry)