import logging import os import os.path from osc import cmdln from osc import core from osc import oscerr from osclib.cache import Cache from osclib.cache_manager import CacheManager from osclib.core import package_list_without_links from osclib.origin import config_load from osclib.origin import origin_find from osclib.origin import config_origin_list from osclib.util import mail_send from shutil import copyfile import time import yaml OSRT_ORIGIN_LOOKUP_TTL = 60 * 60 * 24 * 7 @cmdln.option('--debug', action='store_true', help='output debug information') @cmdln.option('--diff', action='store_true', help='diff against previous report') @cmdln.option('--dry', action='store_true', help='perform a dry-run where applicable') @cmdln.option('--force-refresh', action='store_true', help='force refresh of data') @cmdln.option('--mail', action='store_true', help='mail report to ') @cmdln.option('--origins-only', action='store_true', help='list origins instead of expanded config') @cmdln.option('-p', '--project', help='project on which to operate (default is openSUSE:Factory)') def do_origin(self, subcmd, opts, *args): """${cmd_name}: tools for working with origin information ${cmd_option_list} config: print expanded OSRT:OriginConfig list: print all packages and their origin package: print the origin of package report: print origin summary report Usage: osc origin config [--origins-only] osc origin list [--force-refresh] osc origin package [--debug] PACKAGE osc origin report [--diff] [--force-refresh] [--mail] """ if len(args) == 0: raise oscerr.WrongArgs('A command must be indicated.') command = args[0] if command not in ['config', 'list', 'package', 'report']: raise oscerr.WrongArgs('Unknown command: {}'.format(command)) if command == 'package' and len(args) < 2: raise oscerr.WrongArgs('A package must be indicated.') level = logging.DEBUG if opts.debug else None logging.basicConfig(level=level, format='[%(levelname).1s] %(message)s') # Allow for determining project from osc store. if not opts.project: if core.is_project_dir('.'): opts.project = core.store_read_project('.') else: opts.project = 'openSUSE:Factory' Cache.init() apiurl = self.get_api_url() config = config_load(apiurl, opts.project) if not config: raise oscerr.WrongArgs('OSRT:OriginConfig attribute missing from {}'.format(opts.project)) function = 'osrt_origin_{}'.format(command) globals()[function](apiurl, opts, *args[1:]) def osrt_origin_config(apiurl, opts, *args): config = config_load(apiurl, opts.project) if opts.origins_only: print('\n'.join(config_origin_list(config))) else: yaml.Dumper.ignore_aliases = lambda *args : True print(yaml.dump(config)) def osrt_origin_lookup_file(previous=False): lookup_name = 'lookup.yaml' if not previous else 'lookup.previous.yaml' cache_dir = CacheManager.directory('origin-manager') return os.path.join(cache_dir, lookup_name) def osrt_origin_lookup(apiurl, project, force_refresh=False, previous=False): lookup_path = osrt_origin_lookup_file(previous) if not force_refresh and os.path.exists(lookup_path): if not previous and time.time() - os.stat(lookup_path).st_mtime > OSRT_ORIGIN_LOOKUP_TTL: return osrt_origin_lookup(apiurl, project, True) with open(lookup_path, 'r') as lookup_stream: lookup = yaml.safe_load(lookup_stream) else: if previous: return None packages = package_list_without_links(apiurl, project) logging.debug('{} packages found in {}'.format(len(packages), project)) lookup = {} for package in packages: lookup[str(package)] = str(origin_find(apiurl, project, package)) if os.path.exists(lookup_path): lookup_path_previous = osrt_origin_lookup_file(True) copyfile(lookup_path, lookup_path_previous) with open(lookup_path, 'w+') as lookup_stream: yaml.dump(lookup, lookup_stream, default_flow_style=False) return lookup def osrt_origin_max_key(dictionary, minimum): return max(len(max(dictionary.keys(), key=len)), minimum) def osrt_origin_list(apiurl, opts, *args): lookup = osrt_origin_lookup(apiurl, opts.project, opts.force_refresh) line_format = '{:<' + str(osrt_origin_max_key(lookup, 7)) + '} {}' print(line_format.format('package', 'origin')) for package, origin in lookup.items(): print(line_format.format(package, origin)) def osrt_origin_package(apiurl, opts, *packages): origin_info = origin_find(apiurl, opts.project, packages[0]) print(origin_info) def osrt_origin_report_count(lookup): origin_count = {} for package, origin in lookup.items(): origin_count.setdefault(origin, 0) origin_count[origin] += 1 return origin_count def osrt_origin_report_count_diff(origin_count, origin_count_previous): origin_count_change = {} for origin, count in origin_count.items(): delta = count - origin_count_previous.get(origin, 0) delta = '+' + str(delta) if delta > 0 else str(delta) origin_count_change[origin] = delta return origin_count_change def osrt_origin_report_diff(lookup, lookup_previous): diff = {} for package, origin in lookup.items(): origin_previous = lookup_previous.get(package) if origin != origin_previous: diff[package] = (origin, origin_previous) return diff def osrt_origin_report(apiurl, opts, *args): lookup = osrt_origin_lookup(apiurl, opts.project, opts.force_refresh) origin_count = osrt_origin_report_count(lookup) columns = ['origin', 'count', 'percent'] column_formats = [ '{:<' + str(osrt_origin_max_key(origin_count, 6)) + '}', '{:>5}', '{:>7}', ] if opts.diff: columns.insert(2, 'change') column_formats.insert(2, '{:>6}') lookup_previous = osrt_origin_lookup(apiurl, opts.project, previous=True) if lookup_previous is not None: origin_count_previous = osrt_origin_report_count(lookup_previous) origin_count_change = osrt_origin_report_count_diff(origin_count, origin_count_previous) package_diff = osrt_origin_report_diff(lookup, lookup_previous) else: origin_count_change = {} package_diff = [] line_format = ' '.join(column_formats) report = [line_format.format(*columns)] total = len(lookup) for origin, count in sorted(origin_count.items(), key=lambda x : x[1], reverse=True): values = [origin, count, round(float(count) / total * 100, 2)] if opts.diff: values.insert(2, origin_count_change.get(origin, 0)) report.append(line_format.format(*values)) if opts.diff and len(package_diff): line_format = '{:<' + str(osrt_origin_max_key(package_diff, 7)) + '} ' + \ ' '.join([column_formats[0]] * 2) report.append('') report.append(line_format.format('package', 'origin', 'origin previous')) for package, origins in sorted(package_diff.items()): report.append(line_format.format(package, *origins)) body = '\n'.join(report) print(body) if opts.mail: mail_send(apiurl, opts.project, 'release-list', '{} origin report'.format(opts.project), body, None, dry=opts.dry)