from __future__ import print_function from datetime import timedelta import json import logging import os import os.path from osc import cmdln from osc import core from osc import oscerr from osc.core import get_request_list from osclib.cache import Cache from osclib.cache_manager import CacheManager from osclib.core import package_list_kind_filtered from osclib.core import project_attribute_list from osclib.core import project_locked from osclib.origin import config_load from osclib.origin import config_origin_list from osclib.origin import origin_find from osclib.origin import origin_history from osclib.origin import origin_potentials from osclib.origin import origin_revision_state from osclib.util import mail_send from shutil import copyfile import sys import time import yaml OSRT_ORIGIN_LOOKUP_TTL = 60 * 60 * 24 * 7 @cmdln.option('--debug', action='store_true', help='output debug information') @cmdln.option('--diff', action='store_true', help='diff against previous report') @cmdln.option('--dry', action='store_true', help='perform a dry-run where applicable') @cmdln.option('--force-refresh', action='store_true', help='force refresh of data') @cmdln.option('--format', default='plain', help='output format') @cmdln.option('--mail', action='store_true', help='mail report to ') @cmdln.option('--origins-only', action='store_true', help='list origins instead of expanded config') @cmdln.option('-p', '--project', help='project on which to operate') def do_origin(self, subcmd, opts, *args): """${cmd_name}: tools for working with origin information ${cmd_option_list} config: print expanded OSRT:OriginConfig history: list requests containing an origin annotation list: print all packages and their origin package: print the origin of package potentials: list potential origins of a package projects: list all projects with an OSRT:OriginConfig attribute report: print origin summary report Usage: osc origin config [--origins-only] osc origin history [--format json|yaml] PACKAGE osc origin list [--force-refresh] [--format json|yaml] osc origin package [--debug] PACKAGE osc origin potentials [--format json|yaml] PACKAGE osc origin projects [--format json|yaml] osc origin report [--diff] [--force-refresh] [--mail] """ if len(args) == 0: raise oscerr.WrongArgs('A command must be indicated.') command = args[0] if command not in ['config', 'history', 'list', 'package', 'potentials', 'projects', 'report']: raise oscerr.WrongArgs('Unknown command: {}'.format(command)) if command == 'package' and len(args) < 2: raise oscerr.WrongArgs('A package must be indicated.') level = logging.DEBUG if opts.debug else None logging.basicConfig(level=level, format='[%(levelname).1s] %(message)s') # Allow for determining project from osc store. if not opts.project and core.is_project_dir('.'): opts.project = core.store_read_project('.') Cache.init() apiurl = self.get_api_url() if command != 'projects': if not opts.project: raise oscerr.WrongArgs('A project must be indicated.') config = config_load(apiurl, opts.project) if not config: raise oscerr.WrongArgs('OSRT:OriginConfig attribute missing from {}'.format(opts.project)) function = 'osrt_origin_{}'.format(command) globals()[function](apiurl, opts, *args[1:]) def osrt_origin_config(apiurl, opts, *args): config = config_load(apiurl, opts.project) if opts.origins_only: print('\n'.join(config_origin_list(config))) else: yaml.Dumper.ignore_aliases = lambda *args : True print(yaml.dump(config)) def osrt_origin_dump(format, data): if format == 'json': print(json.dumps(data)) elif format == 'yaml': print(yaml.dump(data)) else: if format != 'plain': print('unknown format: {}'.format(format), file=sys.stderr) return False return True def osrt_origin_history(apiurl, opts, *packages): config = config_load(apiurl, opts.project) history = origin_history(apiurl, opts.project, packages[0], config['review-user']) if osrt_origin_dump(opts.format, history): return line_format = '{:<50} {:<10} {:>7}' print(line_format.format('origin', 'state', 'request')) for record in history: print(line_format.format(record['origin'], record['state'], record['request'])) def osrt_origin_lookup_file(project, previous=False): parts = [project, 'yaml'] if previous: parts.insert(1, 'previous') lookup_name = '.'.join(parts) cache_dir = CacheManager.directory('origin-manager') return os.path.join(cache_dir, lookup_name) def osrt_origin_lookup(apiurl, project, force_refresh=False, previous=False, quiet=False): locked = project_locked(apiurl, project) if locked: force_refresh = False lookup_path = osrt_origin_lookup_file(project, previous) if not force_refresh and os.path.exists(lookup_path): if not locked and not previous: # Force refresh of lookup information if expried. if time.time() - os.stat(lookup_path).st_mtime > OSRT_ORIGIN_LOOKUP_TTL: return osrt_origin_lookup(apiurl, project, True) with open(lookup_path, 'r') as lookup_stream: lookup = yaml.safe_load(lookup_stream) if not isinstance(next(iter(lookup.values())), dict): # Convert flat format to dictionary. for package, origin in lookup.items(): lookup[package] = {'origin': origin} else: if previous: return None packages = package_list_kind_filtered(apiurl, project) lookup = {} for package in packages: origin_info = origin_find(apiurl, project, package) lookup[str(package)] = { 'origin': str(origin_info), 'revisions': origin_revision_state(apiurl, project, package, origin_info), } if os.path.exists(lookup_path): lookup_path_previous = osrt_origin_lookup_file(project, True) copyfile(lookup_path, lookup_path_previous) with open(lookup_path, 'w+') as lookup_stream: yaml.dump(lookup, lookup_stream, default_flow_style=False) if not previous and not quiet: dt = timedelta(seconds=time.time() - os.stat(lookup_path).st_mtime) print('# generated {} ago'.format(dt), file=sys.stderr) return lookup def osrt_origin_max_key(dictionary, minimum): return max(len(max(dictionary.keys(), key=len)), minimum) def osrt_origin_list(apiurl, opts, *args): lookup = osrt_origin_lookup(apiurl, opts.project, opts.force_refresh, quiet=opts.format != 'plain') if opts.format != 'plain': # Suppliment data with request information. requests = get_request_list(apiurl, opts.project, None, None, ['new', 'review'], 'submit') requests.extend(get_request_list(apiurl, opts.project, None, None, ['new', 'review'], 'delete')) requests_map = {} for request in requests: for action in request.actions: requests_map[action.tgt_package] = request.reqid # Convert data from lookup to list. out = [] for package, details in sorted(lookup.items()): out.append({ 'package': package, 'origin': details['origin'], 'revisions': details.get('revisions', []), 'request': requests_map.get(package), }) osrt_origin_dump(opts.format, out) return line_format = '{:<' + str(osrt_origin_max_key(lookup, 7)) + '} {}' print(line_format.format('package', 'origin')) for package, details in sorted(lookup.items()): print(line_format.format(package, details['origin'])) def osrt_origin_package(apiurl, opts, *packages): origin_info = origin_find(apiurl, opts.project, packages[0]) print(origin_info) def osrt_origin_potentials(apiurl, opts, *packages): potentials = origin_potentials(apiurl, opts.project, packages[0]) if opts.format != 'plain': out = [] for origin, version in potentials.items(): out.append({'origin': origin, 'version': version}) osrt_origin_dump(opts.format, out) return line_format = '{:<50} {}' print(line_format.format('origin', 'version')) for origin, version in potentials.items(): print(line_format.format(origin, version)) def osrt_origin_projects(apiurl, opts, *args): projects = list(project_attribute_list(apiurl, 'OSRT:OriginConfig')) if osrt_origin_dump(opts.format, projects): return for project in sorted(projects): print(project) def osrt_origin_report_count(lookup): origin_count = {} for package, details in lookup.items(): origin_count.setdefault(details['origin'], 0) origin_count[details['origin']] += 1 return origin_count def osrt_origin_report_count_diff(origin_count, origin_count_previous): origin_count_change = {} for origin, count in origin_count.items(): delta = count - origin_count_previous.get(origin, 0) delta = '+' + str(delta) if delta > 0 else str(delta) origin_count_change[origin] = delta return origin_count_change def osrt_origin_report_diff(lookup, lookup_previous): diff = {} for package, details in lookup.items(): origin_previous = lookup_previous.get(package, {}).get('origin') if details['origin'] != origin_previous: diff[package] = (details['origin'], origin_previous) return diff def osrt_origin_report(apiurl, opts, *args): lookup = osrt_origin_lookup(apiurl, opts.project, opts.force_refresh) origin_count = osrt_origin_report_count(lookup) columns = ['origin', 'count', 'percent'] column_formats = [ '{:<' + str(osrt_origin_max_key(origin_count, 6)) + '}', '{:>5}', '{:>7}', ] if opts.diff: columns.insert(2, 'change') column_formats.insert(2, '{:>6}') lookup_previous = osrt_origin_lookup(apiurl, opts.project, previous=True) if lookup_previous is not None: origin_count_previous = osrt_origin_report_count(lookup_previous) origin_count_change = osrt_origin_report_count_diff(origin_count, origin_count_previous) package_diff = osrt_origin_report_diff(lookup, lookup_previous) else: origin_count_change = {} package_diff = [] line_format = ' '.join(column_formats) report = [line_format.format(*columns)] total = len(lookup) for origin, count in sorted(origin_count.items(), key=lambda x : x[1], reverse=True): values = [origin, count, round(float(count) / total * 100, 2)] if opts.diff: values.insert(2, origin_count_change.get(origin, 0)) report.append(line_format.format(*values)) if opts.diff and len(package_diff): line_format = '{:<' + str(osrt_origin_max_key(package_diff, 7)) + '} ' + \ ' '.join([column_formats[0]] * 2) report.append('') report.append(line_format.format('package', 'origin', 'origin previous')) for package, origins in sorted(package_diff.items()): report.append(line_format.format(package, *origins)) body = '\n'.join(report) print(body) if opts.mail: mail_send(apiurl, opts.project, 'release-list', '{} origin report'.format(opts.project), body, None, dry=opts.dry)