diff --git a/osc-staging.py b/osc-staging.py index c14287be..7a023f31 100644 --- a/osc-staging.py +++ b/osc-staging.py @@ -104,6 +104,10 @@ def _full_project_name(self, project): @cmdln.option('--filter-by', action='append', help='xpath by which to filter requests') @cmdln.option('--group-by', action='append', help='xpath by which to group requests') @cmdln.option('-i', '--interactive', action='store_true', help='interactively modify selection proposal') +@cmdln.option('-n', '--non-interactive', action='store_true', help='do not ask anything, use default answers') +@cmdln.option('--merge', action='store_true', help='propose merge where applicable and store details to allow future merges') +@cmdln.option('--try-strategies', action='store_true', default=False, help='apply strategies and keep any with desireable outcome') +@cmdln.option('--strategy', help='apply a specific strategy') def do_staging(self, subcmd, opts, *args): """${cmd_name}: Commands to work with staging projects @@ -212,8 +216,11 @@ def do_staging(self, subcmd, opts, *args): osc staging ignore [-m MESSAGE] REQUEST... osc staging unignore [--cleanup] REQUEST...|all osc staging list [--supersede] [PACKAGE...] - osc staging select [--no-freeze] [--move [--from PROJECT] STAGING REQUEST... - osc staging select [--no-freeze] [[--interactive] [--filter-by...] [--group-by...]] [STAGING...] [REQUEST...] + osc staging select [--no-freeze] [--move [--from PROJECT]] STAGING REQUEST... + osc staging select [--no-freeze] [--interactive|--non-interactive] + [--filter-by...] [--group-by...] + [--merge] [--try-strategies] [--strategy] + [STAGING...] [REQUEST...] osc staging unselect REQUEST... osc staging unlock osc staging repair REQUEST... @@ -347,24 +354,40 @@ def do_staging(self, subcmd, opts, *args): print('--move and --from must be used with explicit staging and request list') return - splitter = RequestSplitter(api, api.get_open_requests(), in_ring=True) - if len(requests) > 0: - splitter.filter_add_requests(requests) - if len(splitter.filters) == 0: - splitter.filter_add('./action[not(@type="add_role" or @type="change_devel")]') - splitter.filter_add('@ignored="false"') - if opts.filter_by: - for filter_by in opts.filter_by: - splitter.filter_add(filter_by) - if opts.group_by: - for group_by in opts.group_by: - splitter.group_by(group_by) - splitter.split() - - result = splitter.propose_assignment(stagings) - if result is not True: - print('Failed to generate proposal: {}'.format(result)) + open_requests = api.get_open_requests() + if len(open_requests) == 0: + print('No open requests to consider') return + + splitter = RequestSplitter(api, open_requests, in_ring=True) + + considerable = splitter.stagings_load(stagings) + if considerable == 0: + print('No considerable stagings on which to act') + return + + if opts.merge: + splitter.merge() + if opts.try_strategies: + splitter.strategies_try() + if len(requests) > 0: + splitter.strategy_do('requests', requests=requests) + if opts.strategy: + splitter.strategy_do(opts.strategy) + elif opts.filter_by or opts.group_by: + kwargs = {} + if opts.filter_by: + kwargs['filters'] = opts.filter_by + if opts.group_by: + kwargs['groups'] = opts.group_by + splitter.strategy_do('custom', **kwargs) + else: + if opts.merge: + # Merge any none strategies before final none strategy. + splitter.merge(strategy_none=True) + splitter.strategy_do('none') + splitter.strategy_do_non_bootstrapped('none') + proposal = splitter.proposal if len(proposal) == 0: print('Empty proposal') @@ -376,10 +399,14 @@ def do_staging(self, subcmd, opts, *args): temp.write('# move requests between stagings or comment/remove them\n') temp.write('# change the target staging for a group\n') temp.write('# stagings\n') + if opts.merge: + temp.write('# - merged: {}\n' + .format(', '.join(sorted(splitter.stagings_mergeable + + splitter.stagings_mergeable_none)))) temp.write('# - considered: {}\n' - .format(', '.join(sorted(splitter.stagings_considerable.keys())))) + .format(', '.join(sorted(splitter.stagings_considerable)))) temp.write('# - remaining: {}\n' - .format(', '.join(sorted(splitter.stagings_available.keys())))) + .format(', '.join(sorted(splitter.stagings_available)))) temp.flush() editor = os.getenv('EDITOR') @@ -389,25 +416,36 @@ def do_staging(self, subcmd, opts, *args): proposal = yaml.safe_load(open(temp.name).read()) + # Filter invalidated groups from proposal. + keys = ['group', 'requests', 'staging', 'strategy'] + for group, info in sorted(proposal.items()): + for key in keys: + if not info.get(key): + del proposal[group] + break + print(yaml.safe_dump(proposal, default_flow_style=False)) print('Accept proposal? [y/n] (y): ', end='') - response = raw_input().lower() - if response != '' and response != 'y': - print('Quit') - return + if opts.non_interactive: + print('y') + else: + response = raw_input().lower() + if response != '' and response != 'y': + print('Quit') + return - for group in sorted(proposal.keys()): - g = proposal[group] - if not g['requests']: - # Skipping since all request removed, presumably in interactive. - continue - - print('Staging {}'.format(g['staging'])) + for group, info in sorted(proposal.items()): + print('Staging {} in {}'.format(group, info['staging'])) # SelectCommand expects strings. - request_ids = map(str, g['requests'].keys()) - target_project = api.prj_from_short(g['staging']) + request_ids = map(str, info['requests'].keys()) + target_project = api.prj_from_short(info['staging']) + + if 'merge' not in info: + # Assume that the original splitter_info is desireable + # and that this staging is simply manual followup. + api.set_splitter_info_in_prj_pseudometa(target_project, info['group'], info['strategy']) SelectCommand(api, target_project) \ .perform(request_ids, opts.move, opts.from_, opts.no_freeze) diff --git a/osclib/accept_command.py b/osclib/accept_command.py index 9b66e8d8..82b59cae 100644 --- a/osclib/accept_command.py +++ b/osclib/accept_command.py @@ -69,6 +69,9 @@ class AcceptCommand(object): message='Accept to %s' % self.api.project) self.create_new_links(self.api.project, req['package'], oldspecs) + # Clear pseudometa since it no longer represents the staging. + self.api.clear_prj_pseudometa(project) + # A single comment should be enough to notify everybody, since # they are already mentioned in the comments created by # select/unselect diff --git a/osclib/request_splitter.py b/osclib/request_splitter.py index 76442cac..06c229dc 100644 --- a/osclib/request_splitter.py +++ b/osclib/request_splitter.py @@ -1,3 +1,4 @@ +import hashlib from lxml import etree as ET class RequestSplitter(object): @@ -5,10 +6,16 @@ class RequestSplitter(object): self.api = api self.requests = requests self.in_ring = in_ring + self.mergeable_build_percent = 80 + self.requests_ignored = self.api.get_ignored_requests() + self.reset() + # after propose_assignment() + self.proposal = {} def reset(self): + self.strategy = None self.filters = [] self.groups = [] @@ -16,8 +23,21 @@ class RequestSplitter(object): self.filtered = [] self.other = [] self.grouped = {} - # after propose_assignment() - self.proposal = {} + + def strategy_set(self, name, **kwargs): + self.reset() + + class_name = 'Strategy{}'.format(name.lower().title()) + cls = globals()[class_name] + self.strategy = cls(**kwargs) + self.strategy.apply(self) + + def strategy_from_splitter_info(self, splitter_info): + strategy = splitter_info['strategy'] + if 'args' in strategy: + self.strategy_set(strategy['name'], **strategy['args']) + else: + self.strategy_set(strategy['name']) def filter_add(self, xpath): self.filters.append(ET.XPath(xpath)) @@ -65,6 +85,10 @@ class RequestSplitter(object): def suppliment(self, request): """ Provide additional information for grouping """ + if request.get('ignored'): + # Only supliment once. + return + target = request.find('./action/target') target_project = target.get('project') target_package = target.get('package') @@ -85,6 +109,8 @@ class RequestSplitter(object): else: request.set('ignored', 'false') + request.set('postponed', 'false') + def ring_get(self, target_package): if self.api.crings: ring = self.api.ring_packages_for_links.get(target_package) @@ -121,87 +147,254 @@ class RequestSplitter(object): return '00' return '__'.join(key) - def propose_stagings_load(self, stagings): - self.stagings_considerable = {} - + def is_staging_bootstrapped(self, project): if self.api.rings: + # Determine if staging is bootstrapped. + meta = self.api.get_prj_meta(project) xpath = 'link[@project="{}"]'.format(self.api.rings[0]) + return meta.find(xpath) is not None + + return False + + def is_staging_mergeable(self, status, pseudometa): + # Mergeable if building and not too far along. + return (len(pseudometa['requests']) > 0 and + 'splitter_info' in pseudometa and + status['overall_state'] == 'building' and + self.api.project_status_build_percent(status) <= self.mergeable_build_percent) + + def staging_status_load(self, project): + status = self.api.project_status(project) + return status, self.api.load_prj_pseudometa(status['description']) + + def is_staging_considerable(self, project, pseudometa): + return (len(pseudometa['requests']) == 0 and + self.api.prj_frozen_enough(project)) + + def stagings_load(self, stagings): + self.stagings = {} + self.stagings_considerable = [] + self.stagings_mergeable = [] + self.stagings_mergeable_none = [] # Use specified list of stagings, otherwise only empty, letter stagings. if len(stagings) == 0: stagings = self.api.get_staging_projects_short() - filter_skip = False - else: - filter_skip = True for staging in stagings: project = self.api.prj_from_short(staging) + status, pseudometa = self.staging_status_load(project) - if not filter_skip: - if len(staging) > 1: - continue - - # TODO Allow stagings that have not finished building by threshold. - if len(self.api.get_prj_pseudometa(project)['requests']) > 0: - continue - - if self.api.rings: - # Determine if staging is bootstrapped. - meta = self.api.get_prj_meta(project) - self.stagings_considerable[staging] = True if meta.find(xpath) is not None else False - else: - self.stagings_considerable[staging] = False - - # Allow both considered and remaining to be accessible after proposal. - self.stagings_available = self.stagings_considerable.copy() - - def propose_assignment(self, stagings): - # Determine available stagings and make working copy. - self.propose_stagings_load(stagings) - - if len(self.grouped) > len(self.stagings_available): - return 'more groups than available stagings' - - # Cycle through all groups and initialize proposal and attempt to assign - # groups that have bootstrap_required. - for group in sorted(self.grouped.keys()): - self.proposal[group] = { - 'bootstrap_required': self.grouped[group]['bootstrap_required'], - 'requests': {}, + # Store information about staging. + self.stagings[staging] = { + 'project': project, + 'bootstrapped': self.is_staging_bootstrapped(project), + 'status': status, + 'pseudometa': pseudometa, } - # Covert request nodes to simple proposal form. - for request in self.grouped[group]['requests']: - self.proposal[group]['requests'][int(request.get('id'))] = request.find('action/target').get('package') + # Decide if staging of interested. + if self.is_staging_mergeable(status, pseudometa): + if pseudometa['splitter_info']['strategy']['name'] == 'none': + self.stagings_mergeable_none.append(staging) + else: + self.stagings_mergeable.append(staging) + elif self.is_staging_considerable(project, pseudometa): + self.stagings_considerable.append(staging) + # Allow both considered and remaining to be accessible after proposal. + self.stagings_available = list(self.stagings_considerable) + + return (len(self.stagings_considerable) + + len(self.stagings_mergeable) + + len(self.stagings_mergeable_none)) + + def propose_assignment(self): + # Attempt to assign groups that have bootstrap_required first. + for group in sorted(self.grouped.keys()): if self.grouped[group]['bootstrap_required']: - self.proposal[group]['staging'] = self.propose_staging(True) - if not self.proposal[group]['staging']: - return 'unable to find enough available bootstrapped stagings' + staging = self.propose_staging(choose_bootstrapped=True) + if staging: + self.requests_assign(group, staging) + else: + self.requests_postpone(group) # Assign groups that do not have bootstrap_required and fallback to a # bootstrapped staging if no non-bootstrapped stagings available. for group in sorted(self.grouped.keys()): if not self.grouped[group]['bootstrap_required']: - self.proposal[group]['staging'] = self.propose_staging(False) - if self.proposal[group]['staging']: + staging = self.propose_staging(choose_bootstrapped=False) + if staging: + self.requests_assign(group, staging) continue - self.proposal[group]['staging'] = self.propose_staging(True) - if not self.proposal[group]['staging']: - return 'unable to find enough available stagings' + staging = self.propose_staging(choose_bootstrapped=True) + if staging: + self.requests_assign(group, staging) + else: + self.requests_postpone(group) - return True + def requests_assign(self, group, staging, merge=False): + # Arbitrary, but descriptive group key for proposal. + key = '{}#{}@{}'.format(len(self.proposal), self.strategy.key, group) + self.proposal[key] = { + 'bootstrap_required': self.grouped[group]['bootstrap_required'], + 'group': group, + 'requests': {}, + 'staging': staging, + 'strategy': self.strategy.info(), + } + if merge: + self.proposal[key]['merge'] = True + + # Covert request nodes to simple proposal form. + for request in self.grouped[group]['requests']: + self.proposal[key]['requests'][int(request.get('id'))] = request.find('action/target').get('package') + self.requests.remove(request) + + return key + + def requests_postpone(self, group): + if self.strategy.name == 'none': + return + + for request in self.grouped[group]['requests']: + request.set('postponed', 'true') def propose_staging(self, choose_bootstrapped): found = False - for staging, bootstrapped in sorted(self.stagings_available.items()): - if choose_bootstrapped == bootstrapped: + for staging in sorted(self.stagings_available): + if choose_bootstrapped == self.stagings[staging]['bootstrapped']: found = True break if found: - del self.stagings_available[staging] + self.stagings_available.remove(staging) return staging return None + + def strategies_try(self): + strategies = ( + 'special', + 'devel', + ) + + map(self.strategy_try, strategies) + + def strategy_try(self, name): + self.strategy_set(name) + self.split() + + groups = self.strategy.desirable(self) + if len(groups) == 0: + return + self.filter_grouped(groups) + + self.propose_assignment() + + def strategy_do(self, name, **kwargs): + self.strategy_set(name, **kwargs) + self.split() + self.propose_assignment() + + def strategy_do_non_bootstrapped(self, name, **kwargs): + self.strategy_set(name, **kwargs) + self.filter_add('./action/target[not(starts-with(@ring, "0"))]') + self.split() + self.propose_assignment() + + def filter_grouped(self, groups): + for group in sorted(self.grouped.keys()): + if group not in groups: + del self.grouped[group] + + def merge_staging(self, staging, pseudometa): + splitter_info = pseudometa['splitter_info'] + self.strategy_from_splitter_info(splitter_info) + + if not self.stagings[staging]['bootstrapped']: + # If when the strategy was first run the resulting staging was not + # bootstrapped then ensure no bootstrapped packages are included. + self.filter_add('./action/target[not(starts-with(@ring, "0"))]') + + self.split() + + group = splitter_info['group'] + if group in self.grouped: + key = self.requests_assign(group, staging, merge=True) + + def merge(self, strategy_none=False): + stagings = self.stagings_mergeable_none if strategy_none else self.stagings_mergeable + for staging in sorted(stagings): + self.merge_staging(staging, self.stagings[staging]['pseudometa']) + + +class Strategy(object): + def __init__(self, **kwargs): + self.kwargs = kwargs + self.name = self.__class__.__name__[8:].lower() + self.key = self.name + if kwargs: + self.key += '_' + hashlib.sha1(str(kwargs)).hexdigest()[:7] + + def info(self): + info = {'name': self.name} + if self.kwargs: + info['args'] = self.kwargs + return info + +class StrategyNone(Strategy): + def apply(self, splitter): + splitter.filter_add('./action[not(@type="add_role" or @type="change_devel")]') + splitter.filter_add('@ignored="false"') + splitter.filter_add('@postponed="false"') + +class StrategyRequests(Strategy): + def apply(self, splitter): + splitter.filter_add_requests(self.kwargs['requests']) + +class StrategyCustom(StrategyNone): + def apply(self, splitter): + if 'filters' not in self.kwargs: + super(StrategyCustom, self).apply(splitter) + else: + map(splitter.filter_add, self.kwargs['filters']) + + if 'groups' in self.kwargs: + map(splitter.group_by, self.kwargs['groups']) + +class StrategyDevel(StrategyNone): + GROUP_MIN = 7 + + def apply(self, splitter): + super(StrategyDevel, self).apply(splitter) + splitter.group_by('./action/target/@devel_project') + + def desirable(self, splitter): + groups = [] + for group, info in sorted(splitter.grouped.items()): + if len(info['requests']) >= self.GROUP_MIN: + groups.append(group) + return groups + +class StrategySpecial(StrategyNone): + PACKAGES = [ + 'boost', + 'gcc', + 'gcc6', + 'gcc7', + 'glibc', + 'kernel-source', + 'python2', + 'python3', + 'util-linux', + ] + + def apply(self, splitter): + super(StrategySpecial, self).apply(splitter) + splitter.filter_add_requests(self.PACKAGES) + splitter.group_by('./action/target/@package') + + def desirable(self, splitter): + return splitter.grouped.keys() diff --git a/osclib/stagingapi.py b/osclib/stagingapi.py index f0dceb0c..160c28c2 100644 --- a/osclib/stagingapi.py +++ b/osclib/stagingapi.py @@ -333,13 +333,16 @@ class StagingAPI(object): :param adi: True for only adi stagings, False for only non-adi stagings, and None for both. """ - prefix = len(self.cstaging) + 1 projects = [] for project in self.get_staging_projects(): if project.endswith(':DVD') or \ (adi is not None and self.is_adi_project(project) != adi): continue - projects.append(self.extract_staging_short(project)) + short = self.extract_staging_short(project) + if adi is False and len(short) > 1: + # Non-letter stagings are not setup for stagingapi. + continue + projects.append(short) return projects def is_adi_project(self, p): @@ -546,6 +549,15 @@ class StagingAPI(object): f = http_GET(url) return ET.parse(f).getroot() + def load_prj_pseudometa(self, description_text): + try: + data = yaml.load(description_text) + except (TypeError, AttributeError): + data = {} + # make sure we have a requests field + data['requests'] = data.get('requests', []) + return data + @memoize(ttl=60, session=True, add_invalidate=True) def get_prj_pseudometa(self, project): """ @@ -561,13 +573,7 @@ class StagingAPI(object): # * broken description # * directly linked packages # * removed linked packages - try: - data = yaml.load(description.text) - except (TypeError, AttributeError): - data = {} - # make sure we have a requests field - data['requests'] = data.get('requests', []) - return data + return self.load_prj_pseudometa(description.text) def set_prj_pseudometa(self, project, meta): """ @@ -577,12 +583,11 @@ class StagingAPI(object): """ # Get current metadata - url = make_meta_url('prj', project, self.apiurl) - root = ET.parse(http_GET(url)).getroot() + root = self.get_prj_meta(project) # Find description description = root.find('description') # Order the requests and replace it with yaml - meta['requests'] = sorted(meta['requests'], key=lambda x: x['id']) + meta['requests'] = sorted(meta.get('requests', []), key=lambda x: x['id']) description.text = yaml.dump(meta) # Find title title = root.find('title') @@ -599,6 +604,9 @@ class StagingAPI(object): # Invalidate here the cache for this stating project self._invalidate_get_prj_pseudometa(project) + def clear_prj_pseudometa(self, project): + self.set_prj_pseudometa(project, {}) + def _add_rq_to_prj_pseudometa(self, project, request_id, package): """ Records request as part of the project within metadata @@ -621,6 +629,14 @@ class StagingAPI(object): data['requests'].append({'id': request_id, 'package': package, 'author': author}) self.set_prj_pseudometa(project, data) + def set_splitter_info_in_prj_pseudometa(self, project, group, strategy_info): + data = self.get_prj_pseudometa(project) + data['splitter_info'] = { + 'group': group, + 'strategy': strategy_info, + } + self.set_prj_pseudometa(project, data) + def get_request_id_for_package(self, project, package): """ Query the request id from meta @@ -719,6 +735,13 @@ class StagingAPI(object): return False + def project_status(self, project): + short = self.extract_staging_short(project) + query = {'format': 'json'} + url = self.makeurl(('project', 'staging_projects', self.project, short), + query=query) + return json.load(self.retried_GET(url)) + def check_project_status(self, project): """ Checks a staging project for acceptance. Use the JSON document @@ -728,15 +751,28 @@ class StagingAPI(object): informations) """ - _prefix = '{}:'.format(self.cstaging) - if project.startswith(_prefix): - project = project.replace(_prefix, '') + status = self.project_status(project) + return status and status['overall_state'] == 'acceptable' - query = {'format': 'json'} - url = self.makeurl(('project', 'staging_projects', self.project, project), - query=query) - result = json.load(self.retried_GET(url)) - return result and result['overall_state'] == 'acceptable' + def project_status_build_percent(self, status): + final, tobuild = self.project_status_build_sum(status) + return (final - tobuild) / float(final) * 100 + + def project_status_build_sum(self, status): + final, tobuild = self.project_status_build_sum_repos(status['building_repositories']) + for subproject in status['subprojects']: + # _, _ += ... would be neat. + _final, _tobuild = self.project_status_build_sum_repos(subproject['building_repositories']) + final += _final + tobuild += _tobuild + return final, tobuild + + def project_status_build_sum_repos(self, repositories): + final = tobuild = 0 + for repo in repositories: + final += int(repo['final']) + tobuild += int(repo['tobuild']) + return final, tobuild def days_since_last_freeze(self, project): """