52eb890d76
Without this, the relative rarer types of requests seen in projects with staging and handled by list command will be included in staging proposal. However, since they are not stageable the select operation will fail. This change ensures that a filter is always present when stageable is True to exclude non-stableable requests. The list command sets stageable to false in order to list out the non-stageable requests of interest. This was not observed in openSUSE since the main non-stageable request was change_devel and that was exluded in StrategyNone. That filter could be replaced with the stageable filter, but having an always on filter seems to make more sense since generally operating in one of two modes.
523 lines
19 KiB
Python
523 lines
19 KiB
Python
from datetime import datetime
|
|
import dateutil.parser
|
|
import hashlib
|
|
from lxml import etree as ET
|
|
from osc import conf
|
|
from osc.core import show_project_meta
|
|
from osclib.core import devel_project_fallback
|
|
from osclib.core import request_age
|
|
import re
|
|
|
|
class RequestSplitter(object):
|
|
def __init__(self, api, requests, in_ring, stageable=True):
|
|
self.api = api
|
|
self.requests = requests
|
|
self.in_ring = in_ring
|
|
self.stageable = stageable
|
|
self.config = conf.config[self.api.project]
|
|
|
|
# 55 minutes to avoid two staging bot loops of 30 minutes
|
|
self.request_age_threshold = int(self.config.get('splitter-request-age-threshold', 55 * 60))
|
|
self.staging_age_max = int(self.config.get('splitter-staging-age-max', 8 * 60 * 60))
|
|
special_packages= self.config.get('splitter-special-packages')
|
|
if special_packages is not None:
|
|
StrategySpecial.PACKAGES = special_packages.split(' ')
|
|
|
|
self.requests_ignored = self.api.get_ignored_requests()
|
|
|
|
self.reset()
|
|
# after propose_assignment()
|
|
self.proposal = {}
|
|
|
|
def reset(self):
|
|
self.strategy = None
|
|
self.filters = []
|
|
self.groups = []
|
|
|
|
# after split()
|
|
self.filtered = []
|
|
self.other = []
|
|
self.grouped = {}
|
|
|
|
if self.stageable:
|
|
# Require requests to be stageable (submit or delete package).
|
|
self.filter_add('./action[@type="submit" or (@type="delete" and ./target[@package])]')
|
|
|
|
def strategy_set(self, name, **kwargs):
|
|
self.reset()
|
|
|
|
class_name = 'Strategy{}'.format(name.lower().title())
|
|
cls = globals()[class_name]
|
|
self.strategy = cls(**kwargs)
|
|
self.strategy.apply(self)
|
|
|
|
def strategy_from_splitter_info(self, splitter_info):
|
|
strategy = splitter_info['strategy']
|
|
if 'args' in strategy:
|
|
self.strategy_set(strategy['name'], **strategy['args'])
|
|
else:
|
|
self.strategy_set(strategy['name'])
|
|
|
|
def filter_add(self, xpath):
|
|
self.filters.append(ET.XPath(xpath))
|
|
|
|
def filter_add_requests(self, requests):
|
|
requests = ' ' + ' '.join(requests) + ' '
|
|
self.filter_add('contains("{requests}", concat(" ", @id, " ")) or '
|
|
'contains("{requests}", concat(" ", ./action/target/@package, " "))'
|
|
.format(requests=requests))
|
|
|
|
def group_by(self, xpath, required=False):
|
|
self.groups.append(ET.XPath(xpath))
|
|
if required:
|
|
self.filter_add(xpath)
|
|
|
|
def filter_only(self):
|
|
ret = []
|
|
for request in self.requests:
|
|
self.supplement(request)
|
|
if self.filter_check(request):
|
|
ret.append(request)
|
|
return ret
|
|
|
|
def split(self):
|
|
for request in self.requests:
|
|
self.supplement(request)
|
|
|
|
if not self.filter_check(request):
|
|
continue
|
|
|
|
ring = request.find('./action/target').get('ring')
|
|
if self.in_ring != (not ring):
|
|
# Request is of desired ring type.
|
|
key = self.group_key_build(request)
|
|
if key not in self.grouped:
|
|
self.grouped[key] = {
|
|
'bootstrap_required': False,
|
|
'requests': [],
|
|
}
|
|
|
|
self.grouped[key]['requests'].append(request)
|
|
|
|
if ring and ring.startswith('0'):
|
|
self.grouped[key]['bootstrap_required'] = True
|
|
else:
|
|
self.other.append(request)
|
|
|
|
def supplement(self, request):
|
|
""" Provide additional information for grouping """
|
|
if request.get('ignored'):
|
|
# Only supplement once.
|
|
return
|
|
|
|
history = request.find('history')
|
|
if history is not None:
|
|
age = request_age(request).total_seconds()
|
|
request.set('aged', str(age >= self.request_age_threshold))
|
|
|
|
request_type = request.find('./action').get('type')
|
|
target = request.find('./action/target')
|
|
target_project = target.get('project')
|
|
target_package = target.get('package')
|
|
devel, _ = devel_project_fallback(self.api.apiurl, target_project, target_package)
|
|
if not devel and request_type == 'submit':
|
|
devel = request.find('./action/source').get('project')
|
|
if devel:
|
|
target.set('devel_project', devel)
|
|
StrategySuper.supplement(request)
|
|
|
|
if target_project == self.api.cnonfree:
|
|
target.set('nonfree', 'nonfree')
|
|
|
|
ring = self.ring_get(target_package)
|
|
if ring:
|
|
target.set('ring', ring)
|
|
elif not self.api.conlyadi and request_type == 'delete':
|
|
# Delete requests should always be considered in a ring.
|
|
target.set('ring', 'delete')
|
|
|
|
request_id = int(request.get('id'))
|
|
if request_id in self.requests_ignored:
|
|
request.set('ignored', str(self.requests_ignored[request_id]))
|
|
else:
|
|
request.set('ignored', 'False')
|
|
|
|
request.set('postponed', 'False')
|
|
|
|
def ring_get(self, target_package):
|
|
if self.api.conlyadi:
|
|
return None
|
|
if self.api.crings:
|
|
ring = self.api.ring_packages_for_links.get(target_package)
|
|
if ring:
|
|
# Cut off *:Rings: prefix.
|
|
return ring[len(self.api.crings)+1:]
|
|
else:
|
|
# Projects not using rings handle all requests as ring requests.
|
|
return self.api.project
|
|
return None
|
|
|
|
def filter_check(self, request):
|
|
for xpath in self.filters:
|
|
if not xpath(request):
|
|
return False
|
|
return True
|
|
|
|
def group_key_build(self, request):
|
|
if len(self.groups) == 0:
|
|
return 'all'
|
|
|
|
key = []
|
|
for xpath in self.groups:
|
|
element = xpath(request)
|
|
if element:
|
|
key.append(element[0])
|
|
if len(key) == 0:
|
|
return '00'
|
|
return '__'.join(key)
|
|
|
|
def is_staging_mergeable(self, status, pseudometa):
|
|
return len(pseudometa['requests']) > 0 and 'splitter_info' in pseudometa
|
|
|
|
def should_staging_merge(self, status, pseudometa, bootstrapped):
|
|
if (not bootstrapped and
|
|
pseudometa['splitter_info']['strategy']['name'] in ('devel', 'super') and
|
|
status['overall_state'] not in ('acceptable', 'review')):
|
|
# Simplistic attempt to allow for followup requests to be staged
|
|
# after age max has been passed while still stopping when ready.
|
|
return True
|
|
|
|
if 'activated' not in pseudometa['splitter_info']:
|
|
# No information on the age of the staging.
|
|
return False
|
|
|
|
# Allows for immediate staging when possible while not blocking requests
|
|
# created shortly after. This method removes the need to wait to create
|
|
# a larger staging at once while not ending up with lots of tiny
|
|
# stagings. As such this handles both high and low request backlogs.
|
|
activated = dateutil.parser.parse(pseudometa['splitter_info']['activated'])
|
|
delta = datetime.utcnow() - activated
|
|
return delta.total_seconds() <= self.staging_age_max
|
|
|
|
def staging_status_load(self, project):
|
|
status = self.api.project_status(project)
|
|
return status, self.api.load_prj_pseudometa(status['description'])
|
|
|
|
def is_staging_considerable(self, project, pseudometa):
|
|
return (len(pseudometa['requests']) == 0 and
|
|
self.api.prj_frozen_enough(project))
|
|
|
|
def stagings_load(self, stagings):
|
|
self.stagings = {}
|
|
self.stagings_considerable = []
|
|
self.stagings_mergeable = []
|
|
self.stagings_mergeable_none = []
|
|
|
|
# Use specified list of stagings, otherwise only empty, letter stagings.
|
|
if len(stagings) == 0:
|
|
whitelist = self.config.get('splitter-whitelist')
|
|
if whitelist:
|
|
stagings = whitelist.split()
|
|
else:
|
|
stagings = self.api.get_staging_projects_short()
|
|
should_always = False
|
|
else:
|
|
# If the an explicit list of stagings was included then always
|
|
# attempt to use even if the normal conditions are not met.
|
|
should_always = True
|
|
|
|
for staging in stagings:
|
|
project = self.api.prj_from_short(staging)
|
|
status, pseudometa = self.staging_status_load(project)
|
|
bootstrapped = self.api.is_staging_bootstrapped(project)
|
|
|
|
# Store information about staging.
|
|
self.stagings[staging] = {
|
|
'project': project,
|
|
'bootstrapped': bootstrapped,
|
|
'status': status,
|
|
'pseudometa': pseudometa,
|
|
}
|
|
|
|
# Decide if staging of interested.
|
|
if self.is_staging_mergeable(status, pseudometa) and (
|
|
should_always or self.should_staging_merge(status, pseudometa, bootstrapped)):
|
|
if pseudometa['splitter_info']['strategy']['name'] == 'none':
|
|
self.stagings_mergeable_none.append(staging)
|
|
else:
|
|
self.stagings_mergeable.append(staging)
|
|
elif self.is_staging_considerable(project, pseudometa):
|
|
self.stagings_considerable.append(staging)
|
|
|
|
# Allow both considered and remaining to be accessible after proposal.
|
|
self.stagings_available = list(self.stagings_considerable)
|
|
|
|
return (len(self.stagings_considerable) +
|
|
len(self.stagings_mergeable) +
|
|
len(self.stagings_mergeable_none))
|
|
|
|
def propose_assignment(self):
|
|
# Attempt to assign groups that have bootstrap_required first.
|
|
for group in sorted(self.grouped.keys()):
|
|
if self.grouped[group]['bootstrap_required']:
|
|
staging = self.propose_staging(choose_bootstrapped=True)
|
|
if staging:
|
|
self.requests_assign(group, staging)
|
|
else:
|
|
self.requests_postpone(group)
|
|
|
|
# Assign groups that do not have bootstrap_required and fallback to a
|
|
# bootstrapped staging if no non-bootstrapped stagings available.
|
|
for group in sorted(self.grouped.keys()):
|
|
if not self.grouped[group]['bootstrap_required']:
|
|
staging = self.propose_staging(choose_bootstrapped=False)
|
|
if staging:
|
|
self.requests_assign(group, staging)
|
|
continue
|
|
|
|
staging = self.propose_staging(choose_bootstrapped=True)
|
|
if staging:
|
|
self.requests_assign(group, staging)
|
|
else:
|
|
self.requests_postpone(group)
|
|
|
|
def requests_assign(self, group, staging, merge=False):
|
|
# Arbitrary, but descriptive group key for proposal.
|
|
key = '{}#{}@{}'.format(len(self.proposal), self.strategy.key, group)
|
|
self.proposal[key] = {
|
|
'bootstrap_required': self.grouped[group]['bootstrap_required'],
|
|
'group': group,
|
|
'requests': {},
|
|
'staging': staging,
|
|
'strategy': self.strategy.info(),
|
|
}
|
|
if merge:
|
|
self.proposal[key]['merge'] = True
|
|
|
|
# Covert request nodes to simple proposal form.
|
|
for request in self.grouped[group]['requests']:
|
|
self.proposal[key]['requests'][int(request.get('id'))] = request.find('action/target').get('package')
|
|
self.requests.remove(request)
|
|
|
|
return key
|
|
|
|
def requests_postpone(self, group):
|
|
if self.strategy.name == 'none':
|
|
return
|
|
|
|
for request in self.grouped[group]['requests']:
|
|
request.set('postponed', 'True')
|
|
|
|
def propose_staging(self, choose_bootstrapped):
|
|
found = False
|
|
for staging in sorted(self.stagings_available):
|
|
if choose_bootstrapped == self.stagings[staging]['bootstrapped']:
|
|
found = True
|
|
break
|
|
|
|
if found:
|
|
self.stagings_available.remove(staging)
|
|
return staging
|
|
|
|
return None
|
|
|
|
def strategies_try(self):
|
|
strategies = (
|
|
'special',
|
|
'quick',
|
|
'super',
|
|
'devel',
|
|
)
|
|
|
|
map(self.strategy_try, strategies)
|
|
|
|
def strategy_try(self, name):
|
|
self.strategy_set(name)
|
|
self.split()
|
|
|
|
groups = self.strategy.desirable(self)
|
|
if len(groups) == 0:
|
|
return
|
|
self.filter_grouped(groups)
|
|
|
|
self.propose_assignment()
|
|
|
|
def strategy_do(self, name, **kwargs):
|
|
self.strategy_set(name, **kwargs)
|
|
self.split()
|
|
self.propose_assignment()
|
|
|
|
def strategy_do_non_bootstrapped(self, name, **kwargs):
|
|
self.strategy_set(name, **kwargs)
|
|
self.filter_add('./action/target[not(starts-with(@ring, "0"))]')
|
|
self.split()
|
|
self.propose_assignment()
|
|
|
|
def filter_grouped(self, groups):
|
|
for group in sorted(self.grouped.keys()):
|
|
if group not in groups:
|
|
del self.grouped[group]
|
|
|
|
def merge_staging(self, staging, pseudometa):
|
|
splitter_info = pseudometa['splitter_info']
|
|
self.strategy_from_splitter_info(splitter_info)
|
|
|
|
if not self.stagings[staging]['bootstrapped']:
|
|
# If when the strategy was first run the resulting staging was not
|
|
# bootstrapped then ensure no bootstrapped packages are included.
|
|
self.filter_add('./action/target[not(starts-with(@ring, "0"))]')
|
|
|
|
self.split()
|
|
|
|
group = splitter_info['group']
|
|
if group in self.grouped:
|
|
key = self.requests_assign(group, staging, merge=True)
|
|
|
|
def merge(self, strategy_none=False):
|
|
stagings = self.stagings_mergeable_none if strategy_none else self.stagings_mergeable
|
|
for staging in sorted(stagings):
|
|
self.merge_staging(staging, self.stagings[staging]['pseudometa'])
|
|
|
|
|
|
class Strategy(object):
|
|
def __init__(self, **kwargs):
|
|
self.kwargs = kwargs
|
|
self.name = self.__class__.__name__[8:].lower()
|
|
self.key = self.name
|
|
if kwargs:
|
|
self.key += '_' + hashlib.sha1(str(kwargs)).hexdigest()[:7]
|
|
|
|
def info(self):
|
|
info = {'name': self.name}
|
|
if self.kwargs:
|
|
info['args'] = self.kwargs
|
|
return info
|
|
|
|
def desirable(self, splitter):
|
|
return splitter.grouped.keys()
|
|
|
|
class StrategyNone(Strategy):
|
|
def apply(self, splitter):
|
|
# All other strategies that inherit this are not restricted by age as
|
|
# the age restriction is used to allow other strategies to be observed.
|
|
if type(self) is StrategyNone:
|
|
splitter.filter_add('@aged="True"')
|
|
splitter.filter_add('@ignored="False"')
|
|
splitter.filter_add('@postponed="False"')
|
|
|
|
class StrategyRequests(Strategy):
|
|
def apply(self, splitter):
|
|
splitter.filter_add_requests(self.kwargs['requests'])
|
|
|
|
class StrategyCustom(StrategyNone):
|
|
def apply(self, splitter):
|
|
if 'filters' not in self.kwargs:
|
|
super(StrategyCustom, self).apply(splitter)
|
|
else:
|
|
map(splitter.filter_add, self.kwargs['filters'])
|
|
|
|
if 'groups' in self.kwargs:
|
|
map(splitter.group_by, self.kwargs['groups'])
|
|
|
|
class StrategyDevel(StrategyNone):
|
|
GROUP_MIN = 7
|
|
GROUP_MIN_MAP = {
|
|
'YaST:Head': 2,
|
|
}
|
|
|
|
def apply(self, splitter):
|
|
super(StrategyDevel, self).apply(splitter)
|
|
splitter.group_by('./action/target/@devel_project', True)
|
|
|
|
def desirable(self, splitter):
|
|
groups = []
|
|
for group, info in sorted(splitter.grouped.items()):
|
|
if len(info['requests']) >= self.GROUP_MIN_MAP.get(group, self.GROUP_MIN):
|
|
groups.append(group)
|
|
return groups
|
|
|
|
class StrategySuper(StrategyDevel):
|
|
# Regex pattern prefix representing super devel projects that should be
|
|
# grouped together. The whole pattern will be used and the last colon
|
|
# stripped, otherwise the first match group.
|
|
PATTERNS = [
|
|
'KDE:',
|
|
'GNOME:',
|
|
'(multimedia):(?:libs|apps)',
|
|
'zypp:'
|
|
]
|
|
|
|
@classmethod
|
|
def init(cls):
|
|
cls.patterns = []
|
|
for pattern in cls.PATTERNS:
|
|
cls.patterns.append(re.compile(pattern))
|
|
|
|
@classmethod
|
|
def supplement(cls, request):
|
|
if not hasattr(cls, 'patterns'):
|
|
cls.init()
|
|
|
|
target = request.find('./action/target')
|
|
devel_project = target.get('devel_project')
|
|
for pattern in cls.patterns:
|
|
match = pattern.match(devel_project)
|
|
if match:
|
|
prefix = match.group(0 if len(match.groups()) == 0 else 1).rstrip(':')
|
|
target.set('devel_project_super', prefix)
|
|
break
|
|
|
|
def apply(self, splitter):
|
|
super(StrategySuper, self).apply(splitter)
|
|
splitter.groups = []
|
|
splitter.group_by('./action/target/@devel_project_super', True)
|
|
|
|
class StrategyQuick(StrategyNone):
|
|
def apply(self, splitter):
|
|
super(StrategyQuick, self).apply(splitter)
|
|
|
|
# Leaper accepted which means any extra reviews have been added.
|
|
splitter.filter_add('./review[@by_user="leaper" and @state="accepted"]')
|
|
|
|
# No @by_project reviews that are not accepted. If not first round stage
|
|
# this should also ignore previous staging project reviews or already
|
|
# accepted human reviews.
|
|
splitter.filter_add('not(./review[@by_project and @state!="accepted"])')
|
|
|
|
# Only allow reviews by whitelisted groups and users as all others will
|
|
# be considered non-quick (like @by_group="legal-auto"). The allowed
|
|
# groups are only those configured as reviewers on the target project.
|
|
meta = ET.fromstringlist(show_project_meta(splitter.api.apiurl, splitter.api.project))
|
|
allowed_groups = meta.xpath('group[@role="reviewer"]/@groupid')
|
|
allowed_users = []
|
|
if 'repo-checker' in splitter.config:
|
|
allowed_users.append(splitter.config['repo-checker'])
|
|
|
|
self.filter_review_whitelist(splitter, 'by_group', allowed_groups)
|
|
self.filter_review_whitelist(splitter, 'by_user', allowed_users)
|
|
|
|
def filter_review_whitelist(self, splitter, attribute, allowed):
|
|
# Rather than generate a bunch of @attribute="allowed[0]" pairs
|
|
# contains is used, but the attribute must be asserted first since
|
|
# concat() loses that requirement.
|
|
allowed = ' ' + ' '.join(allowed) + ' '
|
|
splitter.filter_add(
|
|
# Assert that no(non-whitelisted and not accepted) review is found.
|
|
'not(./review[@{attribute} and '
|
|
'not(contains("{allowed}", concat(" ", @{attribute}, " "))) and '
|
|
'@state!="accepted"])'.format(attribute=attribute, allowed=allowed))
|
|
|
|
class StrategySpecial(StrategyNone):
|
|
# Configurable via splitter-special-packages.
|
|
PACKAGES = [
|
|
'gcc',
|
|
'gcc8',
|
|
'glibc',
|
|
'kernel-source',
|
|
]
|
|
|
|
def apply(self, splitter):
|
|
super(StrategySpecial, self).apply(splitter)
|
|
splitter.filter_add_requests(self.PACKAGES)
|
|
splitter.group_by('./action/target/@package')
|