From 8109772372e2b4d6af1f1be69236faef1f304b9f Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Apr 2017 14:40:34 -0500 Subject: [PATCH 1/5] stagingapi: get_open_requests(): replace query string with dictionary. --- osclib/stagingapi.py | 6 +++--- tests/obs.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/osclib/stagingapi.py b/osclib/stagingapi.py index 8ebc0af2..f2895ed0 100644 --- a/osclib/stagingapi.py +++ b/osclib/stagingapi.py @@ -559,14 +559,14 @@ class StagingAPI(object): requests = [] # xpath query, using the -m, -r, -s options - where = "@by_group='{}'+and+@state='new'".format(self.cstaging_group) + where = "@by_group='{}' and @state='new'".format(self.cstaging_group) projects = [format(self.project)] if self.cnonfree: projects.append(self.cnonfree) targets = ["target[@project='{}']".format(p) for p in projects] - query = "match=state/@name='review'+and+review[{}]+and+({})".format( - where, '+or+'.join(targets)) + query = {'match': "state/@name='review' and review[{}] and ({})".format( + where, ' or '.join(targets))} url = self.makeurl(['search', 'request'], query) f = http_GET(url) root = ET.parse(f).getroot() diff --git a/tests/obs.py b/tests/obs.py index 49e765ac..eaa2a210 100644 --- a/tests/obs.py +++ b/tests/obs.py @@ -18,6 +18,7 @@ import os import re import string import time +import urllib2 import urlparse import xml.etree.cElementTree as ET @@ -720,7 +721,7 @@ class OBS(object): @GET('/search/request') def search_request(self, request, uri, headers): """Return a search result for /search/request.""" - query = urlparse.urlparse(uri).query + query = urllib2.unquote(urlparse.urlparse(uri).query) assert query in ( "match=state/@name='review'+and+review[@by_group='factory-staging'+and+@state='new']+and+(target[@project='openSUSE:Factory']+or+target[@project='openSUSE:Factory:NonFree'])", "match=state/@name='review'+and+review[@by_user='factory-repo-checker'+and+@state='new']+and+(target[@project='openSUSE:Factory']+or+target[@project='openSUSE:Factory:NonFree'])" From 06d93b7344bc8b7221e06be85b0b1aac238fbcb4 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Apr 2017 14:41:17 -0500 Subject: [PATCH 2/5] stagingapi: get_open_requests() provide query_extra option. --- osclib/stagingapi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/osclib/stagingapi.py b/osclib/stagingapi.py index f2895ed0..a5a48c24 100644 --- a/osclib/stagingapi.py +++ b/osclib/stagingapi.py @@ -549,7 +549,7 @@ class StagingAPI(object): self.save_file_content('{}:Staging'.format(self.project), 'dashboard', 'ignored_requests', ignore) @memoize(session=True, add_invalidate=True) - def get_open_requests(self): + def get_open_requests(self, query_extra=None): """ Get all requests with open review for staging project that are not yet included in any staging project @@ -567,6 +567,8 @@ class StagingAPI(object): query = {'match': "state/@name='review' and review[{}] and ({})".format( where, ' or '.join(targets))} + if query_extra is not None: + query.update(query_extra) url = self.makeurl(['search', 'request'], query) f = http_GET(url) root = ET.parse(f).getroot() From 4bea74ae3ad84331be390e65cf62e07466c51a6c Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Apr 2017 15:39:47 -0500 Subject: [PATCH 3/5] request_splitter: s/suppliment/supplement/(). --- osclib/request_splitter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/osclib/request_splitter.py b/osclib/request_splitter.py index f96e9b8b..f95e8958 100644 --- a/osclib/request_splitter.py +++ b/osclib/request_splitter.py @@ -54,14 +54,14 @@ class RequestSplitter(object): def filter_only(self): ret = [] for request in self.requests: - self.suppliment(request) + self.supplement(request) if self.filter_check(request): ret.append(request) return ret def split(self): for request in self.requests: - self.suppliment(request) + self.supplement(request) if not self.filter_check(request): continue @@ -83,7 +83,7 @@ class RequestSplitter(object): else: self.other.append(request) - def suppliment(self, request): + def supplement(self, request): """ Provide additional information for grouping """ if request.get('ignored'): # Only supliment once. From 50a83d1108fc26f61cd15d1cd5f9348c5c1de62d Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Apr 2017 15:44:05 -0500 Subject: [PATCH 4/5] request_splitter: use capital for boolean strings in supplement(). Given that str(boolean) is used a capital is sometimes present, but should be consistent to remove the possibility for issues. --- osclib/request_splitter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/osclib/request_splitter.py b/osclib/request_splitter.py index f95e8958..822df6ef 100644 --- a/osclib/request_splitter.py +++ b/osclib/request_splitter.py @@ -107,9 +107,9 @@ class RequestSplitter(object): if request_id in self.requests_ignored: request.set('ignored', str(self.requests_ignored[request_id])) else: - request.set('ignored', 'false') + request.set('ignored', 'False') - request.set('postponed', 'false') + request.set('postponed', 'False') def ring_get(self, target_package): if self.api.crings: @@ -259,7 +259,7 @@ class RequestSplitter(object): return for request in self.grouped[group]['requests']: - request.set('postponed', 'true') + request.set('postponed', 'True') def propose_staging(self, choose_bootstrapped): found = False @@ -347,8 +347,8 @@ class Strategy(object): class StrategyNone(Strategy): def apply(self, splitter): splitter.filter_add('./action[not(@type="add_role" or @type="change_devel")]') - splitter.filter_add('@ignored="false"') - splitter.filter_add('@postponed="false"') + splitter.filter_add('@ignored="False"') + splitter.filter_add('@postponed="False"') class StrategyRequests(Strategy): def apply(self, splitter): From ef2bc900615f24f406be19b24ecf6a74f6b0b7ce Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Apr 2017 15:52:00 -0500 Subject: [PATCH 5/5] request_splitter: provide and utilize an aged attribute on requests. --- osc-staging.py | 2 +- osclib/request_splitter.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/osc-staging.py b/osc-staging.py index 655d15e6..e08ce4e6 100644 --- a/osc-staging.py +++ b/osc-staging.py @@ -387,7 +387,7 @@ def do_staging(self, subcmd, opts, *args): print('--move and --from must be used with explicit staging and request list') return - open_requests = api.get_open_requests() + open_requests = api.get_open_requests({'withhistory': 1}) if len(open_requests) == 0: print('No open requests to consider') return diff --git a/osclib/request_splitter.py b/osclib/request_splitter.py index 822df6ef..39b45e5f 100644 --- a/osclib/request_splitter.py +++ b/osclib/request_splitter.py @@ -1,3 +1,5 @@ +from datetime import datetime +import dateutil.parser import hashlib from lxml import etree as ET @@ -7,6 +9,8 @@ class RequestSplitter(object): self.requests = requests self.in_ring = in_ring self.mergeable_build_percent = 80 + # 55 minutes to avoid two staging bot loops of 30 minutes + self.age_threshold = 55 * 60 self.requests_ignored = self.api.get_ignored_requests() @@ -89,6 +93,12 @@ class RequestSplitter(object): # Only supliment once. return + history = request.find('history') + if history is not None: + created = dateutil.parser.parse(request.find('history').get('when')) + delta = datetime.utcnow() - created + request.set('aged', str(delta.total_seconds() > self.age_threshold)) + target = request.find('./action/target') target_project = target.get('project') target_package = target.get('package') @@ -347,6 +357,10 @@ class Strategy(object): class StrategyNone(Strategy): def apply(self, splitter): splitter.filter_add('./action[not(@type="add_role" or @type="change_devel")]') + # All other strategies that inherit this are not restricted by age as + # the age restriction is used to allow other strategies to be observed. + if type(self) is StrategyNone: + splitter.filter_add('@aged="True"') splitter.filter_add('@ignored="False"') splitter.filter_add('@postponed="False"')