From 226ad12f4220c68b1f42537c01672205d640920a Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 6 Nov 2019 11:21:35 -0600 Subject: [PATCH 01/10] osclib/core: support package in attribute_value_{load,save}() functions. --- osclib/core.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/osclib/core.py b/osclib/core.py index 457b49be..05ea4005 100644 --- a/osclib/core.py +++ b/osclib/core.py @@ -361,8 +361,9 @@ def package_list_kind_filtered(apiurl, project, kinds_allowed=['source']): yield package -def attribute_value_load(apiurl, project, name, namespace='OSRT'): - url = makeurl(apiurl, ['source', project, '_attribute', namespace + ':' + name]) +def attribute_value_load(apiurl, project, name, namespace='OSRT', package=None): + path = list(filter(None, ['source', project, package, '_attribute', namespace + ':' + name])) + url = makeurl(apiurl, path) try: root = ETL.parse(http_GET(url)).getroot() @@ -389,7 +390,7 @@ def attribute_value_load(apiurl, project, name, namespace='OSRT'): # `api -T $xml /attribute/OSRT/$NEWATTRIBUTE/_meta` # # Remember to create for both OBS and IBS as necessary. -def attribute_value_save(apiurl, project, name, value, namespace='OSRT'): +def attribute_value_save(apiurl, project, name, value, namespace='OSRT', package=None): root = ET.Element('attributes') attribute = ET.SubElement(root, 'attribute') @@ -399,7 +400,7 @@ def attribute_value_save(apiurl, project, name, value, namespace='OSRT'): ET.SubElement(attribute, 'value').text = value # The OBS API of attributes is super strange, POST to update. - url = makeurl(apiurl, ['source', project, '_attribute']) + url = makeurl(apiurl, list(filter(None, ['source', project, package, '_attribute']))) http_POST(url, data=ET.tostring(root)) @memoize(session=True) From 13fa93cedebae47dd194c8b8e00e3cbb2766f37d Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 6 Nov 2019 11:21:35 -0600 Subject: [PATCH 02/10] osclib/core: provide attribute_value_delete(). --- osclib/core.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/osclib/core.py b/osclib/core.py index 05ea4005..f05e06bd 100644 --- a/osclib/core.py +++ b/osclib/core.py @@ -13,6 +13,7 @@ from osc.core import get_binarylist from osc.core import get_commitlog from osc.core import get_dependson from osc.core import get_request_list +from osc.core import http_DELETE from osc.core import http_GET from osc.core import http_POST from osc.core import http_PUT @@ -403,6 +404,10 @@ def attribute_value_save(apiurl, project, name, value, namespace='OSRT', package url = makeurl(apiurl, list(filter(None, ['source', project, package, '_attribute']))) http_POST(url, data=ET.tostring(root)) +def attribute_value_delete(apiurl, project, name, namespace='OSRT', package=None): + http_DELETE(makeurl( + apiurl, list(filter(None, ['source', project, package, '_attribute', namespace + ':' + name])))) + @memoize(session=True) def _repository_path_expand(apiurl, project, repo): """Recursively list underlying projects.""" From 818685166ffadd0763838df5430e03cb8e36cf3b Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 6 Nov 2019 11:21:35 -0600 Subject: [PATCH 03/10] osclib/core: provide package_source_{changed,age}() functions. --- osclib/core.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/osclib/core.py b/osclib/core.py index f05e06bd..1e08b9c0 100644 --- a/osclib/core.py +++ b/osclib/core.py @@ -1,6 +1,7 @@ from collections import namedtuple from collections import OrderedDict from datetime import datetime +from datetime import timezone from dateutil.parser import parse as date_parse import re import socket @@ -526,6 +527,14 @@ def project_meta_revision(apiurl, project): apiurl, project, '_project', None, format='xml', meta=True)) return int(root.find('logentry').get('revision')) +def package_source_changed(apiurl, project, package): + url = makeurl(apiurl, ['source', project, package, '_history'], {'limit': 1}) + root = ETL.parse(http_GET(url)).getroot() + return datetime.fromtimestamp(int(root.find('revision/time').text), timezone.utc).replace(tzinfo=None) + +def package_source_age(apiurl, project, package): + return datetime.utcnow() - package_source_changed(apiurl, project, package) + def entity_exists(apiurl, project, package=None): try: http_GET(makeurl(apiurl, list(filter(None, ['source', project, package])) + ['_meta'])) From 4ac724b712b2f1e1022e04bb3ef6b3359433301a Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 6 Nov 2019 11:21:35 -0600 Subject: [PATCH 04/10] osclib/core: request_action_simple_list(): include full history. Without this the requests are not nearly as useful when processing. --- osclib/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osclib/core.py b/osclib/core.py index 1e08b9c0..ec00da5d 100644 --- a/osclib/core.py +++ b/osclib/core.py @@ -968,7 +968,7 @@ def request_action_simple_list(apiurl, project, package, states, request_type): # Disable including source project in get_request_list() query. before = conf.config['include_request_from_project'] conf.config['include_request_from_project'] = False - requests = get_request_list(apiurl, project, package, None, states, request_type) + requests = get_request_list(apiurl, project, package, None, states, request_type, withfullhistory=True) conf.config['include_request_from_project'] = before for request in requests: From 48fc39aba0ca687d2048171a852231b5ba239251 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 6 Nov 2019 11:21:35 -0600 Subject: [PATCH 05/10] osclib/core: request_create_submit(): provide supersede flag. --- osclib/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/osclib/core.py b/osclib/core.py index ec00da5d..d71c3e99 100644 --- a/osclib/core.py +++ b/osclib/core.py @@ -999,7 +999,7 @@ def request_action_list_source(apiurl, project, package, states=['new', 'review' def request_create_submit(apiurl, source_project, source_package, target_project, target_package=None, message=None, revision=None, - ignore_if_any_request=False): + ignore_if_any_request=False, supersede=True): """ ignore_if_any_request: ignore source changes and do not submit if any prior requests """ @@ -1017,6 +1017,8 @@ def request_create_submit(apiurl, source_project, source_package, apiurl, target_project, target_package, REQUEST_STATES_MINUS_ACCEPTED, ['submit']): if ignore_if_any_request: return False + if not supersede and request.state.name in ('new', 'review'): + return False source_hash_pending = package_source_hash( apiurl, action.src_project, action.src_package, action.src_rev) From 8cdd550247d07450dcb991d857b20a0865af7964 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 6 Nov 2019 11:21:35 -0600 Subject: [PATCH 06/10] osclib/core: request_create_submit(): provide frequency option. --- osclib/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/osclib/core.py b/osclib/core.py index d71c3e99..5be2c890 100644 --- a/osclib/core.py +++ b/osclib/core.py @@ -999,7 +999,7 @@ def request_action_list_source(apiurl, project, package, states=['new', 'review' def request_create_submit(apiurl, source_project, source_package, target_project, target_package=None, message=None, revision=None, - ignore_if_any_request=False, supersede=True): + ignore_if_any_request=False, supersede=True, frequency=None): """ ignore_if_any_request: ignore source changes and do not submit if any prior requests """ @@ -1019,6 +1019,8 @@ def request_create_submit(apiurl, source_project, source_package, return False if not supersede and request.state.name in ('new', 'review'): return False + if frequency and request_age(request).total_seconds() < frequency: + return False source_hash_pending = package_source_hash( apiurl, action.src_project, action.src_package, action.src_rev) From e771dc653f18649453ebbf1e734ce3eed726cc79 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 6 Nov 2019 11:21:35 -0600 Subject: [PATCH 07/10] osclib/origin: provide automatic update mode controls. --- dist/obs/OSRT:OriginUpdateDelay.xml | 5 ++ dist/obs/OSRT:OriginUpdateFrequency.xml | 5 ++ dist/obs/OSRT:OriginUpdateSupersede.xml | 5 ++ osclib/origin.py | 51 +++++++++++- tests/origin_tests.py | 102 ++++++++++++++++++++++++ 5 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 dist/obs/OSRT:OriginUpdateDelay.xml create mode 100644 dist/obs/OSRT:OriginUpdateFrequency.xml create mode 100644 dist/obs/OSRT:OriginUpdateSupersede.xml diff --git a/dist/obs/OSRT:OriginUpdateDelay.xml b/dist/obs/OSRT:OriginUpdateDelay.xml new file mode 100644 index 00000000..0a56de7b --- /dev/null +++ b/dist/obs/OSRT:OriginUpdateDelay.xml @@ -0,0 +1,5 @@ + + OriginManager update delay frequency in seconds (minimum time since source change) + 1 + + diff --git a/dist/obs/OSRT:OriginUpdateFrequency.xml b/dist/obs/OSRT:OriginUpdateFrequency.xml new file mode 100644 index 00000000..8351de23 --- /dev/null +++ b/dist/obs/OSRT:OriginUpdateFrequency.xml @@ -0,0 +1,5 @@ + + OriginManager update frequency in seconds (minimum time since last request) + 1 + + diff --git a/dist/obs/OSRT:OriginUpdateSupersede.xml b/dist/obs/OSRT:OriginUpdateSupersede.xml new file mode 100644 index 00000000..183b67a1 --- /dev/null +++ b/dist/obs/OSRT:OriginUpdateSupersede.xml @@ -0,0 +1,5 @@ + + OriginManager allowed to supersede existing requests (true or false) + 1 + + diff --git a/osclib/origin.py b/osclib/origin.py index 3197e46b..4ebe46a9 100644 --- a/osclib/origin.py +++ b/osclib/origin.py @@ -3,10 +3,12 @@ from collections import namedtuple import logging from osc.core import get_request_list from osclib.conf import Config +from osclib.conf import str2bool from osclib.core import attribute_value_load from osclib.core import devel_project_get from osclib.core import devel_projects from osclib.core import entity_exists +from osclib.core import package_source_age from osclib.core import package_source_hash from osclib.core import package_source_hash_history from osclib.core import package_version @@ -41,6 +43,9 @@ POLICY_DEFAULTS = { 'additional_reviews': [], 'automatic_updates': True, 'automatic_updates_initial': False, + 'automatic_updates_supersede': True, + 'automatic_updates_delay': 0, + 'automatic_updates_frequency': 0, 'maintainer_review_always': False, 'maintainer_review_initial': True, 'pending_submission_allow': False, @@ -673,15 +678,28 @@ def origin_update(apiurl, target_project, package): if not policy['automatic_updates']: return False + mode = origin_update_mode(apiurl, target_project, package, policy, origin_info.project) + if mode['skip']: + return False + + age = package_source_age(apiurl, origin_info.project, package).total_seconds() + if age < int(mode['delay']): + return False + + supersede = str2bool(str(mode['supersede'])) + frequency = int(mode['frequency']) + if policy['pending_submission_allow']: - request_id = origin_update_pending(apiurl, origin_info.project, package, target_project) + request_id = origin_update_pending( + apiurl, origin_info.project, package, target_project, supersede, frequency) if request_id: return request_id message = 'Newer source available from package origin.' - return request_create_submit(apiurl, origin_info.project, package, target_project, message=message) + return request_create_submit(apiurl, origin_info.project, package, target_project, message=message, + supersede=supersede, frequency=frequency) -def origin_update_pending(apiurl, origin_project, package, target_project): +def origin_update_pending(apiurl, origin_project, package, target_project, supersede, frequency): apiurl_remote, project_remote = project_remote_apiurl(apiurl, origin_project) request_actions = request_action_list_source( apiurl_remote, project_remote, package, include_release=True) @@ -690,10 +708,35 @@ def origin_update_pending(apiurl, origin_project, package, target_project): message = 'Newer pending source available from package origin. See {}.'.format(identifier) src_project = project_remote_prefixed(apiurl, apiurl_remote, action.src_project) return request_create_submit(apiurl, src_project, action.src_package, - target_project, package, message=message, revision=action.src_rev) + target_project, package, message=message, revision=action.src_rev, + supersede=supersede, frequency=frequency) return False +def origin_update_mode(apiurl, target_project, package, policy, origin_project): + values = {} + for key in ('skip', 'supersede', 'delay', 'frequency'): + attribute = 'OriginUpdate{}'.format(key.capitalize()) + for project in (origin_project, target_project): + for package_attribute in (package, None): + value = attribute_value_load(apiurl, project, attribute, package=package_attribute) + if value is not None: + values[key] = value + break + + if key in values: + break + + if key in values: + continue + + if key == 'skip': + values[key] = not policy['automatic_updates'] + else: + values[key] = policy[f'automatic_updates_{key}'] + + return values + @memoize(session=True) def origin_updatable(apiurl): """ List of origin managed projects that can be updated. """ diff --git a/tests/origin_tests.py b/tests/origin_tests.py index ef82b00a..aa63bb7c 100644 --- a/tests/origin_tests.py +++ b/tests/origin_tests.py @@ -1,7 +1,9 @@ +from datetime import datetime from osc.core import change_review_state from osc.core import copy_pac as copy_package from osc.core import get_request from osclib.comments import CommentAPI +from osclib.core import attribute_value_delete from osclib.core import attribute_value_save from osclib.core import devel_project_get from osclib.core import request_create_change_devel @@ -14,6 +16,7 @@ from osclib.origin import NAME from osclib.origin import origin_annotation_load from osclib.origin import origin_find from osclib.origin import origin_update +import time import yaml from . import OBSLocal @@ -70,6 +73,26 @@ class TestOrigin(OBSLocal.TestCase): self.assertTrue(type(annotation_actual) is dict) self.assertEqual(annotation_actual, annotation) + def _assertUpdate(self, package, desired): + memoize_session_reset() + self.osc_user(self.bot_user) + request_future = origin_update(self.wf.apiurl, self.wf.project, package) + if desired: + self.assertNotEqual(request_future, False) + request_id = request_future.print_and_create() + else: + self.assertEqual(request_future, False) + request_id = None + self.osc_user_pop() + + return request_id + + def assertUpdate(self, package): + return self._assertUpdate(package, True) + + def assertNoUpdate(self, package): + return self._assertUpdate(package, False) + def accept_fallback_review(self, request_id): self.osc_user(self.review_user) change_review_state(apiurl=self.wf.apiurl, @@ -77,6 +100,12 @@ class TestOrigin(OBSLocal.TestCase): by_group=self.review_group, message='approved') self.osc_user_pop() + def waitDelta(self, start, delay): + delta = (datetime.now() - start).total_seconds() + sleep = max(delay - delta, 0) + 1 + print('sleep', sleep) + time.sleep(sleep) + def testRequestMinAge(self): self.origin_config_write([]) @@ -392,3 +421,76 @@ class TestOrigin(OBSLocal.TestCase): request_future = origin_update(self.wf.apiurl, self.wf.project, package2) self.assertEqual(request_future, False) self.osc_user_pop() + + def test_automatic_update_modes(self): + self.remote_config_set_age_minimum() + + upstream1_project = self.randomString('upstream1') + package1 = self.randomString('package1') + + target_package1 = self.wf.create_package(self.target_project, package1) + upstream1_package1 = self.wf.create_package(upstream1_project, package1) + + upstream1_package1.create_commit() + copy_package(self.wf.apiurl, upstream1_project, package1, + self.wf.apiurl, self.target_project, package1) + + attribute_value_save(self.wf.apiurl, upstream1_project, 'ApprovedRequestSource', '', 'OBS') + self.wf.create_attribute_type('OSRT', 'OriginUpdateSkip', 0) + + def config_write(delay=0, supersede=True, frequency=0): + self.origin_config_write([ + {upstream1_project: { + 'automatic_updates_delay': delay, + 'automatic_updates_supersede': supersede, + 'automatic_updates_frequency': frequency, + }}, + ]) + + # Default config with fresh commit. + config_write() + upstream1_package1.create_commit() + + # Check the full order of precidence available to mode attributes. + for project in (upstream1_project, self.target_project): + for package in (package1, None): + # Ensure no update is triggered due to OSRT:OriginUpdateSkip. + attribute_value_save(self.wf.apiurl, project, 'OriginUpdateSkip', '', package=package) + self.assertNoUpdate(package1) + attribute_value_delete(self.wf.apiurl, project, 'OriginUpdateSkip', package=package) + + # Configure a delay, make commit, and ensure no update until delayed. + delay = 17 # Allow enough time for API speed fluctuation. + config_write(delay=delay) + upstream1_package1.create_commit() + start = datetime.now() + + self.assertNoUpdate(package1) + self.waitDelta(start, delay) + request_id_package1_1 = self.assertUpdate(package1) + + # Configure no supersede and ensure no update generated for new commit. + config_write(supersede=False) + upstream1_package1.create_commit() + self.assertNoUpdate(package1) + + # Accept request and ensure update since no request to supersede. + self.assertReviewBot(request_id_package1_1, self.bot_user, 'new', 'accepted') + request_state_change(self.wf.apiurl, request_id_package1_1, 'accepted') + + request_id_package1_2 = self.assertUpdate(package1) + + # Track time since last request created for testing frequency. + start = datetime.now() + + # Configure frequency (removes supersede=False). + config_write(frequency=delay) + + upstream1_package1.create_commit() + self.assertNoUpdate(package1) + + # Fresh commit should not impact frequency which only looks at requests. + self.waitDelta(start, delay) + upstream1_package1.create_commit() + + request_id_package1_3 = self.assertUpdate(package1) From af68882f02113809a7e308c10e64c96bd9ed426e Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Thu, 7 Nov 2019 08:47:07 -0600 Subject: [PATCH 08/10] dist/ci/docker-compoose-test: include which test file is be executed. --- dist/ci/docker-compose-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/ci/docker-compose-test.sh b/dist/ci/docker-compose-test.sh index 2420d712..e6199200 100755 --- a/dist/ci/docker-compose-test.sh +++ b/dist/ci/docker-compose-test.sh @@ -17,6 +17,7 @@ for file in tests/*_tests.py; do if test -f /code/travis.settings; then COVER_ARGS="--with-coverage --cover-package=. --cover-inclusive" fi + echo "running tests from $file..." run_as_tester nosetests $COVER_ARGS -c .noserc -s $file done From 57ebf5a5fee6960046077465e329e83101861f15 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Thu, 7 Nov 2019 08:53:58 -0600 Subject: [PATCH 09/10] tests/OBSLocal: Request: print message once created. --- tests/OBSLocal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/OBSLocal.py b/tests/OBSLocal.py index a0513133..8cba1d00 100644 --- a/tests/OBSLocal.py +++ b/tests/OBSLocal.py @@ -480,6 +480,9 @@ class Request(object): dst_project=self.target_project) self.revoked = False + print('created submit request {}/{} -> {}'.format( + self.source_package.project.name, self.source_package.name, self.target_project)) + def __del__(self): self.revoke() From c34a58061b911fb5c2900aab45fe10d4203df732 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Thu, 7 Nov 2019 10:12:07 -0600 Subject: [PATCH 10/10] tests/OBSLocal: randomString() use fixed length of 2. Reduce random consumption as test environment lacks input. --- tests/OBSLocal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/OBSLocal.py b/tests/OBSLocal.py index 8cba1d00..e7932153 100644 --- a/tests/OBSLocal.py +++ b/tests/OBSLocal.py @@ -158,7 +158,7 @@ class TestCase(unittest.TestCase): if prefix and not prefix.endswith('_'): prefix += '_' if not length: - length = random.randint(10, 30) + length = 2 return prefix + ''.join([random.choice(string.ascii_letters) for i in range(length)])