Merge pull request #2296 from jberry-suse/origin-manager-update-frequency

osclib/origin: provide automatic update mode controls.

dist/ci/docker-compose-test.sh

@@ -17,6 +17,7 @@ for file in tests/*_tests.py; do
   if test -f /code/travis.settings; then
     COVER_ARGS="--with-coverage --cover-package=. --cover-inclusive"
   fi
+  echo "running tests from $file..."
   run_as_tester nosetests $COVER_ARGS -c .noserc -s $file
 done
 

dist/obs/OSRT:OriginUpdateDelay.xml

@@ -0,0 +1,5 @@
+<definition name="OSRT:OriginUpdateDelay" namespace="OSRT">
+  <description>OriginManager update delay frequency in seconds (minimum time since source change)</description>
+  <count>1</count>
+  <modifiable_by role="maintainer"/>
+</definition>

dist/obs/OSRT:OriginUpdateFrequency.xml

@@ -0,0 +1,5 @@
+<definition name="OSRT:OriginUpdateFrequency" namespace="OSRT">
+  <description>OriginManager update frequency in seconds (minimum time since last request)</description>
+  <count>1</count>
+  <modifiable_by role="maintainer"/>
+</definition>

dist/obs/OSRT:OriginUpdateSupersede.xml

@@ -0,0 +1,5 @@
+<definition name="OSRT:OSRT:OriginUpdateSupersede" namespace="OSRT">
+  <description>OriginManager allowed to supersede existing requests (true or false)</description>
+  <count>1</count>
+  <modifiable_by role="maintainer"/>
+</definition>

osclib/core.py

@@ -1,6 +1,7 @@
 from collections import namedtuple
 from collections import OrderedDict
 from datetime import datetime
+from datetime import timezone
 from dateutil.parser import parse as date_parse
 import re
 import socket
@@ -13,6 +14,7 @@ from osc.core import get_binarylist
 from osc.core import get_commitlog
 from osc.core import get_dependson
 from osc.core import get_request_list
+from osc.core import http_DELETE
 from osc.core import http_GET
 from osc.core import http_POST
 from osc.core import http_PUT
@@ -361,8 +363,9 @@ def package_list_kind_filtered(apiurl, project, kinds_allowed=['source']):
 
         yield package
 
-def attribute_value_load(apiurl, project, name, namespace='OSRT'):
-    url = makeurl(apiurl, ['source', project, '_attribute', namespace + ':' + name])
+def attribute_value_load(apiurl, project, name, namespace='OSRT', package=None):
+    path = list(filter(None, ['source', project, package, '_attribute', namespace + ':' + name]))
+    url = makeurl(apiurl, path)
 
     try:
         root = ETL.parse(http_GET(url)).getroot()
@@ -389,7 +392,7 @@ def attribute_value_load(apiurl, project, name, namespace='OSRT'):
 #   `api -T $xml /attribute/OSRT/$NEWATTRIBUTE/_meta`
 #
 # Remember to create for both OBS and IBS as necessary.
-def attribute_value_save(apiurl, project, name, value, namespace='OSRT'):
+def attribute_value_save(apiurl, project, name, value, namespace='OSRT', package=None):
     root = ET.Element('attributes')
 
     attribute = ET.SubElement(root, 'attribute')
@@ -399,9 +402,13 @@ def attribute_value_save(apiurl, project, name, value, namespace='OSRT'):
     ET.SubElement(attribute, 'value').text = value
 
     # The OBS API of attributes is super strange, POST to update.
-    url = makeurl(apiurl, ['source', project, '_attribute'])
+    url = makeurl(apiurl, list(filter(None, ['source', project, package, '_attribute'])))
     http_POST(url, data=ET.tostring(root))
 
+def attribute_value_delete(apiurl, project, name, namespace='OSRT', package=None):
+    http_DELETE(makeurl(
+        apiurl, list(filter(None, ['source', project, package, '_attribute', namespace + ':' + name]))))
+
 @memoize(session=True)
 def _repository_path_expand(apiurl, project, repo):
     """Recursively list underlying projects."""
@@ -520,6 +527,14 @@ def project_meta_revision(apiurl, project):
         apiurl, project, '_project', None, format='xml', meta=True))
     return int(root.find('logentry').get('revision'))
 
+def package_source_changed(apiurl, project, package):
+    url = makeurl(apiurl, ['source', project, package, '_history'], {'limit': 1})
+    root = ETL.parse(http_GET(url)).getroot()
+    return datetime.fromtimestamp(int(root.find('revision/time').text), timezone.utc).replace(tzinfo=None)
+
+def package_source_age(apiurl, project, package):
+    return datetime.utcnow() - package_source_changed(apiurl, project, package)
+
 def entity_exists(apiurl, project, package=None):
     try:
         http_GET(makeurl(apiurl, list(filter(None, ['source', project, package])) + ['_meta']))
@@ -953,7 +968,7 @@ def request_action_simple_list(apiurl, project, package, states, request_type):
     # Disable including source project in get_request_list() query.
     before = conf.config['include_request_from_project']
     conf.config['include_request_from_project'] = False
-    requests = get_request_list(apiurl, project, package, None, states, request_type)
+    requests = get_request_list(apiurl, project, package, None, states, request_type, withfullhistory=True)
     conf.config['include_request_from_project'] = before
 
     for request in requests:
@@ -984,7 +999,7 @@ def request_action_list_source(apiurl, project, package, states=['new', 'review'
 
 def request_create_submit(apiurl, source_project, source_package,
                           target_project, target_package=None, message=None, revision=None,
-                          ignore_if_any_request=False):
+                          ignore_if_any_request=False, supersede=True, frequency=None):
     """
     ignore_if_any_request: ignore source changes and do not submit if any prior requests
     """
@@ -1002,6 +1017,10 @@ def request_create_submit(apiurl, source_project, source_package,
         apiurl, target_project, target_package, REQUEST_STATES_MINUS_ACCEPTED, ['submit']):
         if ignore_if_any_request:
             return False
+        if not supersede and request.state.name in ('new', 'review'):
+            return False
+        if frequency and request_age(request).total_seconds() < frequency:
+            return False
 
         source_hash_pending = package_source_hash(
             apiurl, action.src_project, action.src_package, action.src_rev)

osclib/origin.py

@@ -3,10 +3,12 @@ from collections import namedtuple
 import logging
 from osc.core import get_request_list
 from osclib.conf import Config
+from osclib.conf import str2bool
 from osclib.core import attribute_value_load
 from osclib.core import devel_project_get
 from osclib.core import devel_projects
 from osclib.core import entity_exists
+from osclib.core import package_source_age
 from osclib.core import package_source_hash
 from osclib.core import package_source_hash_history
 from osclib.core import package_version
@@ -41,6 +43,9 @@ POLICY_DEFAULTS = {
     'additional_reviews': [],
     'automatic_updates': True,
     'automatic_updates_initial': False,
+    'automatic_updates_supersede': True,
+    'automatic_updates_delay': 0,
+    'automatic_updates_frequency': 0,
     'maintainer_review_always': False,
     'maintainer_review_initial': True,
     'pending_submission_allow': False,
@@ -673,15 +678,28 @@ def origin_update(apiurl, target_project, package):
     if not policy['automatic_updates']:
         return False
 
+    mode = origin_update_mode(apiurl, target_project, package, policy, origin_info.project)
+    if mode['skip']:
+        return False
+
+    age = package_source_age(apiurl, origin_info.project, package).total_seconds()
+    if age < int(mode['delay']):
+        return False
+
+    supersede = str2bool(str(mode['supersede']))
+    frequency = int(mode['frequency'])
+
     if policy['pending_submission_allow']:
-        request_id = origin_update_pending(apiurl, origin_info.project, package, target_project)
+        request_id = origin_update_pending(
+            apiurl, origin_info.project, package, target_project, supersede, frequency)
         if request_id:
             return request_id
 
     message = 'Newer source available from package origin.'
-    return request_create_submit(apiurl, origin_info.project, package, target_project, message=message)
+    return request_create_submit(apiurl, origin_info.project, package, target_project, message=message,
+                                 supersede=supersede, frequency=frequency)
 
-def origin_update_pending(apiurl, origin_project, package, target_project):
+def origin_update_pending(apiurl, origin_project, package, target_project, supersede, frequency):
     apiurl_remote, project_remote = project_remote_apiurl(apiurl, origin_project)
     request_actions = request_action_list_source(
         apiurl_remote, project_remote, package, include_release=True)
@@ -690,10 +708,35 @@ def origin_update_pending(apiurl, origin_project, package, target_project):
         message = 'Newer pending source available from package origin. See {}.'.format(identifier)
         src_project = project_remote_prefixed(apiurl, apiurl_remote, action.src_project)
         return request_create_submit(apiurl, src_project, action.src_package,
-                                     target_project, package, message=message, revision=action.src_rev)
+                                     target_project, package, message=message, revision=action.src_rev,
+                                     supersede=supersede, frequency=frequency)
 
     return False
 
+def origin_update_mode(apiurl, target_project, package, policy, origin_project):
+    values = {}
+    for key in ('skip', 'supersede', 'delay', 'frequency'):
+        attribute = 'OriginUpdate{}'.format(key.capitalize())
+        for project in (origin_project, target_project):
+            for package_attribute in (package, None):
+                value = attribute_value_load(apiurl, project, attribute, package=package_attribute)
+                if value is not None:
+                    values[key] = value
+                    break
+
+            if key in values:
+                break
+
+        if key in values:
+            continue
+
+        if key == 'skip':
+            values[key] = not policy['automatic_updates']
+        else:
+            values[key] = policy[f'automatic_updates_{key}']
+
+    return values
+
 @memoize(session=True)
 def origin_updatable(apiurl):
     """ List of origin managed projects that can be updated. """

tests/OBSLocal.py

@@ -158,7 +158,7 @@ class TestCase(unittest.TestCase):
         if prefix and not prefix.endswith('_'):
             prefix += '_'
         if not length:
-            length = random.randint(10, 30)
+            length = 2
         return prefix + ''.join([random.choice(string.ascii_letters) for i in range(length)])
 
 
@@ -480,6 +480,9 @@ class Request(object):
                                  dst_project=self.target_project)
         self.revoked = False
 
+        print('created submit request {}/{} -> {}'.format(
+            self.source_package.project.name, self.source_package.name, self.target_project))
+
     def __del__(self):
         self.revoke()
 

tests/origin_tests.py

@@ -1,7 +1,9 @@
+from datetime import datetime
 from osc.core import change_review_state
 from osc.core import copy_pac as copy_package
 from osc.core import get_request
 from osclib.comments import CommentAPI
+from osclib.core import attribute_value_delete
 from osclib.core import attribute_value_save
 from osclib.core import devel_project_get
 from osclib.core import request_create_change_devel
@@ -14,6 +16,7 @@ from osclib.origin import NAME
 from osclib.origin import origin_annotation_load
 from osclib.origin import origin_find
 from osclib.origin import origin_update
+import time
 import yaml
 from . import OBSLocal
 
@@ -70,6 +73,26 @@ class TestOrigin(OBSLocal.TestCase):
         self.assertTrue(type(annotation_actual) is dict)
         self.assertEqual(annotation_actual, annotation)
 
+    def _assertUpdate(self, package, desired):
+        memoize_session_reset()
+        self.osc_user(self.bot_user)
+        request_future = origin_update(self.wf.apiurl, self.wf.project, package)
+        if desired:
+            self.assertNotEqual(request_future, False)
+            request_id = request_future.print_and_create()
+        else:
+            self.assertEqual(request_future, False)
+            request_id = None
+        self.osc_user_pop()
+
+        return request_id
+
+    def assertUpdate(self, package):
+        return self._assertUpdate(package, True)
+
+    def assertNoUpdate(self, package):
+        return self._assertUpdate(package, False)
+
     def accept_fallback_review(self, request_id):
         self.osc_user(self.review_user)
         change_review_state(apiurl=self.wf.apiurl,
@@ -77,6 +100,12 @@ class TestOrigin(OBSLocal.TestCase):
                             by_group=self.review_group, message='approved')
         self.osc_user_pop()
 
+    def waitDelta(self, start, delay):
+        delta = (datetime.now() - start).total_seconds()
+        sleep = max(delay - delta, 0) + 1
+        print('sleep', sleep)
+        time.sleep(sleep)
+
     def testRequestMinAge(self):
         self.origin_config_write([])
 
@@ -392,3 +421,76 @@ class TestOrigin(OBSLocal.TestCase):
         request_future = origin_update(self.wf.apiurl, self.wf.project, package2)
         self.assertEqual(request_future, False)
         self.osc_user_pop()
+
+    def test_automatic_update_modes(self):
+        self.remote_config_set_age_minimum()
+
+        upstream1_project = self.randomString('upstream1')
+        package1 = self.randomString('package1')
+
+        target_package1 = self.wf.create_package(self.target_project, package1)
+        upstream1_package1 = self.wf.create_package(upstream1_project, package1)
+
+        upstream1_package1.create_commit()
+        copy_package(self.wf.apiurl, upstream1_project, package1,
+                     self.wf.apiurl, self.target_project, package1)
+
+        attribute_value_save(self.wf.apiurl, upstream1_project, 'ApprovedRequestSource', '', 'OBS')
+        self.wf.create_attribute_type('OSRT', 'OriginUpdateSkip', 0)
+
+        def config_write(delay=0, supersede=True, frequency=0):
+            self.origin_config_write([
+                {upstream1_project: {
+                    'automatic_updates_delay': delay,
+                    'automatic_updates_supersede': supersede,
+                    'automatic_updates_frequency': frequency,
+                }},
+            ])
+
+        # Default config with fresh commit.
+        config_write()
+        upstream1_package1.create_commit()
+
+        # Check the full order of precidence available to mode attributes.
+        for project in (upstream1_project, self.target_project):
+            for package in (package1, None):
+                # Ensure no update is triggered due to OSRT:OriginUpdateSkip.
+                attribute_value_save(self.wf.apiurl, project, 'OriginUpdateSkip', '', package=package)
+                self.assertNoUpdate(package1)
+                attribute_value_delete(self.wf.apiurl, project, 'OriginUpdateSkip', package=package)
+
+        # Configure a delay, make commit, and ensure no update until delayed.
+        delay = 17  # Allow enough time for API speed fluctuation.
+        config_write(delay=delay)
+        upstream1_package1.create_commit()
+        start = datetime.now()
+
+        self.assertNoUpdate(package1)
+        self.waitDelta(start, delay)
+        request_id_package1_1 = self.assertUpdate(package1)
+
+        # Configure no supersede and ensure no update generated for new commit.
+        config_write(supersede=False)
+        upstream1_package1.create_commit()
+        self.assertNoUpdate(package1)
+
+        # Accept request and ensure update since no request to supersede.
+        self.assertReviewBot(request_id_package1_1, self.bot_user, 'new', 'accepted')
+        request_state_change(self.wf.apiurl, request_id_package1_1, 'accepted')
+
+        request_id_package1_2 = self.assertUpdate(package1)
+
+        # Track time since last request created for testing frequency.
+        start = datetime.now()
+
+        # Configure frequency (removes supersede=False).
+        config_write(frequency=delay)
+
+        upstream1_package1.create_commit()
+        self.assertNoUpdate(package1)
+
+        # Fresh commit should not impact frequency which only looks at requests.
+        self.waitDelta(start, delay)
+        upstream1_package1.create_commit()
+
+        request_id_package1_3 = self.assertUpdate(package1)