TTM: Don't set 'published' snapshot before it's done

Publish stage of the TTM will wait for the ToTest project to be done
before setting 'published' - and that's what the Release stage waits
for.

ttm/cli.py

@@ -15,6 +15,7 @@ import logging
 import ToolBase
 import cmdln
 
+from ttm.manager import QAResult
 from ttm.releaser import ToTestReleaser
 from ttm.publisher import ToTestPublisher
 
@@ -25,7 +26,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
     def __init__(self, *args, **kwargs):
         ToolBase.CommandLineInterface.__init__(self, args, kwargs)
 
-    @cmdln.option('--force', action='store_true', help="Just release, don't check")
+    @cmdln.option('--force', action='store_true', help="Just publish, don't check")
     def do_publish(self, subcmd, opts, project):
         """${cmd_name}: check and publish ToTest
 
@@ -35,6 +36,16 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
 
         ToTestPublisher(self.tool).publish(project, opts.force)
 
+    @cmdln.option('--force', action='store_true', help="Just update status")
+    def do_wait_for_published(self, subcmd, opts, project):
+        """${cmd_name}: wait for ToTest to contain publishing status and publisher finished
+
+        ${cmd_usage}
+        ${cmd_option_list}
+        """
+
+        ToTestPublisher(self.tool).wait_for_published(project, opts.force)
+
     @cmdln.option('--force', action='store_true', help="Just release, don't check")
     def do_release(self, subcmd, opts, project):
         """${cmd_name}: check and release from project to ToTest
@@ -52,5 +63,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
         ${cmd_option_list}
         """
 
-        ToTestPublisher(self.tool).publish(project)
+
+        if ToTestPublisher(self.tool).publish(project) == QAResult.passed:
+            ToTestPublisher(self.tool).wait_for_published(project)
         ToTestReleaser(self.tool).release(project)

ttm/manager.py

@@ -15,6 +15,7 @@ import ToolBase
 import logging
 import re
 import yaml
+from enum import Enum
 from xml.etree import cElementTree as ET
 from osclib.stagingapi import StagingAPI
 try:
@@ -28,6 +29,19 @@ from ttm.totest import ToTest
 class NotFoundException(Exception):
     pass
 
+class QAResult(Enum):
+    inprogress = 1
+    failed = 2
+    passed = 3
+
+    def __str__(self):
+        if self.value == QAResult.inprogress:
+            return 'inprogress'
+        elif self.value == QAResult.failed:
+            return 'failed'
+        else:
+            return 'passed'
+
 class ToTestManager(ToolBase.ToolBase):
 
     def __init__(self, tool):
@@ -95,11 +109,16 @@ class ToTestManager(ToolBase.ToolBase):
                 return result.group(1)
         raise NotFoundException("can't find %s ftp version" % project)
 
-    # we don't lock the access to this attribute as the times these
+    # make sure to update the attribute as atomar as possible - as such
-    # snapshots are greatly different
+    # only update the snapshot and don't erase anything else. The snapshots
+    # have very different update times within the pipeline, so there is
+    # normally no chance that releaser and publisher overwrite states
     def update_status(self, status, snapshot):
         status_dict = self.get_status_dict()
-        if not self.dryrun and status_dict.get(status, '') != snapshot:
+        if self.dryrun:
+            self.logger.info('setting {} snapshot to {}'.format(status, snapshot))
+            return
+        if status_dict.get(status, '') != snapshot:
             status_dict[status] = snapshot
             text = yaml.safe_dump(status_dict)
             self.api.attribute_value_save('ToTestManagerStatus', text)
@@ -111,7 +130,7 @@ class ToTestManager(ToolBase.ToolBase):
         return dict()
 
     def get_status(self, status):
-        return self.get_status_dict().get(status, '')
+        return self.get_status_dict().get(status)
 
     def release_package(self, project, package, set_release=None, repository=None,
                          target_project=None, target_repository=None):
@@ -136,3 +155,32 @@ class ToTestManager(ToolBase.ToolBase):
         else:
             self.api.retried_POST(url)
 
+    def all_repos_done(self, project, codes=None):
+        """Check the build result of the project and only return True if all
+        repos of that project are either published or unpublished
+
+        """
+
+        # coolo's experience says that 'finished' won't be
+        # sufficient here, so don't try to add it :-)
+        codes = ['published', 'unpublished'] if not codes else codes
+
+        url = self.api.makeurl(
+            ['build', project, '_result'], {'code': 'failed'})
+        f = self.api.retried_GET(url)
+        root = ET.parse(f).getroot()
+        ready = True
+        for repo in root.findall('result'):
+            # ignore ports. 'factory' is used by arm for repos that are not
+            # meant to use the totest manager.
+            if repo.get('repository') in ('ports', 'factory', 'images_staging'):
+                continue
+            if repo.get('dirty', '') == 'true':
+                self.logger.info('%s %s %s -> %s' % (repo.get('project'),
+                                                repo.get('repository'), repo.get('arch'), 'dirty'))
+                ready = False
+            if repo.get('code') not in codes:
+                self.logger.info('%s %s %s -> %s' % (repo.get('project'),
+                                                repo.get('repository'), repo.get('arch'), repo.get('code')))
+                ready = False
+        return ready

ttm/publisher.py

@@ -10,21 +10,18 @@
 # (C) 2019 coolo@suse.de, SUSE
 # Distribute under GPLv2 or GPLv3
 
+
 import json
 import re
 import yaml
 import pika
+import time
 
 import osc
 from osc.core import makeurl
-from ttm.manager import ToTestManager, NotFoundException
+from ttm.manager import ToTestManager, NotFoundException, QAResult
 from openqa_client.client import OpenQA_Client
 
-# QA Results
-QA_INPROGRESS = 1
-QA_FAILED = 2
-QA_PASSED = 3
-
 class ToTestPublisher(ToTestManager):
 
     def __init__(self, tool):
@@ -40,7 +37,7 @@ class ToTestPublisher(ToTestManager):
         """Analyze the openQA jobs of a given snapshot Returns a QAResult"""
 
         if snapshot is None:
-            return QA_FAILED
+            return QAResult.failed
 
         jobs = self.find_openqa_results(snapshot)
 
@@ -49,7 +46,7 @@ class ToTestPublisher(ToTestManager):
 
         if len(jobs) < self.project.jobs_num:  # not yet scheduled
             self.logger.warning('we have only %s jobs' % len(jobs))
-            return QA_INPROGRESS
+            return QAResult.inprogress
 
         in_progress = False
         for job in jobs:
@@ -121,12 +118,12 @@ class ToTestPublisher(ToTestManager):
         self.save_issues_to_ignore()
 
         if len(self.failed_relevant_jobs) > 0:
-            return QA_FAILED
+            return QAResult.failed
 
         if in_progress:
-            return QA_INPROGRESS
+            return QAResult.inprogress
 
-        return QA_PASSED
+        return QAResult.passed
 
     def send_amqp_event(self, current_snapshot, current_result):
         amqp_url = osc.conf.config.get('ttm_amqp_url')
@@ -135,7 +132,7 @@ class ToTestPublisher(ToTestManager):
             return
 
         self.logger.debug('Sending AMQP message')
-        inf = re.sub(r'ed$', '', self._result2str(current_result))
+        inf = re.sub(r'ed$', '', str(current_result))
         msg_topic = '%s.ttm.build.%s' % (self.project.base.lower(), inf)
         msg_body = json.dumps({
             'build': current_snapshot,
@@ -173,26 +170,26 @@ class ToTestPublisher(ToTestManager):
         group_id = self.openqa_group_id()
 
         if self.get_status('publishing') == current_snapshot or self.get_status('published') == current_snapshot:
-            self.logger.info('{} is already published'.format(current_snapshot))
+            self.logger.info('{} is already publishing'.format(current_snapshot))
-            return
+            return QAResult.inprogress
 
         self.update_pinned_descr = False
         current_result = self.overall_result(current_snapshot)
         current_qa_version = self.current_qa_version()
 
-        self.logger.info('current_snapshot %s: %s' %
+        self.logger.info('current_snapshot {}: {}'.format(current_snapshot, str(current_result)))
-                    (current_snapshot, self._result2str(current_result)))
+        self.logger.debug('current_qa_version {}'.format(current_qa_version))
-        self.logger.debug('current_qa_version %s', current_qa_version)
 
         self.send_amqp_event(current_snapshot, current_result)
 
-        if current_result == QA_FAILED:
+        if current_result == QAResult.failed:
             self.update_status('failed', current_snapshot)
+            return QAResult.failed
         else:
             self.update_status('failed', '')
 
-        if current_result != QA_PASSED:
+        if current_result != QAResult.passed:
-            return
+            return QAResult.inprogress
 
         if current_qa_version != current_snapshot:
             # We reached a very bad status: openQA testing is 'done', but not of the same version
@@ -204,7 +201,22 @@ class ToTestPublisher(ToTestManager):
         self.publish_factory_totest()
         self.write_version_to_dashboard('snapshot', current_snapshot)
         self.update_status('publishing', current_snapshot)
-        # for now we don't wait
+        return QAResult.passed
+
+    def wait_for_published(self, project, force=False):
+        self.setup(project)
+
+        if not force:
+            wait_time = 20
+            while not self.all_repos_done(self.project.test_project):
+                self.logger.info('{} is still not published, waiting {} seconds'.format(self.project.test_project, wait_time))
+                time.sleep(wait_time)
+
+        current_snapshot = self.get_status('publishing')
+        if self.dryrun:
+            self.logger.info('Publisher finished, updating published snpashot to {}'.format(current_snapshot))
+            return
+
         self.update_status('published', current_snapshot)
         group_id = self.openqa_group_id()
         if not group_id:
@@ -231,14 +243,6 @@ class ToTestPublisher(ToTestManager):
             jobs.append(job)
         return jobs
 
-    def _result2str(self, result):
-        if result == QA_INPROGRESS:
-            return 'inprogress'
-        elif result == QA_FAILED:
-            return 'failed'
-        else:
-            return 'passed'
-
     def add_published_tag(self, group_id, snapshot):
         if self.dryrun:
             return
@@ -301,13 +305,13 @@ class ToTestPublisher(ToTestManager):
 
     def publish_factory_totest(self):
         self.logger.info('Publish test project content')
+        if self.dryrun or self.project.do_not_release:
+            return
         if self.project.container_products:
             self.logger.info('Releasing container products from ToTest')
             for container in self.project.container_products:
                 self.release_package(self.project.test_project, container.package,
                                       repository=self.project.totest_container_repo)
-        if not (self.dryrun or self.project.do_not_release):
+        self.api.switch_flag_in_prj(
-            self.api.switch_flag_in_prj(
+            self.project.test_project, flag='publish', state='enable',
-                self.project.test_project, flag='publish', state='enable',
+            repository=self.project.product_repo)
-                repository=self.project.product_repo)
-

ttm/releaser.py

@@ -27,33 +27,25 @@ class ToTestReleaser(ToTestManager):
     def release(self, project, force=False):
         self.setup(project)
 
-        current_snapshot = self.get_status('testing')
+        testing_snapshot = self.get_status('testing')
         new_snapshot = self.version_from_project()
 
         # not overwriting
-        if new_snapshot == current_snapshot:
+        if new_snapshot == testing_snapshot:
             self.logger.debug('no change in snapshot version')
             return
 
-        if current_snapshot:
+        if testing_snapshot != self.get_status('failed') and testing_snapshot != self.get_status('published'):
-            testing_snapshot = self.get_status('testing')
+            self.logger.debug('Snapshot {} is still in progress'.format(testing_snapshot))
-            if testing_snapshot != self.get_status('failed') and testing_snapshot != self.get_status('publishing'):
+            return
-                self.logger.debug('Snapshot {} is still in progress'.format(testing_snapshot))
-                return
 
-        self.logger.info('current_snapshot %s', current_snapshot)
+        self.logger.info('testing snapshot %s', testing_snapshot)
-        self.logger.debug('new_snapshot %s', new_snapshot)
+        self.logger.debug('new snapshot %s', new_snapshot)
 
         if not self.is_snapshotable():
             self.logger.debug('not snapshotable')
             return
 
-        if not force and not self.all_repos_done(self.project.test_project):
-            self.logger.debug("not all repos done, can't release")
-            # the repos have to be done, otherwise we better not touch them
-            # with a new release
-            return
-
         self.update_totest(new_snapshot)
         self.update_status('testing', new_snapshot)
         self.update_status('failed', '')
@@ -83,36 +75,6 @@ class ToTestReleaser(ToTestManager):
         return self.iso_build_version(self.project.name, self.project.image_products[0].package,
                                       arch=self.project.image_products[0].archs[0])
 
-    def all_repos_done(self, project, codes=None):
-        """Check the build result of the project and only return True if all
-        repos of that project are either published or unpublished
-
-        """
-
-        # coolo's experience says that 'finished' won't be
-        # sufficient here, so don't try to add it :-)
-        codes = ['published', 'unpublished'] if not codes else codes
-
-        url = self.api.makeurl(
-            ['build', project, '_result'], {'code': 'failed'})
-        f = self.api.retried_GET(url)
-        root = ET.parse(f).getroot()
-        ready = True
-        for repo in root.findall('result'):
-            # ignore ports. 'factory' is used by arm for repos that are not
-            # meant to use the totest manager.
-            if repo.get('repository') in ('ports', 'factory', 'images_staging'):
-                continue
-            if repo.get('dirty', '') == 'true':
-                self.logger.info('%s %s %s -> %s' % (repo.get('project'),
-                                                repo.get('repository'), repo.get('arch'), 'dirty'))
-                ready = False
-            if repo.get('code') not in codes:
-                self.logger.info('%s %s %s -> %s' % (repo.get('project'),
-                                                repo.get('repository'), repo.get('arch'), repo.get('code')))
-                ready = False
-        return ready
-
     def maxsize_for_package(self, package):
         if re.match(r'.*-mini-.*', package):
             return 737280000  # a CD needs to match
@@ -267,4 +229,3 @@ class ToTestReleaser(ToTestManager):
                                         repository=self.project.product_repo)
 
         self._release(set_release=release)
-