Merge pull request #2617 from openSUSE/test_factory_workflow

Test to showcase a submit request to Factory

docs/processes.md

@@ -38,6 +38,10 @@ Currently, the plugin relies on the OBS capabilities to implement [staging
 workflows](https://github.com/openSUSE/open-build-service/wiki/Staging-Workflow), extending and
 adapting them to the (open)SUSE use case.
 
+This [testcase](../tests/factory_submit_request_test.py) showcases the whole submission process
+explaining how the different reviews are created and processed by OBS and by the involved bots and
+release tools.
+
 ## SUSE Linux Enterprise Development
 
 The SUSE Linux Enterprise distribution is built in a SUSE-internal instance of OBS usually referred

tests/OBSLocal.py

@@ -157,6 +157,11 @@ class TestCase(unittest.TestCase):
         if comment:
             self.assertEqual(review.comment, comment)
 
+    def assertRequestState(self, rid, **kwargs):
+        request = get_request(self.apiurl, rid)
+        for key, value in kwargs.items():
+            self.assertEqual(getattr(request.state, key), value)
+
     def randomString(self, prefix='', length=None):
         if prefix and not prefix.endswith('_'):
             prefix += '_'

tests/factory_submit_request_test.py

@@ -0,0 +1,193 @@
+import logging
+from . import OBSLocal
+import random
+import os
+
+# Needed to mock LegalAuto
+from osc.core import change_review_state
+from mock import MagicMock
+
+# Import the involved staging commands
+from osclib.freeze_command import FreezeCommand
+from osclib.select_command import SelectCommand
+from osclib.accept_command import AcceptCommand
+
+# Import the involved bots
+from check_source import CheckSource
+legal_auto = __import__("legal-auto") # Needed because of the dash in the filename
+LegalAuto = legal_auto.LegalAuto
+
+
+PROJECT = 'openSUSE:Factory'
+DEVEL_PROJECT = 'devel:drinking'
+STAGING_PROJECT_NAME = 'openSUSE:Factory:Staging:A'
+HUMAN_REVIEWER = 'factory-cop'
+FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'fixtures')
+
+class TestFactorySubmitRequest(OBSLocal.TestCase):
+    """Tests for the whole lifecycle of submit requests in Factory
+
+    This test is intended to showcase the typical workflow of new submit request for a ring package
+    in Factory. Good for newcommers and to serve as a reference to create tests for similar
+    scenarios.
+
+    The goal is not to test all possible combinations of things that could go wrong with every
+    review bot. Please use separate per-bot tests (like check_source_test.py) for that.
+
+    This is also useful as smoke test, to check that all the pieces keep working together.
+    """
+
+    def setUp(self):
+        super(TestFactorySubmitRequest, self).setUp()
+
+        # Setup the basic scenario, with manual reviewers, staging projects, rings and wine as
+        # example package (wine is in ring1, see OBSLocal.StagingWorkflow.setup_rings)
+        self.wf = OBSLocal.StagingWorkflow(PROJECT)
+        self.__setup_review_team()
+        self.__setup_devel_package('wine')
+        self.wf.setup_rings(devel_project=DEVEL_PROJECT)
+
+        # Relax the requisites to send a submit request to Factory,
+        # so the CheckSource bot is easier to please
+        self.wf.remote_config_set({'required-source-maintainer': ''})
+
+        # Setup the different bots typically used for Factory
+        self.review_bots = {}
+        self.__setup_review_bot('factory-auto', CheckSource)
+        self.__setup_review_bot('licensedigger', LegalAuto)
+
+        # Sorry, but LegalAuto is simply too hard to test while keeping this test readable,
+        # see the description of __mock_licendigger for more rationale
+        self.__mock_licensedigger()
+
+        # The staging project must be frozen in order to move packages into it
+        FreezeCommand(self.wf.api).perform(STAGING_PROJECT_NAME)
+
+        # Create the submit request
+        self.request = self.wf.create_submit_request(DEVEL_PROJECT, 'wine')
+
+    def tearDown(self):
+        super().tearDown()
+        del self.wf
+
+    def project(self):
+        return self.wf.projects[PROJECT]
+
+    def devel_project(self):
+        return self.wf.projects[DEVEL_PROJECT]
+
+    def test_happy_path(self):
+        """Tests the ideal case in which all bots are happy and the request successfully goes
+        through staging"""
+        # Initial state: reviews have been created for the bots and for the staging workflow
+        reqid = self.request.reqid
+        self.assertReview(reqid, by_user=('factory-auto', 'new'))
+        self.assertReview(reqid, by_user=('licensedigger', 'new'))
+        self.assertReview(reqid, by_group=('factory-staging', 'new'))
+
+        # Let bots come into play
+        self.__execute_review_bot('factory-auto', [reqid])
+        self.__execute_review_bot('licensedigger', [reqid])
+
+        # Bots are happy, now it's time for manual review (requested by the bots) and
+        # for the staging work
+        self.assertReview(reqid, by_user=('factory-auto', 'accepted'))
+        self.assertReview(reqid, by_user=('licensedigger', 'accepted'))
+        self.assertReview(reqid, by_group=('opensuse-review-team', 'new'))
+        self.assertReview(reqid, by_group=('factory-staging', 'new'))
+
+        # Let's first accept the manual review
+        change_review_state(
+            apiurl = self.wf.apiurl, reqid = reqid,
+            newstate = 'accepted', by_group='opensuse-review-team'
+        )
+
+        # Now only the staging workflow is pending
+        self.assertReview(reqid, by_user=('factory-auto', 'accepted'))
+        self.assertReview(reqid, by_user=('licensedigger', 'accepted'))
+        self.assertReview(reqid, by_group=('opensuse-review-team', 'accepted'))
+        self.assertReview(reqid, by_group=('factory-staging', 'new'))
+
+        # Let's put the request into the staging project
+        SelectCommand(self.wf.api, STAGING_PROJECT_NAME).perform(['wine'])
+
+        # The factory-staging review is now accepted and a new review associated to the
+        # staging project has been created
+        self.assertReview(reqid, by_group=('factory-staging', 'accepted'))
+        self.assertReview(reqid, by_project=(STAGING_PROJECT_NAME, 'new'))
+
+        # Let's say everything looks good in the staging project and it can be accepted
+        AcceptCommand(self.wf.api).accept_all([STAGING_PROJECT_NAME], True)
+
+        # Finally, all the reviews are accepted: one for each bot, one for manual review and
+        # two for the staging project (one as a consequence of selecting the package into a
+        # staging project and the other as a consequence of accepting the staging)
+        self.assertReview(reqid, by_user=('factory-auto', 'accepted'))
+        self.assertReview(reqid, by_user=('licensedigger', 'accepted'))
+        self.assertReview(reqid, by_group=('opensuse-review-team', 'accepted'))
+        self.assertReview(reqid, by_group=('factory-staging', 'accepted'))
+        self.assertReview(reqid, by_project=(STAGING_PROJECT_NAME, 'accepted'))
+
+        # So it's time to accept the request
+        self.request.change_state('accepted')
+        self.assertRequestState(reqid, name='accepted')
+
+    def __setup_devel_package(self, pkg_name):
+        pkg = self.wf.create_package(DEVEL_PROJECT, pkg_name)
+        pkg.commit_files(os.path.join(FIXTURES, 'packages', pkg_name))
+
+    def __setup_review_team(self):
+        """Creates the review team with some user on it
+
+        According to the default configuration for Factory, the CheckSource bot must create a review
+        for the group 'opensuse-review-team' for each request that passes the automatic checks.
+        That behavior can be configured with the following two parameters: 'review-team' and
+        'check-source-add-review-team'. This function ensures the test can work with the default
+        configuration, to serve as a realistic example.
+        """
+        self.wf.create_user(HUMAN_REVIEWER)
+        self.wf.create_group('opensuse-review-team', users=[HUMAN_REVIEWER])
+
+    def __setup_review_bot(self, user, bot_class):
+        """Instantiates a bot and adds the associated user as reviewer of PROJECT
+
+        :param user: user to create for the bot
+        :type user: str
+        :param bot_class: type of bot to setup
+        """
+        self.wf.create_user(user)
+        self.project().add_reviewers(users = [user])
+
+        bot_name = self.__generate_bot_name(user)
+        bot = bot_class(self.wf.apiurl, user=user, logger=logging.getLogger(bot_name))
+        bot.bot_name = bot_name
+
+        self.review_bots[user] = bot
+
+    def __execute_review_bot(self, user, requests):
+        bot = self.review_bots[user]
+        bot.set_request_ids(requests)
+        bot.check_requests()
+
+    def __generate_bot_name(self, user):
+        """Used to ensure different test runs operate in unique namespace."""
+
+        return '::'.join([type(self).__name__, user, str(random.getrandbits(8))])
+
+    def __mock_licensedigger(self):
+        """Mocks the execution of the LegalAuto bot, so it always succeeds and accepts the review
+
+        Setting up a bot and then just mocking its whole execution may look pointless, but this
+        testcase was conceived as a showcase of the Factory workflow, so all relevant bots should be
+        represented. Unfortunatelly, LegalAuto is not written to be testable and it's very dependant
+        on external components. Hopefully this whole mock could be removed in the future.
+        """
+        bot = self.review_bots['licensedigger']
+        bot.check_requests = MagicMock(side_effect=self.__accept_license)
+
+    def __accept_license(self):
+        """See :func:`__mock_licensedigger`"""
+        change_review_state(
+            apiurl = self.wf.apiurl, reqid = self.request.reqid,
+            newstate = 'accepted', by_user='licensedigger'
+        )

tests/fixtures/packages/wine/wine.changes

@@ -0,0 +1,5 @@
+-------------------------------------------------------------------
+Thu Aug  4 00:06:66 UTC 2021 - Wine Drinker <WineIs@NotAnEmulator.com>
+
+- Initial version.
+- 1

tests/fixtures/packages/wine/wine.spec

@@ -0,0 +1,16 @@
+#
+# Copyright (c) 2021 SUSE LLC
+#
+# This file is under MIT license
+
+
+Name:           wine
+Version:        1
+Release:        0
+Summary:        Wine
+License:        GPL-2.0-only
+URL:            https://www.winehq.org/
+Source:         wine.tar.gz
+BuildArch:      noarch
+
+%changelog