Merge pull request #2336 from coolo/fix_accept

Fix accept
2019-11-24 20:00:24 +01:00 · 2019-11-24 20:00:24 +01:00 · 886dc106eb
commit 886dc106eb
parent 9008d7e265 8576f22876
5 changed files with 128 additions and 156 deletions
--- a/osc-staging.py
+++ b/osc-staging.py
@ -420,19 +420,7 @@ def do_staging(self, subcmd, opts, *args):
                    api.days_since_last_freeze(prj)))
        elif cmd == 'accept':
            cmd = AcceptCommand(api)
-            for prj in args[1:]:
-                if cmd.perform(api.prj_from_letter(prj), opts.force):
-                    cmd.reset_rebuild_data(prj)
-                else:
-                    return
-                if not opts.no_cleanup:
-                    if api.item_exists(api.prj_from_letter(prj)):
-                        cmd.cleanup(api.prj_from_letter(prj))
-            cmd.accept_other_new()
-            if opts.project.startswith('openSUSE:'):
-                cmd.update_factory_version()
-                if api.item_exists(api.crebuild):
-                    cmd.sync_buildfailures()
+            cmd.accept_all(args[1:], opts.force, not opts.no_cleanup)
        elif cmd == 'unselect':
            UnselectCommand(api).perform(args[1:], opts.cleanup, opts.message)
        elif cmd == 'select':
--- a/osclib/accept_command.py
+++ b/osclib/accept_command.py
@ -8,7 +8,7 @@ from xml.etree import cElementTree as ET

 from osc.core import change_request_state, show_package_meta, wipebinaries
 from osc.core import http_GET, http_PUT, http_DELETE, http_POST
-from osc.core import delete_package, search, set_devel_project
+from osc.core import delete_package, search
 from osc.core import Request
 from osc.util.helper import decode_it
 from osclib.core import attribute_value_save
@ -24,24 +24,20 @@ class AcceptCommand(object):
        self.api = api

    def find_new_requests(self, project):
-        query = "match=state/@name='new'+and+action/target/@project='{}'".format(project)
-        url = self.api.makeurl(['search', 'request'], query)
+        match = f"state/@name='new' and action/target/@project='{project}'"
+        url = self.api.makeurl(['search', 'request'], { 'match': match })

        f = http_GET(url)
        root = ET.parse(f).getroot()

        rqs = []
        for rq in root.findall('request'):
-            pkgs = []
-            act_type = None
-            actions = rq.findall('action')
-            for action in actions:
-                act_type = action.get('type')
-                targets = action.findall('target')
-                for t in targets:
-                    pkgs.append(str(t.get('package')))
-
-            rqs.append({'id': int(rq.get('id')), 'packages': pkgs, 'type': act_type})
+            for action in rq.findall('action'):
+                for t in action.findall('target'):
+                    rqs.append({'id': int(rq.get('id')),
+                                'package': str(t.get('package')),
+                                'type': action.get('type')})
+                    break
        return rqs

    def reset_rebuild_data(self, project):
@ -55,59 +51,91 @@ class AcceptCommand(object):
                stg.find('rebuild').text = 'unknown'
                stg.find('supportpkg').text = ''

-        # reset accpted staging project rebuild state to unknown and clean up
+        # reset accepted staging project rebuild state to unknown and clean up
        # supportpkg list
        content = ET.tostring(root)
        if content != data:
            self.api.pseudometa_file_save('support_pkg_rebuild', content, 'accept command update')

-    def perform(self, project, force=False):
-        """Accept the staging project for review and submit to Factory /
-        Leap ...
+    def delete_linked(self):
+        for package in self.requests['delete']:
+            for link in self.api.linked_packages(package):
+                if link['project'] in self.api.rings or link['project'] == self.api.project:
+                    print(f"delete {link['project']}/{link['package']}")
+                    delete_package(self.api.apiurl, link['project'], link['package'],
+                                   msg="remove link while accepting delete of {}".format(package))

-        Then disable the build to disabled
-        :param project: staging project we are working with
+    def accept_all(self, projects, force=False, cleanup=True):
+        self.requests = { 'delete': [], 'submit': [] }
+        staging_packages = {}

-        """
+        for prj in projects:
+            project = self.api.prj_from_letter(prj)

-        status = self.api.check_project_status(project)
+            status = self.api.project_status(project)
+            if status.get('state') != 'acceptable' and not force:
+                print('The project "{}" is not yet acceptable.'.format(project))
+                #return False

-        if not status:
-            print('The project "{}" is not yet acceptable.'.format(project))
-            if not force:
-                return False
+            staging_packages[project] = []
+            for request in status.findall('staged_requests/request'):
+                self.requests[request.get('type')].append(request.get('package'))
+                staging_packages[project].append(request.get('package'))

-        status = self.api.project_status(project)
-        packages = []
+        other_new = self.find_new_requests(self.api.project)
+        for req in other_new:
+            self.requests[req['type']].append(req['package'])

-        rf = RequestFinder(self.api)
-        oldspecs = {}
-        for req in status.findall('staged_requests/request'):
-            packages.append(req.get('package'))
+        print('delete links to packages pending deletion...')
+        self.delete_linked()

-            print('Checking file list of {}'.format(req.get('package')))
-            os = self.api.get_filelist_for_package(pkgname=req.get('package'),
-                                                   project=self.api.project,
-                                                   extension='spec')
-            oldspecs[req.get('package')] = os
-            #self.create_new_links(self.api.project, req['package'], oldspecs)
+        opts = {}
+        if force:
+            opts['force'] = '1'

-        print(oldspecs)
+        print('triggering staging accepts...')
+        for prj in projects:
+            project = self.api.prj_from_letter(prj)

-        u = self.api.makeurl(['staging', self.api.project, 'staging_projects', project, 'accept'])
-        f = http_POST(u)
+            u = self.api.makeurl(['staging', self.api.project, 'staging_projects', project, 'accept'], opts)
+            f = http_POST(u)

-        while self.api.project_status(project, reload=True, requests=False).get('state') != 'empty':
-            time.sleep(1)
+        for req in other_new:
+            print(f"Accepting request {req['id']}: {req['package']}")
+            change_request_state(self.api.apiurl, str(req['id']), 'accepted', message='Accept to %s' % self.api.project)

-        self.api.accept_status_comment(project, packages)
-        self.api.staging_deactivate(project)
+        for prj in projects:
+            project = self.api.prj_from_letter(prj)
+            print(f'waiting for staging project {project} to be accepted')
+
+            while True:
+                status = self.api.project_status(project, reload=True)
+                if status.get('state') == 'empty':
+                    break
+                print('{} requests still staged - waiting'.format(status.find('staged_requests').get('count')))
+                time.sleep(1)
+
+            self.api.accept_status_comment(project, staging_packages[project])
+            self.api.staging_deactivate(project)
+
+            self.reset_rebuild_data(prj)
+
+            if cleanup:
+                self.cleanup(project)
+
+        for package in self.requests['submit']:
+            self.fix_linking_packages(package)
+
+        if self.api.project.startswith('openSUSE:'):
+            self.update_factory_version()
+            if self.api.item_exists(self.api.crebuild):
+                self.sync_buildfailures()

        return True

    def cleanup(self, project):
        if not self.api.item_exists(project):
-            return False
+            return

        pkglist = self.api.list_packages(project)
        clean_list = set(pkglist) - set(self.api.cnocleanup_packages)
@ -116,66 +144,28 @@ class AcceptCommand(object):
            print("[cleanup] deleted %s/%s" % (project, package))
            delete_package(self.api.apiurl, project, package, force=True, msg="autocleanup")

-        # wipe Test-DVD binaries and breaks kiwi build
-        if project.startswith('openSUSE:'):
-            for package in pkglist:
-                if package.startswith('Test-DVD-'):
-                    # intend to break the kiwi file
-                    arch = package.split('-')[-1]
-                    fakepkgname = 'I-am-breaks-kiwi-build'
-                    oldkiwifile = source_file_load(self.api.apiurl, project, package, 'PRODUCT-'+arch+'.kiwi')
-                    if oldkiwifile is not None:
-                        newkiwifile = re.sub(r'<repopackage name="openSUSE-release"/>', '<repopackage name="%s"/>' % fakepkgname, oldkiwifile)
-                        source_file_save(self.api.apiurl, project, package, 'PRODUCT-' + arch + '.kiwi', newkiwifile)
+        return

-                    # do wipe binary now
-                    query = { 'cmd': 'wipe' }
-                    query['package'] = package
-                    query['repository'] = 'images'
+    def fix_linking_packages(self, package):
+        project = self.api.project
+        file_list = self.api.get_filelist_for_package(package, project)
+        # ignore
+        if '_multibuild' in file_list or '_link' in file_list:
+            return
+        needed_links = set()
+        for file in file_list:
+            if file.endswith('.spec') and file != f'{package}.spec':
+                needed_links.add(file[:-5])
+        local_links = set()
+        for link in self.api.linked_packages(package):
+            if link['project'] == project:
+                local_links.add(link['package'])

-                    url = self.api.makeurl(['build', project], query)
-                    try:
-                        http_POST(url)
-                    except HTTPError as err:
-                        # failed to wipe isos but we can just continue
-                        pass
-
-        return True
-
-    def accept_other_new(self):
-        changed = False
-
-        rqlist = self.find_new_requests(self.api.project)
-        for req in rqlist:
-            oldspecs = self.api.get_filelist_for_package(pkgname=req['packages'][0], project=self.api.project, extension='spec')
-            print('Accepting request %d: %s' % (req['id'], ','.join(req['packages'])))
-            if req['type'] == 'delete':
-                # Remove devel project/package tag before accepting the request
-                self.remove_obsoleted_develtag(self.api.project, req['packages'][0])
-            change_request_state(self.api.apiurl, str(req['id']), 'accepted', message='Accept to %s' % self.api.project)
-            # Check if all .spec files of the package we just accepted has a package container to build
-            self.create_new_links(self.api.project, req['packages'][0], oldspecs)
-            changed = True
-
-        return changed
-
-    def remove_obsoleted_develtag(self, project, package):
-        xpath = {
-            'package': "@project='%s' and devel/@project=@project and devel/@package='%s'" % (project, package),
-        }
-        collection = search(self.api.apiurl, **xpath)['package']
-        for pkg in collection.findall('package'):
-            set_devel_project(self.api.apiurl, project, pkg.attrib['name'], devprj=None)
-
-    def create_new_links(self, project, pkgname, oldspeclist):
-        filelist = self.api.get_filelist_for_package(pkgname=pkgname, project=project, extension='spec')
-        removedspecs = set(oldspeclist) - set(filelist)
-        for spec in removedspecs:
-            # Deleting all the packages that no longer have a .spec file
-            url = self.api.makeurl(['source', project, spec[:-5]])
-            print("Deleting package %s from project %s" % (spec[:-5], project))
+        # Deleting all the packages that no longer have a .spec file
+        for link in local_links - needed_links:
+            print(f"Deleting package {project}/{link}")
            try:
-                http_DELETE(url)
+                delete_package(self.api.apiurl, project, link, msg=f"No longer linking to {package}")
            except HTTPError as err:
                if err.code == 404:
                    # the package link was not yet created, which was likely a mistake from earlier
@ -185,37 +175,28 @@ class AcceptCommand(object):
                    raise

            # Remove package from Rings in case 2nd specfile was removed
-            if self.api.ring_packages.get(spec[:-5]):
-                delete_package(self.api.apiurl, self.api.ring_packages.get(spec[:-5]), spec[:-5], force=True, msg="Cleanup package in Rings")
+            if self.api.ring_packages.get(link):
+                delete_package(self.api.apiurl, self.api.ring_packages.get(link), link, force=True, msg="Cleanup package in Rings")

-        if len(filelist) > 1:
+        for link in needed_links - local_links:
            # There is more than one .spec file in the package; link package containers as needed
-            origmeta = source_file_load(self.api.apiurl, project, pkgname, '_meta')
-            for specfile in filelist:
-                package = specfile[:-5]  # stripping .spec off the filename gives the packagename
-                if package == pkgname:
-                    # This is the original package and does not need to be linked to itself
-                    continue
-                # Check if the target package already exists, if it does not, we get a HTTP error 404 to catch
-                if not self.api.item_exists(project, package):
-                    print("Creating new package %s linked to %s" % (package, pkgname))
-                    # new package does not exist. Let's link it with new metadata
-                    newmeta = re.sub(r'(<package.*name=.){}'.format(pkgname),
-                                     r'\1{}'.format(package),
-                                     origmeta)
-                    newmeta = re.sub(r'<devel.*>',
-                                     r'<devel package="{}"/>'.format(pkgname),
-                                     newmeta)
-                    newmeta = re.sub(r'<bcntsynctag>.*</bcntsynctag>',
-                                     r'',
-                                     newmeta)
-                    newmeta = re.sub(r'</package>',
-                                     r'<bcntsynctag>{}</bcntsynctag></package>'.format(pkgname),
-                                     newmeta)
-                    source_file_save(self.api.apiurl, project, package, '_meta', newmeta)
-                    link = "<link package=\"{}\" cicount=\"copy\" />".format(pkgname)
-                    source_file_save(self.api.apiurl, project, package, '_link', link)
-        return True
+            meta = ET.fromstring(source_file_load(self.api.apiurl, project, package, '_meta'))
+            print(f"Creating new link {link}->{package}")
+
+            meta.attrib['name'] = link
+            bcnt = meta.find('bcntsynctag')
+            if bcnt is None:
+                bcnt = ET.SubElement(meta, 'bcntsynctag')
+            bcnt.text = package
+            devel = meta.find('devel')
+            if devel is None:
+                devel = ET.SubElement(meta, 'devel')
+            devel.attrib['project'] = project
+            devel.attrib['package'] = package
+
+            source_file_save(self.api.apiurl, project, link, '_meta', ET.tostring(meta))
+            xml = f"<link package='{package}' cicount='copy' />"
+            source_file_save(self.api.apiurl, project, link, '_link', xml)

    def update_version_attribute(self, project, version):
        version_attr = attribute_value_load(self.api.apiurl, project, 'ProductVersion')
--- a/osclib/conf.py
+++ b/osclib/conf.py
@ -156,6 +156,7 @@ DEFAULT = {
        'staging-required-checks-adi': '',
        'installcheck-ignore-duplicated-binaries': '',
        'onlyadi': '',
+        'nocleanup-packages': '',
        'rings': '',
        'rebuild': '',
        'product': '',
--- a/osclib/stagingapi.py
+++ b/osclib/stagingapi.py
@ -847,16 +847,6 @@ class StagingAPI(object):
        url = self.makeurl(paths, opts)
        return ET.parse(self.retried_GET(url)).getroot()

-    def check_project_status(self, project):
-        """
-        Checks a staging project for acceptance.
-        :param project: project to check
-        :return true (ok)/false (empty prj) or list of strings with
-                informations)
-
-        """
-        return self.project_status(project, requests=False).get('state') == 'acceptable'
-
    def project_status_build_percent(self, status):
        final, tobuild = self.project_status_build_sum(status)
        return final / float(final + tobuild) * 100
@ -966,6 +956,18 @@ class StagingAPI(object):

        return ret

+    def linked_packages(self, package, project=None):
+        if not project:
+            project=self.project
+
+        url = self.makeurl(['source', project, package], { 'cmd': 'showlinked' })
+        f = http_POST(url)
+        root = ET.parse(f).getroot()
+        result = []
+        for package in root.findall('package'):
+            result.append({'project': package.get('project'), 'package': package.get('name')})
+        return result
+
    def create_and_wipe_package(self, project, package):
        """
        Helper function for delete requests
--- a/tests/accept_tests.py
+++ b/tests/accept_tests.py
@ -28,7 +28,7 @@ class TestAccept(unittest.TestCase):
    def test_accept_comments(self):
        wf = self.setup_wf()

-        self.assertEqual(True, AcceptCommand(wf.api).perform(self.prj))
+        self.assertEqual(True, AcceptCommand(wf.api).accept_all(['B']))

        # Comments are cleared up
        accepted_comments = self.c_api.get_comments(project_name=self.prj)
@ -40,7 +40,7 @@ class TestAccept(unittest.TestCase):
        # snipe out cleanup to see the comments before the final countdown
        wf.api.staging_deactivate = MagicMock(return_value=True)

-        self.assertEqual(True, AcceptCommand(wf.api).perform(self.prj))
+        self.assertEqual(True, AcceptCommand(wf.api).accept_all(['B']))

        comments = self.c_api.get_comments(project_name=self.prj)
        self.assertGreater(len(comments), len(self.comments))