Merge pull request #518 from lnussel/fixes

Tons of fixes

ReviewBot.py

@@ -83,9 +83,6 @@ class ReviewBot(object):
 
     def check_requests(self):
 
-        by_user = self.fallback_user
-        by_group = self.fallback_group
-
         for req in self.requests:
             self.logger.debug("checking %s"%req.reqid)
             good = self.check_one_request(req)
@@ -98,28 +95,36 @@ class ReviewBot(object):
             if good is None:
                 self.logger.info("%s ignored"%req.reqid)
             elif good:
-                self.logger.info("%s is good"%req.reqid)
                 self._set_review(req, 'accepted')
             else:
-                if self.review_mode == 'fallback-onfail':
+                self._set_review(req, 'declined')
-                    self.logger.info("%s needs fallback reviewer"%req.reqid)
-                    self.add_review(req, by_group=by_group, by_user=by_user)
-                    self._set_review(req, 'accepted')
-                else:
-                    self.logger.info("%s is not acceptable"%req.reqid)
-                    self._set_review(req, 'declined')
 
     def _set_review(self, req, state):
         doit = self.can_accept_review(req.reqid)
         if doit is None:
            self.logger.info("can't change state, %s does not have the reviewer"%(req.reqid))
 
+        newstate = state
+
+        by_user = self.fallback_user
+        by_group = self.fallback_group
+
+        if state == 'declined':
+            if self.review_mode == 'fallback-onfail':
+                self.logger.info("%s needs fallback reviewer"%req.reqid)
+                self.add_review(req, by_group=by_group, by_user=by_user)
+                newstate = 'accepted'
+        elif self.review_mode == 'fallback-always':
+            self.add_review(req, by_group=by_group, by_user=by_user)
+
+        self.logger.info("%s %s"%(req.reqid, state))
+
         if doit == True:
             self.logger.debug("setting %s to %s"%(req.reqid, state))
             if not self.dryrun:
                 msg = self.review_messages[state] if state in self.review_messages else state
                 osc.core.change_review_state(apiurl = self.apiurl,
-                        reqid = req.reqid, newstate = state, 
+                        reqid = req.reqid, newstate = newstate,
                         by_group=self.review_group,
                         by_user=self.review_user, message=msg)
         else:

check_maintenance_incidents.py

@@ -94,7 +94,37 @@ class MaintenanceChecker(ReviewBot.ReviewBot):
 
     # check if pkgname was submitted by the correct maintainer. If not, set
     # self.needs_maintainer_review
-    def _check_maintainer_review_needed(self, req, pkgname, author):
+    def _check_maintainer_review_needed(self, req, a):
+        author = req.get_creator()
+        if a.type == 'maintenance_incident':
+            # check if there is a link and use that or the real package
+            # name as src_packge may end with something like
+            # .openSUSE_XX.Y_Update
+            pkgname = a.src_package
+            (linkprj, linkpkg) = self._get_linktarget(a.src_project, pkgname)
+            if linkpkg is not None:
+                pkgname = linkpkg
+            if pkgname == 'patchinfo':
+                return None
+
+            project = a.tgt_releaseproject
+        else:
+            pkgname = a.tgt_package
+            project = a.tgt_project
+
+        if project.startswith('openSUSE:Leap:'):
+            mapping = MaintenanceChecker._get_lookup_yml(self.apiurl, project)
+            if mapping is None:
+                self.logger.error("error loading mapping for {}".format(project))
+            elif not pkgname in mapping:
+                self.logger.debug("{} not tracked".format(pkgname))
+            else:
+                origin = mapping[pkgname]
+                self.logger.debug("{} comes from {}, submitted from {}".format(pkgname, origin, a.src_project))
+                if origin.startswith('SUSE:SLE-12') and a.src_project.startswith('SUSE:SLE-12'):
+                    self.logger.info("{} submitted from {}, no maintainer review needed".format(pkgname, a.src_project))
+                    return
+
         maintainers = set(self._maintainers(pkgname))
         if maintainers:
             known_maintainer = False
@@ -115,33 +145,8 @@ class MaintenanceChecker(ReviewBot.ReviewBot):
             self.needs_maintainer_review.add(pkgname)
 
     def check_action_maintenance_incident(self, req, a):
-        author = req.get_creator()
-        # check if there is a link and use that or the real package
-        # name as src_packge may end with something like
-        # .openSUSE_XX.Y_Update
-        pkgname = a.src_package
-        (linkprj, linkpkg) = self._get_linktarget(a.src_project, pkgname)
-        if linkpkg is not None:
-            pkgname = linkpkg
-        if pkgname == 'patchinfo':
-            return None
 
-        skip_maintainer_review = False
+        self._check_maintainer_review_needed(req, a)
-        if a.tgt_releaseproject.startswith('openSUSE:Leap:'):
-            mapping = MaintenanceChecker._get_lookup_yml(self.apiurl, a.tgt_releaseproject)
-            if mapping is None:
-                self.logger.error("error loading mapping for {}".format(a.tgt_releaseproject))
-            elif not pkgname in mapping:
-                self.logger.error("{} not tracked".format(pkgname))
-            else:
-                origin = mapping[pkgname]
-                self.logger.debug("{} comes from {}, submitted from {}".format(pkgname, origin, a.src_project))
-                if origin.startswith('SUSE:SLE-12') and origin == a.src_project:
-                    skip_maintainer_review = True
-                    self.logger.info("{} submitted from {}, no maintainer review needed".format(pkgname, a.src_project))
-
-        if not skip_maintainer_review:
-            self._check_maintainer_review_needed(req, pkgname, author)
 
         if a.tgt_releaseproject.startswith("openSUSE:Backports:"):
             self.add_factory_source = True
@@ -149,10 +154,8 @@ class MaintenanceChecker(ReviewBot.ReviewBot):
         return True
 
     def check_action_submit(self, req, a):
-        author = req.get_creator()
-        pkgname = a.tgt_package
 
-        self._check_maintainer_review_needed(req, pkgname, author)
+        self._check_maintainer_review_needed(req, a)
 
         return True
 

check_source_in_factory.py

@@ -33,8 +33,10 @@ except ImportError:
 import osc.conf
 import osc.core
 import urllib2
+import yaml
 import ReviewBot
 
+
 class FactorySourceChecker(ReviewBot.ReviewBot):
     """ this review bot checks if the sources of a submission are
     either in Factory or a request for Factory with the same sources
@@ -50,6 +52,16 @@ class FactorySourceChecker(ReviewBot.ReviewBot):
             self.factory = "openSUSE:Factory"
         ReviewBot.ReviewBot.__init__(self, *args, **kwargs)
         self.review_messages = { 'accepted' : 'ok', 'declined': 'the package needs to be accepted in Factory first' }
+        self.lookup = None
+
+    def parse_lookup(self, project):
+        self.lookup = yaml.safe_load(self._load_lookup_file(project))
+
+    def _load_lookup_file(self, prj):
+        if prj.endswith(':NonFree'):
+            prj = prj[:-len(':NonFree')]
+        return osc.core.http_GET(osc.core.makeurl(self.apiurl,
+                                ['source', prj, '00Meta', 'lookup.yml']))
 
     def check_source_submission(self, src_project, src_package, src_rev, target_project, target_package):
         self.logger.info("%s/%s@%s -> %s/%s"%(src_project, src_package, src_rev, target_project, target_package))
@@ -71,24 +83,35 @@ class FactorySourceChecker(ReviewBot.ReviewBot):
 
         return good
 
+    def _package_get_upstream_project(self, package):
+        """ return project where the specified pacakge is supposed to come
+        from. Either by lookup table or self.factory """
+        if self.lookup and package in self.lookup:
+            return self.lookup[package]
+
+        return self.factory
+
     def _check_factory(self, rev, package):
         """check if factory sources contain the package and revision. check head and history"""
-        self.logger.debug("checking %s in %s"%(package, self.factory))
+        project = self._package_get_upstream_project(package)
+        if project is None:
+            return False
+        self.logger.debug("checking %s in %s"%(package, project))
         try:
-            si = osc.core.show_package_meta(self.apiurl, self.factory, package)
+            si = osc.core.show_package_meta(self.apiurl, project, package)
         except (urllib2.HTTPError, urllib2.URLError):
             si = None
         if si is None:
             self.logger.debug("new package")
             return None
         else:
-            si = self.get_sourceinfo(self.factory, package)
+            si = self.get_sourceinfo(project, package)
             if rev == si.verifymd5:
                 self.logger.debug("srcmd5 matches")
                 return True
 
         self.logger.debug("%s not the latest version, checking history", rev)
-        u = osc.core.makeurl(self.apiurl, [ 'source', self.factory, package, '_history' ], { 'limit': '5' })
+        u = osc.core.makeurl(self.apiurl, [ 'source', project, package, '_history' ], { 'limit': '5' })
         try:
             r = osc.core.http_GET(u)
         except urllib2.HTTPError, e:
@@ -111,7 +134,11 @@ class FactorySourceChecker(ReviewBot.ReviewBot):
     def _check_requests(self, rev, package):
         self.logger.debug("checking requests")
         try:
-            requests = osc.core.get_request_list(self.apiurl, self.factory, package, None, ['new', 'review'], 'submit')
+            project = self._package_get_upstream_project(package)
+            if project is None:
+                self.logger.error("no upstream project found for {}, can't check requests".format(package))
+                return None
+            requests = osc.core.get_request_list(self.apiurl, project, package, None, ['new', 'review'], 'submit')
         except (urllib2.HTTPError, urllib2.URLError):
             self.logger.debug("none request")
             return None
@@ -125,7 +152,7 @@ class FactorySourceChecker(ReviewBot.ReviewBot):
                         self.logger.debug("request ok")
                         return True
                     elif req.state.name == 'review':
-                        self.logger.debug("request still in review")
+                        self.logger.info("request still in review")
                         return None
                     else:
                         self.logger.error("request in state %s not expected"%req.state.name)
@@ -140,6 +167,7 @@ class CommandLineInterface(ReviewBot.CommandLineInterface):
     def get_optparser(self):
         parser = ReviewBot.CommandLineInterface.get_optparser(self)
         parser.add_option("--factory", metavar="project", help="the openSUSE Factory project")
+        parser.add_option("--lookup", metavar="project", help="use lookup file")
 
         return parser
 
@@ -152,12 +180,17 @@ class CommandLineInterface(ReviewBot.CommandLineInterface):
         if user is None:
             user = osc.conf.get_apiurl_usr(apiurl)
 
-        return FactorySourceChecker(apiurl = apiurl, \
+        bot = FactorySourceChecker(apiurl = apiurl, \
                 factory = self.options.factory, \
                 dryrun = self.options.dry, \
                 user = user, \
                 logger = self.logger)
 
+        if self.options.lookup:
+            bot.parse_lookup(self.options.lookup)
+
+        return bot
+
 if __name__ == "__main__":
     app = CommandLineInterface()
     sys.exit( app.main() )

create_test_Leap:42.2_dvd-1.testcase

@@ -21,6 +21,11 @@ job install name screen
 job install name alsa-utils
 job install name yast2-nfs-client
 job install name kernel-default
+job install name kexec-tools
+
+#ifdef __x86_64__
+job install name mokutil
+#endif
 
 job lock name bash-completion
 job lock name bash-doc

create_test_Leap:42.2_dvd-2.testcase

@@ -32,6 +32,7 @@ job install name xdelta
 job install name kate
 job install name sddm
 job install name kernel-default
+job install name kexec-tools
 
 job lock name gtk2-branding-upstream
 job lock name gdm-branding-upstream
@@ -48,6 +49,7 @@ job install provides pattern() = laptop
 job install provides pattern() = office
 job install name MozillaThunderbird
 job install name libreoffice
+job install name mokutil
 job lock name libgcc_s1-32bit
 job install provides virtualbox-guest-kmp
 #endif

leaper.py

@@ -0,0 +1,120 @@
+#!/usr/bin/python
+# Copyright (c) 2014 SUSE Linux Products GmbH
+# Copyright (c) 2016 SUSE LLC
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from pprint import pprint
+import os, sys, re
+import logging
+from optparse import OptionParser
+import cmdln
+
+try:
+    from xml.etree import cElementTree as ET
+except ImportError:
+    import cElementTree as ET
+
+import osc.conf
+import osc.core
+import urllib2
+import yaml
+import ReviewBot
+from check_maintenance_incidents import MaintenanceChecker
+from check_source_in_factory import FactorySourceChecker
+
+class Leaper(ReviewBot.ReviewBot):
+
+    def __init__(self, *args, **kwargs):
+        ReviewBot.ReviewBot.__init__(self, *args, **kwargs)
+        self.maintbot = MaintenanceChecker(*args, **kwargs)
+        # for FactorySourceChecker
+        self.factory = FactorySourceChecker(*args, **kwargs)
+        self.factory.parse_lookup('openSUSE:Leap:42.2')
+
+    def check_source_submission(self, src_project, src_package, src_rev, target_project, target_package):
+        return self.factory.check_source_submission(src_project, src_package, src_rev, target_project, target_package)
+
+    def check_one_request(self, req):
+        self.review_messages = self.DEFAULT_REVIEW_MESSAGES.copy()
+
+        if len(req.actions) != 1:
+            msg = "only one action per request please"
+            self.review_messages['declined'] = msg
+            return False
+
+        # if the fallback reviewer created the request she probably
+        # knows what she does :-)
+        if self.fallback_user and req.get_creator() == self.fallback_user:
+            self.logger.debug("skip fallback review")
+            return True
+
+        has_upstream_sources = ReviewBot.ReviewBot.check_one_request(self, req)
+        has_correct_maintainer = self.maintbot.check_one_request(req)
+
+        # not reviewed yet?
+        if has_upstream_sources is None:
+            return None
+
+        self.logger.debug("upstream sources: {}, maintainer ok: {}".format(has_upstream_sources, has_correct_maintainer))
+
+        if has_upstream_sources != True or has_correct_maintainer != True:
+            if has_upstream_sources != True:
+                self.review_messages['declined'] += '\nOrigin project changed'
+            # shouldn't happen actually
+            if has_correct_maintainer != True:
+                self.review_messages['declined'] += '\nMaintainer check failed'
+            return False
+
+        return True
+
+class CommandLineInterface(ReviewBot.CommandLineInterface):
+
+    def __init__(self, *args, **kwargs):
+        ReviewBot.CommandLineInterface.__init__(self, args, kwargs)
+
+    def get_optparser(self):
+        parser = ReviewBot.CommandLineInterface.get_optparser(self)
+
+        return parser
+
+    def setup_checker(self):
+
+        apiurl = osc.conf.config['apiurl']
+        if apiurl is None:
+            raise osc.oscerr.ConfigError("missing apiurl")
+        user = self.options.user
+        group = self.options.group
+        # if no args are given, use the current oscrc "owner"
+        if user is None and group is None:
+            user = osc.conf.get_apiurl_usr(apiurl)
+
+        bot = Leaper(apiurl = apiurl, \
+                dryrun = self.options.dry, \
+                user = user, \
+                group = group, \
+                logger = self.logger)
+
+        return bot
+
+if __name__ == "__main__":
+    app = CommandLineInterface()
+    sys.exit( app.main() )
+
+# vim: sw=4 et

manager_42.py

@@ -51,6 +51,8 @@ class Manager42(object):
         self.caching = caching
         self.apiurl = osc.conf.config['apiurl']
         self.project_preference_order = [
+                #'SUSE:SLE-12-SP2:Update',
+                'SUSE:SLE-12-SP2:GA',
                 'SUSE:SLE-12-SP1:Update',
                 'SUSE:SLE-12-SP1:GA',
                 'SUSE:SLE-12:Update',
@@ -58,6 +60,7 @@ class Manager42(object):
                 'openSUSE:Leap:42.1:Update',
                 'openSUSE:Leap:42.1',
                 'openSUSE:Factory',
+                'openSUSE:Leap:42.2:SLE-workarounds'
                 ]
 
         self.parse_lookup()
@@ -243,21 +246,25 @@ class Manager42(object):
             srcmd5, rev = self.check_source_in_project(lproject, package, root.get('verifymd5'))
             if srcmd5:
                 logger.debug("{} lookup from {} is correct".format(package, lproject))
-                return
+                # if it's from Factory we check if the package can be found elsewhere meanwhile
-            if lproject == 'openSUSE:Factory':
+                if lproject != 'openSUSE:Factory':
+                    return
+            elif lproject == 'openSUSE:Factory' and not package in self.packages[lproject]:
                 his = self.get_package_history(lproject, package, deleted=True)
                 if his:
                     logger.debug("{} got dropped from {}".format(package, lproject))
-                    return
 
         logger.debug("check where %s came from", package)
         foundit = False
         for project in self.project_preference_order:
             srcmd5, rev = self.check_source_in_project(project, package, root.get('verifymd5'))
             if srcmd5:
-                logger.info('{} -> {} (was {})'.format(package, project, lproject))
+                if project != lproject:
-                self.lookup[package] = project
+                    logger.info('{} -> {} (was {})'.format(package, project, lproject))
-                self.lookup_changes += 1
+                    self.lookup[package] = project
+                    self.lookup_changes += 1
+                else:
+                    logger.debug('{} still coming from {}'.format(package, project))
                 foundit = True
                 break
 

osc-staging.py

@@ -87,6 +87,8 @@ def _full_project_name(self, project):
               help='force the select command ignoring the time from the last freeze')
 @cmdln.option('--no-cleanup', dest='no_cleanup', action='store_true',
               help='do not cleanup remaining packages in staging projects after accept')
+@cmdln.option('--no-bootstrap', dest='bootstrap', action='store_false', default=True,
+              help='do not update bootstrap-copy when freezing')
 def do_staging(self, subcmd, opts, *args):
     """${cmd_name}: Commands to work with staging projects
 
@@ -120,7 +122,7 @@ def do_staging(self, subcmd, opts, *args):
         osc staging accept [--force] [LETTER...]
         osc staging check [--old] REPO
         osc staging cleanup_rings
-        osc staging freeze PROJECT...
+        osc staging freeze [--no-boostrap] PROJECT...
         osc staging frozenage PROJECT...
         osc staging list [--supersede]
         osc staging select [--no-freeze] [--move [--from PROJECT]] LETTER REQUEST...
@@ -172,19 +174,14 @@ def do_staging(self, subcmd, opts, *args):
             CheckCommand(api).perform(prj, opts.old)
         elif cmd == 'freeze':
             for prj in args[1:]:
-                FreezeCommand(api).perform(api.prj_from_letter(prj))
+                FreezeCommand(api).perform(api.prj_from_letter(prj), copy_bootstrap = opts.bootstrap )
         elif cmd == 'frozenage':
             for prj in args[1:]:
                 print "%s last frozen %0.1f days ago" % (api.prj_from_letter(prj), api.days_since_last_freeze(api.prj_from_letter(prj)))
         elif cmd == 'acheck':
             # Is it safe to accept? Meaning: /totest contains what it should and is not dirty
             version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64")
-            skip_totest = False
+            if version_totest:
-            if not version_totest:
-                # SLE don't have totest repository and openSUSE-release.rpm
-                skip_totest = api.item_exists(api.project, "release-notes-sles")
-
-            if not skip_totest:
                 version_openqa = api.load_file_content("%s:Staging" % api.project, "dashboard", "version_totest")
                 totest_dirty = api.is_repo_dirty(api.project, 'totest')
                 print "version_openqa: %s / version_totest: %s / totest_dirty: %s\n" % (version_openqa, version_totest, totest_dirty)
@@ -193,12 +190,8 @@ def do_staging(self, subcmd, opts, *args):
         elif cmd == 'accept':
             # Is it safe to accept? Meaning: /totest contains what it should and is not dirty
             version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64")
-            skip_totest = False
-            if not version_totest:
-                # SLE don't have totest repository and openSUSE-release.rpm
-                skip_totest = api.item_exists(api.project, "release-notes-sles")
 
-            if skip_totest or opts.force:
+            if version_totest or opts.force:
                 # SLE does not have a totest_version or openqa_version - ignore it
                 version_openqa = version_totest
                 totest_dirty   = False
@@ -209,7 +202,7 @@ def do_staging(self, subcmd, opts, *args):
             if version_openqa == version_totest and not totest_dirty:
                 cmd = AcceptCommand(api)
                 for prj in args[1:]:
-                    if not cmd.perform(api.prj_from_letter(prj)):
+                    if not cmd.perform(api.prj_from_letter(prj), opts.force):
                         return
                     if not opts.no_cleanup:
                         if api.item_exists(api.prj_from_letter(prj)):

osclib/accept_command.py

@@ -34,7 +34,7 @@ class AcceptCommand(object):
             rqs.append({'id': int(rq.get('id')), 'packages': pkgs})
         return rqs
 
-    def perform(self, project):
+    def perform(self, project, force=False):
         """Accept the staging project for review and submit to Factory /
         openSUSE 13.2 ...
 
@@ -47,7 +47,8 @@ class AcceptCommand(object):
 
         if not status:
             print('The project "{}" is not yet acceptable.'.format(project))
-            return False
+            if not force:
+                return False
 
         meta = self.api.get_prj_pseudometa(project)
         requests = []

osclib/adi_command.py

@@ -40,14 +40,6 @@ class AdiCommand:
         for p in self.api.get_adi_projects():
             self.check_adi_project(p)
 
-    def get_devel_project(self, project, package):
-        m = show_package_meta(self.api.apiurl, project, package)
-        node = ET.fromstring(''.join(m)).find('devel')
-        if node is None:
-            return None
-        else:
-            return node.get('project')
-
     def create_new_adi(self, wanted_requests, by_dp=False, split=False):
         all_requests = self.api.get_open_requests()
 
@@ -94,12 +86,12 @@ class AdiCommand:
                     non_ring_requests[request_id] = [request_id]
                 else:
                     if by_dp:
-                        devel_project = self.get_devel_project(source_project, source_package)
+                        devel_project = self.api.get_devel_project(source_project, source_package)
                         # try target pacakge in Factory
                         # this is a bit against Leap development in case submissions is from Update,
                         # or any other project than Factory
                         if devel_project is None and self.api.project.startswith('openSUSE:'):
-                            devel_project = self.get_devel_project('openSUSE:Factory', target_package)
+                            devel_project = self.api.get_devel_project('openSUSE:Factory', target_package)
                         if devel_project is not None:
                             source_project = devel_project
 

osclib/cleanup_rings.py

@@ -28,11 +28,36 @@ class CleanupRings(object):
         f = http_GET(url)
         root = ET.parse(f).getroot()
         for si in root.findall('sourceinfo'):
-            linked = si.find('linked')
+            links = si.findall('linked')
-            if linked is not None and linked.get('project') != self.api.project:
+            pkg = si.get('package')
-                if not linked.get('project').startswith(self.api.crings):
+            if links is None or len(links) == 0:
-                    print "#not linking to base: ", self.api.crings, linked.get('project')
+                print '# {} not a link'.format(pkg)
-                self.links[linked.get('package')] = si.get('package')
+            else:
+                linked = links[0]
+                dprj = linked.get('project')
+                dpkg = linked.get('package')
+                if dprj != self.api.project:
+                    if not dprj.startswith(self.api.crings):
+                        print "#{} not linking to base {} but {}".format(pkg, self.api.project, dprj)
+                    self.links[dpkg] = pkg
+                # multi spec package must link to ring
+                elif len(links) > 1:
+                    mainpkg = links[1].get('package')
+                    mainprj = links[1].get('project')
+                    if mainprj != self.api.project:
+                        print '# FIXME: {} links to {}'.format(pkg, mainprj)
+                    else:
+                        destring = None
+                        if mainpkg in self.api.ring_packages:
+                            destring = self.api.ring_packages[mainpkg]
+                        if not destring:
+                            print '# {} links to {} but is not in a ring'.format(pkg, mainpkg)
+                            print "osc linkpac {}/{} {}/{}".format(mainprj, mainpkg, prj, mainpkg)
+                        else:
+                            if pkg != 'glibc.i686': # FIXME: ugly exception
+                                print "osc linkpac -f {}/{} {}/{}".format(destring, mainpkg, prj, pkg)
+                                self.links[mainpkg] = pkg
+
 
     def fill_pkgdeps(self, prj, repo, arch):
         url = makeurl(self.api.apiurl, ['build', prj, repo, arch, '_builddepinfo'])
@@ -58,7 +83,8 @@ class CleanupRings(object):
             source = package.find('source').text
             for pkg in package.findall('pkgdep'):
                 if pkg.text not in self.bin2src:
-                    print('Package {} not found in place'.format(pkg.text))
+                    if not pkg.text.startswith('texlive-'): # XXX: texlive bullshit packaging
+                        print('Package {} not found in place'.format(pkg.text))
                     continue
                 b = self.bin2src[pkg.text]
                 self.pkgdeps[b] = source
@@ -78,7 +104,7 @@ class CleanupRings(object):
                     return False
 
         self.find_inner_ring_links(prj)
-        for arch in [ 'x86_64', 'ppc64le' ]:
+        for arch in self.api.cstaging_dvd_archs:
             self.fill_pkgdeps(prj, 'standard', arch)
 
             if prj == '{}:1-MinimalX'.format(self.api.crings):
@@ -117,6 +143,8 @@ class CleanupRings(object):
 
         for source in self.sources:
             if source not in self.pkgdeps and source not in self.links:
+                if source.startswith('texlive-specs-'): # XXX: texlive bullshit packaging
+                    continue
                 print('osc rdelete -m cleanup {} {}'.format(prj, source))
                 if nextprj:
                     print('osc linkpac {} {} {}').format(self.api.project, source, nextprj)

osclib/conf.py

@@ -32,8 +32,9 @@ from osc import conf
 DEFAULT = {
     r'openSUSE:(?P<project>[-\w\d:.]+)': {
         'staging': 'openSUSE:%(project)s:Staging',
-        'staging-group': '%(project.lower)s-staging',
+        'staging-group': 'factory-staging',
         'staging-archs': 'i586 x86_64 ppc64le',
+        'staging-dvd-archs': 'x86_64 ppc64le',
         'nocleanup-packages': 'Test-DVD-x86_64 Test-DVD-ppc64le bootstrap-copy',
         'rings': 'openSUSE:%(project)s:Rings',
         'nonfree': 'openSUSE:%(project)s:NonFree',
@@ -47,6 +48,7 @@ DEFAULT = {
         'staging': 'SUSE:%(project)s:Staging',
         'staging-group': 'sle-staging-managers',  # '%(project.lower)s-staging',
         'staging-archs': 'i586 x86_64',
+        'staging-dvd-archs': 'x86_64',
         'nocleanup-packages': 'Test-DVD-x86_64 sles-release',
         'rings': None,
         'nonfree': None,
@@ -110,7 +112,7 @@ class Config(object):
         params = [set(d) for d in DEFAULT.values()]
         params = reduce(operator.__and__, params)
         if not all(p in conf.config[self.project] for p in params):
-            msg = 'Please, add [%s] section in %s' % (self.project, self.conf_file)
+            msg = 'Please, add [%s] section in %s, see %s for details' % (self.project, self.conf_file, __file__)
             raise Exception(msg)
 
     def read_section(self, section, defaults):

osclib/freeze_command.py

@@ -116,7 +116,7 @@ class FreezeCommand(object):
                     return False
         return True
 
-    def perform(self, prj):
+    def perform(self, prj, copy_bootstrap=True):
         self.prj = prj
         self.set_links()
 
@@ -129,16 +129,17 @@ class FreezeCommand(object):
         if not self.is_bootstrap():
             return
 
-        self.set_bootstrap_copy()
+        if copy_bootstrap:
-        self.create_bootstrap_aggregate()
+            self.set_bootstrap_copy()
-        print("waiting for scheduler to disable...")
+            self.create_bootstrap_aggregate()
-        while not self.verify_bootstrap_copy_codes(['disabled']):
+            print("waiting for scheduler to disable...")
-            time.sleep(1)
+            while not self.verify_bootstrap_copy_codes(['disabled']):
-        self.build_switch_bootstrap_copy('enable')
+                time.sleep(1)
-        print("waiting for scheduler to copy...")
+            self.build_switch_bootstrap_copy('enable')
-        while not self.verify_bootstrap_copy_codes(['finished', 'succeeded']):
+            print("waiting for scheduler to copy...")
-            time.sleep(1)
+            while not self.verify_bootstrap_copy_codes(['finished', 'succeeded']):
-        self.build_switch_bootstrap_copy('disable')
+                time.sleep(1)
+            self.build_switch_bootstrap_copy('disable')
 
         # Update the version information found in the Test-DVD package, to match openSUSE-release
         if self.api.item_exists(prj, "openSUSE-release"):
@@ -256,7 +257,10 @@ class FreezeCommand(object):
                 # print(package, linked.get('package'), linked.get('project'))
                 f = self.api.retried_GET(url)
                 proot = ET.parse(f).getroot()
-                ET.SubElement(flink, 'package', {'name': package, 'srcmd5': proot.get('lsrcmd5'), 'vrev': si.get('vrev')})
+                lsrcmd5 = proot.get('lsrcmd5')
+                if lsrcmd5 is None:
+                    raise Exception("{}/{} is not a link but we expected one".format(self.api.project, package))
+                ET.SubElement(flink, 'package', {'name': package, 'srcmd5': lsrcmd5, 'vrev': si.get('vrev')})
                 return package
         if package in ['rpmlint-mini-AGGR']:
             return package  # we should not freeze aggregates

osclib/list_command.py

@@ -26,6 +26,7 @@ class ListCommand:
         non_ring_packages = []
         change_devel_requests = {}
 
+        result = {}
         for request in requests:
             # Consolidate all data from request
             request_id = int(request.get('id'))
@@ -48,6 +49,9 @@ class ListCommand:
             # package
             if self.api.crings:
                 ring = self.api.ring_packages.get(target_package)
+                if ring:
+                    # cut off *:Rings: prefix
+                    ring = ring[len(self.api.crings)+1:]
             else:
                 ring = self.api.project
 
@@ -56,15 +60,23 @@ class ListCommand:
                 if ring:
                     ring = ring + " (delete request)"
                 else:
-                    ring = '{} is non-ring package (delete request)'.format(target_package)
+                    ring = '(delete request)'
 
             # This condition is quite moot as we dispatched stuff
             # above anyway
             if ring:
-                print('Request({}): {} -> {}'.format(request_id, target_package, ring))
+                devel = self.api.get_devel_project("openSUSE:Factory", target_package)
+                if devel is None:
+                    devel = '00'
+                result.setdefault(devel, []).append('sr#{}: {:<30} -> {}'.format(request_id, target_package, ring))
             else:
                 non_ring_packages.append(target_package)
 
+        for prj in sorted(result.keys()):
+            print prj
+            for line in result[prj]:
+                print ' ', line
+
         if len(non_ring_packages):
             print "Not in a ring:", ' '.join(sorted(non_ring_packages))
         if len(change_devel_requests):

osclib/select_command.py

@@ -101,6 +101,8 @@ class SelectCommand(object):
                 msg = msg.format(fprj, self.target_project)
             print(msg)
             return True
+        elif supersede:
+            print('"{} ({}) supersedes {}'.format(request, supersede[1], supersede[0]))
         else:
             raise oscerr.WrongArgs('Arguments for select are not correct.')
 

osclib/stagingapi.py

@@ -53,7 +53,8 @@ class StagingAPI(object):
         # Store some prefix / data used in the code.
         self.cstaging = conf.config[project]['staging']
         self.cstaging_group = conf.config[project]['staging-group']
-        self.cstaging_archs = conf.config[project]['staging-archs'].split(' ')
+        self.cstaging_archs = conf.config[project]['staging-archs'].split()
+        self.cstaging_dvd_archs = conf.config[project]['staging-dvd-archs'].split()
         self.cstaging_nocleanup = conf.config[project]['nocleanup-packages'].split()
         self.crings = conf.config[project]['rings']
         self.cnonfree = conf.config[project]['nonfree']
@@ -63,6 +64,7 @@ class StagingAPI(object):
         self.user = conf.get_apiurl_usr(apiurl)
         self._ring_packages = None
         self._packages_staged = None
+        self._package_metas = dict()
 
         # If the project support rings, inititialize some variables.
         if self.crings:
@@ -802,6 +804,7 @@ class StagingAPI(object):
         if not force_enable_build:
             if self.crings and not self.ring_packages.get(tar_pkg) and not self.is_adi_project(project):
                 disable_build = True
+                logging.warning("{}/{} not in ring, build disabled".format(project, tar_pkg))
             else:
                 project = self.map_ring_package_to_subject(project, tar_pkg)
 
@@ -1229,3 +1232,24 @@ class StagingAPI(object):
             return True
         else:
             return False
+
+    # from manager_42
+    def _fill_package_meta(self, project):
+        url = makeurl(self.apiurl, ['search', 'package'], "match=[@project='%s']" % project)
+        root = ET.parse(self.retried_GET(url))
+        for p in root.findall('package'):
+            name = p.attrib['name']
+            self._package_metas.setdefault(project, {})[name] = p
+
+    def get_devel_project(self, project, package):
+        if not project in self._package_metas:
+            self._fill_package_meta(project)
+
+        if not package in self._package_metas[project]:
+            return None
+
+        node = self._package_metas[project][package].find('devel')
+        if node is None:
+            return None
+
+        return node.get('project')

totest-manager.py

@@ -366,6 +366,7 @@ class ToTestBase(object):
 
         logger.info('current_snapshot %s: %s'%(current_snapshot, self._result2str(current_result)))
         logger.debug('new_snapshot %s', new_snapshot)
+        logger.debug('current_qa_version %s', current_qa_version)
 
         snapshotable = self.factory_snapshottable()
         logger.debug("snapshotable: %s", snapshotable)

update_crawler.py

@@ -60,6 +60,8 @@ class UpdateCrawler(object):
         self.filter_lookup = set()
         self.caching = False
         self.dryrun = False
+        self.skipped = {}
+        self.submit_new = {}
 
         self.parse_lookup()
 
@@ -94,6 +96,14 @@ class UpdateCrawler(object):
                 return self.retried_GET(url)
             raise e
 
+    def get_project_meta(self, prj):
+        url = makeurl(self.apiurl, ['source', prj, '_meta'])
+        return self.cached_GET(url)
+
+    def is_maintenance_project(self, prj):
+        root = ET.fromstring(self.get_project_meta(prj))
+        return root.get('kind', None) == 'maintenance_release'
+
     def _meta_get_packagelist(self, prj, deleted=None, expand=False):
 
         query = {}
@@ -128,8 +138,8 @@ class UpdateCrawler(object):
                 ret[package.get('package')] = package
         return ret
 
-    def _submitrequest(self, src_project, src_package, rev, dst_project,
+    def _find_existing_request(self, src_project, src_package, rev, dst_project,
-                       dst_package, msg):
+                       dst_package):
         """Create a submit request."""
         states = ['new', 'review', 'declined', 'revoked']
         reqs = osc.core.get_exact_request_list(self.apiurl,
@@ -145,32 +155,37 @@ class UpdateCrawler(object):
                 if a.to_xml().find('source').get('rev') == rev:
                     logging.debug('{}: found existing request {}'.format(dst_package, r.reqid))
                     foundrev = True
+        return foundrev
+
+    def _submitrequest(self, src_project, src_package, rev, dst_project,
+                       dst_package, msg):
         res = 0
-        if not foundrev:
+        print "creating submit request", src_project, src_package, rev, dst_project, dst_package
-            print "creating submit request", src_project, src_package, rev, dst_project, dst_package
+        if not self.dryrun:
-            if not self.dryrun:
+            res = osc.core.create_submit_request(self.apiurl,
-                res = osc.core.create_submit_request(self.apiurl,
+                                                 src_project,
-                                                     src_project,
+                                                 src_package,
-                                                     src_package,
+                                                 dst_project,
-                                                     dst_project,
+                                                 dst_package,
-                                                     dst_package,
+                                                 orev=rev,
-                                                     orev=rev,
+                                                 message=msg)
-                                                     message=msg)
         return res
 
-    def submitrequest(self, src_project, src_package, rev, dst_package):
+    def submitrequest(self, src_project, src_package, rev, dst_package, origin):
         """Create a submit request using the osc.commandline.Osc class."""
         dst_project = self.to_prj
         msg = 'Automatic request from %s by UpdateCrawler' % src_project
-        return self._submitrequest(src_project, src_package, rev, dst_project,
+        if not self._find_existing_request(src_project, src_package, rev, dst_project, dst_package):
+            return self._submitrequest(src_project, src_package, rev, dst_project,
                                    dst_package, msg)
+        return 0
 
     def is_source_innerlink(self, project, package):
         try:
-            root = ET.parse(
+            root = ET.fromstring(
-                http_GET(makeurl(self.apiurl,
+                self.cached_GET(makeurl(self.apiurl,
                                  ['source', project, package, '_link']
-                ))).getroot()
+                )))
             if root.get('project') is None and root.get('cicount'):
                 return True
         except urllib2.HTTPError, err:
@@ -183,8 +198,11 @@ class UpdateCrawler(object):
         self.lookup = yaml.safe_load(self._load_lookup_file())
 
     def _load_lookup_file(self):
+        prj = self.to_prj
+        if prj.endswith(':NonFree'):
+            prj = prj[:-len(':NonFree')]
         return self.cached_GET(makeurl(self.apiurl,
-                                ['source', self.to_prj, '00Meta', 'lookup.yml']))
+                                ['source', prj, '00Meta', 'lookup.yml']))
 
     def follow_link(self, project, package, rev, verifymd5):
         #print "follow", project, package, rev
@@ -210,32 +228,54 @@ class UpdateCrawler(object):
         return (project, package, rev)
 
     def update_targets(self, targets, sources):
-        for package, sourceinfo in sources.items():
-            if package.startswith('patchinfo.'):
-                continue
 
-            if self.filter_lookup and not self.lookup.get(package, '') in self.filter_lookup:
+        # special case maintenance project. Only consider main
+        # package names. The code later follows the link in the
+        # source project then.
+        if self.is_maintenance_project(self.from_prj):
+            mainpacks = set()
+            for package, sourceinfo in sources.items():
+                if package.startswith('patchinfo.'):
+                    continue
+                files = set([node.text for node in sourceinfo.findall('filename')])
+                if '{}.spec'.format(package) in files:
+                    mainpacks.add(package)
+
+            sources = { package: sourceinfo for package, sourceinfo in sources.iteritems() if package in mainpacks }
+
+        for package, sourceinfo in sources.items():
+
+            origin = self.lookup.get(package, '')
+            if self.filter_lookup and not origin in self.filter_lookup:
+                if not origin.startswith('subpackage of'):
+                    self.skipped.setdefault(origin, set()).add(package)
                 continue
 
             if not package in targets:
-                logging.debug('Package %s not found in targets' % (package))
+                if not self.submit_new:
-                continue
+                    logging.info('Package %s not found in targets' % (package))
+                    continue
 
-            targetinfo = targets[package]
+                if self.is_source_innerlink(self.from_prj, package):
+                    logging.debug('Package %s is sub package' % (package))
+                    continue
 
-            #if package != 'openssl':
+            else:
-            #    continue
+                targetinfo = targets[package]
 
-            # Compare verifymd5
+                #if package != 'openssl':
-            md5_from = sourceinfo.get('verifymd5')
+                #    continue
-            md5_to = targetinfo.get('verifymd5')
-            if md5_from == md5_to:
-                #logging.info('Package %s not marked for update' % package)
-                continue
 
-            if self.is_source_innerlink(self.to_prj, package):
+                # Compare verifymd5
-                logging.debug('Package %s is sub package' % (package))
+                md5_from = sourceinfo.get('verifymd5')
-                continue
+                md5_to = targetinfo.get('verifymd5')
+                if md5_from == md5_to:
+                    #logging.info('Package %s not marked for update' % package)
+                    continue
+
+                if self.is_source_innerlink(self.to_prj, package):
+                    logging.debug('Package %s is sub package' % (package))
+                    continue
 
 #            this makes only sense if we look at the expanded view
 #            and want to submit from proper project
@@ -248,13 +288,12 @@ class UpdateCrawler(object):
                                                                  sourceinfo.get('srcmd5'),
                                                                  sourceinfo.get('verifymd5'))
 
-            res = self.submitrequest(src_project, src_package, src_rev, package)
+            res = self.submitrequest(src_project, src_package, src_rev, package, origin)
             if res:
                 logging.info('Created request %s for %s' % (res, package))
             elif res != 0:
                 logging.error('Error creating the request for %s' % package)
 
-
     def crawl(self, packages):
         """Main method of the class that run the crawler."""
         targets = self.get_source_infos(self.to_prj, packages)
@@ -270,8 +309,10 @@ def main(args):
     uc = UpdateCrawler(args.from_prj, args.to_prj)
     uc.caching = args.cache_requests
     uc.dryrun = args.dry
+    uc.submit_new = args.new
     if args.only_from:
-        uc.filter_lookup.add(args.only_from)
+        for prj in args.only_from:
+            uc.filter_lookup.add(prj)
 
     given_packages = args.packages
     if not given_packages:
@@ -281,6 +322,12 @@ def main(args):
             given_packages = uc.latest_packages()
     uc.crawl(given_packages)
 
+    if uc.skipped:
+        from pprint import pformat
+        logging.debug("skipped packages: %s", pformat(uc.skipped))
+
+
+
 if __name__ == '__main__':
     description = 'Create update SRs for Leap.'
     parser = argparse.ArgumentParser(description=description)
@@ -297,9 +344,10 @@ if __name__ == '__main__':
     parser.add_argument('-t', '--to', dest='to_prj', metavar='PROJECT',
                         help='project where to submit the updates to (default: %s)' % OPENSUSE,
                         default=OPENSUSE)
-    parser.add_argument('--only-from', dest='only_from', metavar='PROJECT',
+    parser.add_argument('--only-from', dest='only_from', metavar='PROJECT', action ='append',
                         help='only submit packages that came from PROJECT')
     parser.add_argument("--osc-debug", action="store_true", help="osc debug output")
+    parser.add_argument("--new", action="store_true", help="also submit new packages")
     parser.add_argument('--cache-requests', action='store_true', default=False,
                         help='cache GET requests. Not recommended for daily use.')
     parser.add_argument("packages", nargs='*', help="packages to check")