Merge pull request #891 from lnussel/biarch

Biarch tool enhancements

ToolBase.py

@@ -37,8 +37,6 @@ from osclib.memoize import memoize
 
 logger = logging.getLogger()
 
-makeurl = osc.core.makeurl
-
 http_GET = osc.core.http_GET
 http_DELETE = osc.core.http_DELETE
 http_POST = osc.core.http_POST
@@ -82,8 +80,14 @@ class ToolBase(object):
         else:
             osc.core.http_PUT(*args, **kwargs)
 
+    def http_POST(self, *args, **kwargs):
+        if self.dryrun:
+            logging.debug("dryrun POST %s %s", args, str(kwargs)[:200])
+        else:
+            osc.core.http_POST(*args, **kwargs)
+
     def get_project_meta(self, prj):
-        url = makeurl(self.apiurl, ['source', prj, '_meta'])
+        url = self.makeurl(['source', prj, '_meta'])
         return self.cached_GET(url)
 
     def _meta_get_packagelist(self, prj, deleted=None, expand=False):
@@ -94,7 +98,7 @@ class ToolBase(object):
         if expand:
             query['expand'] = 1
 
-        u = makeurl(self.apiurl, ['source', prj], query)
+        u = self.makeurl(['source', prj], query)
         return self.cached_GET(u)
 
     def meta_get_packagelist(self, prj, deleted=None, expand=False):
@@ -103,8 +107,7 @@ class ToolBase(object):
 
     # FIXME: duplicated from manager_42
     def latest_packages(self, project):
-        data = self.cached_GET(makeurl(self.apiurl,
+        data = self.cached_GET(self.makeurl(['project', 'latest_commits', project]))
-                                       ['project', 'latest_commits', project]))
         lc = ET.fromstring(data)
         packages = set()
         for entry in lc.findall('{http://www.w3.org/2005/Atom}entry'):
@@ -113,6 +116,15 @@ class ToolBase(object):
                 packages.add(title[3:].split(' ')[0])
         return sorted(packages)
 
+    def makeurl(self, l, query=None):
+        """
+        Wrapper around osc's makeurl passing our apiurl
+        :return url made for l and query
+        """
+        query = [] if not query else query
+        return osc.core.makeurl(self.apiurl, l, query)
+
+
     def process(self, packages):
         """ reimplement this """
         True

biarchtool.py

@@ -30,8 +30,6 @@ import osc.core
 
 import ToolBase
 
-makeurl = osc.core.makeurl
-
 logger = logging.getLogger()
 
 FACTORY = "openSUSE:Factory"
@@ -42,58 +40,104 @@ class BiArchTool(ToolBase.ToolBase):
         ToolBase.ToolBase.__init__(self)
         self.project = project
         self.biarch_packages = None
+        self._has_baselibs = dict()
+        self.packages = []
+        self.arch = 'i586'
+        self.rdeps = None
+        self.whitelist = {
+                'i586': set([
+                    'ovmf'
+                    ]),
+                }
+
+    def has_baselibs(self, package):
+        if package in self._has_baselibs:
+            return self._has_baselibs[package]
+
+        ret = False
+        files = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, package])))
+        for n in files.findall("./entry[@name='baselibs.conf']"):
+            logger.debug('%s has baselibs', package)
+            ret = True
+            return True
+        self._has_baselibs[package] = ret
+        return ret
+
+    def has_baselibs_recursive(self, package):
+        r = self.has_baselibs(package)
+        if not r and package in self.rdeps:
+            for p in self.rdeps[package]:
+                r = self.has_baselibs_recursive(p)
+                if r:
+                    break
+        return r
 
     def _init_biarch_packages(self):
         if self.biarch_packages is None:
             self.biarch_packages = set(self.meta_get_packagelist("%s:Rings:0-Bootstrap"%self.project))
             self.biarch_packages |= set(self.meta_get_packagelist("%s:Rings:1-MinimalX"%self.project))
 
-    def enable_baselibs_packages(self, packages, force=False):
+        self._init_rdeps()
-        self._init_biarch_packages()
+
+    def _init_rdeps(self):
+        if self.rdeps is not None:
+            return
+        self.rdeps = dict()
+        url = self.makeurl(['build', self.project, 'standard', self.arch, '_builddepinfo' ], {'view':'revpkgnames'})
+        x = ET.fromstring(self.cached_GET(url))
+        for pnode in x.findall('package'):
+            name = pnode.get('name')
+            for depnode in pnode.findall('pkgdep'):
+                depname = depnode.text
+                self.rdeps.setdefault(name, set()).add(depname)
+
+    def select_packages(self, packages):
+        if packages == '__all__':
+            self.packages = self.meta_get_packagelist(self.project)
+        elif packages == '__latest__':
+            self.packages = self._filter_packages_by_time(self.latest_packages(self.project))
+        else:
+            self.packages = packages
+
+    # check when _product was last changed, eg by packagelist
+    # generator. Yield only packges that got checked in after that
+    # point in time.
+    def _filter_packages_by_time(self, packages):
+        x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, '_product', '_history'], {'limit':'1'})))
+        producttime = int(x.find('./revision/time').text)
         for pkg in packages:
+            x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, pkg, '_history'], {'rev':'1'})))
+            packagetime = int(x.find('./revision/time').text)
+#            if producttime > packagetime:
+#                continue
+            yield pkg
+
+    def remove_explicit_enable(self):
+
+        self._init_biarch_packages()
+
+        resulturl = self.makeurl(['build', self.project, '_result'])
+        result = ET.fromstring(self.cached_GET(resulturl))
+
+        packages = set()
+
+        for n in result.findall("./result[@arch='{}']/status".format(self.arch)):
+            if n.get('code') not in ('disabled', 'excluded'):
+                #logger.debug('%s %s', n.get('package'), n.get('code'))
+                packages.add(n.get('package'))
+
+        for pkg in sorted(packages):
+            changed = False
+
             logger.debug("processing %s", pkg)
-            pkgmetaurl = makeurl(self.apiurl, ['source', self.project, pkg, '_meta'])
+            pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta'])
             pkgmeta = ET.fromstring(self.cached_GET(pkgmetaurl))
-            is_enabled = None
-            is_disabled = None
-            has_baselibs = None
-            must_enable = None
-            changed = None
 
-            if force:
+            for build in pkgmeta.findall("./build"):
-                must_enable = True
+                for n in build.findall("./enable[@arch='{}']".format(self.arch)):
-
+                    logger.debug("disable %s", pkg)
-            for n in pkgmeta.findall("./build/enable[@arch='i586']"):
+                    build.remove(n)
-                is_enabled = True
-                break
-            for n in pkgmeta.findall("./build/disable[@arch='i586']"):
-                is_disabled = True
-                break
-            if pkg in self.biarch_packages:
-                logger.debug('%s is known biarch package', pkg)
-                must_enable = True
-            else:
-                files = ET.fromstring(self.cached_GET(makeurl(self.apiurl, ['source', self.project, pkg])))
-                for n in files.findall("./entry[@name='baselibs.conf']"):
-                    has_baselibs = True
-                    logger.debug('%s has baselibs', pkg)
-                    break
-            if has_baselibs:
-                must_enable = True
-
-            if must_enable:
-                if is_disabled:
-                    logger.warn('%s should be enabled but is disabled', pkg)
-                if not is_enabled:
-                    logger.info('enabling %s for biarch', pkg)
-                    bn = pkgmeta.find('build')
-                    if bn is None:
-                        bn = ET.SubElement(pkgmeta, 'build')
-                    ET.SubElement(bn, 'enable', { 'arch' : 'i586' })
                     changed = True
-            else:
-                if is_enabled:
-                    logger.warn("%s enabled or biarch without need", pkg)
 
             if changed:
                 try:
@@ -103,6 +147,122 @@ class BiArchTool(ToolBase.ToolBase):
                 except urllib2.HTTPError, e:
                     logger.error('failed to update %s: %s', pkg, e)
 
+    def add_explicit_disable(self, wipebinaries=False):
+
+        self._init_biarch_packages()
+
+        resulturl = self.makeurl(['source', self.project])
+        result = ET.fromstring(self.cached_GET(resulturl))
+
+        for pkg in self.packages:
+
+            changed = False
+
+            logger.debug("processing %s", pkg)
+            pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta'])
+            pkgmeta = ET.fromstring(self.cached_GET(pkgmetaurl))
+
+            build = pkgmeta.findall("./build")
+            if not build:
+                logger.debug('disable %s for %s', pkg, self.arch)
+                bn = pkgmeta.find('build')
+                if bn is None:
+                    bn = ET.SubElement(pkgmeta, 'build')
+                ET.SubElement(bn, 'disable', { 'arch' : self.arch })
+                changed = True
+
+            if changed:
+                try:
+                    self.http_PUT(pkgmetaurl, data=ET.tostring(pkgmeta))
+                    if self.caching:
+                        self._invalidate__cached_GET(pkgmetaurl)
+                    if wipebinaries:
+                        self.http_POST(self.makeurl(['build', self.project], {
+                            'cmd' : 'wipe',
+                            'arch': self.arch,
+                            'package' : pkg }))
+                except urllib2.HTTPError, e:
+                    logger.error('failed to update %s: %s', pkg, e)
+
+
+    def enable_baselibs_packages(self, force=False, wipebinaries=False):
+        self._init_biarch_packages()
+        for pkg in self.packages:
+            logger.debug("processing %s", pkg)
+            pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta'])
+            pkgmeta = ET.fromstring(self.cached_GET(pkgmetaurl))
+            is_enabled = None
+            is_disabled = None
+            has_baselibs = None
+            must_disable = None
+            changed = None
+
+            if force:
+                must_disable = False
+
+            for n in pkgmeta.findall("./build/enable[@arch='{}']".format(self.arch)):
+                is_enabled = True
+                break
+            for n in pkgmeta.findall("./build/disable[@arch='{}']".format(self.arch)):
+                is_disabled = True
+                break
+            if pkg in self.biarch_packages:
+                logger.debug('%s is known biarch package', pkg)
+                must_disable = False
+            elif pkg in self.whitelist[self.arch]:
+                logger.debug('%s is whitelisted', pkg)
+                must_disable = False
+            else:
+                has_baselibs = self.has_baselibs_recursive(pkg)
+                if has_baselibs:
+                    must_disable = False
+                else:
+                    must_disable = True
+
+            if must_disable == False:
+                if is_disabled:
+                    logger.info('enabling %s for %s', pkg, self.arch)
+                    for build in pkgmeta.findall("./build"):
+                        for n in build.findall("./disable[@arch='{}']".format(self.arch)):
+                            build.remove(n)
+                            changed = True
+                    if changed == False:
+                        logger.error('build tag not found in %s/%s!?', pkg, self.arch)
+                else:
+                    logger.debug('%s already enabled for %s', pkg, self.arch)
+            elif must_disable == True:
+                if not is_disabled:
+                    logger.info('disabling %s for %s', pkg, self.arch)
+                    bn = pkgmeta.find('build')
+                    if bn is None:
+                        bn = ET.SubElement(pkgmeta, 'build')
+                    ET.SubElement(bn, 'disable', { 'arch' : self.arch })
+                    changed = True
+                else:
+                    logger.debug('%s already disabled for %s', pkg, self.arch)
+
+            if is_enabled:
+                logger.info('removing explicit enable %s for %s', pkg, self.arch)
+                for build in pkgmeta.findall("./build"):
+                    for n in build.findall("./enable[@arch='{}']".format(self.arch)):
+                        build.remove(n)
+                        changed = True
+                if changed == False:
+                    logger.error('build tag not found in %s/%s!?', pkg, self.arch)
+
+            if changed:
+                try:
+                    self.http_PUT(pkgmetaurl, data=ET.tostring(pkgmeta))
+                    if self.caching:
+                        self._invalidate__cached_GET(pkgmetaurl)
+                    if must_disable and wipebinaries:
+                        self.http_POST(self.makeurl(['build', self.project], {
+                            'cmd' : 'wipe',
+                            'arch': self.arch,
+                            'package' : pkg }))
+                except urllib2.HTTPError, e:
+                    logger.error('failed to update %s: %s', pkg, e)
+
 class CommandLineInterface(ToolBase.CommandLineInterface):
 
     def __init__(self, *args, **kwargs):
@@ -119,18 +279,18 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
         tool = BiArchTool(self.options.project)
         return tool
 
-    def _packages(self, all, packages):
+    def _select_packages(self, all, packages):
-        if not packages:
+        if packages:
-            if all:
+            self.tool.select_packages(packages)
-                packages = self.tool.meta_get_packagelist(self.tool.project)
+        elif all:
+            self.tool.select_packages('__all__')
         else:
-                packages = self.tool.latest_packages(self.tool.project)
+            self.tool.select_packages('__latest__')
-
-        return packages
 
     @cmdln.option('-n', '--interval', metavar="minutes", type="int", help="periodic interval in minutes")
     @cmdln.option('-a', '--all', action='store_true', help='process all packages')
     @cmdln.option('-f', '--force', action='store_true', help='enable in any case')
+    @cmdln.option('--wipe', action='store_true', help='also wipe binaries')
     def do_enable_baselibs_packages(self, subcmd, opts, *packages):
         """${cmd_name}: enable build for packages in Ring 0 or 1 or with
         baselibs.conf
@@ -139,7 +299,34 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
         ${cmd_option_list}
         """
         def work():
-            self.tool.enable_baselibs_packages(self._packages(opts.all, packages), opts.force)
+            self._select_packages(opts.all, packages)
+            self.tool.enable_baselibs_packages(force=opts.force, wipebinaries=opts.wipe)
+
+        self.runner(work, opts.interval)
+
+    @cmdln.option('-a', '--all', action='store_true', help='process all packages')
+    def do_remove_explicit_enable(self, subcmd, opts, *packages):
+        """${cmd_name}: remove all explicit enable tags from packages
+
+        ${cmd_usage}
+        ${cmd_option_list}
+        """
+
+        self.tool.remove_explicit_enable()
+
+    @cmdln.option('-a', '--all', action='store_true', help='process all packages')
+    @cmdln.option('-n', '--interval', metavar="minutes", type="int", help="periodic interval in minutes")
+    @cmdln.option('--wipe', action='store_true', help='also wipe binaries')
+    def do_add_explicit_disable(self, subcmd, opts, *packages):
+        """${cmd_name}: add explicit disable to all packages
+
+        ${cmd_usage}
+        ${cmd_option_list}
+        """
+
+        def work():
+            self._select_packages(opts.all, packages)
+            self.tool.add_explicit_disable(wipebinaries=opts.wipe)
 
         self.runner(work, opts.interval)