diff --git a/ToolBase.py b/ToolBase.py index c0e6e6bb..0e2c2f8e 100644 --- a/ToolBase.py +++ b/ToolBase.py @@ -37,8 +37,6 @@ from osclib.memoize import memoize logger = logging.getLogger() -makeurl = osc.core.makeurl - http_GET = osc.core.http_GET http_DELETE = osc.core.http_DELETE http_POST = osc.core.http_POST @@ -82,8 +80,14 @@ class ToolBase(object): else: osc.core.http_PUT(*args, **kwargs) + def http_POST(self, *args, **kwargs): + if self.dryrun: + logging.debug("dryrun POST %s %s", args, str(kwargs)[:200]) + else: + osc.core.http_POST(*args, **kwargs) + def get_project_meta(self, prj): - url = makeurl(self.apiurl, ['source', prj, '_meta']) + url = self.makeurl(['source', prj, '_meta']) return self.cached_GET(url) def _meta_get_packagelist(self, prj, deleted=None, expand=False): @@ -94,7 +98,7 @@ class ToolBase(object): if expand: query['expand'] = 1 - u = makeurl(self.apiurl, ['source', prj], query) + u = self.makeurl(['source', prj], query) return self.cached_GET(u) def meta_get_packagelist(self, prj, deleted=None, expand=False): @@ -103,8 +107,7 @@ class ToolBase(object): # FIXME: duplicated from manager_42 def latest_packages(self, project): - data = self.cached_GET(makeurl(self.apiurl, - ['project', 'latest_commits', project])) + data = self.cached_GET(self.makeurl(['project', 'latest_commits', project])) lc = ET.fromstring(data) packages = set() for entry in lc.findall('{http://www.w3.org/2005/Atom}entry'): @@ -113,6 +116,15 @@ class ToolBase(object): packages.add(title[3:].split(' ')[0]) return sorted(packages) + def makeurl(self, l, query=None): + """ + Wrapper around osc's makeurl passing our apiurl + :return url made for l and query + """ + query = [] if not query else query + return osc.core.makeurl(self.apiurl, l, query) + + def process(self, packages): """ reimplement this """ True diff --git a/biarchtool.py b/biarchtool.py index 2bfb7cf7..86c7cbaf 100755 --- a/biarchtool.py +++ b/biarchtool.py @@ -30,8 +30,6 @@ import osc.core import ToolBase -makeurl = osc.core.makeurl - logger = logging.getLogger() FACTORY = "openSUSE:Factory" @@ -42,58 +40,104 @@ class BiArchTool(ToolBase.ToolBase): ToolBase.ToolBase.__init__(self) self.project = project self.biarch_packages = None + self._has_baselibs = dict() + self.packages = [] + self.arch = 'i586' + self.rdeps = None + self.whitelist = { + 'i586': set([ + 'ovmf' + ]), + } + + def has_baselibs(self, package): + if package in self._has_baselibs: + return self._has_baselibs[package] + + ret = False + files = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, package]))) + for n in files.findall("./entry[@name='baselibs.conf']"): + logger.debug('%s has baselibs', package) + ret = True + return True + self._has_baselibs[package] = ret + return ret + + def has_baselibs_recursive(self, package): + r = self.has_baselibs(package) + if not r and package in self.rdeps: + for p in self.rdeps[package]: + r = self.has_baselibs_recursive(p) + if r: + break + return r def _init_biarch_packages(self): if self.biarch_packages is None: self.biarch_packages = set(self.meta_get_packagelist("%s:Rings:0-Bootstrap"%self.project)) self.biarch_packages |= set(self.meta_get_packagelist("%s:Rings:1-MinimalX"%self.project)) - def enable_baselibs_packages(self, packages, force=False): - self._init_biarch_packages() + self._init_rdeps() + + def _init_rdeps(self): + if self.rdeps is not None: + return + self.rdeps = dict() + url = self.makeurl(['build', self.project, 'standard', self.arch, '_builddepinfo' ], {'view':'revpkgnames'}) + x = ET.fromstring(self.cached_GET(url)) + for pnode in x.findall('package'): + name = pnode.get('name') + for depnode in pnode.findall('pkgdep'): + depname = depnode.text + self.rdeps.setdefault(name, set()).add(depname) + + def select_packages(self, packages): + if packages == '__all__': + self.packages = self.meta_get_packagelist(self.project) + elif packages == '__latest__': + self.packages = self._filter_packages_by_time(self.latest_packages(self.project)) + else: + self.packages = packages + + # check when _product was last changed, eg by packagelist + # generator. Yield only packges that got checked in after that + # point in time. + def _filter_packages_by_time(self, packages): + x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, '_product', '_history'], {'limit':'1'}))) + producttime = int(x.find('./revision/time').text) for pkg in packages: + x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, pkg, '_history'], {'rev':'1'}))) + packagetime = int(x.find('./revision/time').text) +# if producttime > packagetime: +# continue + yield pkg + + def remove_explicit_enable(self): + + self._init_biarch_packages() + + resulturl = self.makeurl(['build', self.project, '_result']) + result = ET.fromstring(self.cached_GET(resulturl)) + + packages = set() + + for n in result.findall("./result[@arch='{}']/status".format(self.arch)): + if n.get('code') not in ('disabled', 'excluded'): + #logger.debug('%s %s', n.get('package'), n.get('code')) + packages.add(n.get('package')) + + for pkg in sorted(packages): + changed = False + logger.debug("processing %s", pkg) - pkgmetaurl = makeurl(self.apiurl, ['source', self.project, pkg, '_meta']) + pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta']) pkgmeta = ET.fromstring(self.cached_GET(pkgmetaurl)) - is_enabled = None - is_disabled = None - has_baselibs = None - must_enable = None - changed = None - if force: - must_enable = True - - for n in pkgmeta.findall("./build/enable[@arch='i586']"): - is_enabled = True - break - for n in pkgmeta.findall("./build/disable[@arch='i586']"): - is_disabled = True - break - if pkg in self.biarch_packages: - logger.debug('%s is known biarch package', pkg) - must_enable = True - else: - files = ET.fromstring(self.cached_GET(makeurl(self.apiurl, ['source', self.project, pkg]))) - for n in files.findall("./entry[@name='baselibs.conf']"): - has_baselibs = True - logger.debug('%s has baselibs', pkg) - break - if has_baselibs: - must_enable = True - - if must_enable: - if is_disabled: - logger.warn('%s should be enabled but is disabled', pkg) - if not is_enabled: - logger.info('enabling %s for biarch', pkg) - bn = pkgmeta.find('build') - if bn is None: - bn = ET.SubElement(pkgmeta, 'build') - ET.SubElement(bn, 'enable', { 'arch' : 'i586' }) + for build in pkgmeta.findall("./build"): + for n in build.findall("./enable[@arch='{}']".format(self.arch)): + logger.debug("disable %s", pkg) + build.remove(n) changed = True - else: - if is_enabled: - logger.warn("%s enabled or biarch without need", pkg) if changed: try: @@ -103,6 +147,122 @@ class BiArchTool(ToolBase.ToolBase): except urllib2.HTTPError, e: logger.error('failed to update %s: %s', pkg, e) + def add_explicit_disable(self, wipebinaries=False): + + self._init_biarch_packages() + + resulturl = self.makeurl(['source', self.project]) + result = ET.fromstring(self.cached_GET(resulturl)) + + for pkg in self.packages: + + changed = False + + logger.debug("processing %s", pkg) + pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta']) + pkgmeta = ET.fromstring(self.cached_GET(pkgmetaurl)) + + build = pkgmeta.findall("./build") + if not build: + logger.debug('disable %s for %s', pkg, self.arch) + bn = pkgmeta.find('build') + if bn is None: + bn = ET.SubElement(pkgmeta, 'build') + ET.SubElement(bn, 'disable', { 'arch' : self.arch }) + changed = True + + if changed: + try: + self.http_PUT(pkgmetaurl, data=ET.tostring(pkgmeta)) + if self.caching: + self._invalidate__cached_GET(pkgmetaurl) + if wipebinaries: + self.http_POST(self.makeurl(['build', self.project], { + 'cmd' : 'wipe', + 'arch': self.arch, + 'package' : pkg })) + except urllib2.HTTPError, e: + logger.error('failed to update %s: %s', pkg, e) + + + def enable_baselibs_packages(self, force=False, wipebinaries=False): + self._init_biarch_packages() + for pkg in self.packages: + logger.debug("processing %s", pkg) + pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta']) + pkgmeta = ET.fromstring(self.cached_GET(pkgmetaurl)) + is_enabled = None + is_disabled = None + has_baselibs = None + must_disable = None + changed = None + + if force: + must_disable = False + + for n in pkgmeta.findall("./build/enable[@arch='{}']".format(self.arch)): + is_enabled = True + break + for n in pkgmeta.findall("./build/disable[@arch='{}']".format(self.arch)): + is_disabled = True + break + if pkg in self.biarch_packages: + logger.debug('%s is known biarch package', pkg) + must_disable = False + elif pkg in self.whitelist[self.arch]: + logger.debug('%s is whitelisted', pkg) + must_disable = False + else: + has_baselibs = self.has_baselibs_recursive(pkg) + if has_baselibs: + must_disable = False + else: + must_disable = True + + if must_disable == False: + if is_disabled: + logger.info('enabling %s for %s', pkg, self.arch) + for build in pkgmeta.findall("./build"): + for n in build.findall("./disable[@arch='{}']".format(self.arch)): + build.remove(n) + changed = True + if changed == False: + logger.error('build tag not found in %s/%s!?', pkg, self.arch) + else: + logger.debug('%s already enabled for %s', pkg, self.arch) + elif must_disable == True: + if not is_disabled: + logger.info('disabling %s for %s', pkg, self.arch) + bn = pkgmeta.find('build') + if bn is None: + bn = ET.SubElement(pkgmeta, 'build') + ET.SubElement(bn, 'disable', { 'arch' : self.arch }) + changed = True + else: + logger.debug('%s already disabled for %s', pkg, self.arch) + + if is_enabled: + logger.info('removing explicit enable %s for %s', pkg, self.arch) + for build in pkgmeta.findall("./build"): + for n in build.findall("./enable[@arch='{}']".format(self.arch)): + build.remove(n) + changed = True + if changed == False: + logger.error('build tag not found in %s/%s!?', pkg, self.arch) + + if changed: + try: + self.http_PUT(pkgmetaurl, data=ET.tostring(pkgmeta)) + if self.caching: + self._invalidate__cached_GET(pkgmetaurl) + if must_disable and wipebinaries: + self.http_POST(self.makeurl(['build', self.project], { + 'cmd' : 'wipe', + 'arch': self.arch, + 'package' : pkg })) + except urllib2.HTTPError, e: + logger.error('failed to update %s: %s', pkg, e) + class CommandLineInterface(ToolBase.CommandLineInterface): def __init__(self, *args, **kwargs): @@ -119,18 +279,18 @@ class CommandLineInterface(ToolBase.CommandLineInterface): tool = BiArchTool(self.options.project) return tool - def _packages(self, all, packages): - if not packages: - if all: - packages = self.tool.meta_get_packagelist(self.tool.project) - else: - packages = self.tool.latest_packages(self.tool.project) - - return packages + def _select_packages(self, all, packages): + if packages: + self.tool.select_packages(packages) + elif all: + self.tool.select_packages('__all__') + else: + self.tool.select_packages('__latest__') @cmdln.option('-n', '--interval', metavar="minutes", type="int", help="periodic interval in minutes") @cmdln.option('-a', '--all', action='store_true', help='process all packages') @cmdln.option('-f', '--force', action='store_true', help='enable in any case') + @cmdln.option('--wipe', action='store_true', help='also wipe binaries') def do_enable_baselibs_packages(self, subcmd, opts, *packages): """${cmd_name}: enable build for packages in Ring 0 or 1 or with baselibs.conf @@ -139,7 +299,34 @@ class CommandLineInterface(ToolBase.CommandLineInterface): ${cmd_option_list} """ def work(): - self.tool.enable_baselibs_packages(self._packages(opts.all, packages), opts.force) + self._select_packages(opts.all, packages) + self.tool.enable_baselibs_packages(force=opts.force, wipebinaries=opts.wipe) + + self.runner(work, opts.interval) + + @cmdln.option('-a', '--all', action='store_true', help='process all packages') + def do_remove_explicit_enable(self, subcmd, opts, *packages): + """${cmd_name}: remove all explicit enable tags from packages + + ${cmd_usage} + ${cmd_option_list} + """ + + self.tool.remove_explicit_enable() + + @cmdln.option('-a', '--all', action='store_true', help='process all packages') + @cmdln.option('-n', '--interval', metavar="minutes", type="int", help="periodic interval in minutes") + @cmdln.option('--wipe', action='store_true', help='also wipe binaries') + def do_add_explicit_disable(self, subcmd, opts, *packages): + """${cmd_name}: add explicit disable to all packages + + ${cmd_usage} + ${cmd_option_list} + """ + + def work(): + self._select_packages(opts.all, packages) + self.tool.add_explicit_disable(wipebinaries=opts.wipe) self.runner(work, opts.interval)