yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #891 from lnussel/biarch

Browse Source 
Biarch tool enhancements
This commit is contained in:
Ludwig Nussel 2017-05-10 11:15:28 +02:00 committed by GitHub
commit 3b30308274
2 changed files with 257 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
ToolBase.py
View File

@ -37,8 +37,6 @@ from osclib.memoize import memoize
logger = logging.getLogger()
makeurl = osc.core.makeurl
http_GET = osc.core.http_GET
http_DELETE = osc.core.http_DELETE
http_POST = osc.core.http_POST
@ -82,8 +80,14 @@ class ToolBase(object):
else:
osc.core.http_PUT(*args, **kwargs)
def http_POST(self, *args, **kwargs):
if self.dryrun:
logging.debug("dryrun POST %s %s", args, str(kwargs)[:200])
else:
osc.core.http_POST(*args, **kwargs)
def get_project_meta(self, prj):
url = makeurl(self.apiurl, ['source', prj, '_meta'])
url = self.makeurl(['source', prj, '_meta'])
return self.cached_GET(url)
def _meta_get_packagelist(self, prj, deleted=None, expand=False):
@ -94,7 +98,7 @@ class ToolBase(object):
if expand:
query['expand'] = 1
u = makeurl(self.apiurl, ['source', prj], query)
u = self.makeurl(['source', prj], query)
return self.cached_GET(u)
def meta_get_packagelist(self, prj, deleted=None, expand=False):
@ -103,8 +107,7 @@ class ToolBase(object):
# FIXME: duplicated from manager_42
def latest_packages(self, project):
data = self.cached_GET(makeurl(self.apiurl,
['project', 'latest_commits', project]))
data = self.cached_GET(self.makeurl(['project', 'latest_commits', project]))
lc = ET.fromstring(data)
packages = set()
for entry in lc.findall('{http://www.w3.org/2005/Atom}entry'):
@ -113,6 +116,15 @@ class ToolBase(object):
packages.add(title[3:].split(' ')[0])
return sorted(packages)
def makeurl(self, l, query=None):
"""
Wrapper around osc's makeurl passing our apiurl
:return url made for l and query
"""
query = [] if not query else query
return osc.core.makeurl(self.apiurl, l, query)
def process(self, packages):
""" reimplement this """
True

291
biarchtool.py
View File

@ -30,8 +30,6 @@ import osc.core
import ToolBase
makeurl = osc.core.makeurl
logger = logging.getLogger()
FACTORY = "openSUSE:Factory"
@ -42,58 +40,104 @@ class BiArchTool(ToolBase.ToolBase):
ToolBase.ToolBase.__init__(self)
self.project = project
self.biarch_packages = None
self._has_baselibs = dict()
self.packages = []
self.arch = 'i586'
self.rdeps = None
self.whitelist = {
'i586': set([
'ovmf'
]),
}
def has_baselibs(self, package):
if package in self._has_baselibs:
return self._has_baselibs[package]
ret = False
files = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, package])))
for n in files.findall("./entry[@name='baselibs.conf']"):
logger.debug('%s has baselibs', package)
ret = True
return True
self._has_baselibs[package] = ret
return ret
def has_baselibs_recursive(self, package):
r = self.has_baselibs(package)
if not r and package in self.rdeps:
for p in self.rdeps[package]:
r = self.has_baselibs_recursive(p)
if r:
break
return r
def _init_biarch_packages(self):
if self.biarch_packages is None:
self.biarch_packages = set(self.meta_get_packagelist("%s:Rings:0-Bootstrap"%self.project))
self.biarch_packages |= set(self.meta_get_packagelist("%s:Rings:1-MinimalX"%self.project))
def enable_baselibs_packages(self, packages, force=False):
self._init_biarch_packages()
self._init_rdeps()
def _init_rdeps(self):
if self.rdeps is not None:
return
self.rdeps = dict()
url = self.makeurl(['build', self.project, 'standard', self.arch, '_builddepinfo' ], {'view':'revpkgnames'})
x = ET.fromstring(self.cached_GET(url))
for pnode in x.findall('package'):
name = pnode.get('name')
for depnode in pnode.findall('pkgdep'):
depname = depnode.text
self.rdeps.setdefault(name, set()).add(depname)
def select_packages(self, packages):
if packages == '__all__':
self.packages = self.meta_get_packagelist(self.project)
elif packages == '__latest__':
self.packages = self._filter_packages_by_time(self.latest_packages(self.project))
else:
self.packages = packages
# check when _product was last changed, eg by packagelist
# generator. Yield only packges that got checked in after that
# point in time.
def _filter_packages_by_time(self, packages):
x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, '_product', '_history'], {'limit':'1'})))
producttime = int(x.find('./revision/time').text)
for pkg in packages:
x = ET.fromstring(self.cached_GET(self.makeurl(['source', self.project, pkg, '_history'], {'rev':'1'})))
packagetime = int(x.find('./revision/time').text)
# if producttime > packagetime:
# continue
yield pkg
def remove_explicit_enable(self):
self._init_biarch_packages()
resulturl = self.makeurl(['build', self.project, '_result'])
result = ET.fromstring(self.cached_GET(resulturl))
packages = set()
for n in result.findall("./result[@arch='{}']/status".format(self.arch)):
if n.get('code') not in ('disabled', 'excluded'):
#logger.debug('%s %s', n.get('package'), n.get('code'))
packages.add(n.get('package'))
for pkg in sorted(packages):
changed = False
logger.debug("processing %s", pkg)
pkgmetaurl = makeurl(self.apiurl, ['source', self.project, pkg, '_meta'])
pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta'])
pkgmeta = ET.fromstring(self.cached_GET(pkgmetaurl))
is_enabled = None
is_disabled = None
has_baselibs = None
must_enable = None
changed = None
if force:
must_enable = True
for n in pkgmeta.findall("./build/enable[@arch='i586']"):
is_enabled = True
break
for n in pkgmeta.findall("./build/disable[@arch='i586']"):
is_disabled = True
break
if pkg in self.biarch_packages:
logger.debug('%s is known biarch package', pkg)
must_enable = True
else:
files = ET.fromstring(self.cached_GET(makeurl(self.apiurl, ['source', self.project, pkg])))
for n in files.findall("./entry[@name='baselibs.conf']"):
has_baselibs = True
logger.debug('%s has baselibs', pkg)
break
if has_baselibs:
must_enable = True
if must_enable:
if is_disabled:
logger.warn('%s should be enabled but is disabled', pkg)
if not is_enabled:
logger.info('enabling %s for biarch', pkg)
bn = pkgmeta.find('build')
if bn is None:
bn = ET.SubElement(pkgmeta, 'build')
ET.SubElement(bn, 'enable', { 'arch' : 'i586' })
for build in pkgmeta.findall("./build"):
for n in build.findall("./enable[@arch='{}']".format(self.arch)):
logger.debug("disable %s", pkg)
build.remove(n)
changed = True
else:
if is_enabled:
logger.warn("%s enabled or biarch without need", pkg)
if changed:
try:
@ -103,6 +147,122 @@ class BiArchTool(ToolBase.ToolBase):
except urllib2.HTTPError, e:
logger.error('failed to update %s: %s', pkg, e)
def add_explicit_disable(self, wipebinaries=False):
self._init_biarch_packages()
resulturl = self.makeurl(['source', self.project])
result = ET.fromstring(self.cached_GET(resulturl))
for pkg in self.packages:
changed = False
logger.debug("processing %s", pkg)
pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta'])
pkgmeta = ET.fromstring(self.cached_GET(pkgmetaurl))
build = pkgmeta.findall("./build")
if not build:
logger.debug('disable %s for %s', pkg, self.arch)
bn = pkgmeta.find('build')
if bn is None:
bn = ET.SubElement(pkgmeta, 'build')
ET.SubElement(bn, 'disable', { 'arch' : self.arch })
changed = True
if changed:
try:
self.http_PUT(pkgmetaurl, data=ET.tostring(pkgmeta))
if self.caching:
self._invalidate__cached_GET(pkgmetaurl)
if wipebinaries:
self.http_POST(self.makeurl(['build', self.project], {
'cmd' : 'wipe',
'arch': self.arch,
'package' : pkg }))
except urllib2.HTTPError, e:
logger.error('failed to update %s: %s', pkg, e)
def enable_baselibs_packages(self, force=False, wipebinaries=False):
self._init_biarch_packages()
for pkg in self.packages:
logger.debug("processing %s", pkg)
pkgmetaurl = self.makeurl(['source', self.project, pkg, '_meta'])
pkgmeta = ET.fromstring(self.cached_GET(pkgmetaurl))
is_enabled = None
is_disabled = None
has_baselibs = None
must_disable = None
changed = None
if force:
must_disable = False
for n in pkgmeta.findall("./build/enable[@arch='{}']".format(self.arch)):
is_enabled = True
break
for n in pkgmeta.findall("./build/disable[@arch='{}']".format(self.arch)):
is_disabled = True
break
if pkg in self.biarch_packages:
logger.debug('%s is known biarch package', pkg)
must_disable = False
elif pkg in self.whitelist[self.arch]:
logger.debug('%s is whitelisted', pkg)
must_disable = False
else:
has_baselibs = self.has_baselibs_recursive(pkg)
if has_baselibs:
must_disable = False
else:
must_disable = True
if must_disable == False:
if is_disabled:
logger.info('enabling %s for %s', pkg, self.arch)
for build in pkgmeta.findall("./build"):
for n in build.findall("./disable[@arch='{}']".format(self.arch)):
build.remove(n)
changed = True
if changed == False:
logger.error('build tag not found in %s/%s!?', pkg, self.arch)
else:
logger.debug('%s already enabled for %s', pkg, self.arch)
elif must_disable == True:
if not is_disabled:
logger.info('disabling %s for %s', pkg, self.arch)
bn = pkgmeta.find('build')
if bn is None:
bn = ET.SubElement(pkgmeta, 'build')
ET.SubElement(bn, 'disable', { 'arch' : self.arch })
changed = True
else:
logger.debug('%s already disabled for %s', pkg, self.arch)
if is_enabled:
logger.info('removing explicit enable %s for %s', pkg, self.arch)
for build in pkgmeta.findall("./build"):
for n in build.findall("./enable[@arch='{}']".format(self.arch)):
build.remove(n)
changed = True
if changed == False:
logger.error('build tag not found in %s/%s!?', pkg, self.arch)
if changed:
try:
self.http_PUT(pkgmetaurl, data=ET.tostring(pkgmeta))
if self.caching:
self._invalidate__cached_GET(pkgmetaurl)
if must_disable and wipebinaries:
self.http_POST(self.makeurl(['build', self.project], {
'cmd' : 'wipe',
'arch': self.arch,
'package' : pkg }))
except urllib2.HTTPError, e:
logger.error('failed to update %s: %s', pkg, e)
class CommandLineInterface(ToolBase.CommandLineInterface):
def __init__(self, *args, **kwargs):
@ -119,18 +279,18 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
tool = BiArchTool(self.options.project)
return tool
def _packages(self, all, packages):
if not packages:
if all:
packages = self.tool.meta_get_packagelist(self.tool.project)
else:
packages = self.tool.latest_packages(self.tool.project)
return packages
def _select_packages(self, all, packages):
if packages:
self.tool.select_packages(packages)
elif all:
self.tool.select_packages('__all__')
else:
self.tool.select_packages('__latest__')
@cmdln.option('-n', '--interval', metavar="minutes", type="int", help="periodic interval in minutes")
@cmdln.option('-a', '--all', action='store_true', help='process all packages')
@cmdln.option('-f', '--force', action='store_true', help='enable in any case')
@cmdln.option('--wipe', action='store_true', help='also wipe binaries')
def do_enable_baselibs_packages(self, subcmd, opts, *packages):
"""${cmd_name}: enable build for packages in Ring 0 or 1 or with
baselibs.conf
@ -139,7 +299,34 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
${cmd_option_list}
"""
def work():
self.tool.enable_baselibs_packages(self._packages(opts.all, packages), opts.force)
self._select_packages(opts.all, packages)
self.tool.enable_baselibs_packages(force=opts.force, wipebinaries=opts.wipe)
self.runner(work, opts.interval)
@cmdln.option('-a', '--all', action='store_true', help='process all packages')
def do_remove_explicit_enable(self, subcmd, opts, *packages):
"""${cmd_name}: remove all explicit enable tags from packages
${cmd_usage}
${cmd_option_list}
"""
self.tool.remove_explicit_enable()
@cmdln.option('-a', '--all', action='store_true', help='process all packages')
@cmdln.option('-n', '--interval', metavar="minutes", type="int", help="periodic interval in minutes")
@cmdln.option('--wipe', action='store_true', help='also wipe binaries')
def do_add_explicit_disable(self, subcmd, opts, *packages):
"""${cmd_name}: add explicit disable to all packages
${cmd_usage}
${cmd_option_list}
"""
def work():
self._select_packages(opts.all, packages)
self.tool.add_explicit_disable(wipebinaries=opts.wipe)
self.runner(work, opts.interval)