yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #518 from lnussel/fixes

Browse Source 
Tons of fixes
This commit is contained in:
Ludwig Nussel 2016-06-08 16:56:16 +02:00
commit cde6f6c07b
18 changed files with 427 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
ReviewBot.py
View File

@ -83,9 +83,6 @@ class ReviewBot(object):
def check_requests(self):
by_user = self.fallback_user
by_group = self.fallback_group
for req in self.requests:
self.logger.debug("checking %s"%req.reqid)
good = self.check_one_request(req)
@ -98,28 +95,36 @@ class ReviewBot(object):
if good is None:
self.logger.info("%s ignored"%req.reqid)
elif good:
self.logger.info("%s is good"%req.reqid)
self._set_review(req, 'accepted')
else:
if self.review_mode == 'fallback-onfail':
self.logger.info("%s needs fallback reviewer"%req.reqid)
self.add_review(req, by_group=by_group, by_user=by_user)
self._set_review(req, 'accepted')
else:
self.logger.info("%s is not acceptable"%req.reqid)
self._set_review(req, 'declined')
self._set_review(req, 'declined')
def _set_review(self, req, state):
doit = self.can_accept_review(req.reqid)
if doit is None:
self.logger.info("can't change state, %s does not have the reviewer"%(req.reqid))
newstate = state
by_user = self.fallback_user
by_group = self.fallback_group
if state == 'declined':
if self.review_mode == 'fallback-onfail':
self.logger.info("%s needs fallback reviewer"%req.reqid)
self.add_review(req, by_group=by_group, by_user=by_user)
newstate = 'accepted'
elif self.review_mode == 'fallback-always':
self.add_review(req, by_group=by_group, by_user=by_user)
self.logger.info("%s %s"%(req.reqid, state))
if doit == True:
self.logger.debug("setting %s to %s"%(req.reqid, state))
if not self.dryrun:
msg = self.review_messages[state] if state in self.review_messages else state
osc.core.change_review_state(apiurl = self.apiurl,
reqid = req.reqid, newstate = state,
reqid = req.reqid, newstate = newstate,
by_group=self.review_group,
by_user=self.review_user, message=msg)
else:

63
check_maintenance_incidents.py
View File

@ -94,7 +94,37 @@ class MaintenanceChecker(ReviewBot.ReviewBot):
# check if pkgname was submitted by the correct maintainer. If not, set
# self.needs_maintainer_review
def _check_maintainer_review_needed(self, req, pkgname, author):
def _check_maintainer_review_needed(self, req, a):
author = req.get_creator()
if a.type == 'maintenance_incident':
# check if there is a link and use that or the real package
# name as src_packge may end with something like
# .openSUSE_XX.Y_Update
pkgname = a.src_package
(linkprj, linkpkg) = self._get_linktarget(a.src_project, pkgname)
if linkpkg is not None:
pkgname = linkpkg
if pkgname == 'patchinfo':
return None
project = a.tgt_releaseproject
else:
pkgname = a.tgt_package
project = a.tgt_project
if project.startswith('openSUSE:Leap:'):
mapping = MaintenanceChecker._get_lookup_yml(self.apiurl, project)
if mapping is None:
self.logger.error("error loading mapping for {}".format(project))
elif not pkgname in mapping:
self.logger.debug("{} not tracked".format(pkgname))
else:
origin = mapping[pkgname]
self.logger.debug("{} comes from {}, submitted from {}".format(pkgname, origin, a.src_project))
if origin.startswith('SUSE:SLE-12') and a.src_project.startswith('SUSE:SLE-12'):
self.logger.info("{} submitted from {}, no maintainer review needed".format(pkgname, a.src_project))
return
maintainers = set(self._maintainers(pkgname))
if maintainers:
known_maintainer = False
@ -115,33 +145,8 @@ class MaintenanceChecker(ReviewBot.ReviewBot):
self.needs_maintainer_review.add(pkgname)
def check_action_maintenance_incident(self, req, a):
author = req.get_creator()
# check if there is a link and use that or the real package
# name as src_packge may end with something like
# .openSUSE_XX.Y_Update
pkgname = a.src_package
(linkprj, linkpkg) = self._get_linktarget(a.src_project, pkgname)
if linkpkg is not None:
pkgname = linkpkg
if pkgname == 'patchinfo':
return None
skip_maintainer_review = False
if a.tgt_releaseproject.startswith('openSUSE:Leap:'):
mapping = MaintenanceChecker._get_lookup_yml(self.apiurl, a.tgt_releaseproject)
if mapping is None:
self.logger.error("error loading mapping for {}".format(a.tgt_releaseproject))
elif not pkgname in mapping:
self.logger.error("{} not tracked".format(pkgname))
else:
origin = mapping[pkgname]
self.logger.debug("{} comes from {}, submitted from {}".format(pkgname, origin, a.src_project))
if origin.startswith('SUSE:SLE-12') and origin == a.src_project:
skip_maintainer_review = True
self.logger.info("{} submitted from {}, no maintainer review needed".format(pkgname, a.src_project))
if not skip_maintainer_review:
self._check_maintainer_review_needed(req, pkgname, author)
self._check_maintainer_review_needed(req, a)
if a.tgt_releaseproject.startswith("openSUSE:Backports:"):
self.add_factory_source = True
@ -149,10 +154,8 @@ class MaintenanceChecker(ReviewBot.ReviewBot):
return True
def check_action_submit(self, req, a):
author = req.get_creator()
pkgname = a.tgt_package
self._check_maintainer_review_needed(req, pkgname, author)
self._check_maintainer_review_needed(req, a)
return True

47
check_source_in_factory.py
View File

@ -33,8 +33,10 @@ except ImportError:
import osc.conf
import osc.core
import urllib2
import yaml
import ReviewBot
class FactorySourceChecker(ReviewBot.ReviewBot):
""" this review bot checks if the sources of a submission are
either in Factory or a request for Factory with the same sources
@ -50,6 +52,16 @@ class FactorySourceChecker(ReviewBot.ReviewBot):
self.factory = "openSUSE:Factory"
ReviewBot.ReviewBot.__init__(self, *args, **kwargs)
self.review_messages = { 'accepted' : 'ok', 'declined': 'the package needs to be accepted in Factory first' }
self.lookup = None
def parse_lookup(self, project):
self.lookup = yaml.safe_load(self._load_lookup_file(project))
def _load_lookup_file(self, prj):
if prj.endswith(':NonFree'):
prj = prj[:-len(':NonFree')]
return osc.core.http_GET(osc.core.makeurl(self.apiurl,
['source', prj, '00Meta', 'lookup.yml']))
def check_source_submission(self, src_project, src_package, src_rev, target_project, target_package):
self.logger.info("%s/%s@%s -> %s/%s"%(src_project, src_package, src_rev, target_project, target_package))
@ -71,24 +83,35 @@ class FactorySourceChecker(ReviewBot.ReviewBot):
return good
def _package_get_upstream_project(self, package):
""" return project where the specified pacakge is supposed to come
from. Either by lookup table or self.factory """
if self.lookup and package in self.lookup:
return self.lookup[package]
return self.factory
def _check_factory(self, rev, package):
"""check if factory sources contain the package and revision. check head and history"""
self.logger.debug("checking %s in %s"%(package, self.factory))
project = self._package_get_upstream_project(package)
if project is None:
return False
self.logger.debug("checking %s in %s"%(package, project))
try:
si = osc.core.show_package_meta(self.apiurl, self.factory, package)
si = osc.core.show_package_meta(self.apiurl, project, package)
except (urllib2.HTTPError, urllib2.URLError):
si = None
if si is None:
self.logger.debug("new package")
return None
else:
si = self.get_sourceinfo(self.factory, package)
si = self.get_sourceinfo(project, package)
if rev == si.verifymd5:
self.logger.debug("srcmd5 matches")
return True
self.logger.debug("%s not the latest version, checking history", rev)
u = osc.core.makeurl(self.apiurl, [ 'source', self.factory, package, '_history' ], { 'limit': '5' })
u = osc.core.makeurl(self.apiurl, [ 'source', project, package, '_history' ], { 'limit': '5' })
try:
r = osc.core.http_GET(u)
except urllib2.HTTPError, e:
@ -111,7 +134,11 @@ class FactorySourceChecker(ReviewBot.ReviewBot):
def _check_requests(self, rev, package):
self.logger.debug("checking requests")
try:
requests = osc.core.get_request_list(self.apiurl, self.factory, package, None, ['new', 'review'], 'submit')
project = self._package_get_upstream_project(package)
if project is None:
self.logger.error("no upstream project found for {}, can't check requests".format(package))
return None
requests = osc.core.get_request_list(self.apiurl, project, package, None, ['new', 'review'], 'submit')
except (urllib2.HTTPError, urllib2.URLError):
self.logger.debug("none request")
return None
@ -125,7 +152,7 @@ class FactorySourceChecker(ReviewBot.ReviewBot):
self.logger.debug("request ok")
return True
elif req.state.name == 'review':
self.logger.debug("request still in review")
self.logger.info("request still in review")
return None
else:
self.logger.error("request in state %s not expected"%req.state.name)
@ -140,6 +167,7 @@ class CommandLineInterface(ReviewBot.CommandLineInterface):
def get_optparser(self):
parser = ReviewBot.CommandLineInterface.get_optparser(self)
parser.add_option("--factory", metavar="project", help="the openSUSE Factory project")
parser.add_option("--lookup", metavar="project", help="use lookup file")
return parser
@ -152,12 +180,17 @@ class CommandLineInterface(ReviewBot.CommandLineInterface):
if user is None:
user = osc.conf.get_apiurl_usr(apiurl)
return FactorySourceChecker(apiurl = apiurl, \
bot = FactorySourceChecker(apiurl = apiurl, \
factory = self.options.factory, \
dryrun = self.options.dry, \
user = user, \
logger = self.logger)
if self.options.lookup:
bot.parse_lookup(self.options.lookup)
return bot
if __name__ == "__main__":
app = CommandLineInterface()
sys.exit( app.main() )

5
create_test_Leap:42.2_dvd-1.testcase
View File

@ -21,6 +21,11 @@ job install name screen
job install name alsa-utils
job install name yast2-nfs-client
job install name kernel-default
job install name kexec-tools
#ifdef __x86_64__
job install name mokutil
#endif
job lock name bash-completion
job lock name bash-doc

2
create_test_Leap:42.2_dvd-2.testcase
View File

@ -32,6 +32,7 @@ job install name xdelta
job install name kate
job install name sddm
job install name kernel-default
job install name kexec-tools
job lock name gtk2-branding-upstream
job lock name gdm-branding-upstream
@ -48,6 +49,7 @@ job install provides pattern() = laptop
job install provides pattern() = office
job install name MozillaThunderbird
job install name libreoffice
job install name mokutil
job lock name libgcc_s1-32bit
job install provides virtualbox-guest-kmp
#endif

120
leaper.py Executable file
View File

@ -0,0 +1,120 @@
#!/usr/bin/python
# Copyright (c) 2014 SUSE Linux Products GmbH
# Copyright (c) 2016 SUSE LLC
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from pprint import pprint
import os, sys, re
import logging
from optparse import OptionParser
import cmdln
try:
from xml.etree import cElementTree as ET
except ImportError:
import cElementTree as ET
import osc.conf
import osc.core
import urllib2
import yaml
import ReviewBot
from check_maintenance_incidents import MaintenanceChecker
from check_source_in_factory import FactorySourceChecker
class Leaper(ReviewBot.ReviewBot):
def __init__(self, *args, **kwargs):
ReviewBot.ReviewBot.__init__(self, *args, **kwargs)
self.maintbot = MaintenanceChecker(*args, **kwargs)
# for FactorySourceChecker
self.factory = FactorySourceChecker(*args, **kwargs)
self.factory.parse_lookup('openSUSE:Leap:42.2')
def check_source_submission(self, src_project, src_package, src_rev, target_project, target_package):
return self.factory.check_source_submission(src_project, src_package, src_rev, target_project, target_package)
def check_one_request(self, req):
self.review_messages = self.DEFAULT_REVIEW_MESSAGES.copy()
if len(req.actions) != 1:
msg = "only one action per request please"
self.review_messages['declined'] = msg
return False
# if the fallback reviewer created the request she probably
# knows what she does :-)
if self.fallback_user and req.get_creator() == self.fallback_user:
self.logger.debug("skip fallback review")
return True
has_upstream_sources = ReviewBot.ReviewBot.check_one_request(self, req)
has_correct_maintainer = self.maintbot.check_one_request(req)
# not reviewed yet?
if has_upstream_sources is None:
return None
self.logger.debug("upstream sources: {}, maintainer ok: {}".format(has_upstream_sources, has_correct_maintainer))
if has_upstream_sources != True or has_correct_maintainer != True:
if has_upstream_sources != True:
self.review_messages['declined'] += '\nOrigin project changed'
# shouldn't happen actually
if has_correct_maintainer != True:
self.review_messages['declined'] += '\nMaintainer check failed'
return False
return True
class CommandLineInterface(ReviewBot.CommandLineInterface):
def __init__(self, *args, **kwargs):
ReviewBot.CommandLineInterface.__init__(self, args, kwargs)
def get_optparser(self):
parser = ReviewBot.CommandLineInterface.get_optparser(self)
return parser
def setup_checker(self):
apiurl = osc.conf.config['apiurl']
if apiurl is None:
raise osc.oscerr.ConfigError("missing apiurl")
user = self.options.user
group = self.options.group
# if no args are given, use the current oscrc "owner"
if user is None and group is None:
user = osc.conf.get_apiurl_usr(apiurl)
bot = Leaper(apiurl = apiurl, \
dryrun = self.options.dry, \
user = user, \
group = group, \
logger = self.logger)
return bot
if __name__ == "__main__":
app = CommandLineInterface()
sys.exit( app.main() )
# vim: sw=4 et

19
manager_42.py
View File

@ -51,6 +51,8 @@ class Manager42(object):
self.caching = caching
self.apiurl = osc.conf.config['apiurl']
self.project_preference_order = [
#'SUSE:SLE-12-SP2:Update',
'SUSE:SLE-12-SP2:GA',
'SUSE:SLE-12-SP1:Update',
'SUSE:SLE-12-SP1:GA',
'SUSE:SLE-12:Update',
@ -58,6 +60,7 @@ class Manager42(object):
'openSUSE:Leap:42.1:Update',
'openSUSE:Leap:42.1',
'openSUSE:Factory',
'openSUSE:Leap:42.2:SLE-workarounds'
]
self.parse_lookup()
@ -243,21 +246,25 @@ class Manager42(object):
srcmd5, rev = self.check_source_in_project(lproject, package, root.get('verifymd5'))
if srcmd5:
logger.debug("{} lookup from {} is correct".format(package, lproject))
return
if lproject == 'openSUSE:Factory':
# if it's from Factory we check if the package can be found elsewhere meanwhile
if lproject != 'openSUSE:Factory':
return
elif lproject == 'openSUSE:Factory' and not package in self.packages[lproject]:
his = self.get_package_history(lproject, package, deleted=True)
if his:
logger.debug("{} got dropped from {}".format(package, lproject))
return
logger.debug("check where %s came from", package)
foundit = False
for project in self.project_preference_order:
srcmd5, rev = self.check_source_in_project(project, package, root.get('verifymd5'))
if srcmd5:
logger.info('{} -> {} (was {})'.format(package, project, lproject))
self.lookup[package] = project
self.lookup_changes += 1
if project != lproject:
logger.info('{} -> {} (was {})'.format(package, project, lproject))
self.lookup[package] = project
self.lookup_changes += 1
else:
logger.debug('{} still coming from {}'.format(package, project))
foundit = True
break

21
osc-staging.py
View File

@ -87,6 +87,8 @@ def _full_project_name(self, project):
help='force the select command ignoring the time from the last freeze')
@cmdln.option('--no-cleanup', dest='no_cleanup', action='store_true',
help='do not cleanup remaining packages in staging projects after accept')
@cmdln.option('--no-bootstrap', dest='bootstrap', action='store_false', default=True,
help='do not update bootstrap-copy when freezing')
def do_staging(self, subcmd, opts, *args):
"""${cmd_name}: Commands to work with staging projects
@ -120,7 +122,7 @@ def do_staging(self, subcmd, opts, *args):
osc staging accept [--force] [LETTER...]
osc staging check [--old] REPO
osc staging cleanup_rings
osc staging freeze PROJECT...
osc staging freeze [--no-boostrap] PROJECT...
osc staging frozenage PROJECT...
osc staging list [--supersede]
osc staging select [--no-freeze] [--move [--from PROJECT]] LETTER REQUEST...
@ -172,19 +174,14 @@ def do_staging(self, subcmd, opts, *args):
CheckCommand(api).perform(prj, opts.old)
elif cmd == 'freeze':
for prj in args[1:]:
FreezeCommand(api).perform(api.prj_from_letter(prj))
FreezeCommand(api).perform(api.prj_from_letter(prj), copy_bootstrap = opts.bootstrap )
elif cmd == 'frozenage':
for prj in args[1:]:
print "%s last frozen %0.1f days ago" % (api.prj_from_letter(prj), api.days_since_last_freeze(api.prj_from_letter(prj)))
elif cmd == 'acheck':
# Is it safe to accept? Meaning: /totest contains what it should and is not dirty
version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64")
skip_totest = False
if not version_totest:
# SLE don't have totest repository and openSUSE-release.rpm
skip_totest = api.item_exists(api.project, "release-notes-sles")
if not skip_totest:
if version_totest:
version_openqa = api.load_file_content("%s:Staging" % api.project, "dashboard", "version_totest")
totest_dirty = api.is_repo_dirty(api.project, 'totest')
print "version_openqa: %s / version_totest: %s / totest_dirty: %s\n" % (version_openqa, version_totest, totest_dirty)
@ -193,12 +190,8 @@ def do_staging(self, subcmd, opts, *args):
elif cmd == 'accept':
# Is it safe to accept? Meaning: /totest contains what it should and is not dirty
version_totest = api.get_binary_version(api.project, "openSUSE-release.rpm", repository="totest", arch="x86_64")
skip_totest = False
if not version_totest:
# SLE don't have totest repository and openSUSE-release.rpm
skip_totest = api.item_exists(api.project, "release-notes-sles")
if skip_totest or opts.force:
if version_totest or opts.force:
# SLE does not have a totest_version or openqa_version - ignore it
version_openqa = version_totest
totest_dirty = False
@ -209,7 +202,7 @@ def do_staging(self, subcmd, opts, *args):
if version_openqa == version_totest and not totest_dirty:
cmd = AcceptCommand(api)
for prj in args[1:]:
if not cmd.perform(api.prj_from_letter(prj)):
if not cmd.perform(api.prj_from_letter(prj), opts.force):
return
if not opts.no_cleanup:
if api.item_exists(api.prj_from_letter(prj)):

5
osclib/accept_command.py
View File

@ -34,7 +34,7 @@ class AcceptCommand(object):
rqs.append({'id': int(rq.get('id')), 'packages': pkgs})
return rqs
def perform(self, project):
def perform(self, project, force=False):
"""Accept the staging project for review and submit to Factory /
openSUSE 13.2 ...
@ -47,7 +47,8 @@ class AcceptCommand(object):
if not status:
print('The project "{}" is not yet acceptable.'.format(project))
return False
if not force:
return False
meta = self.api.get_prj_pseudometa(project)
requests = []

12
osclib/adi_command.py
View File

@ -40,14 +40,6 @@ class AdiCommand:
for p in self.api.get_adi_projects():
self.check_adi_project(p)
def get_devel_project(self, project, package):
m = show_package_meta(self.api.apiurl, project, package)
node = ET.fromstring(''.join(m)).find('devel')
if node is None:
return None
else:
return node.get('project')
def create_new_adi(self, wanted_requests, by_dp=False, split=False):
all_requests = self.api.get_open_requests()
@ -94,12 +86,12 @@ class AdiCommand:
non_ring_requests[request_id] = [request_id]
else:
if by_dp:
devel_project = self.get_devel_project(source_project, source_package)
devel_project = self.api.get_devel_project(source_project, source_package)
# try target pacakge in Factory
# this is a bit against Leap development in case submissions is from Update,
# or any other project than Factory
if devel_project is None and self.api.project.startswith('openSUSE:'):
devel_project = self.get_devel_project('openSUSE:Factory', target_package)
devel_project = self.api.get_devel_project('openSUSE:Factory', target_package)
if devel_project is not None:
source_project = devel_project

42
osclib/cleanup_rings.py
View File

@ -28,11 +28,36 @@ class CleanupRings(object):
f = http_GET(url)
root = ET.parse(f).getroot()
for si in root.findall('sourceinfo'):
linked = si.find('linked')
if linked is not None and linked.get('project') != self.api.project:
if not linked.get('project').startswith(self.api.crings):
print "#not linking to base: ", self.api.crings, linked.get('project')
self.links[linked.get('package')] = si.get('package')
links = si.findall('linked')
pkg = si.get('package')
if links is None or len(links) == 0:
print '# {} not a link'.format(pkg)
else:
linked = links[0]
dprj = linked.get('project')
dpkg = linked.get('package')
if dprj != self.api.project:
if not dprj.startswith(self.api.crings):
print "#{} not linking to base {} but {}".format(pkg, self.api.project, dprj)
self.links[dpkg] = pkg
# multi spec package must link to ring
elif len(links) > 1:
mainpkg = links[1].get('package')
mainprj = links[1].get('project')
if mainprj != self.api.project:
print '# FIXME: {} links to {}'.format(pkg, mainprj)
else:
destring = None
if mainpkg in self.api.ring_packages:
destring = self.api.ring_packages[mainpkg]
if not destring:
print '# {} links to {} but is not in a ring'.format(pkg, mainpkg)
print "osc linkpac {}/{} {}/{}".format(mainprj, mainpkg, prj, mainpkg)
else:
if pkg != 'glibc.i686': # FIXME: ugly exception
print "osc linkpac -f {}/{} {}/{}".format(destring, mainpkg, prj, pkg)
self.links[mainpkg] = pkg
def fill_pkgdeps(self, prj, repo, arch):
url = makeurl(self.api.apiurl, ['build', prj, repo, arch, '_builddepinfo'])
@ -58,7 +83,8 @@ class CleanupRings(object):
source = package.find('source').text
for pkg in package.findall('pkgdep'):
if pkg.text not in self.bin2src:
print('Package {} not found in place'.format(pkg.text))
if not pkg.text.startswith('texlive-'): # XXX: texlive bullshit packaging
print('Package {} not found in place'.format(pkg.text))
continue
b = self.bin2src[pkg.text]
self.pkgdeps[b] = source
@ -78,7 +104,7 @@ class CleanupRings(object):
return False
self.find_inner_ring_links(prj)
for arch in [ 'x86_64', 'ppc64le' ]:
for arch in self.api.cstaging_dvd_archs:
self.fill_pkgdeps(prj, 'standard', arch)
if prj == '{}:1-MinimalX'.format(self.api.crings):
@ -117,6 +143,8 @@ class CleanupRings(object):
for source in self.sources:
if source not in self.pkgdeps and source not in self.links:
if source.startswith('texlive-specs-'): # XXX: texlive bullshit packaging
continue
print('osc rdelete -m cleanup {} {}'.format(prj, source))
if nextprj:
print('osc linkpac {} {} {}').format(self.api.project, source, nextprj)

6
osclib/conf.py
View File

@ -32,8 +32,9 @@ from osc import conf
DEFAULT = {
r'openSUSE:(?P<project>[-\w\d:.]+)': {
'staging': 'openSUSE:%(project)s:Staging',
'staging-group': '%(project.lower)s-staging',
'staging-group': 'factory-staging',
'staging-archs': 'i586 x86_64 ppc64le',
'staging-dvd-archs': 'x86_64 ppc64le',
'nocleanup-packages': 'Test-DVD-x86_64 Test-DVD-ppc64le bootstrap-copy',
'rings': 'openSUSE:%(project)s:Rings',
'nonfree': 'openSUSE:%(project)s:NonFree',
@ -47,6 +48,7 @@ DEFAULT = {
'staging': 'SUSE:%(project)s:Staging',
'staging-group': 'sle-staging-managers', # '%(project.lower)s-staging',
'staging-archs': 'i586 x86_64',
'staging-dvd-archs': 'x86_64',
'nocleanup-packages': 'Test-DVD-x86_64 sles-release',
'rings': None,
'nonfree': None,
@ -110,7 +112,7 @@ class Config(object):
params = [set(d) for d in DEFAULT.values()]
params = reduce(operator.__and__, params)
if not all(p in conf.config[self.project] for p in params):
msg = 'Please, add [%s] section in %s' % (self.project, self.conf_file)
msg = 'Please, add [%s] section in %s, see %s for details' % (self.project, self.conf_file, __file__)
raise Exception(msg)
def read_section(self, section, defaults):

28
osclib/freeze_command.py
View File

@ -116,7 +116,7 @@ class FreezeCommand(object):
return False
return True
def perform(self, prj):
def perform(self, prj, copy_bootstrap=True):
self.prj = prj
self.set_links()
@ -129,16 +129,17 @@ class FreezeCommand(object):
if not self.is_bootstrap():
return
self.set_bootstrap_copy()
self.create_bootstrap_aggregate()
print("waiting for scheduler to disable...")
while not self.verify_bootstrap_copy_codes(['disabled']):
time.sleep(1)
self.build_switch_bootstrap_copy('enable')
print("waiting for scheduler to copy...")
while not self.verify_bootstrap_copy_codes(['finished', 'succeeded']):
time.sleep(1)
self.build_switch_bootstrap_copy('disable')
if copy_bootstrap:
self.set_bootstrap_copy()
self.create_bootstrap_aggregate()
print("waiting for scheduler to disable...")
while not self.verify_bootstrap_copy_codes(['disabled']):
time.sleep(1)
self.build_switch_bootstrap_copy('enable')
print("waiting for scheduler to copy...")
while not self.verify_bootstrap_copy_codes(['finished', 'succeeded']):
time.sleep(1)
self.build_switch_bootstrap_copy('disable')
# Update the version information found in the Test-DVD package, to match openSUSE-release
if self.api.item_exists(prj, "openSUSE-release"):
@ -256,7 +257,10 @@ class FreezeCommand(object):
# print(package, linked.get('package'), linked.get('project'))
f = self.api.retried_GET(url)
proot = ET.parse(f).getroot()
ET.SubElement(flink, 'package', {'name': package, 'srcmd5': proot.get('lsrcmd5'), 'vrev': si.get('vrev')})
lsrcmd5 = proot.get('lsrcmd5')
if lsrcmd5 is None:
raise Exception("{}/{} is not a link but we expected one".format(self.api.project, package))
ET.SubElement(flink, 'package', {'name': package, 'srcmd5': lsrcmd5, 'vrev': si.get('vrev')})
return package
if package in ['rpmlint-mini-AGGR']:
return package # we should not freeze aggregates

16
osclib/list_command.py
View File

@ -26,6 +26,7 @@ class ListCommand:
non_ring_packages = []
change_devel_requests = {}
result = {}
for request in requests:
# Consolidate all data from request
request_id = int(request.get('id'))
@ -48,6 +49,9 @@ class ListCommand:
# package
if self.api.crings:
ring = self.api.ring_packages.get(target_package)
if ring:
# cut off *:Rings: prefix
ring = ring[len(self.api.crings)+1:]
else:
ring = self.api.project
@ -56,15 +60,23 @@ class ListCommand:
if ring:
ring = ring + " (delete request)"
else:
ring = '{} is non-ring package (delete request)'.format(target_package)
ring = '(delete request)'
# This condition is quite moot as we dispatched stuff
# above anyway
if ring:
print('Request({}): {} -> {}'.format(request_id, target_package, ring))
devel = self.api.get_devel_project("openSUSE:Factory", target_package)
if devel is None:
devel = '00'
result.setdefault(devel, []).append('sr#{}: {:<30} -> {}'.format(request_id, target_package, ring))
else:
non_ring_packages.append(target_package)
for prj in sorted(result.keys()):
print prj
for line in result[prj]:
print ' ', line
if len(non_ring_packages):
print "Not in a ring:", ' '.join(sorted(non_ring_packages))
if len(change_devel_requests):

2
osclib/select_command.py
View File

@ -101,6 +101,8 @@ class SelectCommand(object):
msg = msg.format(fprj, self.target_project)
print(msg)
return True
elif supersede:
print('"{} ({}) supersedes {}'.format(request, supersede[1], supersede[0]))
else:
raise oscerr.WrongArgs('Arguments for select are not correct.')

26
osclib/stagingapi.py
View File

@ -53,7 +53,8 @@ class StagingAPI(object):
# Store some prefix / data used in the code.
self.cstaging = conf.config[project]['staging']
self.cstaging_group = conf.config[project]['staging-group']
self.cstaging_archs = conf.config[project]['staging-archs'].split(' ')
self.cstaging_archs = conf.config[project]['staging-archs'].split()
self.cstaging_dvd_archs = conf.config[project]['staging-dvd-archs'].split()
self.cstaging_nocleanup = conf.config[project]['nocleanup-packages'].split()
self.crings = conf.config[project]['rings']
self.cnonfree = conf.config[project]['nonfree']
@ -63,6 +64,7 @@ class StagingAPI(object):
self.user = conf.get_apiurl_usr(apiurl)
self._ring_packages = None
self._packages_staged = None
self._package_metas = dict()
# If the project support rings, inititialize some variables.
if self.crings:
@ -802,6 +804,7 @@ class StagingAPI(object):
if not force_enable_build:
if self.crings and not self.ring_packages.get(tar_pkg) and not self.is_adi_project(project):
disable_build = True
logging.warning("{}/{} not in ring, build disabled".format(project, tar_pkg))
else:
project = self.map_ring_package_to_subject(project, tar_pkg)
@ -1229,3 +1232,24 @@ class StagingAPI(object):
return True
else:
return False
# from manager_42
def _fill_package_meta(self, project):
url = makeurl(self.apiurl, ['search', 'package'], "match=[@project='%s']" % project)
root = ET.parse(self.retried_GET(url))
for p in root.findall('package'):
name = p.attrib['name']
self._package_metas.setdefault(project, {})[name] = p
def get_devel_project(self, project, package):
if not project in self._package_metas:
self._fill_package_meta(project)
if not package in self._package_metas[project]:
return None
node = self._package_metas[project][package].find('devel')
if node is None:
return None
return node.get('project')

1
totest-manager.py
View File

@ -366,6 +366,7 @@ class ToTestBase(object):
logger.info('current_snapshot %s: %s'%(current_snapshot, self._result2str(current_result)))
logger.debug('new_snapshot %s', new_snapshot)
logger.debug('current_qa_version %s', current_qa_version)
snapshotable = self.factory_snapshottable()
logger.debug("snapshotable: %s", snapshotable)

128
update_crawler.py
View File

@ -60,6 +60,8 @@ class UpdateCrawler(object):
self.filter_lookup = set()
self.caching = False
self.dryrun = False
self.skipped = {}
self.submit_new = {}
self.parse_lookup()
@ -94,6 +96,14 @@ class UpdateCrawler(object):
return self.retried_GET(url)
raise e
def get_project_meta(self, prj):
url = makeurl(self.apiurl, ['source', prj, '_meta'])
return self.cached_GET(url)
def is_maintenance_project(self, prj):
root = ET.fromstring(self.get_project_meta(prj))
return root.get('kind', None) == 'maintenance_release'
def _meta_get_packagelist(self, prj, deleted=None, expand=False):
query = {}
@ -128,8 +138,8 @@ class UpdateCrawler(object):
ret[package.get('package')] = package
return ret
def _submitrequest(self, src_project, src_package, rev, dst_project,
dst_package, msg):
def _find_existing_request(self, src_project, src_package, rev, dst_project,
dst_package):
"""Create a submit request."""
states = ['new', 'review', 'declined', 'revoked']
reqs = osc.core.get_exact_request_list(self.apiurl,
@ -145,32 +155,37 @@ class UpdateCrawler(object):
if a.to_xml().find('source').get('rev') == rev:
logging.debug('{}: found existing request {}'.format(dst_package, r.reqid))
foundrev = True
return foundrev
def _submitrequest(self, src_project, src_package, rev, dst_project,
dst_package, msg):
res = 0
if not foundrev:
print "creating submit request", src_project, src_package, rev, dst_project, dst_package
if not self.dryrun:
res = osc.core.create_submit_request(self.apiurl,
src_project,
src_package,
dst_project,
dst_package,
orev=rev,
message=msg)
print "creating submit request", src_project, src_package, rev, dst_project, dst_package
if not self.dryrun:
res = osc.core.create_submit_request(self.apiurl,
src_project,
src_package,
dst_project,
dst_package,
orev=rev,
message=msg)
return res
def submitrequest(self, src_project, src_package, rev, dst_package):
def submitrequest(self, src_project, src_package, rev, dst_package, origin):
"""Create a submit request using the osc.commandline.Osc class."""
dst_project = self.to_prj
msg = 'Automatic request from %s by UpdateCrawler' % src_project
return self._submitrequest(src_project, src_package, rev, dst_project,
if not self._find_existing_request(src_project, src_package, rev, dst_project, dst_package):
return self._submitrequest(src_project, src_package, rev, dst_project,
dst_package, msg)
return 0
def is_source_innerlink(self, project, package):
try:
root = ET.parse(
http_GET(makeurl(self.apiurl,
root = ET.fromstring(
self.cached_GET(makeurl(self.apiurl,
['source', project, package, '_link']
))).getroot()
)))
if root.get('project') is None and root.get('cicount'):
return True
except urllib2.HTTPError, err:
@ -183,8 +198,11 @@ class UpdateCrawler(object):
self.lookup = yaml.safe_load(self._load_lookup_file())
def _load_lookup_file(self):
prj = self.to_prj
if prj.endswith(':NonFree'):
prj = prj[:-len(':NonFree')]
return self.cached_GET(makeurl(self.apiurl,
['source', self.to_prj, '00Meta', 'lookup.yml']))
['source', prj, '00Meta', 'lookup.yml']))
def follow_link(self, project, package, rev, verifymd5):
#print "follow", project, package, rev
@ -210,32 +228,54 @@ class UpdateCrawler(object):
return (project, package, rev)
def update_targets(self, targets, sources):
for package, sourceinfo in sources.items():
if package.startswith('patchinfo.'):
continue
if self.filter_lookup and not self.lookup.get(package, '') in self.filter_lookup:
# special case maintenance project. Only consider main
# package names. The code later follows the link in the
# source project then.
if self.is_maintenance_project(self.from_prj):
mainpacks = set()
for package, sourceinfo in sources.items():
if package.startswith('patchinfo.'):
continue
files = set([node.text for node in sourceinfo.findall('filename')])
if '{}.spec'.format(package) in files:
mainpacks.add(package)
sources = { package: sourceinfo for package, sourceinfo in sources.iteritems() if package in mainpacks }
for package, sourceinfo in sources.items():
origin = self.lookup.get(package, '')
if self.filter_lookup and not origin in self.filter_lookup:
if not origin.startswith('subpackage of'):
self.skipped.setdefault(origin, set()).add(package)
continue
if not package in targets:
logging.debug('Package %s not found in targets' % (package))
continue
if not self.submit_new:
logging.info('Package %s not found in targets' % (package))
continue
targetinfo = targets[package]
if self.is_source_innerlink(self.from_prj, package):
logging.debug('Package %s is sub package' % (package))
continue
#if package != 'openssl':
# continue
else:
targetinfo = targets[package]
# Compare verifymd5
md5_from = sourceinfo.get('verifymd5')
md5_to = targetinfo.get('verifymd5')
if md5_from == md5_to:
#logging.info('Package %s not marked for update' % package)
continue
#if package != 'openssl':
# continue
if self.is_source_innerlink(self.to_prj, package):
logging.debug('Package %s is sub package' % (package))
continue
# Compare verifymd5
md5_from = sourceinfo.get('verifymd5')
md5_to = targetinfo.get('verifymd5')
if md5_from == md5_to:
#logging.info('Package %s not marked for update' % package)
continue
if self.is_source_innerlink(self.to_prj, package):
logging.debug('Package %s is sub package' % (package))
continue
# this makes only sense if we look at the expanded view
# and want to submit from proper project
@ -248,13 +288,12 @@ class UpdateCrawler(object):
sourceinfo.get('srcmd5'),
sourceinfo.get('verifymd5'))
res = self.submitrequest(src_project, src_package, src_rev, package)
res = self.submitrequest(src_project, src_package, src_rev, package, origin)
if res:
logging.info('Created request %s for %s' % (res, package))
elif res != 0:
logging.error('Error creating the request for %s' % package)
def crawl(self, packages):
"""Main method of the class that run the crawler."""
targets = self.get_source_infos(self.to_prj, packages)
@ -270,8 +309,10 @@ def main(args):
uc = UpdateCrawler(args.from_prj, args.to_prj)
uc.caching = args.cache_requests
uc.dryrun = args.dry
uc.submit_new = args.new
if args.only_from:
uc.filter_lookup.add(args.only_from)
for prj in args.only_from:
uc.filter_lookup.add(prj)
given_packages = args.packages
if not given_packages:
@ -281,6 +322,12 @@ def main(args):
given_packages = uc.latest_packages()
uc.crawl(given_packages)
if uc.skipped:
from pprint import pformat
logging.debug("skipped packages: %s", pformat(uc.skipped))
if __name__ == '__main__':
description = 'Create update SRs for Leap.'
parser = argparse.ArgumentParser(description=description)
@ -297,9 +344,10 @@ if __name__ == '__main__':
parser.add_argument('-t', '--to', dest='to_prj', metavar='PROJECT',
help='project where to submit the updates to (default: %s)' % OPENSUSE,
default=OPENSUSE)
parser.add_argument('--only-from', dest='only_from', metavar='PROJECT',
parser.add_argument('--only-from', dest='only_from', metavar='PROJECT', action ='append',
help='only submit packages that came from PROJECT')
parser.add_argument("--osc-debug", action="store_true", help="osc debug output")
parser.add_argument("--new", action="store_true", help="also submit new packages")
parser.add_argument('--cache-requests', action='store_true', default=False,
help='cache GET requests. Not recommended for daily use.')
parser.add_argument("packages", nargs='*', help="packages to check")