Better filtering, new packages
- allow filtering from multiple projects - option to also submit new packaes - print skipped packages
This commit is contained in:
Ludwig Nussel
parent
919009aaf3
commit
5be0c17f51
@ -60,6 +60,8 @@ class UpdateCrawler(object):
|
||||
self.filter_lookup = set()
|
||||
self.caching = False
|
||||
self.dryrun = False
|
||||
self.skipped = {}
|
||||
self.submit_new = {}
|
||||
|
||||
self.parse_lookup()
|
||||
|
||||
@ -136,8 +138,8 @@ class UpdateCrawler(object):
|
||||
ret[package.get('package')] = package
|
||||
return ret
|
||||
|
||||
def _submitrequest(self, src_project, src_package, rev, dst_project,
|
||||
dst_package, msg):
|
||||
def _find_existing_request(self, src_project, src_package, rev, dst_project,
|
||||
dst_package):
|
||||
"""Create a submit request."""
|
||||
states = ['new', 'review', 'declined', 'revoked']
|
||||
reqs = osc.core.get_exact_request_list(self.apiurl,
|
||||
@ -153,25 +155,30 @@ class UpdateCrawler(object):
|
||||
if a.to_xml().find('source').get('rev') == rev:
|
||||
logging.debug('{}: found existing request {}'.format(dst_package, r.reqid))
|
||||
foundrev = True
|
||||
return foundrev
|
||||
|
||||
def _submitrequest(self, src_project, src_package, rev, dst_project,
|
||||
dst_package, msg):
|
||||
res = 0
|
||||
if not foundrev:
|
||||
print "creating submit request", src_project, src_package, rev, dst_project, dst_package
|
||||
if not self.dryrun:
|
||||
res = osc.core.create_submit_request(self.apiurl,
|
||||
src_project,
|
||||
src_package,
|
||||
dst_project,
|
||||
dst_package,
|
||||
orev=rev,
|
||||
message=msg)
|
||||
print "creating submit request", src_project, src_package, rev, dst_project, dst_package
|
||||
if not self.dryrun:
|
||||
res = osc.core.create_submit_request(self.apiurl,
|
||||
src_project,
|
||||
src_package,
|
||||
dst_project,
|
||||
dst_package,
|
||||
orev=rev,
|
||||
message=msg)
|
||||
return res
|
||||
|
||||
def submitrequest(self, src_project, src_package, rev, dst_package):
|
||||
def submitrequest(self, src_project, src_package, rev, dst_package, origin):
|
||||
"""Create a submit request using the osc.commandline.Osc class."""
|
||||
dst_project = self.to_prj
|
||||
msg = 'Automatic request from %s by UpdateCrawler' % src_project
|
||||
return self._submitrequest(src_project, src_package, rev, dst_project,
|
||||
if not self._find_existing_request(src_project, src_package, rev, dst_project, dst_package):
|
||||
return self._submitrequest(src_project, src_package, rev, dst_project,
|
||||
dst_package, msg)
|
||||
return 0
|
||||
|
||||
def is_source_innerlink(self, project, package):
|
||||
try:
|
||||
@ -238,28 +245,37 @@ class UpdateCrawler(object):
|
||||
|
||||
for package, sourceinfo in sources.items():
|
||||
|
||||
if self.filter_lookup and not self.lookup.get(package, '') in self.filter_lookup:
|
||||
origin = self.lookup.get(package, '')
|
||||
if self.filter_lookup and not origin in self.filter_lookup:
|
||||
if not origin.startswith('subpackage of'):
|
||||
self.skipped.setdefault(origin, set()).add(package)
|
||||
continue
|
||||
|
||||
if not package in targets:
|
||||
logging.info('Package %s not found in targets' % (package))
|
||||
continue
|
||||
if not self.submit_new:
|
||||
logging.info('Package %s not found in targets' % (package))
|
||||
continue
|
||||
|
||||
targetinfo = targets[package]
|
||||
if self.is_source_innerlink(self.from_prj, package):
|
||||
logging.debug('Package %s is sub package' % (package))
|
||||
continue
|
||||
|
||||
#if package != 'openssl':
|
||||
# continue
|
||||
else:
|
||||
targetinfo = targets[package]
|
||||
|
||||
# Compare verifymd5
|
||||
md5_from = sourceinfo.get('verifymd5')
|
||||
md5_to = targetinfo.get('verifymd5')
|
||||
if md5_from == md5_to:
|
||||
#logging.info('Package %s not marked for update' % package)
|
||||
continue
|
||||
#if package != 'openssl':
|
||||
# continue
|
||||
|
||||
if self.is_source_innerlink(self.to_prj, package):
|
||||
logging.debug('Package %s is sub package' % (package))
|
||||
continue
|
||||
# Compare verifymd5
|
||||
md5_from = sourceinfo.get('verifymd5')
|
||||
md5_to = targetinfo.get('verifymd5')
|
||||
if md5_from == md5_to:
|
||||
#logging.info('Package %s not marked for update' % package)
|
||||
continue
|
||||
|
||||
if self.is_source_innerlink(self.to_prj, package):
|
||||
logging.debug('Package %s is sub package' % (package))
|
||||
continue
|
||||
|
||||
# this makes only sense if we look at the expanded view
|
||||
# and want to submit from proper project
|
||||
@ -272,13 +288,12 @@ class UpdateCrawler(object):
|
||||
sourceinfo.get('srcmd5'),
|
||||
sourceinfo.get('verifymd5'))
|
||||
|
||||
res = self.submitrequest(src_project, src_package, src_rev, package)
|
||||
res = self.submitrequest(src_project, src_package, src_rev, package, origin)
|
||||
if res:
|
||||
logging.info('Created request %s for %s' % (res, package))
|
||||
elif res != 0:
|
||||
logging.error('Error creating the request for %s' % package)
|
||||
|
||||
|
||||
def crawl(self, packages):
|
||||
"""Main method of the class that run the crawler."""
|
||||
targets = self.get_source_infos(self.to_prj, packages)
|
||||
@ -294,8 +309,10 @@ def main(args):
|
||||
uc = UpdateCrawler(args.from_prj, args.to_prj)
|
||||
uc.caching = args.cache_requests
|
||||
uc.dryrun = args.dry
|
||||
uc.submit_new = args.new
|
||||
if args.only_from:
|
||||
uc.filter_lookup.add(args.only_from)
|
||||
for prj in args.only_from:
|
||||
uc.filter_lookup.add(prj)
|
||||
|
||||
given_packages = args.packages
|
||||
if not given_packages:
|
||||
@ -305,6 +322,12 @@ def main(args):
|
||||
given_packages = uc.latest_packages()
|
||||
uc.crawl(given_packages)
|
||||
|
||||
if uc.skipped:
|
||||
from pprint import pformat
|
||||
logging.debug("skipped packages: %s", pformat(uc.skipped))
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
description = 'Create update SRs for Leap.'
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
@ -321,9 +344,10 @@ if __name__ == '__main__':
|
||||
parser.add_argument('-t', '--to', dest='to_prj', metavar='PROJECT',
|
||||
help='project where to submit the updates to (default: %s)' % OPENSUSE,
|
||||
default=OPENSUSE)
|
||||
parser.add_argument('--only-from', dest='only_from', metavar='PROJECT',
|
||||
parser.add_argument('--only-from', dest='only_from', metavar='PROJECT', action ='append',
|
||||
help='only submit packages that came from PROJECT')
|
||||
parser.add_argument("--osc-debug", action="store_true", help="osc debug output")
|
||||
parser.add_argument("--new", action="store_true", help="also submit new packages")
|
||||
parser.add_argument('--cache-requests', action='store_true', default=False,
|
||||
help='cache GET requests. Not recommended for daily use.')
|
||||
parser.add_argument("packages", nargs='*', help="packages to check")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user