326a087168
With the python shebang remaining as Python 2, switch the scripts to using Python 3. This also allows us to clean up some imports.
373 lines
15 KiB
Python
Executable File
373 lines
15 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
import argparse
|
|
import itertools
|
|
import logging
|
|
import sys
|
|
import time
|
|
from urllib.error import HTTPError
|
|
from xml.etree import cElementTree as ET
|
|
|
|
import osc.conf
|
|
import osc.core
|
|
import rpm
|
|
import yaml
|
|
import re
|
|
from urllib import quote_plus
|
|
|
|
from osclib.memoize import memoize
|
|
from osclib.conf import Config
|
|
from osclib.core import devel_project_get
|
|
from osclib.stagingapi import StagingAPI
|
|
|
|
OPENSUSE = 'openSUSE:Leap:42.3'
|
|
FACTORY = 'openSUSE:Factory'
|
|
SLE = 'SUSE:SLE-12-SP2:Update'
|
|
|
|
makeurl = osc.core.makeurl
|
|
http_GET = osc.core.http_GET
|
|
|
|
# http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks-in-python
|
|
def chunks(l, n):
|
|
""" Yield successive n-sized chunks from l.
|
|
"""
|
|
for i in range(0, len(l), n):
|
|
yield l[i:i + n]
|
|
|
|
class UpdateCrawler(object):
|
|
def __init__(self, from_prj, to_prj):
|
|
self.from_prj = from_prj
|
|
self.to_prj = to_prj
|
|
self.apiurl = osc.conf.config['apiurl']
|
|
self.debug = osc.conf.config['debug']
|
|
self.filter_lookup = set()
|
|
self.caching = False
|
|
self.dryrun = False
|
|
self.skipped = {}
|
|
self.submit_new = {}
|
|
self.api = StagingAPI(
|
|
osc.conf.config['apiurl'], project = to_prj)
|
|
|
|
self.parse_lookup()
|
|
|
|
# FIXME: duplicated from manager_42
|
|
def latest_packages(self):
|
|
apiurl = self.apiurl
|
|
prj = self.from_prj
|
|
if prj.startswith('openSUSE.org:'):
|
|
apiurl = 'https://api.opensuse.org'
|
|
prj = prj[len('openSUSE.org:'):]
|
|
data = self.cached_GET(makeurl(apiurl,
|
|
['project', 'latest_commits', prj]))
|
|
lc = ET.fromstring(data)
|
|
packages = set()
|
|
for entry in lc.findall('{http://www.w3.org/2005/Atom}entry'):
|
|
title = entry.find('{http://www.w3.org/2005/Atom}title').text
|
|
if title.startswith('In '):
|
|
packages.add(title[3:].split(' ')[0])
|
|
return sorted(packages)
|
|
|
|
@memoize()
|
|
def _cached_GET(self, url):
|
|
return self.retried_GET(url).read()
|
|
|
|
def cached_GET(self, url):
|
|
if self.caching:
|
|
return self._cached_GET(url)
|
|
return self.retried_GET(url).read()
|
|
|
|
def retried_GET(self, url):
|
|
try:
|
|
return http_GET(url)
|
|
except HTTPError as e:
|
|
if 500 <= e.code <= 599:
|
|
print('Retrying {}'.format(url))
|
|
time.sleep(1)
|
|
return self.retried_GET(url)
|
|
raise e
|
|
|
|
def get_project_meta(self, prj):
|
|
url = makeurl(self.apiurl, ['source', prj, '_meta'])
|
|
return self.cached_GET(url)
|
|
|
|
def is_maintenance_project(self, prj):
|
|
root = ET.fromstring(self.get_project_meta(prj))
|
|
return root.get('kind', None) == 'maintenance_release'
|
|
|
|
def _meta_get_packagelist(self, prj, deleted=None, expand=False):
|
|
|
|
query = {}
|
|
if deleted:
|
|
query['deleted'] = 1
|
|
if expand:
|
|
query['expand'] = 1
|
|
|
|
u = osc.core.makeurl(self.apiurl, ['source', prj], query)
|
|
return self.cached_GET(u)
|
|
|
|
def meta_get_packagelist(self, prj, deleted=None, expand=False):
|
|
root = ET.fromstring(self._meta_get_packagelist(prj, deleted, expand))
|
|
return [ node.get('name') for node in root.findall('entry') if not node.get('name') == '000product' and not node.get('name').startswith('patchinfo.') ]
|
|
|
|
def _get_source_infos(self, project, packages):
|
|
query = ['view=info']
|
|
if packages:
|
|
query += ['package=%s' % quote_plus(p) for p in packages]
|
|
|
|
return self.cached_GET(makeurl(self.apiurl,
|
|
['source', project],
|
|
query))
|
|
|
|
def get_source_infos(self, project, packages):
|
|
ret = dict()
|
|
for pkg_chunks in chunks(sorted(packages), 50):
|
|
root = ET.fromstring(self._get_source_infos(project, pkg_chunks))
|
|
for package in root.findall('sourceinfo'):
|
|
if package.findall('error'):
|
|
continue
|
|
ret[package.get('package')] = package
|
|
return ret
|
|
|
|
def _get_source_package(self, project, package, revision):
|
|
opts = { 'view': 'info' }
|
|
if revision:
|
|
opts['rev'] = revision
|
|
return self.cached_GET(makeurl(self.apiurl,
|
|
['source', project, package], opts))
|
|
|
|
def _find_existing_request(self, src_project, src_package, rev, dst_project,
|
|
dst_package):
|
|
"""Create a submit request."""
|
|
states = ['new', 'review', 'declined', 'revoked', 'superseded']
|
|
reqs = osc.core.get_exact_request_list(self.apiurl,
|
|
src_project,
|
|
dst_project,
|
|
src_package,
|
|
dst_package,
|
|
req_type='submit',
|
|
req_state=states)
|
|
foundrev = False
|
|
for r in reqs:
|
|
for a in r.actions:
|
|
srcrev = a.src_rev
|
|
# sometimes requests only contain the decimal revision
|
|
if re.match(r'^\d+$', srcrev) is not None:
|
|
xml = ET.fromstring(self._get_source_package(src_project, src_package, srcrev))
|
|
srcrev = xml.get('verifymd5')
|
|
logging.debug('rev {}'.format(srcrev))
|
|
if srcrev == rev:
|
|
logging.debug('{}: found existing request {} {}/{}'.format(dst_package, r.reqid, a.src_project, src_project))
|
|
foundrev = True
|
|
return foundrev
|
|
|
|
def _submitrequest(self, src_project, src_package, rev, dst_project,
|
|
dst_package, msg):
|
|
res = 0
|
|
print("creating submit request", src_project, src_package, rev, dst_project, dst_package)
|
|
if not self.dryrun:
|
|
res = osc.core.create_submit_request(self.apiurl,
|
|
src_project,
|
|
src_package,
|
|
dst_project,
|
|
dst_package,
|
|
orev=rev,
|
|
message=msg)
|
|
return res
|
|
|
|
def submitrequest(self, src_project, src_package, rev, dst_package, origin):
|
|
"""Create a submit request using the osc.commandline.Osc class."""
|
|
dst_project = self.to_prj
|
|
msg = 'Automatic request from %s by UpdateCrawler' % src_project
|
|
if not self._find_existing_request(src_project, src_package, rev, dst_project, dst_package):
|
|
return self._submitrequest(src_project, src_package, rev, dst_project,
|
|
dst_package, msg)
|
|
return 0
|
|
|
|
def is_source_innerlink(self, project, package):
|
|
try:
|
|
root = ET.fromstring(
|
|
self.cached_GET(makeurl(self.apiurl,
|
|
['source', project, package, '_link']
|
|
)))
|
|
if root.get('project') is None and root.get('cicount'):
|
|
return True
|
|
except HTTPError as err:
|
|
# if there is no link, it can't be a link
|
|
if err.code == 404:
|
|
return False
|
|
raise
|
|
|
|
def parse_lookup(self):
|
|
self.lookup = yaml.safe_load(self._load_lookup_file())
|
|
|
|
def _load_lookup_file(self):
|
|
prj = self.to_prj
|
|
return self.cached_GET(makeurl(self.apiurl,
|
|
['source', prj, '00Meta', 'lookup.yml']))
|
|
|
|
def follow_link(self, project, package, rev, verifymd5):
|
|
#print "follow", project, package, rev
|
|
# verify it's still the same package
|
|
xml = ET.fromstring(self._get_source_package(project, package, rev))
|
|
if xml.get('verifymd5') != verifymd5:
|
|
return None
|
|
xml = ET.fromstring(self.cached_GET(makeurl(self.apiurl,
|
|
['source', project, package],
|
|
{
|
|
'rev': rev
|
|
})))
|
|
linkinfo = xml.find('linkinfo')
|
|
if not linkinfo is None:
|
|
ret = self.follow_link(linkinfo.get('project'), linkinfo.get('package'), linkinfo.get('srcmd5'), verifymd5)
|
|
if ret:
|
|
project, package, rev = ret
|
|
return (project, package, rev)
|
|
|
|
def update_targets(self, targets, sources):
|
|
|
|
# special case maintenance project. Only consider main
|
|
# package names. The code later follows the link in the
|
|
# source project then.
|
|
if self.is_maintenance_project(self.from_prj):
|
|
mainpacks = set()
|
|
for package, sourceinfo in sources.items():
|
|
if package.startswith('patchinfo.'):
|
|
continue
|
|
files = set([node.text for node in sourceinfo.findall('filename')])
|
|
if '{}.spec'.format(package) in files:
|
|
mainpacks.add(package)
|
|
|
|
sources = { package: sourceinfo for package, sourceinfo in sources.iteritems() if package in mainpacks }
|
|
|
|
for package, sourceinfo in sources.items():
|
|
|
|
origin = self.lookup.get(package, '')
|
|
if origin.startswith('Devel;'):
|
|
(dummy, origin, dummy) = origin.split(';')
|
|
|
|
if self.filter_lookup and not origin in self.filter_lookup:
|
|
if not origin.startswith('subpackage of'):
|
|
self.skipped.setdefault(origin, set()).add(package)
|
|
continue
|
|
|
|
if not package in targets:
|
|
if not self.submit_new:
|
|
logging.info('Package %s not found in targets' % (package))
|
|
continue
|
|
|
|
if self.is_source_innerlink(self.from_prj, package):
|
|
logging.debug('Package %s is sub package' % (package))
|
|
continue
|
|
|
|
else:
|
|
targetinfo = targets[package]
|
|
|
|
# XXX: make more generic :-)
|
|
devel_prj = devel_project_get(self.apiurl, FACTORY, package)
|
|
if devel_prj == 'devel:languages:haskell':
|
|
logging.info('skipping haskell package %s' % package)
|
|
continue
|
|
|
|
# Compare verifymd5
|
|
md5_from = sourceinfo.get('verifymd5')
|
|
md5_to = targetinfo.get('verifymd5')
|
|
if md5_from == md5_to:
|
|
#logging.info('Package %s not marked for update' % package)
|
|
continue
|
|
|
|
if self.is_source_innerlink(self.to_prj, package):
|
|
logging.debug('Package %s is sub package' % (package))
|
|
continue
|
|
|
|
# this makes only sense if we look at the expanded view
|
|
# and want to submit from proper project
|
|
# originproject = default_origin
|
|
# if not sourceinfo.find('originproject') is None:
|
|
# originproject = sourceinfo.find('originproject').text
|
|
# logging.warn('changed originproject for {} to {}'.format(package, originproject))
|
|
|
|
src_project, src_package, src_rev = self.follow_link(self.from_prj, package,
|
|
sourceinfo.get('srcmd5'),
|
|
sourceinfo.get('verifymd5'))
|
|
|
|
res = self.submitrequest(src_project, src_package, src_rev, package, origin)
|
|
if res:
|
|
logging.info('Created request %s for %s' % (res, package))
|
|
elif res != 0:
|
|
logging.error('Error creating the request for %s' % package)
|
|
|
|
def crawl(self, packages):
|
|
"""Main method of the class that run the crawler."""
|
|
targets = self.get_source_infos(self.to_prj, packages)
|
|
sources = self.get_source_infos(self.from_prj, packages)
|
|
self.update_targets(targets, sources)
|
|
|
|
def main(args):
|
|
# Configure OSC
|
|
osc.conf.get_config(override_apiurl=args.apiurl)
|
|
osc.conf.config['debug'] = args.osc_debug
|
|
|
|
# initialize stagingapi config
|
|
Config(osc.conf.config['apiurl'], args.to_prj)
|
|
|
|
uc = UpdateCrawler(args.from_prj, args.to_prj)
|
|
uc.caching = args.cache_requests
|
|
uc.dryrun = args.dry
|
|
uc.submit_new = args.new
|
|
if args.only_from:
|
|
for prj in args.only_from:
|
|
uc.filter_lookup.add(prj)
|
|
|
|
given_packages = args.packages
|
|
if not given_packages:
|
|
if args.all:
|
|
given_packages = uc.meta_get_packagelist(args.from_prj)
|
|
else:
|
|
given_packages = uc.latest_packages()
|
|
uc.crawl(given_packages)
|
|
|
|
if uc.skipped:
|
|
from pprint import pformat
|
|
logging.debug("skipped packages: %s", pformat(uc.skipped))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
description = 'Create update SRs for Leap.'
|
|
parser = argparse.ArgumentParser(description=description)
|
|
parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL')
|
|
parser.add_argument('-d', '--debug', action='store_true',
|
|
help='print info useful for debuging')
|
|
parser.add_argument('-a', '--all', action='store_true',
|
|
help='check all packages')
|
|
parser.add_argument('-n', '--dry', action='store_true',
|
|
help='dry run, no POST, PUT, DELETE')
|
|
parser.add_argument('-f', '--from', dest='from_prj', metavar='PROJECT',
|
|
help='project where to get the updates (default: %s)' % SLE,
|
|
default=SLE)
|
|
parser.add_argument('-t', '--to', dest='to_prj', metavar='PROJECT',
|
|
help='project where to submit the updates to (default: %s)' % OPENSUSE,
|
|
default=OPENSUSE)
|
|
parser.add_argument('--only-from', dest='only_from', metavar='PROJECT', action ='append',
|
|
help='only submit packages that came from PROJECT')
|
|
parser.add_argument("--osc-debug", action="store_true", help="osc debug output")
|
|
parser.add_argument("--new", action="store_true", help="also submit new packages")
|
|
parser.add_argument('--cache-requests', action='store_true', default=False,
|
|
help='cache GET requests. Not recommended for daily use.')
|
|
parser.add_argument("packages", nargs='*', help="packages to check")
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Set logging configuration
|
|
logging.basicConfig(level=logging.DEBUG if args.debug
|
|
else logging.INFO)
|
|
|
|
if args.dry:
|
|
def dryrun(t, *args, **kwargs):
|
|
return lambda *args, **kwargs: logging.debug("dryrun %s %s %s", t, args, str(kwargs)[:200])
|
|
|
|
http_POST = dryrun('POST')
|
|
http_PUT = dryrun('PUT')
|
|
http_DELETE = dryrun('DELETE')
|
|
|
|
sys.exit(main(args))
|