yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Some basic refactorization. Add outliers detector from Base:build.

Browse Source
This commit is contained in:
Alberto Planas 2013-06-20 17:30:54 +02:00
parent 27f6ac587f
commit 4573afb109
1 changed files with 108 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

148
osc-check_repo.py
View File

@ -3,31 +3,52 @@
# Distribute under GPLv2 or GPLv3
#
# Copy this script to ~/.osc-plugins/ or /var/lib/osc-plugins .
# Then try to run 'osc checker --help' to see the usage.
# Then try to run 'osc check_repo --help' to see the usage.
import socket
import os
import traceback
import re
import subprocess
import shutil
from urllib import quote_plus
import urllib2
def _check_repo_change_review_state(self, opts, id, newstate, message='', supersed=None):
from xml.etree import cElementTree as ET
from osc import oscerr
from osc.core import (get_binary_file,
get_buildinfo,
http_GET,
http_POST,
makeurl,
Request)
def _check_repo_change_review_state(self, opts, id_, newstate, message='', supersed=None):
""" taken from osc/osc/core.py, improved:
- verbose option added,
- empty by_user=& removed.
- numeric id can be int().
"""
query = {'cmd': 'changereviewstate', 'newstate': newstate }
query['by_user']= 'factory-repo-checker'
if supersed: query['superseded_by'] = supersed
# if message: query['comment'] = message
u = makeurl(opts.apiurl, ['request', str(id)], query=query)
query = {
'cmd': 'changereviewstate',
'newstate': newstate,
'by_user': 'factory-repo-checker',
}
if supersed:
query['superseded_by'] = supersed
# if message:
# query['comment'] = message
code = 404
u = makeurl(opts.apiurl, ['request', str(id_)], query=query)
try:
f = http_POST(u, data=message)
root = ET.parse(f).getroot()
return root.attrib['code']
except urllib2.HTTPError:
print "error", u
return 404
code = root.attrib['code']
except urllib2.HTTPError, e:
print 'ERROR in URL %s [%s]'%(u, e)
return code
def _check_repo_find_submit_request(self, opts, project, package):
xpath = "(action/target/@project='%s' and action/target/@package='%s' and action/@type='submit' and (state/@name='new' or state/@name='review' or state/@name='accepted'))" % (project, package)
@ -44,6 +65,7 @@ def _check_repo_find_submit_request(self, opts, project, package):
return int(r.reqid)
return None
def _check_repo_fetch_group(self, opts, group):
if opts.groups.get(group): return
u = makeurl(opts.apiurl, ['request', str(group)])
@ -56,6 +78,7 @@ def _check_repo_fetch_group(self, opts, group):
opts.grouped[id] = group
opts.groups[group] = a
def _check_repo_avoid_wrong_friends(self, prj, repo, arch, pkg, opts):
try:
url = makeurl(opts.apiurl, ["build", prj, repo, arch, pkg])
@ -68,11 +91,12 @@ def _check_repo_avoid_wrong_friends(self, prj, repo, arch, pkg, opts):
return False
return True
def _check_repo_one_request(self, rq, opts):
class CheckRepoPackage:
def __repr__(self):
return "[%d:%s/%s]" % (int(self.request),self.sproject, self.spackage)
return '[%d:%s/%s]' % (int(self.request), self.sproject, self.spackage)
def __init__(self):
self.updated = False
@ -81,7 +105,6 @@ def _check_repo_one_request(self, rq, opts):
id = int(rq.get('id'))
approved_actions = 0
actions = rq.findall('action')
if len(actions) != 1:
msg = "only one action per request is supported - create a group instead: https://github.com/SUSE/hackweek/wiki/Improved-Factory-devel-project-submission-workflow"
@ -181,9 +204,10 @@ def _check_repo_one_request(self, rq, opts):
sp.rev = root.attrib['srcmd5']
return packs
def _check_repo_buildsuccess(self, p, opts):
try:
url = makeurl(opts.apiurl, ['build', p.sproject, "_result?lastsuccess&package=%s&pathproject=%s&srcmd5=%s" % (quote_plus(p.spackage),quote_plus(p.tproject),p.rev)])
url = makeurl(opts.apiurl, ['build', p.sproject, "_result?lastsuccess&package=%s&pathproject=%s&srcmd5=%s" % (quote_plus(p.spackage), quote_plus(p.tproject), p.rev)])
root = ET.parse(http_GET(url)).getroot()
except urllib2.HTTPError:
print "error", url
@ -197,7 +221,6 @@ def _check_repo_buildsuccess(self, p, opts):
alldisabled = True
foundbuilding = None
foundfailed = None
found64 = True
tocheckrepos = []
for repo in root.findall('repository'):
@ -283,6 +306,7 @@ def _check_repo_buildsuccess(self, p, opts):
return True
def _check_repo_repo_list(self, prj, repo, arch, pkg, opts):
url = makeurl(opts.apiurl, ['build', prj, repo, arch, pkg])
files = []
@ -304,15 +328,17 @@ def _check_repo_repo_list(self, prj, repo, arch, pkg, opts):
if result.group(4) == 'src':
continue
files.append((fn, pname, result.group(4)))
except urllib2.HTTPError, err:
pass
except urllib2.HTTPError:
print "error", url
return files
def _check_repo_get_binary(self, apiurl, prj, repo, arch, package, file, target):
if os.path.exists(target):
return
get_binary_file(apiurl, prj, repo, arch, file, package = package, target_filename = target)
def _check_repo_download(self, p, destdir, opts):
if p.build_excluded:
return [], []
@ -356,6 +382,32 @@ def _check_repo_download(self, p, destdir, opts):
toignore.append(fn[1])
return toignore, downloads
def _get_build_deps(self, prj, repo, arch, pkg, opts):
xml = get_buildinfo(opts.apiurl, prj, pkg, repo, arch)
root = ET.fromstring(xml)
return [e.attrib['name'] for e in root.findall('bdep')]
def _get_base_build_bin(self, opts):
"""Get Base:build pagacke list"""
binaries = {}
for arch in ('x86_64', 'i586'):
url = makeurl(opts.apiurl, ['/build/Base:build/standard/%s/_repository'%arch,])
f = http_GET(url)
root = ET.parse(f).getroot()
binaries[arch] = set([e.attrib['filename'][:-4] for e in root.findall('binary')])
return binaries
def _get_base_build_src(self, opts):
"""Get Base:build pagacke list"""
url = makeurl(opts.apiurl, ['/source/Base:build',])
f = http_GET(url)
root = ET.parse(f).getroot()
return set([e.attrib['name'] for e in root.findall('entry')])
def _check_repo_group(self, id, reqs, opts):
print "\ncheck group", reqs
for p in reqs:
@ -395,6 +447,20 @@ def _check_repo_group(self, id, reqs, opts):
downloads.extend(d)
toignore.extend(i)
# Get all the Base:build packages (source and binary)
base_build_bin = self._get_base_build_bin(opts)
base_build_src = self._get_base_build_src(opts)
for p in reqs:
# Be sure that if the package is in Base:build, all the
# dependecies are also in Base:build
if p.spackage in base_build_src:
# TODO - Check all the arch for this package
for arch in ('x86_64', 'i586'):
build_deps = set(self._get_build_deps(p.sproject, p.goodrepo, arch, p.spackage, opts))
outliers = build_deps - base_build_bin[arch]
if outliers:
print 'Outliers (%s)'%arch, outliers
for p in reqs:
smissing = []
for package in p.missings:
@ -451,60 +517,56 @@ def _check_repo_group(self, id, reqs, opts):
p.updated = True
shutil.rmtree(destdir)
def _check_repo_fetch_request(self, id, opts):
url = makeurl(opts.apiurl, ['request', str(id)])
def _check_repo_fetch_request(self, id_, opts):
url = makeurl(opts.apiurl, ['request', str(id_)])
f = http_GET(url)
xml = ET.parse(f)
root = xml.getroot()
return self._check_repo_one_request(root, opts)
def do_check_repo(self, subcmd, opts, *args):
"""${cmd_name}: checker review of submit requests.
Usage:
osc checker [OPT] [list] [FILTER|PACKAGE_SRC]
osc check_repo [OPT] [list] [FILTER|PACKAGE_SRC]
Shows pending review requests and their current state.
${cmd_option_list}
"""
if len(args) == 0:
raise oscerr.WrongArgs("Please give a subcommand to 'osc checker' or try 'osc help checker'")
raise oscerr.WrongArgs("Please give a subcommand to 'osc check_repo' or try 'osc help check_repo'")
opts.mode = ''
opts.groups = {}
opts.grouped = {}
opts.verbose = False
from pprint import pprint
opts.apiurl = self.get_api_url()
tmphome = None
if args[0] == 'skip':
for id in args[1:]:
self._check_repo_change_review_state(opts, id, 'accepted', message="skip review")
for id_ in args[1:]:
self._check_repo_change_review_state(opts, id_, 'accepted', message='skip review')
return
ids = {}
for a in args:
if (re.match('\d+', a)):
ids[a] = 1
ids = [arg for arg in args if arg.isdigit()]
packs = []
if (not len(ids)):
if not ids:
# xpath query, using the -m, -r, -s options
where = "@by_user='factory-repo-checker'+and+@state='new'"
url = makeurl(opts.apiurl, ['search','request'], "match=state/@name='review'+and+review["+where+"]")
url = makeurl(opts.apiurl, ['search', 'request'],
"match=state/@name='review'+and+review["+where+"]")
f = http_GET(url)
root = ET.parse(f).getroot()
for rq in root.findall('request'):
packs.extend(self._check_repo_one_request(rq, opts))
else:
# we have a list, use them.
for id in ids.keys():
packs.extend(self._check_repo_fetch_request(id, opts))
for id_ in ids:
packs.extend(self._check_repo_fetch_request(id_, opts))
groups = {}
for p in packs:
@ -512,11 +574,5 @@ def do_check_repo(self, subcmd, opts, *args):
a.append(p)
groups[p.group] = a
for id, reqs in groups.items():
self._check_repo_group(id, reqs, opts)
#Local Variables:
#mode: python
#py-indent-offset: 4
#tab-width: 8
#End:
for id_, reqs in groups.items():
self._check_repo_group(id_, reqs, opts)