435 lines
15 KiB
Python
435 lines
15 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# (C) 2014 mhrusecky@suse.cz, openSUSE.org
|
|
# (C) 2014 tchvatal@suse.cz, openSUSE.org
|
|
# Distribute under GPLv2 or GPLv3
|
|
|
|
import logging
|
|
from xml.etree import cElementTree as ET
|
|
|
|
import yaml
|
|
|
|
from osc import oscerr
|
|
from osc.core import delete_package
|
|
from osc.core import get_request
|
|
from osc.core import make_meta_url
|
|
from osc.core import makeurl
|
|
from osc.core import metafile
|
|
from osc.core import http_GET
|
|
from osc.core import http_POST
|
|
from osc.core import http_PUT
|
|
from osc.core import link_pac
|
|
|
|
|
|
class StagingAPI(object):
|
|
"""
|
|
Class containing various api calls to work with staging projects.
|
|
"""
|
|
|
|
def __init__(self, apiurl):
|
|
"""
|
|
Initialize instance variables
|
|
"""
|
|
|
|
self.apiurl = apiurl
|
|
self.rings = ['openSUSE:Factory:Rings:0-Bootstrap',
|
|
'openSUSE:Factory:Rings:1-MinimalX']
|
|
self.ring_packages = self._generate_ring_packages()
|
|
|
|
|
|
def _generate_ring_packages(self):
|
|
"""
|
|
Generate dictionary with names of the rings
|
|
:return dictionary with ring names
|
|
"""
|
|
|
|
ret = {}
|
|
|
|
for prj in self.rings:
|
|
url = makeurl(self.apiurl, ['source', prj])
|
|
root = http_GET(url)
|
|
for entry in ET.parse(root).getroot().findall('entry'):
|
|
ret[entry.attrib['name']] = prj
|
|
return ret
|
|
|
|
|
|
def get_package_information(self, project, pkgname):
|
|
"""
|
|
Get the revision packagename and source project to copy from
|
|
based on content provided
|
|
:param project: the project we are having the package in
|
|
:param pkgname: name of the package we want to identify
|
|
:return dict ( project, package, revision, md5sum )
|
|
"""
|
|
|
|
package_info = {}
|
|
|
|
url = makeurl(self.apiurl, ['source', project, pkgname])
|
|
content = http_GET(url)
|
|
root = ET.parse(content).getroot().find('linkinfo')
|
|
package_info['srcmd5'] = root.attrib['srcmd5']
|
|
package_info['rev'] = root.attrib['rev']
|
|
package_info['project'] = root.attrib['project']
|
|
package_info['package'] = root.attrib['package']
|
|
|
|
return package_info
|
|
|
|
|
|
def move_between_project(self, source_project, package, destination_project):
|
|
"""
|
|
Move selected package from one staging to another
|
|
:param source_project: Source project
|
|
:param package: Source package
|
|
:param destination_project: Destination project
|
|
"""
|
|
|
|
# Get the relevant information about source
|
|
meta = self.get_prj_pseudometa(source_project)
|
|
req_id = -1
|
|
for req in meta['requests']:
|
|
if req['package'] == package:
|
|
req_id = req['id']
|
|
if req_id == -1:
|
|
raise oscerr.WrongArgs("Couldn't find request for package {0} in project {1}".format(package,source_project))
|
|
|
|
# Copy the package
|
|
self.rq_to_prj(req_id, destination_project)
|
|
# Delete the old one
|
|
self.rm_from_prj(package, source_project, 'Moved to {0}'.format(destination_project))
|
|
|
|
def get_staging_projects(self):
|
|
"""
|
|
Get all current running staging projects
|
|
:return list of known staging projects
|
|
"""
|
|
|
|
projects = []
|
|
|
|
url = makeurl(self.apiurl, ['search', 'project',
|
|
'id?match=starts-with(@name,\'openSUSE:Factory:Staging:\')'])
|
|
projxml = http_GET(url)
|
|
root = ET.parse(projxml).getroot()
|
|
for val in root.findall('project'):
|
|
projects.append(val.get('name'))
|
|
return projects
|
|
|
|
|
|
def staging_change_review_state(self, request_id, newstate, message):
|
|
"""
|
|
Change review state of the staging request
|
|
:param request_id: id of the request
|
|
:param newstate: state of the new request
|
|
:param message: message for the review
|
|
"""
|
|
""" taken from osc/osc/core.py, improved:
|
|
- verbose option added,
|
|
- empty by_user=& removed.
|
|
- numeric id can be int().
|
|
"""
|
|
query = {
|
|
'cmd': 'changereviewstate',
|
|
'newstate': newstate,
|
|
'by_group': 'factory-staging',
|
|
'comment': message
|
|
}
|
|
|
|
url = makeurl(self.apiurl, ['request', str(request_id)], query=query)
|
|
http_POST(url, data=message)
|
|
|
|
|
|
def accept_non_ring_request(self, request):
|
|
"""
|
|
Accept review of requests that are not yet in any ring so we
|
|
don't delay their testing.
|
|
:param request: request to check
|
|
"""
|
|
|
|
# Consolidate all data from request
|
|
request_id = int(request.get('id'))
|
|
action = request.findall('action')
|
|
if not action:
|
|
raise oscerr.WrongArgs('Request {0} has no action'.format(request_id))
|
|
# we care only about first action
|
|
action = action[0]
|
|
|
|
# Where are we targeting the package
|
|
target_project = action.find('target').get('project')
|
|
target_package = action.find('target').get('package')
|
|
|
|
# If the values are empty it is no error
|
|
if not target_project or not target_package:
|
|
logging.info('no target/package in request {0}, action {1}; '.format(request_id, action))
|
|
|
|
# Verify the package ring
|
|
ring = self.ring_packages.get(target_package, None)
|
|
if not ring:
|
|
# accept the request here
|
|
message = "No need for staging, not in tested ring project."
|
|
self.staging_change_review_state(request_id, 'accepted', message)
|
|
|
|
|
|
def get_open_requests(self):
|
|
"""
|
|
Get all requests with open review for staging project
|
|
that are not yet included in any staging project
|
|
:return list of pending open review requests
|
|
"""
|
|
|
|
requests = []
|
|
|
|
# xpath query, using the -m, -r, -s options
|
|
where = "@by_group='factory-staging'+and+@state='new'"
|
|
|
|
url = makeurl(self.apiurl, ['search','request'], "match=state/@name='review'+and+review["+where+"]")
|
|
f = http_GET(url)
|
|
root = ET.parse(f).getroot()
|
|
|
|
for rq in root.findall('request'):
|
|
requests.append(rq)
|
|
|
|
return requests
|
|
|
|
|
|
def dispatch_open_requests(self):
|
|
"""
|
|
Verify all requests and dispatch them to staging projects or approve them
|
|
"""
|
|
|
|
# get all current pending requests
|
|
requests = self.get_open_requests()
|
|
# check if we can reduce it down by accepting some
|
|
for rq in requests:
|
|
self.accept_non_ring_request(rq)
|
|
|
|
# FIXME: dispatch to various staging projects automatically
|
|
|
|
|
|
def get_prj_pseudometa(self, project):
|
|
"""
|
|
Gets project data from YAML in project description
|
|
:param project: project to read data from
|
|
:return structured object with metadata
|
|
"""
|
|
|
|
url = make_meta_url('prj', project, self.apiurl)
|
|
data = http_GET(url).readlines()
|
|
root = ET.fromstring(''.join(data))
|
|
description = root.find('description')
|
|
# If YAML parsing fails, load default
|
|
# FIXME: Better handling of errors
|
|
# * broken description
|
|
# * directly linked packages
|
|
# * removed linked packages
|
|
try:
|
|
data = yaml.load(description.text)
|
|
data['requests']
|
|
except:
|
|
data = yaml.load('requests: []')
|
|
return data
|
|
|
|
|
|
def set_prj_pseudometa(self, project, meta):
|
|
"""
|
|
Sets project description to the YAML of the provided object
|
|
:param project: project to save into
|
|
:param meta: data to save
|
|
"""
|
|
|
|
# Get current metadata
|
|
url = make_meta_url('prj', project, self.apiurl)
|
|
data = http_GET(url).readlines()
|
|
root = ET.fromstring(''.join(data))
|
|
# Find description
|
|
description = root.find('description')
|
|
# Replace it with yaml
|
|
description.text = yaml.dump(meta)
|
|
# Find title
|
|
title = root.find('title')
|
|
# Put something nice into title as well
|
|
new_title = []
|
|
for request in meta['requests']:
|
|
new_title.append(request['package'])
|
|
nt = ', '.join(sorted(new_title))
|
|
title.text = nt[:240]
|
|
# Write XML back
|
|
url = make_meta_url('prj',project, self.apiurl, force=True)
|
|
f = metafile(url, ET.tostring(root))
|
|
http_PUT(f.url, file=f.filename)
|
|
|
|
|
|
def _add_rq_to_prj_pseudometa(self, project, request_id, package):
|
|
"""
|
|
Records request as part of the project within metadata
|
|
:param project: project to record into
|
|
:param request_id: request id to record
|
|
:param package: package the request is about
|
|
"""
|
|
|
|
data = self.get_prj_pseudometa(project)
|
|
append = True
|
|
for request in data['requests']:
|
|
if request['package'] == package:
|
|
request['id'] = request_id
|
|
append = False
|
|
if append:
|
|
data['requests'].append( { 'id': request_id, 'package': package} )
|
|
self.set_prj_pseudometa(project, data)
|
|
# FIXME Add sr to group request as well
|
|
|
|
def _remove_rq_from_prj_pseudometa(self, project, package):
|
|
"""
|
|
Delete request from the project pseudometa
|
|
:param project: project to remove from
|
|
:param package: package we want to remove from meta
|
|
"""
|
|
|
|
data = self.get_prj_pseudometa(project)
|
|
data['requests'] = filter(lambda x: x['package'] != package, data['requests'])
|
|
self.set_prj_pseudometa(project, data)
|
|
# FIXME Add sr to group request as well
|
|
|
|
def rm_from_prj(self, package, project, msg = None):
|
|
"""
|
|
Delete request from the project
|
|
:param project: project to remove from
|
|
:param package: package we want to remove
|
|
:param msg: message for the log
|
|
"""
|
|
|
|
self._remove_rq_from_prj_pseudometa(project, package)
|
|
delete_package(self.apiurl, project, package, force=True, msg=msg)
|
|
|
|
def create_package_container(self, project, package, disable_build = False):
|
|
"""
|
|
Creates a package container without any fields in project/package
|
|
:param project: project to create it
|
|
:param package: package name
|
|
:param disable_build: should the package be created with build flag disabled
|
|
"""
|
|
dst_meta = '<package name="%s"><title/><description/></package>' % package
|
|
if disable_build:
|
|
root = ET.fromstring(dst_meta)
|
|
elm = ET.SubElement(root, 'build')
|
|
ET.SubElement(elm, 'disable')
|
|
dst_meta = ET.tostring(root)
|
|
|
|
url = makeurl(self.apiurl, ['source', project, package, '_meta'] )
|
|
http_PUT(url, data=dst_meta)
|
|
|
|
def check_project_status(self, project):
|
|
"""
|
|
Checks whether everything is built in project
|
|
:param project: project to check
|
|
"""
|
|
# Get build results
|
|
query = {}
|
|
query['lastbuild'] = 1
|
|
u = makeurl(self.apiurl, ['build', project, '_result'], query=query)
|
|
f = http_GET(u)
|
|
root = ET.fromstring(''.join(f.readlines()))
|
|
|
|
# Check them
|
|
broken = []
|
|
working = []
|
|
# Iterate through repositories
|
|
for results in root.findall('result'):
|
|
if results.get("state") not in [ "published", "unpublished" ]:
|
|
working.append({"path": "{0}/{1}".format(results.get("repository"), results.get("arch")), "state": results.get("state")})
|
|
# Iterate through packages
|
|
for node in results:
|
|
result = node.get("code")
|
|
# Skip not built (yet)
|
|
if result in [ "blocked", "building", "disabled" "excluded", "finished", "unknown", "unpublished", "published" ]:
|
|
continue
|
|
# Find broken
|
|
if result in [ "broken", "failed", "unresolvable" ]:
|
|
broken.append({"pkg": node.get("package"), "state" : result, "path" : "{0}/{1}".format(results.get("repository"),results.get("arch"))})
|
|
|
|
# Print the results
|
|
if len(working) == 0 and len(broken) == 0:
|
|
print "Everything is green!"
|
|
else:
|
|
if len(working) != 0:
|
|
print "Following repositories are still building:"
|
|
for i in working:
|
|
print " {0}: {1}".format(i['path'], i['state'])
|
|
print
|
|
if len(broken) != 0:
|
|
print "Following packages are broken:"
|
|
for i in broken:
|
|
print " {0} ({1}): {2}".format(i['pkg'], i['path'], i['state'])
|
|
print
|
|
print "Found errors in staging project {0}!".format(project)
|
|
|
|
|
|
def rq_to_prj(self, request_id, project):
|
|
"""
|
|
Links request to project - delete or submit
|
|
:param request_id: request to link
|
|
:param project: project to link into
|
|
"""
|
|
# read info from sr
|
|
tar_pkg = None
|
|
|
|
req = get_request(self.apiurl, str(request_id))
|
|
if not req:
|
|
raise oscerr.WrongArgs("Request {0} not found".format(request_id))
|
|
|
|
act = req.get_actions("submit")
|
|
if act:
|
|
tar_pkg = self.submit_to_prj(act[0], project)
|
|
|
|
act = req.get_actions("delete")
|
|
if act:
|
|
tar_pkg = self.delete_to_prj(act[0], project)
|
|
|
|
if not tar_pkg:
|
|
raise oscerr.WrongArgs("Request {0} is not a submit or delete request".format(request_id))
|
|
|
|
# register the package name
|
|
self._add_rq_to_prj_pseudometa(project, int(request_id), tar_pkg)
|
|
|
|
def delete_to_prj(self, act, project):
|
|
"""
|
|
Hides Package in project
|
|
:param act: action for delete request
|
|
:param project: project to hide in
|
|
"""
|
|
|
|
tar_pkg = act.tgt_package
|
|
|
|
# create build disabled package
|
|
self.create_package_container(project, tar_pkg, disable_build=True)
|
|
# now trigger wipebinaries to emulate a delete
|
|
url = makeurl(self.apiurl, ['build', project], { 'cmd': 'wipe', 'package': tar_pkg } )
|
|
http_POST(url)
|
|
|
|
return tar_pkg
|
|
|
|
def submit_to_prj(self, act, project):
|
|
"""
|
|
Links sources from request to project
|
|
:param act: action for submit request
|
|
:param project: project to link into
|
|
"""
|
|
|
|
src_prj = act.src_project
|
|
src_rev = act.src_rev
|
|
src_pkg = act.src_package
|
|
tar_pkg = act.tgt_package
|
|
|
|
self.create_package_container(project, tar_pkg)
|
|
|
|
# expand the revision to a md5
|
|
url = makeurl(self.apiurl, ['source', src_prj, src_pkg], { 'rev': src_rev, 'expand': 1 })
|
|
f = http_GET(url)
|
|
root = ET.parse(f).getroot()
|
|
src_rev = root.attrib['srcmd5']
|
|
src_vrev = root.attrib['vrev']
|
|
|
|
# link stuff - not using linkpac because linkpac copies meta from source
|
|
root = ET.Element('link', package=src_pkg, project=src_prj, rev=src_rev, vrev=src_vrev)
|
|
url = makeurl(self.apiurl, ['source', project, tar_pkg, '_link'])
|
|
http_PUT(url, data=ET.tostring(root))
|
|
return tar_pkg
|