yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor configuration parameters into configuration file.

Browse Source
This commit is contained in:
Alberto Planas 2015-02-19 10:57:55 +01:00
parent 4aad2749ec
commit c9d1e06ca8
24 changed files with 374 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
openqa-comments.py
View File

@ -20,6 +20,7 @@ from collections import defaultdict
import json
from osclib.comments import CommentAPI
from osclib.conf import Config
from osclib.stagingapi import StagingAPI
import osc
@ -41,13 +42,13 @@ class OpenQAReport(object):
def _openQA_url(self, job):
test_name = job['name'].split('-')[-1]
link = 'https://openqa.opensuse.org/tests/%s' % job['id']
link = '%s/tests/%s' % (self.api.copenqa, job['id'])
text = '[%s](%s)' % (test_name, link)
return text
def _openQA_module_url(self, job, module):
link = 'https://openqa.opensuse.org/tests/%s/modules/%s/steps/1' % (
job['id'], module['name']
link = '%s/tests/%s/modules/%s/steps/1' % (
self.api.copenqa, job['id'], module['name']
)
text = '[%s](%s)' % (module['name'], link)
return text
@ -58,13 +59,13 @@ class OpenQAReport(object):
return safe_margin <= time_delta
def get_info(self, project):
_prefix = 'openSUSE:{}:Staging:'.format(self.api.opensuse)
_prefix = '{}:'.format(self.api.cstaging)
if project.startswith(_prefix):
project = project.replace(_prefix, '')
query = {'format': 'json'}
url = api.makeurl(('project', 'staging_projects',
'openSUSE:%s' % api.opensuse, project), query=query)
api.project, project), query=query)
info = json.load(api.retried_GET(url))
return info
@ -216,7 +217,8 @@ if __name__ == '__main__':
if args.force:
MARGIN_HOURS = 0
api = StagingAPI(osc.conf.config['apiurl'], args.project)
Config('openSUSE:%s' % args.project)
api = StagingAPI(osc.conf.config['apiurl'], 'openSUSE:%s' % args.project)
openQA = OpenQAReport(api)
if args.staging:

6
osc-check_repo.py
View File

@ -71,7 +71,7 @@ def _check_repo_download(self, request):
if request.error:
return set()
staging_prefix = 'openSUSE:{}:Staging:'.format(self.checkrepo.opensuse)
staging_prefix = '{}:'.format(self.checkrepo.staging.cstaging)
if staging_prefix in str(request.group):
pkglist = self.checkrepo.get_package_list_from_repository(
request.group, 'standard', arch,
@ -359,7 +359,7 @@ def _check_repo_group(self, id_, requests, debug=False):
# Detect if this error message comes from a staging project.
# Store it in the repo_checker_error, that is the text that
# will be published in the error message.
staging_prefix = 'openSUSE:{}:Staging:'.format(self.checkrepo.opensuse)
staging_prefix = '{}:'.format(self.checkrepo.staging.cstaging)
if staging_prefix in project_repo[0]:
repo_checker_error = stdoutdata
if not any(staging_prefix in p_r[0] for p_r in execution_plan):
@ -402,7 +402,7 @@ def _check_repo_group(self, id_, requests, debug=False):
def _mirror_full(self, plugin_dir, repo_dir):
"""Call bs_mirrorfull script to mirror packages."""
url = 'https://build.opensuse.org/build/openSUSE:%s/%s/x86_64' % (self.checkrepo.opensuse, 'standard')
url = 'https://build.opensuse.org/build/%s/%s/x86_64' % (self.checkrepo.project, 'standard')
if not os.path.exists(repo_dir):
os.mkdir(repo_dir)

5
osc-staging.py
View File

@ -17,6 +17,7 @@
import os
import os.path
import sys
import warnings
from osc import cmdln
from osc import oscerr
@ -46,7 +47,7 @@ def _print_version(self):
def _full_project_name(self, project):
"""Deduce the full project name."""
if ':' in project:
if project.startswith(('openSUSE', 'SUSE')):
return project
if 'Factory' in project or 'openSUSE' in project:
@ -56,7 +57,7 @@ def _full_project_name(self, project):
return 'SUSE:%s' % project
# If we can't guess, raise a Warning
raise Warning('% project not recognized.' % project)
warnings.warn('% project not recognized.' % project)
return project

57
osclib/accept_command.py
View File

@ -1,4 +1,5 @@
import re
import warnings
from xml.etree import cElementTree as ET
from osc.core import change_request_state
@ -56,15 +57,15 @@ class AcceptCommand(object):
msg = 'Accepting staging review for {}'.format(req['package'])
print(msg)
oldspecs = self.api.get_filelist_for_package(pkgname=req['package'], project='openSUSE:{}'.format(self.api.opensuse), extension='spec')
change_request_state(self.api.apiurl, str(req['id']), 'accepted', message='Accept to %s' % self.api.opensuse)
self.create_new_links('openSUSE:{}'.format(self.api.opensuse), req['package'], oldspecs)
oldspecs = self.api.get_filelist_for_package(pkgname=req['package'], project=self.api.project, extension='spec')
change_request_state(self.api.apiurl, str(req['id']), 'accepted', message='Accept to %s' % self.api.project)
self.create_new_links(self.api.project, req['package'], oldspecs)
# A single comment should be enough to notify everybody, since
# they are already mentioned in the comments created by
# select/unselect
pkg_list = ", ".join(packages)
cmmt = 'Project "{}" accepted. The following packages have been submitted to {}: {}.'.format(project, self.api.opensuse, pkg_list)
cmmt = 'Project "{}" accepted. The following packages have been submitted to {}: {}.'.format(project, self.api.project, pkg_list)
self.comment.add_comment(project_name=project, comment=cmmt)
# XXX CAUTION - AFAIK the 'accept' command is expected to clean the messages here.
@ -78,14 +79,16 @@ class AcceptCommand(object):
def accept_other_new(self):
changed = False
rqlist = self.find_new_requests('openSUSE:{}'.format(self.api.opensuse))
rqlist += self.find_new_requests('openSUSE:{}:NonFree'.format(self.api.opensuse))
rqlist = self.find_new_requests(self.api.project)
if self.api.cnonfree:
rqlist += self.find_new_requests(self.api.cnonfree)
for req in rqlist:
oldspecs = self.api.get_filelist_for_package(pkgname=req['packages'][0], project='openSUSE:{}'.format(self.api.opensuse), extension='spec')
oldspecs = self.api.get_filelist_for_package(pkgname=req['packages'][0], project=self.api.project, extension='spec')
print 'Accepting request %d: %s' % (req['id'], ','.join(req['packages']))
change_request_state(self.api.apiurl, str(req['id']), 'accepted', message='Accept to %s' % self.api.opensuse)
change_request_state(self.api.apiurl, str(req['id']), 'accepted', message='Accept to %s' % self.api.project)
# Check if all .spec files of the package we just accepted has a package container to build
self.create_new_links('openSUSE:{}'.format(self.api.opensuse), req['packages'][0], oldspecs)
self.create_new_links(self.api.project, req['packages'][0], oldspecs)
changed = True
return changed
@ -128,9 +131,19 @@ class AcceptCommand(object):
return True
def update_factory_version(self):
"""Update openSUSE (Factory, 13.2, ...) version if is necessary."""
project = 'openSUSE:{}'.format(self.api.opensuse)
url = self.api.makeurl(['source', project, '_product', 'openSUSE.product'])
"""Update project (Factory, 13.2, ...) version if is necessary."""
# XXX TODO - This method have `factory` in the name. Can be
# missleading.
# If thereis not product defined for this project, show the
# warning and return.
if not self.api.cproduct:
warnings.warn('There is not product defined in the configuration file.')
return
project = self.api.project
url = self.api.makeurl(['source', project, '_product', self.api.cproduct])
product = http_GET(url).read()
curr_version = date.today().strftime('%Y%m%d')
@ -140,20 +153,22 @@ class AcceptCommand(object):
http_PUT(url + '?comment=Update+version', data=new_product)
def sync_buildfailures(self):
"""Trigger rebuild of packages that failed build in either
openSUSE:Factory or openSUSE:Factory:Rebuild, but not the other
Helps over the fact that openSUSE:Factory uses rebuild=local,
thus sometimes 'hiding' build failures."""
"""
Trigger rebuild of packages that failed build in either
openSUSE:Factory or openSUSE:Factory:Rebuild, but not the
other Helps over the fact that openSUSE:Factory uses
rebuild=local, thus sometimes 'hiding' build failures.
"""
for arch in ["x86_64","i586"]:
fact_result = self.api.get_prj_results('openSUSE:Factory', arch)
for arch in ["x86_64", "i586"]:
fact_result = self.api.get_prj_results(self.api.project, arch)
fact_result = self.api.check_pkgs(fact_result)
rebuild_result = self.api.get_prj_results('openSUSE:Factory:Rebuild', arch)
rebuild_result = self.api.get_prj_results(self.api.crebuild, arch)
rebuild_result = self.api.check_pkgs(rebuild_result)
result = set(rebuild_result) ^ set(fact_result)
print sorted(result)
for package in result:
self.api.rebuild_pkg(package, 'openSUSE:Factory', arch, None)
self.api.rebuild_pkg(package, 'openSUSE:Factory:Rebuild', arch, None)
self.api.rebuild_pkg(package, self.api.project, arch, None)
self.api.rebuild_pkg(package, self.api.crebuild, arch, None)

21
osclib/check_command.py
View File

@ -74,14 +74,15 @@ class CheckCommand(object):
break
# openQA results
if not project['openqa_jobs']:
report.append(' - No openQA result yet')
for job in project['openqa_jobs']:
if job['result'] != 'passed':
qa_result = job['result'] if job['result'] != 'none' else 'running'
report.append(" - openQA's overall status is %s for https://openqa.opensuse.org/tests/%s" % (qa_result, job['id']))
report.extend(' %s: fail' % module['name'] for module in job['modules'] if module['result'] == 'fail')
break
if self.api.copenqa:
if not project['openqa_jobs']:
report.append(' - No openQA result yet')
for job in project['openqa_jobs']:
if job['result'] != 'passed':
qa_result = job['result'] if job['result'] != 'none' else 'running'
report.append(" - openQA's overall status is %s for %s/tests/%s" % (qa_result, self.api.copenqa, job['id']))
report.extend(' %s: fail' % module['name'] for module in job['modules'] if module['result'] == 'fail')
break
subproject = project['subproject']
if subproject:
@ -109,10 +110,10 @@ class CheckCommand(object):
query = {'format': 'json'}
if project:
url = self.api.makeurl(('project', 'staging_projects', 'openSUSE:%s' % self.api.opensuse,
url = self.api.makeurl(('project', 'staging_projects', self.api.project,
project), query=query)
else:
url = self.api.makeurl(('project', 'staging_projects', 'openSUSE:%s' % self.api.opensuse),
url = self.api.makeurl(('project', 'staging_projects', self.api.project),
query=query)
info = json.load(self.api.retried_GET(url))
if not project:

10
osclib/checkrepo.py
View File

@ -137,11 +137,11 @@ class Request(object):
class CheckRepo(object):
def __init__(self, apiurl, opensuse='Factory', readonly=False, force_clean=False, debug=False):
def __init__(self, apiurl, project='Factory', readonly=False, force_clean=False, debug=False):
"""CheckRepo constructor."""
self.apiurl = apiurl
self.opensuse = opensuse
self.staging = StagingAPI(apiurl, opensuse)
self.project = 'openSUSE:%s' % project
self.staging = StagingAPI(apiurl, self.project)
self.pkgcache = PkgCache(BINCACHE, force_clean=force_clean)
@ -241,8 +241,8 @@ class CheckRepo(object):
"""Search pending requests to review."""
requests = []
review = "@by_user='factory-repo-checker'+and+@state='new'"
target = "@project='openSUSE:{}'".format(self.opensuse)
target_nf = "@project='openSUSE:{}:NonFree'".format(self.opensuse)
target = "@project='{}'".format(self.project)
target_nf = "@project='{}'".format(self.staging.cnonfree)
try:
url = makeurl(self.apiurl, ('search', 'request'),
"match=state/@name='review'+and+review[%s]+and+(target[%s]+or+target[%s])" % (

22
osclib/cleanup_rings.py
View File

@ -13,11 +13,11 @@ class CleanupRings(object):
self.links = {}
def perform(self):
self.check_depinfo_ring('openSUSE:{}:Rings:0-Bootstrap'.format(self.api.opensuse),
'openSUSE:{}:Rings:1-MinimalX'.format(self.api.opensuse))
self.check_depinfo_ring('openSUSE:{}:Rings:1-MinimalX'.format(self.api.opensuse),
'openSUSE:{}:Rings:2-TestDVD'.format(self.api.opensuse))
self.check_depinfo_ring('openSUSE:{}:Rings:2-TestDVD'.format(self.api.opensuse), None)
self.check_depinfo_ring('{}:0-Bootstrap'.format(self.api.crings),
'{}:1-MinimalX'.format(self.api.crings))
self.check_depinfo_ring('{}:1-MinimalX'.format(self.api.crings),
'{}:2-TestDVD'.format(self.api.crings))
self.check_depinfo_ring('{}:2-TestDVD'.format(self.api.crings), None)
def find_inner_ring_links(self, prj):
query = {
@ -29,8 +29,8 @@ class CleanupRings(object):
root = ET.parse(f).getroot()
for si in root.findall('sourceinfo'):
linked = si.find('linked')
if linked is not None and linked.get('project') != 'openSUSE:{}'.format(self.api.opensuse):
if not linked.get('project').startswith('openSUSE:{}:Rings:'.format(self.api.opensuse)):
if linked is not None and linked.get('project') != self.api.project:
if not linked.get('project').startswith(self.api.crings):
print(ET.tostring(si))
self.links[linked.get('package')] = si.get('package')
@ -81,7 +81,7 @@ class CleanupRings(object):
self.find_inner_ring_links(prj)
self.fill_pkgdeps(prj, 'standard', 'x86_64')
if prj == 'openSUSE:{}:Rings:1-MinimalX'.format(self.api.opensuse):
if prj == '{}:1-MinimalX'.format(self.api.crings):
url = makeurl(self.api.apiurl, ['build', prj, 'images', 'x86_64', 'Test-DVD-x86_64', '_buildinfo'])
root = ET.parse(http_GET(url)).getroot()
for bdep in root.findall('bdep'):
@ -93,7 +93,7 @@ class CleanupRings(object):
b = self.bin2src[b]
self.pkgdeps[b] = 'MYdvd'
if prj == 'openSUSE:{}:Rings:2-TestDVD'.format(self.api.opensuse):
if prj == '{}:2-TestDVD'.format(self.api.crings):
url = makeurl(self.api.apiurl, ['build', prj, 'images', 'x86_64', 'Test-DVD-x86_64', '_buildinfo'])
root = ET.parse(http_GET(url)).getroot()
for bdep in root.findall('bdep'):
@ -105,7 +105,7 @@ class CleanupRings(object):
b = self.bin2src[b]
self.pkgdeps[b] = 'MYdvd2'
if prj == 'openSUSE:{}:Rings:0-Bootstrap'.format(self.api.opensuse):
if prj == '{}:0-Bootstrap'.format(self.api.crings):
url = makeurl(self.api.apiurl, ['build', prj, 'standard', '_buildconfig'])
for line in http_GET(url).read().split('\n'):
if line.startswith('Preinstall:') or line.startswith('Support:'):
@ -119,4 +119,4 @@ class CleanupRings(object):
if source not in self.pkgdeps and source not in self.links:
print('osc rdelete -m cleanup {} {}'.format(prj, source))
if nextprj:
print('osc linkpac openSUSE:{} {} {}').format(self.api.opensuse, source, nextprj)
print('osc linkpac {} {} {}').format(self.api.project, source, nextprj)

38
osclib/conf.py
View File

@ -1,4 +1,4 @@
# Copyright (C) 2015 SUSE Linux Products GmbH
# Copyright (C) 2015 SUSE Linux GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -22,14 +22,35 @@ import re
from osc import conf
# Sane defatuls for openSUSE and SUSE. The string interpolation rule
# is as this:
#
# * %(project)s to replace the name of the project.
# * %(project.lower)s to replace the lower case version of the name of
# the project.
DEFAULT = {
r'openSUSE:(?P<project>[-\w\d]+)': {
'staging': 'openSUSE:%(project)s:Staging',
'staging-group': '%(project.lower)s-staging',
'rings': 'openSUSE:%(project)s:Rings',
'nonfree': 'openSUSE:%(project)s:NonFree',
'rebuild': 'openSUSE:%(project)s:Rebuild',
'product': 'openSUSE.product',
'openqa': 'https://openqa.opensuse.org',
'lock': 'openSUSE:%(project)s:Staging',
'lock-ns': 'openSUSE',
},
r'SUSE:(?P<project>[-\w\d]+)': {
'staging': 'SUSE:%(project)s:GA:Staging',
'lock': 'SUSE:%(project)s:GA:Staging',
'staging': 'SUSE:%(project)s:Staging',
'staging-group': '%(project.lower)s-staging',
'rings': None,
'nonfree': None,
'rebuild': None,
'product': None,
'openqa': None,
'lock': None,
'lock-ns': 'OBS',
}
}
@ -41,6 +62,7 @@ DEFAULT = {
# [openSUSE:Factory]
#
# staging = openSUSE:Factory:Staging
# rings = openSUSE:Factory:Rings
# lock = openSUSE:Factory:Staging
#
@ -69,9 +91,13 @@ class Config(object):
match = re.match(prj_pattern, self.project)
if match:
project = match.group('project')
defaults = {
k: v % {'project': project} for k, v in DEFAULT[prj_pattern].items() if v
}
for k, v in DEFAULT[prj_pattern].items():
if isinstance(v, basestring) and '%(project)s' in v:
defaults[k] = v % {'project': project}
elif isinstance(v, basestring) and '%(project.lower)s' in v:
defaults[k] = v % {'project.lower': project.lower()}
else:
defaults[k] = v
break
# Update the configuration, only when it is necessary

25
osclib/cycle.py
View File

@ -232,17 +232,17 @@ class CycleDetector(object):
"""Detect cycles in a specific repository."""
if not project:
project = 'openSUSE:{}'.format(self.api.opensuse)
project = self.api.project
# filter submit requests
requests = [rq for rq in requests if rq.action_type == 'submit' and not rq.updated]
# Detect cycles - We create the full graph from _builddepinfo.
factory_graph = self._get_builddepinfo_graph(project, repository, arch)
factory_cycles = factory_graph.cycles()
project_graph = self._get_builddepinfo_graph(project, repository, arch)
project_cycles = project_graph.cycles()
# This graph will be updated for every request
current_graph = deepcopy(factory_graph)
current_graph = deepcopy(project_graph)
subpkgs = current_graph.subpkgs
@ -291,15 +291,16 @@ class CycleDetector(object):
# packages. We need to inform about this, so this can become
# a warning instead of an error.
#
# To do that, we store in `factory_cycles_pkgs` all the
# factory cycles as a set of packages, so we can check in the
# new cycle (also as a set of package) is included here.
factory_cycles_pkgs = {frozenset(cycle) for cycle in factory_cycles}
# To do that, we store in `project_cycles_pkgs` all the
# project (i.e Factory) cycles as a set of packages, so we can
# check in the new cycle (also as a set of package) is
# included here.
project_cycles_pkgs = {frozenset(cycle) for cycle in project_cycles}
for cycle in current_graph.cycles():
if cycle not in factory_cycles:
factory_edges = set((u, v) for u in cycle for v in factory_graph.edges(u) if v in cycle)
if cycle not in project_cycles:
project_edges = set((u, v) for u in cycle for v in project_graph.edges(u) if v in cycle)
current_edges = set((u, v) for u in cycle for v in current_graph.edges(u) if v in cycle)
current_pkgs = set(cycle)
yield (cycle,
sorted(current_edges - factory_edges),
current_pkgs in factory_cycles_pkgs)
sorted(current_edges - project_edges),
current_pkgs in project_cycles_pkgs)

28
osclib/freeze_command.py
View File

@ -1,3 +1,19 @@
# Copyright (C) 2015 SUSE Linux GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import time
import re
from xml.etree import cElementTree as ET
@ -34,7 +50,7 @@ class FreezeCommand(object):
self.create_bootstrap_aggregate_file()
def bootstrap_packages(self):
url = self.api.makeurl(['source', 'openSUSE:{}:Rings:0-Bootstrap'.format(self.api.opensuse)])
url = self.api.makeurl(['source', '{}:0-Bootstrap'.format(self.api.crings)])
f = self.api.retried_GET(url)
root = ET.parse(f).getroot()
l = list()
@ -51,7 +67,7 @@ class FreezeCommand(object):
root = ET.Element('aggregatelist')
a = ET.SubElement(root, 'aggregate',
{'project': 'openSUSE:{}:Rings:0-Bootstrap'.format(self.api.opensuse)})
{'project': '{}:0-Bootstrap'.format(self.api.crings)})
for package in self.bootstrap_packages():
p = ET.SubElement(a, 'package')
@ -147,7 +163,7 @@ class FreezeCommand(object):
root = ET.Element('project', {'name': prj})
ET.SubElement(root, 'title')
ET.SubElement(root, 'description')
links = self.projectlinks or ['openSUSE:{}:Rings:1-MinimalX'.format(self.api.opensuse)]
links = self.projectlinks or ['{}:1-MinimalX'.format(self.api.crings)]
for lprj in links:
ET.SubElement(root, 'link', {'project': lprj})
f = ET.SubElement(root, 'build')
@ -161,7 +177,7 @@ class FreezeCommand(object):
ET.SubElement(f, 'enable')
r = ET.SubElement(root, 'repository', {'name': 'bootstrap_copy'})
ET.SubElement(r, 'path', {'project': 'openSUSE:{}:Staging'.format(self.api.opensuse), 'repository': 'standard'})
ET.SubElement(r, 'path', {'project': self.api.cstaging, 'repository': 'standard'})
a = ET.SubElement(r, 'arch')
a.text = 'i586'
a = ET.SubElement(r, 'arch')
@ -182,7 +198,7 @@ class FreezeCommand(object):
return ET.tostring(root)
def freeze_prjlinks(self):
sources = dict()
sources = {}
flink = ET.Element('frozenlinks')
for lprj in self.projectlinks:
@ -211,7 +227,7 @@ class FreezeCommand(object):
for linked in si.findall('linked'):
if linked.get('project') in self.projectlinks:
# take the unexpanded md5 from Factory / 13.2 link
url = self.api.makeurl(['source', 'openSUSE:{}'.format(self.api.opensuse), package],
url = self.api.makeurl(['source', self.api.project, package],
{'view': 'info', 'nofilename': '1'})
# print(package, linked.get('package'), linked.get('project'))
f = self.api.retried_GET(url)

24
osclib/obslock.py
View File

@ -1,4 +1,4 @@
# Copyright (C) 2015 SUSE Linux Products GmbH
# Copyright (C) 2015 SUSE Linux GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -16,6 +16,7 @@
from datetime import datetime
import time
import warnings
from xml.etree import cElementTree as ET
from osc import conf
@ -30,6 +31,8 @@ class OBSLock(object):
def __init__(self, apiurl, project, ttl=3600):
self.apiurl = apiurl
self.project = project
self.lock = conf.config[project]['lock']
self.ns = conf.config[project]['lock-ns']
# TTL is measured in seconds
self.ttl = ttl
self.user = conf.config['api_host_options'][apiurl]['user']
@ -50,7 +53,7 @@ class OBSLock(object):
return user, ts
def _read(self):
url = makeurl(self.apiurl, ['source', self.project, '_attribute', 'openSUSE:LockedBy'])
url = makeurl(self.apiurl, ['source', self.lock, '_attribute', '%s:LockedBy' % self.ns])
root = ET.parse(http_GET(url)).getroot()
signature = None
try:
@ -60,16 +63,22 @@ class OBSLock(object):
return signature
def _write(self, signature):
url = makeurl(self.apiurl, ['source', self.project, '_attribute', 'openSUSE:LockedBy'])
url = makeurl(self.apiurl, ['source', self.lock, '_attribute', '%s:LockedBy' % self.ns])
data = """
<attributes>
<attribute namespace='openSUSE' name='LockedBy'>
<attribute namespace='%s' name='LockedBy'>
<value>%s</value>
</attribute>
</attributes>""" % signature
</attributes>""" % (self.ns, signature)
http_POST(url, data=data)
def acquire(self):
# If the project doesn't have locks configured, raise a
# Warning (but continue the operation)
if not self.lock:
warnings.warn('Locking attribute is not found. Create one to avoid race conditions.')
return self
user, ts = self._parse(self._read())
if user and ts:
now = datetime.utcnow()
@ -89,6 +98,11 @@ class OBSLock(object):
return self
def release(self):
# If the project do not have locks configured, simply ignore
# the operation.
if not self.lock:
return
user, ts = self._parse(self._read())
if user == self.user:
self._write('')

16
osclib/request_finder.py
View File

@ -51,9 +51,9 @@ class RequestFinder(object):
return None
project = root.find('action').find('target').get('project')
if (project != 'openSUSE:{}'.format(self.api.opensuse) and not project.startswith('openSUSE:{}:Staging:'.format(self.api.opensuse))):
msg = 'Request {} is not for openSUSE:{}, but for {}'
msg = msg.format(request_id, self.api.opensuse, project)
if (project != self.api.project and not project.startswith(self.api.cstaging)):
msg = 'Request {} is not for {}, but for {}'
msg = msg.format(request_id, self.api.project, project)
raise oscerr.WrongArgs(msg)
self.srs[int(request_id)] = {'project': project}
@ -65,8 +65,8 @@ class RequestFinder(object):
:param package: name of the package
"""
query = 'states=new,review,declined&project=openSUSE:{}&view=collection&package={}'
query = query.format(self.api.opensuse, urllib2.quote(package))
query = 'states=new,review,declined&project={}&view=collection&package={}'
query = query.format(self.api.project, urllib2.quote(package))
url = makeurl(self.api.apiurl, ['request'], query)
f = http_GET(url)
@ -82,7 +82,7 @@ class RequestFinder(object):
request = int(sr.get('id'))
state = sr.find('state').get('name')
self.srs[request] = {'project': 'openSUSE:{}'.format(self.api.opensuse), 'state': state}
self.srs[request] = {'project': self.api.project, 'state': state}
if last_rq:
if self.srs[last_rq]['state'] == 'declined':
@ -107,7 +107,7 @@ class RequestFinder(object):
:param source_project: name of the source project
"""
query = 'states=new,review&project=openSUSE:{}&view=collection'.format(self.api.opensuse)
query = 'states=new,review&project={}&view=collection'.format(self.api.project)
url = makeurl(self.api.apiurl, ['request'], query)
f = http_GET(url)
root = ET.parse(f).getroot()
@ -119,7 +119,7 @@ class RequestFinder(object):
if src is not None and src.get('project') == source_project:
request = int(sr.attrib['id'])
state = sr.find('state').get('name')
self.srs[request] = {'project': 'openSUSE:{}'.format(self.api.opensuse), 'state': state}
self.srs[request] = {'project': self.api.project, 'state': state}
ret = True
return ret

94
osclib/stagingapi.py
View File

@ -1,8 +1,18 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2015 SUSE Linux GmbH
#
# (C) 2014 mhrusecky@suse.cz, openSUSE.org
# (C) 2014 tchvatal@suse.cz, openSUSE.org
# Distribute under GPLv2 or GPLv3
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import json
import logging
@ -13,6 +23,7 @@ from xml.etree import cElementTree as ET
import yaml
from osc import conf
from osc import oscerr
from osc.core import change_review_state
from osc.core import delete_package
@ -31,21 +42,31 @@ class StagingAPI(object):
Class containing various api calls to work with staging projects.
"""
def __init__(self, apiurl, opensuse='Factory'):
"""
Initialize instance variables
"""
def __init__(self, apiurl, project):
"""Initialize instance variables."""
self.apiurl = apiurl
self.opensuse = opensuse
self.rings = (
'openSUSE:{}:Rings:0-Bootstrap'.format(self.opensuse),
'openSUSE:{}:Rings:1-MinimalX'.format(self.opensuse),
'openSUSE:{}:Rings:2-TestDVD'.format(self.opensuse)
)
self.ring_packages = self._generate_ring_packages()
self.packages_staged = self._get_staged_requests()
self.project = project
# Store some prefix / data used in the code.
self.cstaging = conf.config[project]['staging']
self.cstaging_group = conf.config[project]['staging-group']
self.crings = conf.config[project]['rings']
self.cnonfree = conf.config[project]['nonfree']
self.crebuild = conf.config[project]['rebuild']
self.cproduct = conf.config[project]['product']
self.copenqa = conf.config[project]['openqa']
# If the project support rings, inititialize some variables.
if self.crings:
self.rings = (
'{}:0-Bootstrap'.format(self.crings),
'{}:1-MinimalX'.format(self.crings),
'{}:2-TestDVD'.format(self.crings)
)
self.ring_packages = self._generate_ring_packages()
self.packages_staged = self._get_staged_requests()
def makeurl(self, l, query=None):
"""
@ -92,12 +113,12 @@ class StagingAPI(object):
"""
ret = {}
for prj in self.rings:
url = self.makeurl(['source', prj])
root = http_GET(url)
for entry in ET.parse(root).getroot().findall('entry'):
pkg = entry.attrib['name']
# XXX TODO - Test-DVD-x86_64 is hardcoded here
if pkg in ret and pkg != 'Test-DVD-x86_64':
msg = '{} is defined in two projects ({} and {})'
raise Exception(msg.format(pkg, ret[pkg], prj))
@ -110,7 +131,7 @@ class StagingAPI(object):
:return dict of staged requests with their project and srid
"""
packages_staged = dict()
packages_staged = {}
for prj in self.get_staging_projects():
meta = self.get_prj_pseudometa(prj)
for req in meta['requests']:
@ -215,7 +236,7 @@ class StagingAPI(object):
projects = []
query = "id?match=starts-with(@name,'openSUSE:{}:Staging:')".format(self.opensuse)
query = "id?match=starts-with(@name,'{}:')".format(self.cstaging)
url = self.makeurl(['search', 'project', query])
projxml = http_GET(url)
root = ET.parse(projxml).getroot()
@ -287,7 +308,7 @@ class StagingAPI(object):
# accept the request here
message = 'No need for staging, not in tested ring projects.'
self.do_change_review_state(request_id, 'accepted', message=message,
by_group='factory-staging')
by_group=self.cstaging_group)
def supseded_request(self, request):
"""
@ -349,12 +370,14 @@ class StagingAPI(object):
requests = []
# xpath query, using the -m, -r, -s options
where = "@by_group='factory-staging'+and+@state='new'"
target = "@project='openSUSE:{}'".format(self.opensuse)
target_nf = "@project='openSUSE:{}:NonFree'".format(self.opensuse)
where = "@by_group='{}'+and+@state='new'".format(self.cstaging_group)
projects = [format(self.project)]
if self.cnonfree:
projects.append(self.cnonfree)
targets = ["target[@project='{}']".format(p) for p in projects]
query = "match=state/@name='review'+and+review[{}]+and+(target[{}]+or+target[{}])".format(
where, target, target_nf)
query = "match=state/@name='review'+and+review[{}]+and+({})".format(
where, '+or+'.join(targets))
url = self.makeurl(['search', 'request'], query)
f = http_GET(url)
root = ET.parse(f).getroot()
@ -557,12 +580,12 @@ class StagingAPI(object):
informations)
"""
_prefix = 'openSUSE:{}:Staging:'.format(self.opensuse)
_prefix = '{}:'.format(self.cstaging)
if project.startswith(_prefix):
project = project.replace(_prefix, '')
query = {'format': 'json'}
url = self.makeurl(('project', 'staging_projects', 'openSUSE:%s' % self.opensuse, project),
url = self.makeurl(('project', 'staging_projects', self.project, project),
query=query)
result = json.load(self.retried_GET(url))
return result['overall_state'] == 'acceptable'
@ -581,7 +604,10 @@ class StagingAPI(object):
return 100000 # quite some!
def check_if_job_is_ok(self, job):
url = 'https://openqa.opensuse.org/tests/{}/file/results.json'.format(job['id'])
if not self.copenqa:
return
url = '{}/tests/{}/file/results.json'.format(self.copenqa, job['id'])
try:
f = urllib2.urlopen(url)
except urllib2.HTTPError:
@ -598,7 +624,7 @@ class StagingAPI(object):
# pprint.pprint(openqa)
# pprint.pprint(job)
if overall != 'ok':
return "openQA's overall status is {} for https://openqa.opensuse.org/tests/{}".format(overall, job['id'])
return "openQA's overall status is {} for {}/tests/{}".format(overall, self.openqa, job['id'])
for module in openqa['testmodules']:
# zypper_in fails at the moment - urgent fix needed
@ -606,7 +632,7 @@ class StagingAPI(object):
continue
if module['name'] in ['kate', 'ooffice', 'amarok', 'thunderbird', 'gnucash']:
continue
return '{} test failed: https://openqa.opensuse.org/tests/{}'.format(module['name'], job['id'])
return '{} test failed: {}/tests/{}'.format(module['name'], self.openqa, job['id'])
return None
def rq_to_prj(self, request_id, project):
@ -643,7 +669,7 @@ class StagingAPI(object):
# now remove the staging checker
self.do_change_review_state(request_id, 'accepted',
by_group='factory-staging',
by_group=self.cstaging_group,
message='Picked {}'.format(project))
return True
@ -659,7 +685,7 @@ class StagingAPI(object):
if project.endswith(':DVD'):
return project # not yet
ring_dvd = 'openSUSE:{}:Rings:2-TestDVD'.format(self.opensuse)
ring_dvd = '{}:2-TestDVD'.format(self.project)
if self.ring_packages.get(pkg) == ring_dvd:
return project + ":DVD"
@ -780,7 +806,7 @@ class StagingAPI(object):
def prj_from_letter(self, letter):
if ':' in letter: # not a letter
return letter
return 'openSUSE:{}:Staging:{}'.format(self.opensuse, letter)
return '{}:{}'.format(self.cstaging, letter)
def list_requests_in_prj(self, project):
where = "@by_project='%s'+and+@state='new'" % project
@ -1048,7 +1074,7 @@ class StagingAPI(object):
return results
def check_pkgs(self, rebuild_list):
url = self.makeurl(['source', 'openSUSE:Factory'])
url = self.makeurl(['source', self.project])
pkglist = []
root = ET.parse(http_GET(url)).getroot()

2
osclib/unselect_command.py
View File

@ -20,7 +20,7 @@ class UnselectCommand(object):
msg = 'Unselecting "{}" from "{}"'.format(request, staging_project)
print(msg)
self.api.rm_from_prj(staging_project, request_id=request)
self.api.add_review(request, by_group='factory-staging', msg='Please recheck')
self.api.add_review(request, by_group=self.api.cstaging_group, msg='Please recheck')
# Notify everybody about the changes
for prj in affected_projects:

26
tests/accept_tests.py
View File

@ -1,16 +1,27 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2015 SUSE Linux GmbH
#
# (C) 2014 tchvatal@suse.cz, openSUSE.org
# Distribute under GPLv2 or later
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import unittest
from obs import APIURL
from obs import OBS
from osclib.accept_command import AcceptCommand
from osclib.stagingapi import StagingAPI
from osclib.conf import Config
from osclib.comments import CommentAPI
from osclib.stagingapi import StagingAPI
class TestAccept(unittest.TestCase):
@ -20,7 +31,8 @@ class TestAccept(unittest.TestCase):
Initialize the configuration
"""
self.obs = OBS()
self.api = StagingAPI(APIURL)
Config('openSUSE:Factory')
self.api = StagingAPI(APIURL, 'openSUSE:Factory')
def test_accept_comments(self):
c_api = CommentAPI(self.api.apiurl)
@ -38,6 +50,6 @@ class TestAccept(unittest.TestCase):
# But the comment was written at some point
self.assertEqual(len(self.obs.comment_bodies), 1)
comment = self.obs.comment_bodies[0]
self.assertTrue('The following packages have been submitted to Factory' in comment)
self.assertTrue('The following packages have been submitted to openSUSE:Factory' in comment)
self.assertTrue('apparmor' in comment)
self.assertTrue('mariadb' in comment)

22
tests/api_tests.py
View File

@ -1,8 +1,18 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2015 SUSE Linux GmbH
#
# (C) 2014 tchvatal@suse.cz, openSUSE.org
# Distribute under GPLv2 or later
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import sys
import unittest
@ -10,6 +20,7 @@ import httpretty
from obs import APIURL
from obs import OBS
from osclib.conf import Config
from osclib.stagingapi import StagingAPI
@ -32,7 +43,8 @@ class TestApiCalls(unittest.TestCase):
"""
self.obs = OBS()
self.api = StagingAPI(APIURL)
Config('openSUSE:Factory')
self.api = StagingAPI(APIURL, 'openSUSE:Factory')
def test_ring_packages(self):
"""

9
tests/check_tests.py
View File

@ -1,7 +1,4 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2014 SUSE Linux Products GmbH
# Copyright (C) 2015 SUSE Linux GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -21,6 +18,7 @@ import unittest
from obs import APIURL
from obs import OBS
from osclib.conf import Config
from osclib.check_command import CheckCommand
from osclib.stagingapi import StagingAPI
@ -74,7 +72,8 @@ class TestCheckCommand(unittest.TestCase):
"""Initialize the configuration."""
self.obs = OBS()
self.stagingapi = StagingAPI(APIURL)
Config('openSUSE:Factory')
self.stagingapi = StagingAPI(APIURL, 'openSUSE:Factory')
self.checkcommand = CheckCommand(self.stagingapi)
def test_check_command_all(self):

7
tests/checkrepo_tests.py
View File

@ -1,7 +1,4 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2014 SUSE Linux Products GmbH
# Copyright (C) 2015 SUSE Linux GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -22,6 +19,7 @@ import unittest
from obs import APIURL
from obs import OBS
from osclib.checkrepo import CheckRepo
from osclib.conf import Config
class TestCheckRepoCalls(unittest.TestCase):
@ -31,6 +29,7 @@ class TestCheckRepoCalls(unittest.TestCase):
"""Initialize the configuration."""
self.obs = OBS()
Config('openSUSE:Factory')
self.checkrepo = CheckRepo(APIURL, force_clean=True)
# Des-memoize some functions
self.checkrepo.build = self.checkrepo._build

18
tests/factory_source_tests.py
View File

@ -1,4 +1,18 @@
#!/usr/bin/python
# Copyright (C) 2015 SUSE Linux GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import unittest
@ -13,9 +27,11 @@ from check_source_in_factory import FactorySourceChecker
APIURL = 'https://testhost.example.com'
FIXTURES = os.path.join(os.getcwd(), 'tests/fixtures')
def rr(s):
return re.compile(re.escape(APIURL + s))
class TestFactorySourceAccept(unittest.TestCase):
def setUp(self):

22
tests/freeze_tests.py
View File

@ -1,8 +1,18 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2015 SUSE Linux GmbH
#
# (C) 2014 tchvatal@suse.cz, openSUSE.org
# Distribute under GPLv2 or later
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import unittest
@ -12,6 +22,7 @@ import tempfile
from obs import APIURL
from obs import OBS
from osclib.conf import Config
from osclib.freeze_command import FreezeCommand
from osclib.stagingapi import StagingAPI
@ -22,7 +33,8 @@ class TestFreeze(unittest.TestCase):
Initialize the configuration
"""
self.obs = OBS()
self.api = StagingAPI(APIURL)
Config('openSUSE:Factory')
self.api = StagingAPI(APIURL, 'openSUSE:Factory')
def _get_fixture_path(self, filename):
"""

18
tests/maintenance_tests.py
View File

@ -1,4 +1,18 @@
#!/usr/bin/python
# Copyright (C) 2015 SUSE Linux GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import unittest
@ -13,9 +27,11 @@ from check_maintenance_incidents import MaintenanceChecker
APIURL = 'https://maintenancetest.example.com'
FIXTURES = os.path.join(os.getcwd(), 'tests/fixtures')
def rr(s):
return re.compile(re.escape(APIURL + s))
class TestMaintenance(unittest.TestCase):
def setUp(self):

16
tests/obs.py
View File

@ -1,3 +1,19 @@
# Copyright (C) 2015 SUSE Linux GmbH
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import re
import string

18
tests/pkgcache_tests.py
View File

@ -1,8 +1,18 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2015 SUSE Linux GmbH
#
# (C) 2014 aplanas@suse.de, openSUSE.org
# Distribute under GPLv2 or later
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import shutil

24
tests/select_tests.py
View File

@ -1,17 +1,28 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2015 SUSE Linux GmbH
#
# (C) 2014 tchvatal@suse.cz, openSUSE.org
# Distribute under GPLv2 or later
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import unittest
from obs import APIURL
from obs import OBS
from osc import oscerr
from osclib.comments import CommentAPI
from osclib.conf import Config
from osclib.select_command import SelectCommand
from osclib.stagingapi import StagingAPI
from osclib.comments import CommentAPI
class TestSelect(unittest.TestCase):
@ -21,7 +32,8 @@ class TestSelect(unittest.TestCase):
Initialize the configuration
"""
self.obs = OBS()
self.api = StagingAPI(APIURL)
Config('openSUSE:Factory')
self.api = StagingAPI(APIURL, 'openSUSE:Factory')
def test_old_frozen(self):
self.assertEqual(self.api.prj_frozen_enough('openSUSE:Factory:Staging:A'), False)