Merge pull request #2993 from Vogtinator/bcirepopublisher
Add BCI repo publishing bot
This commit is contained in:
GitHub
commit
3301485864
@ -127,6 +127,25 @@ pipelines:
|
||||
osc -A https://api.suse.de/ api "/build/$PRJ/_result?view=summary&repository=images" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED
|
||||
done
|
||||
|
||||
SLE_BCI_15SP3.RepoPublisher:
|
||||
group: BCI
|
||||
lock_behavior: unlockWhenFinished
|
||||
materials:
|
||||
git:
|
||||
git: https://github.com/openSUSE/openSUSE-release-tools.git
|
||||
environment_variables:
|
||||
OSC_CONFIG: /home/go/config/oscrc-totest-manager
|
||||
BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}'
|
||||
stages:
|
||||
- Run:
|
||||
approval: manual
|
||||
resources:
|
||||
- staging-bot
|
||||
tasks:
|
||||
- script: |
|
||||
export PYTHONPATH=scripts
|
||||
./scripts/gocd/bci_repo_publish.py -A https://api.suse.de --verbose --debug run "--token=$BCI_TOKEN" 15-SP3
|
||||
|
||||
SLE_BCI_15SP4.RelPkgs:
|
||||
group: BCI
|
||||
lock_behavior: unlockWhenFinished
|
||||
@ -212,6 +231,25 @@ pipelines:
|
||||
osc -A https://api.suse.de/ api "/build/$PRJ/_result?view=summary&repository=images" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED
|
||||
done
|
||||
|
||||
SLE_BCI_15SP4.RepoPublisher:
|
||||
group: BCI
|
||||
lock_behavior: unlockWhenFinished
|
||||
materials:
|
||||
git:
|
||||
git: https://github.com/openSUSE/openSUSE-release-tools.git
|
||||
environment_variables:
|
||||
OSC_CONFIG: /home/go/config/oscrc-totest-manager
|
||||
BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}'
|
||||
stages:
|
||||
- Run:
|
||||
approval: manual
|
||||
resources:
|
||||
- staging-bot
|
||||
tasks:
|
||||
- script: |
|
||||
export PYTHONPATH=scripts
|
||||
./scripts/gocd/bci_repo_publish.py -A https://api.suse.de --verbose --debug run "--token=$BCI_TOKEN" 15-SP4
|
||||
|
||||
SLE_BCI_15SP5.RelPkgs:
|
||||
group: BCI
|
||||
lock_behavior: unlockWhenFinished
|
||||
@ -297,3 +335,22 @@ pipelines:
|
||||
osc -A https://api.suse.de/ api "/build/$PRJ/_result?view=summary&repository=images" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED
|
||||
done
|
||||
|
||||
SLE_BCI_15SP5.RepoPublisher:
|
||||
group: BCI
|
||||
lock_behavior: unlockWhenFinished
|
||||
materials:
|
||||
git:
|
||||
git: https://github.com/openSUSE/openSUSE-release-tools.git
|
||||
environment_variables:
|
||||
OSC_CONFIG: /home/go/config/oscrc-totest-manager
|
||||
BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}'
|
||||
stages:
|
||||
- Run:
|
||||
approval: manual
|
||||
resources:
|
||||
- staging-bot
|
||||
tasks:
|
||||
- script: |
|
||||
export PYTHONPATH=scripts
|
||||
./scripts/gocd/bci_repo_publish.py -A https://api.suse.de --verbose --debug run "--token=$BCI_TOKEN" 15-SP5
|
||||
|
||||
|
||||
@ -126,4 +126,23 @@ pipelines:
|
||||
done
|
||||
osc -A https://api.suse.de/ api "/build/$PRJ/_result?view=summary&repository=images" | grep "result project" | grep 'code="published" state="published">' && echo PUBLISHED
|
||||
done
|
||||
|
||||
SLE_BCI_15<%= sp %>.RepoPublisher:
|
||||
group: BCI
|
||||
lock_behavior: unlockWhenFinished
|
||||
materials:
|
||||
git:
|
||||
git: https://github.com/openSUSE/openSUSE-release-tools.git
|
||||
environment_variables:
|
||||
OSC_CONFIG: /home/go/config/oscrc-totest-manager
|
||||
BCI_TOKEN: '{{SECRET:[opensuse.secrets][BCI_TOKEN]}}'
|
||||
stages:
|
||||
- Run:
|
||||
approval: manual
|
||||
resources:
|
||||
- staging-bot
|
||||
tasks:
|
||||
- script: |
|
||||
export PYTHONPATH=scripts
|
||||
./scripts/gocd/bci_repo_publish.py -A https://api.suse.de --verbose --debug run "--token=$BCI_TOKEN" 15-<%= sp %>
|
||||
<% end %>
|
||||
|
||||
199
gocd/bci_repo_publish.py
Executable file
199
gocd/bci_repo_publish.py
Executable file
@ -0,0 +1,199 @@
|
||||
#!/usr/bin/python3
|
||||
# (c) 2023 fvogt@suse.de
|
||||
# GPL-2.0-or-later
|
||||
|
||||
# This is a "mini ttm" for the BCI repo. Differences:
|
||||
# * No :ToTest staging area
|
||||
# * Only the repo is built, so it needs BCI specific openQA queries
|
||||
# * The publishing location is arch specific
|
||||
# * Uses a token for releasing to the publishing project
|
||||
|
||||
|
||||
import cmdln
|
||||
import logging
|
||||
import ToolBase
|
||||
import requests
|
||||
import sys
|
||||
import time
|
||||
import re
|
||||
from lxml import etree as ET
|
||||
from openqa_client.client import OpenQA_Client
|
||||
from osc.core import http_GET, makeurl
|
||||
|
||||
|
||||
class BCIRepoPublisher(ToolBase.ToolBase):
|
||||
def __init__(self):
|
||||
ToolBase.ToolBase.__init__(self)
|
||||
self.logger = logging.getLogger(__name__)
|
||||
self.openqa = OpenQA_Client(server='https://openqa.suse.de')
|
||||
|
||||
def version_of_product(self, project, package, repo, arch):
|
||||
"""Get the build version of the given product build, based on the binary name."""
|
||||
url = makeurl(self.apiurl, ['build', project, repo, arch, package])
|
||||
root = ET.parse(http_GET(url)).getroot()
|
||||
for binary in root.findall('binary'):
|
||||
result = re.match(r'.*-Build(.*)-Media1.report', binary.get('filename'))
|
||||
if result:
|
||||
return result.group(1)
|
||||
|
||||
raise RuntimeError(f"Failed to get version of {project}/{package}")
|
||||
|
||||
def mtime_of_product(self, project, package, repo, arch):
|
||||
"""Get the build time stamp of the given product, based on _buildenv."""
|
||||
url = makeurl(self.apiurl, ['build', project, repo, arch, package])
|
||||
root = ET.parse(http_GET(url)).getroot()
|
||||
mtime = root.xpath('/binarylist/binary[@filename = "_buildenv"]/@mtime')
|
||||
return mtime[0]
|
||||
|
||||
def openqa_jobs_for_product(self, arch, version, build):
|
||||
"""Query openQA for all relevant jobs"""
|
||||
values = {
|
||||
'group': 'BCI repo',
|
||||
'flavor': 'BCI-Repo-Updates',
|
||||
'arch': arch,
|
||||
'version': version,
|
||||
'build': build,
|
||||
'scope': 'current',
|
||||
'latest': '1',
|
||||
}
|
||||
return self.openqa.openqa_request('GET', 'jobs', values)['jobs']
|
||||
|
||||
def is_repo_published(self, project, repo):
|
||||
"""Validates that the given prj/repo is fully published and all builds
|
||||
have succeeded."""
|
||||
url = makeurl(self.apiurl, ['build', project, '_result'],
|
||||
{'view': 'summary', 'repository': repo})
|
||||
root = ET.parse(http_GET(url)).getroot()
|
||||
for result in root.findall('result'):
|
||||
if result.get('dirty', 'false') != 'false':
|
||||
return False
|
||||
if result.get('code') != 'published' or result.get('state') != 'published':
|
||||
return False
|
||||
|
||||
for statuscount in root.findall('statuscount'):
|
||||
if statuscount.get('code') not in ('succeeded', 'disabled', 'excluded'):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def run(self, version, token=None):
|
||||
build_prj = f'SUSE:SLE-{version}:Update:BCI'
|
||||
|
||||
if not self.is_repo_published(build_prj, 'images'):
|
||||
self.logger.info(f'{build_prj}/images not successfully built')
|
||||
return
|
||||
|
||||
# Build the list of packages with metainfo
|
||||
packages = []
|
||||
# As long as it's the same everywhere, hardcoding this list here
|
||||
# is easier and safer than trying to derive it from the package list.
|
||||
for arch in ('aarch64', 'ppc64le', 's390x', 'x86_64'):
|
||||
packages.append({
|
||||
'arch': arch,
|
||||
'name': f'000product:SLE_BCI-ftp-POOL-{arch}',
|
||||
'build_prj': build_prj,
|
||||
'publish_prj': f'SUSE:Products:SLE-BCI:{version}:{arch}'
|
||||
})
|
||||
|
||||
# Fetch the build numbers of built products.
|
||||
# After release, the BuildXXX part vanishes, so the mtime has to be
|
||||
# used instead for comparing built and published binaries.
|
||||
for pkg in packages:
|
||||
pkg['built_version'] = self.version_of_product(pkg['build_prj'], pkg['name'],
|
||||
'images', 'local')
|
||||
pkg['built_mtime'] = self.mtime_of_product(pkg['build_prj'], pkg['name'],
|
||||
'images', 'local')
|
||||
pkg['published_mtime'] = self.mtime_of_product(pkg['publish_prj'], pkg['name'],
|
||||
'images', 'local')
|
||||
|
||||
# Verify that the builds for all archs are in sync
|
||||
built_versions = {pkg['built_version'] for pkg in packages}
|
||||
if len(built_versions) != 1:
|
||||
# This should not be the case if everything is built and idle
|
||||
self.logger.warning(f'Different builds found - not releasing: {packages}')
|
||||
return
|
||||
|
||||
# Compare versions
|
||||
newer_version_available = [int(pkg['built_mtime']) > int(pkg['published_mtime'])
|
||||
for pkg in packages]
|
||||
if not any(newer_version_available):
|
||||
self.logger.info('Current build already published, nothing to do.')
|
||||
return
|
||||
|
||||
# Check openQA results
|
||||
openqa_passed = True
|
||||
for pkg in packages:
|
||||
passed = 0
|
||||
pending = 0
|
||||
failed = 0
|
||||
for job in self.openqa_jobs_for_product(arch=pkg['arch'], version=version,
|
||||
build=pkg['built_version']):
|
||||
if job['result'] in ('passed', 'softfailed'):
|
||||
passed += 1
|
||||
elif job['result'] == 'none':
|
||||
self.logger.info(f'https://openqa.suse.de/tests/{job["id"]} pending')
|
||||
pending += 1
|
||||
else:
|
||||
self.logger.warning(f'https://openqa.suse.de/tests/{job["id"]} failed')
|
||||
failed += 1
|
||||
|
||||
if passed == 0 or pending > 0 or failed > 0:
|
||||
openqa_passed = False
|
||||
|
||||
if not openqa_passed:
|
||||
self.logger.info('No positive result from openQA (yet)')
|
||||
return
|
||||
|
||||
# Trigger publishing
|
||||
if token is None:
|
||||
self.logger.warning('Would publish now, but no token specified')
|
||||
return
|
||||
|
||||
for pkg in packages:
|
||||
self.logger.info(f'Releasing {pkg["name"]}...')
|
||||
params = {
|
||||
'project': pkg['build_prj'], 'package': pkg['name'],
|
||||
'filter_source_repository': 'images',
|
||||
'targetproject': pkg['publish_prj'], 'targetrepository': 'images'
|
||||
}
|
||||
url = makeurl(self.apiurl, ['trigger', 'release'], params)
|
||||
# No bindings for using tokens yet, so do the request manually
|
||||
req = requests.post(url, headers={'Authorization': f'Token {token}'})
|
||||
if req.status_code != 200:
|
||||
raise RuntimeError(f'Releasing failed: {req.text}')
|
||||
|
||||
self.logger.info('Waiting for publishing to finish')
|
||||
for pkg in packages:
|
||||
while not self.is_repo_published(pkg['publish_prj'], 'images'):
|
||||
self.logger.debug(f'Waiting for {pkg["publish_prj"]}')
|
||||
time.sleep(20)
|
||||
|
||||
|
||||
class CommandLineInterface(ToolBase.CommandLineInterface):
|
||||
def __init__(self, *args, **kwargs):
|
||||
ToolBase.CommandLineInterface.__init__(self, args, kwargs)
|
||||
|
||||
def setup_tool(self):
|
||||
tool = BCIRepoPublisher()
|
||||
if self.options.debug:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
elif self.options.verbose:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
return tool
|
||||
|
||||
@cmdln.option('--token', help='The token for publishing. Does a dry run if not given.')
|
||||
def do_run(self, subcmd, opts, project):
|
||||
"""${cmd_name}: run the BCI repo publisher for the specified project,
|
||||
e.g. 15-SP3
|
||||
|
||||
${cmd_usage}
|
||||
${cmd_option_list}
|
||||
"""
|
||||
|
||||
self.tool.run(project, token=opts.token)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli = CommandLineInterface()
|
||||
sys.exit(cli.main())
|
||||
Loading…
x
Reference in New Issue
Block a user