pkglistgen: migrate bash scripts to python.
This commit is contained in:
Jimmy Berry
parent
facb6fecf6
commit
5e9392f4a0
2
Makefile
2
Makefile
@ -9,7 +9,7 @@ pkgdata_BINS = \
|
||||
leaper.py \
|
||||
manager_42.py \
|
||||
metrics.py \
|
||||
pkglistgen.sh \
|
||||
pkglistgen.py \
|
||||
repo_checker.py \
|
||||
suppkg_rebuild.py \
|
||||
totest-manager.py \
|
||||
|
||||
5
dist/package/openSUSE-release-tools.spec
vendored
5
dist/package/openSUSE-release-tools.spec
vendored
@ -483,11 +483,6 @@ fi
|
||||
%files pkglistgen
|
||||
%defattr(-,root,root,-)
|
||||
%{_bindir}/osrt-pkglistgen
|
||||
%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0
|
||||
%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0-all
|
||||
%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Rings
|
||||
%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Staging
|
||||
%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh
|
||||
%{_unitdir}/osrt-pkglistgen@.service
|
||||
%{_unitdir}/osrt-pkglistgen@.timer
|
||||
|
||||
|
||||
@ -69,6 +69,11 @@ DEFAULT = {
|
||||
# check_source.py
|
||||
# review-team optionally added by leaper.py.
|
||||
'repo-checker': 'repo-checker',
|
||||
'pkglistgen-archs': 'x86_64',
|
||||
'pkglistgen-archs-ports': 'aarch64',
|
||||
'pkglistgen-locals-from': 'openSUSE-product',
|
||||
'pkglistgen-include-suggested': '1',
|
||||
'pkglistgen-delete-kiwis': 'openSUSE-ftp-ftp-x86_64.kiwi openSUSE-cd-mini-x86_64.kiwi',
|
||||
},
|
||||
r'SUSE:(?P<project>SLE-15.*$)': {
|
||||
'staging': 'SUSE:%(project)s:Staging',
|
||||
|
||||
241
pkglistgen.py
241
pkglistgen.py
@ -24,13 +24,22 @@
|
||||
# TODO: solve all devel packages to include
|
||||
from __future__ import print_function
|
||||
|
||||
import copy
|
||||
from lxml import etree as ET
|
||||
from collections import namedtuple
|
||||
import sys
|
||||
import cmdln
|
||||
import logging
|
||||
import urllib2
|
||||
import osc.core
|
||||
from osc.core import checkout_package
|
||||
from osc.core import http_GET
|
||||
from osc.core import makeurl
|
||||
from osc.core import Package
|
||||
from osc.core import show_results_meta
|
||||
from osc.core import undelete_package
|
||||
from osc import conf
|
||||
from osclib.conf import Config
|
||||
from osclib.stagingapi import StagingAPI
|
||||
import glob
|
||||
import solv
|
||||
from pprint import pprint, pformat
|
||||
@ -44,6 +53,7 @@ from StringIO import StringIO
|
||||
import gzip
|
||||
import tempfile
|
||||
import random
|
||||
import shutil
|
||||
import string
|
||||
|
||||
import ToolBase
|
||||
@ -55,6 +65,7 @@ logger = logging.getLogger()
|
||||
|
||||
ARCHITECTURES = ['x86_64', 'ppc64le', 's390x', 'aarch64']
|
||||
DEFAULT_REPOS = ("openSUSE:Factory/standard")
|
||||
PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product'
|
||||
|
||||
class Group(object):
|
||||
|
||||
@ -627,6 +638,7 @@ class PkgListGen(ToolBase.ToolBase):
|
||||
fh.write(" \n")
|
||||
|
||||
class CommandLineInterface(ToolBase.CommandLineInterface):
|
||||
SCOPES = ['all', 'target', 'rings', 'staging', 'ports']
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
ToolBase.CommandLineInterface.__init__(self, args, kwargs)
|
||||
@ -929,6 +941,233 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
|
||||
self.tool._collect_unsorted_packages(modules)
|
||||
self.tool._write_all_groups()
|
||||
|
||||
@cmdln.option('-f', '--force', action='store_true', help='continue even if build is in progress')
|
||||
@cmdln.option('-p', '--project', help='target project')
|
||||
@cmdln.option('-s', '--scope', default='all', help='scope on which to operate ({})'.format(', '.join(SCOPES)))
|
||||
def do_update_and_solve(self, subcmd, opts):
|
||||
"""${cmd_name}: update and solve for given scope
|
||||
|
||||
${cmd_usage}
|
||||
${cmd_option_list}
|
||||
"""
|
||||
|
||||
if not opts.project:
|
||||
raise ValueError('project is required')
|
||||
if opts.scope not in self.SCOPES:
|
||||
raise ValueError('scope must be one of: {}'.format(', '.join(self.SCOPES)))
|
||||
|
||||
if opts.scope == 'all':
|
||||
for scope in self.SCOPES[1:]:
|
||||
opts.scope = scope
|
||||
self.do_update_and_solve(subcmd, copy.deepcopy(opts))
|
||||
return
|
||||
|
||||
# Store target project as opts.project will contain subprojects.
|
||||
target_project = opts.project
|
||||
|
||||
config = Config(target_project)
|
||||
apiurl = conf.config['apiurl']
|
||||
api = StagingAPI(apiurl, target_project)
|
||||
config.apply_remote(api)
|
||||
|
||||
target_config = conf.config[target_project]
|
||||
archs_key = 'pkglistgen-archs' if opts.scope != 'ports' else 'pkglistgen-archs-ports'
|
||||
if archs_key in target_config:
|
||||
self.options.architectures = target_config.get(archs_key).split(' ')
|
||||
main_repo = target_config['main-repo']
|
||||
|
||||
if opts.scope == 'target':
|
||||
self.options.repos = ['/'.join([target_project, main_repo])]
|
||||
self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts)
|
||||
return
|
||||
elif opts.scope == 'ports':
|
||||
# TODO Continue supporting #1297, but should be abstracted.
|
||||
main_repo = 'ports'
|
||||
opts.project += ':Ports'
|
||||
self.options.repos = ['/'.join([opts.project, main_repo])]
|
||||
self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts)
|
||||
return
|
||||
elif opts.scope == 'rings':
|
||||
opts.project = api.rings[1]
|
||||
self.options.repos = [
|
||||
'/'.join([api.rings[1], main_repo]),
|
||||
'/'.join([api.rings[0], main_repo]),
|
||||
]
|
||||
self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts)
|
||||
|
||||
opts.project = api.rings[2]
|
||||
self.options.repos.insert(0, '/'.join([api.rings[2], main_repo]))
|
||||
self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts, skip_release=True)
|
||||
return
|
||||
elif opts.scope == 'staging':
|
||||
letters = api.get_staging_projects_short()
|
||||
for letter in letters:
|
||||
opts.project = api.prj_from_short(letter)
|
||||
self.options.repos = ['/'.join([opts.project, main_repo])]
|
||||
|
||||
if not api.is_staging_bootstrapped(opts.project):
|
||||
self.options.repos.append('/'.join([opts.project, 'bootstrap_copy']))
|
||||
|
||||
# DVD project first since it depends on main.
|
||||
if api.rings:
|
||||
opts_dvd = copy.deepcopy(opts)
|
||||
opts.project += ':DVD'
|
||||
self.options.repos.insert(0, '/'.join([opts.project, main_repo]))
|
||||
self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts_dvd, skip_release=True)
|
||||
|
||||
self.update_and_solve_target(apiurl, target_project, target_config, main_repo, opts)
|
||||
return
|
||||
|
||||
def update_and_solve_target(self, apiurl, target_project, target_config, main_repo, opts,
|
||||
skip_release=False):
|
||||
group = target_config.get('pkglistgen-group', '000package-groups')
|
||||
product = target_config.get('pkglistgen-product', '000product')
|
||||
release = target_config.get('pkglistgen-release', '000release-packages')
|
||||
|
||||
url = makeurl(apiurl, ['source', opts.project])
|
||||
packages = ET.parse(http_GET(url)).getroot()
|
||||
if packages.find('entry[@name="{}"]'.format(product)) is None:
|
||||
undelete_package(apiurl, opts.project, product, 'revive')
|
||||
# TODO disable build.
|
||||
print('{} undeleted, skip dvd until next cycle'.format(product))
|
||||
return
|
||||
elif not opts.force:
|
||||
root = ET.fromstringlist(show_results_meta(apiurl, opts.project, product,
|
||||
repository=[main_repo], multibuild=True))
|
||||
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
|
||||
print('{}/{} build in progress'.format(opts.project, product))
|
||||
return
|
||||
|
||||
checkout_list = [group, product]
|
||||
if not skip_release:
|
||||
checkout_list.append(release)
|
||||
|
||||
if packages.find('entry[@name="{}"]'.format(release)) is None:
|
||||
undelete_package(apiurl, opts.project, product, 'revive')
|
||||
print('{} undeleted, skip dvd until next cycle'.format(release))
|
||||
return
|
||||
|
||||
# Cache dir specific to hostname and project.
|
||||
host = urlparse.urlparse(apiurl).hostname
|
||||
cache_dir = os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
|
||||
cache_dir = os.path.join(cache_dir, 'opensuse-packagelists', host, opts.project)
|
||||
|
||||
if os.path.exists(cache_dir):
|
||||
shutil.rmtree(cache_dir)
|
||||
os.makedirs(cache_dir)
|
||||
|
||||
group_dir = os.path.join(cache_dir, group)
|
||||
product_dir = os.path.join(cache_dir, product)
|
||||
release_dir = os.path.join(cache_dir, release)
|
||||
|
||||
for package in checkout_list:
|
||||
checkout_package(apiurl, opts.project, package, expand_link=True, prj_dir=cache_dir)
|
||||
|
||||
if not skip_release:
|
||||
self.unlink_all_except(release_dir)
|
||||
self.unlink_all_except(product_dir)
|
||||
self.copy_directory_contents(group_dir, product_dir,
|
||||
['supportstatus.txt', 'groups.yml', 'package-groups.changes'])
|
||||
self.change_extension(product_dir, '.spec.in', '.spec')
|
||||
|
||||
self.options.output_dir = product_dir
|
||||
self.postoptparse()
|
||||
|
||||
self.do_update('update', opts)
|
||||
|
||||
opts.ignore_recommended = bool(target_config.get('pkglistgen-include-recommended'))
|
||||
opts.include_suggested = bool(target_config.get('pkglistgen-include-suggested'))
|
||||
opts.locales_from = target_config.get('pkglistgen-locals-from')
|
||||
self.do_solve('solve', opts)
|
||||
|
||||
delete_products = target_config.get('pkglistgen-delete-products', '').split(' ')
|
||||
self.unlink_list(product_dir, delete_products)
|
||||
|
||||
for product_file in glob.glob(os.path.join(product_dir, '*.product')):
|
||||
print(subprocess.check_output(
|
||||
[PRODUCT_SERVICE, product_file, product_dir, opts.project]))
|
||||
|
||||
delete_kiwis = target_config.get('pkglistgen-delete-kiwis', '').split(' ')
|
||||
self.unlink_list(product_dir, delete_kiwis)
|
||||
|
||||
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
|
||||
if skip_release:
|
||||
self.unlink_list(None, spec_files)
|
||||
else:
|
||||
self.move_list(spec_files, release_dir)
|
||||
|
||||
self.multibuild_from_glob(product_dir, '*.kiwi')
|
||||
self.build_stub(product_dir, 'kiwi')
|
||||
self.commit_package(product_dir)
|
||||
|
||||
if not skip_release:
|
||||
self.multibuild_from_glob(release_dir, '*.spec')
|
||||
self.build_stub(release_dir, 'spec')
|
||||
self.commit_package(release_dir)
|
||||
|
||||
def move_list(self, file_list, destination):
|
||||
for name in file_list:
|
||||
os.rename(name, os.path.join(destination, os.path.basename(name)))
|
||||
|
||||
def unlink_list(self, path, names):
|
||||
for name in names:
|
||||
if path is None:
|
||||
name_path = name
|
||||
else:
|
||||
name_path = os.path.join(path, name)
|
||||
|
||||
if os.path.isfile(name_path):
|
||||
os.unlink(name_path)
|
||||
|
||||
def unlink_all_except(self, path, ignore_list=['_service'], ignore_hidden=True):
|
||||
for name in os.listdir(path):
|
||||
if name in ignore_list or (ignore_hidden and name.startswith('.')):
|
||||
continue
|
||||
|
||||
name_path = os.path.join(path, name)
|
||||
if os.path.isfile(name_path):
|
||||
os.unlink(name_path)
|
||||
|
||||
def copy_directory_contents(self, source, destination, ignore_list=[]):
|
||||
for name in os.listdir(source):
|
||||
name_path = os.path.join(source, name)
|
||||
if name in ignore_list or not os.path.isfile(name_path):
|
||||
continue
|
||||
|
||||
shutil.copy(name_path, os.path.join(destination, name))
|
||||
|
||||
def change_extension(self, path, original, final):
|
||||
for name in glob.glob(os.path.join(path, '*{}'.format(original))):
|
||||
# Assumes the extension is only found at the end.
|
||||
os.rename(name, name.replace(original, final))
|
||||
|
||||
def multibuild_from_glob(self, destination, pathname):
|
||||
root = ET.Element('multibuild')
|
||||
for name in glob.glob(os.path.join(destination, pathname)):
|
||||
package = ET.SubElement(root, 'package')
|
||||
package.text = os.path.splitext(os.path.basename(name))[0]
|
||||
|
||||
with open(os.path.join(destination, '_multibuild'), 'w+b') as f:
|
||||
f.write(ET.tostring(root, pretty_print=True))
|
||||
|
||||
def build_stub(self, destination, extension):
|
||||
f = file(os.path.join(destination, '.'.join(['stub', extension])), 'w+')
|
||||
f.write('# prevent building single {} files twice\n'.format(extension))
|
||||
f.write('Name: stub\n')
|
||||
f.write('Version: 0.0\n')
|
||||
f.close()
|
||||
|
||||
def commit_package(self, path):
|
||||
package = Package(path)
|
||||
if self.options.dry:
|
||||
for i in package.get_diff():
|
||||
print(''.join(i))
|
||||
else:
|
||||
# No proper API function to perform the same operation.
|
||||
print(subprocess.check_output(
|
||||
' '.join(['cd', path, '&&', 'osc', 'addremove']), shell=True))
|
||||
package.commit(msg='Automatic update')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app = CommandLineInterface()
|
||||
|
||||
146
pkglistgen.sh
146
pkglistgen.sh
@ -1,146 +0,0 @@
|
||||
#!/bin/bash
|
||||
# FIXME: 000package-groups is not frozen, osc up doesn't do anything
|
||||
# when updated in underlying project
|
||||
|
||||
set -e
|
||||
shopt -s nullglob
|
||||
|
||||
self=$(readlink -e $(type -p "$0"))
|
||||
|
||||
: ${project:=openSUSE:Factory}
|
||||
: ${api:=api.opensuse.org}
|
||||
: ${repos:=$project/standard}
|
||||
: ${productrepo:=standard}
|
||||
: ${arch:=x86_64}
|
||||
|
||||
groups="000package-groups"
|
||||
product="000product"
|
||||
releases="000release-packages"
|
||||
|
||||
cachedir=${XDG_CACHE_HOME:-~/.cache}/opensuse-packagelists/$api/$project
|
||||
todo=("$product" "$groups")
|
||||
solveargs=()
|
||||
|
||||
if [ -n "$IGNORE_RECOMMENDED" ]; then
|
||||
solveargs+=('--ignore-recommended')
|
||||
fi
|
||||
if [ -n "$INCLUDE_SUGGESTED" ]; then
|
||||
solveargs+=('--include-suggested')
|
||||
fi
|
||||
if [ -n "$LOCALES_FROM" ]; then
|
||||
solveargs+=('--locales-from', "$LOCALES_FROM")
|
||||
fi
|
||||
|
||||
_osc=`type -p osc`
|
||||
osc()
|
||||
{
|
||||
"$_osc" -A "https://$api" "$@"
|
||||
}
|
||||
|
||||
checkin() {
|
||||
if [ -n "$dryrun" ]; then
|
||||
osc diff
|
||||
else
|
||||
osc addremove
|
||||
osc ci -m "Automatic update"
|
||||
fi
|
||||
}
|
||||
|
||||
if ! osc api "/source/$project/" | grep -q "$product" ; then
|
||||
osc undelete -m revive "$project/$product"
|
||||
# FIXME: build disable it
|
||||
echo "$product undeleted, skip dvd until next cycle"
|
||||
exit 0
|
||||
elif [ -z "$FORCE" ]; then
|
||||
bs_status=`osc api "/build/$project/_result?package=$product&repository=$productrepo"`
|
||||
if echo "${bs_status}" | grep -q 'building\|dirty'; then
|
||||
echo "$project build in progress, skipping."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
mkdir -p "$cachedir"
|
||||
cd "$cachedir"
|
||||
|
||||
if [ -z "$skip_releases" ]; then
|
||||
todo+=("$releases")
|
||||
if ! osc api "/source/$project/" | grep -q "$releases" ; then
|
||||
osc undelete -m revive "$project/$releases"
|
||||
echo "$releases undeleted, skip dvd until next cycle"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
# update package checkouts
|
||||
for i in "${todo[@]}"; do
|
||||
if [ ! -e "$i" ]; then
|
||||
osc co -c "$project/$i"
|
||||
fi
|
||||
pushd "$i"
|
||||
if ! osc status; then
|
||||
# merge conflict etc, try to check out new
|
||||
popd
|
||||
rm -rf "$i"
|
||||
osc co -c "$project/$i"
|
||||
else
|
||||
osc up
|
||||
popd
|
||||
fi
|
||||
done
|
||||
|
||||
[ -z "$releases" ] || rm -f "$cachedir/$releases"/*
|
||||
cd "$cachedir/$product"
|
||||
rm -f -- *
|
||||
cp .osc/_service .
|
||||
cp "$cachedir/$groups"/* .
|
||||
rm -f supportstatus.txt groups.yml package-groups.changes
|
||||
for i in *.spec.in; do
|
||||
mv -v $i "${i%.in}"
|
||||
done
|
||||
if ! ${self%.sh}.py -i "$cachedir/$groups" -r $repos -o . -a $arch update; then
|
||||
echo "no change in packages"
|
||||
fi
|
||||
${self%.sh}.py -i "$cachedir/$groups" -r $repos -o . -a $arch solve "${solveargs[@]}"
|
||||
for i in $delete_products; do
|
||||
rm -vf -- "$i"
|
||||
done
|
||||
for i in *.product; do
|
||||
/usr/lib/obs/service/create_single_product $PWD/$i $PWD $(cat .osc/_project)
|
||||
done
|
||||
for i in $delete_kiwis; do
|
||||
rm -vf -- "$i"
|
||||
done
|
||||
if [ -z "$skip_releases" ]; then
|
||||
mv -v *.spec "$cachedir/$releases"
|
||||
else
|
||||
rm -vf *.spec
|
||||
fi
|
||||
echo '<multibuild>' > _multibuild
|
||||
for file in *.kiwi; do
|
||||
container="${file##*/}"
|
||||
container="${container%.kiwi}"
|
||||
echo " <package>${container}</package>" >> _multibuild
|
||||
done
|
||||
echo '</multibuild>' >> _multibuild
|
||||
cat << EOF > stub.kiwi
|
||||
# prevent building single kiwi files twice
|
||||
Name: stub
|
||||
Version: 0.0
|
||||
EOF
|
||||
checkin
|
||||
|
||||
if [ -z "$skip_releases" ]; then
|
||||
cd "$cachedir/$releases"
|
||||
echo '<multibuild>' > _multibuild
|
||||
for file in *.spec; do
|
||||
container="${file##*/}"
|
||||
container="${container%.spec}"
|
||||
echo " <package>${container}</package>" >> _multibuild
|
||||
done
|
||||
echo '</multibuild>' >> _multibuild
|
||||
cat <<-EOF > stub.spec
|
||||
# prevent building single spec files twice
|
||||
Name: stub
|
||||
Version: 0.0
|
||||
EOF
|
||||
checkin
|
||||
fi
|
||||
@ -1,3 +0,0 @@
|
||||
#!/bin/bash
|
||||
export project=openSUSE:Leap:15.0
|
||||
osrt-pkglistgen
|
||||
@ -1,6 +0,0 @@
|
||||
#!/bin/bash
|
||||
export LOCALES_FROM=openSUSE-product
|
||||
export INCLUDE_SUGGESTED=1
|
||||
osrt-pkglistgen-openSUSE:Leap:15.0
|
||||
osrt-pkglistgen-openSUSE:Leap:15.0:Rings
|
||||
osrt-pkglistgen-openSUSE:Leap:15.0:Staging
|
||||
@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
self=$(readlink $(type -p "$0"))
|
||||
export project=openSUSE:Leap:15.0:Ports
|
||||
export repos=$project/ports
|
||||
export arch=aarch64
|
||||
export productrepo=ports
|
||||
osrt-pkglistgen
|
||||
@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
main=openSUSE:Leap:15.0
|
||||
export delete_kiwis="openSUSE-ftp-ftp-x86_64.kiwi openSUSE-cd-mini-x86_64.kiwi"
|
||||
|
||||
export project=$main:Rings:1-MinimalX
|
||||
export repos=$project/standard,$main:Rings:0-Bootstrap/standard
|
||||
osrt-pkglistgen
|
||||
|
||||
export project=$main:Rings:2-TestDVD
|
||||
export repos=$main:Rings:2-TestDVD/standard,$repos
|
||||
export skip_releases=1
|
||||
osrt-pkglistgen
|
||||
@ -1,20 +0,0 @@
|
||||
#!/bin/bash
|
||||
self=$(readlink $(type -p "$0"))
|
||||
main=openSUSE:Leap:15.0
|
||||
: ${letter:=A B C D E}
|
||||
|
||||
export delete_kiwis="openSUSE-ftp-ftp-x86_64.kiwi openSUSE-cd-mini-x86_64.kiwi"
|
||||
for l in $letter; do
|
||||
export project=$main:Staging:$l
|
||||
echo "checking $project..."
|
||||
export repos=$project/standard
|
||||
if [ "$l" != A -a "$l" != B ]; then
|
||||
repos="$repos,$project/bootstrap_copy"
|
||||
fi
|
||||
|
||||
# DVD project first as it depends on the project below, so might look
|
||||
# busy if we update the other one first
|
||||
project=$project:DVD repos=$project/standard,$repos skip_releases=1 osrt-pkglistgen
|
||||
|
||||
osrt-pkglistgen
|
||||
done
|
||||
@ -3,7 +3,7 @@ Description=openSUSE Release Tools: generate package lists for %i
|
||||
|
||||
[Service]
|
||||
User=osrt-repo-checker
|
||||
ExecStart=/usr/bin/osrt-pkglistgen-%i-all
|
||||
ExecStart=/usr/bin/osrt-pkglistgen -p "%i"
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user