Add skippkg-finder.py to overwrite NON_FTP_PACKAGES for obsoleted and unneeded package
This commit is contained in:
parent
43b07171ad
commit
70acfda962
372
skippkg-finder.py
Executable file
372
skippkg-finder.py
Executable file
@ -0,0 +1,372 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from urllib.error import HTTPError
|
||||
|
||||
import re
|
||||
from lxml import etree as ET
|
||||
from collections import namedtuple
|
||||
|
||||
import osc.conf
|
||||
import osc.core
|
||||
from osc.core import http_GET
|
||||
from osc.core import makeurl
|
||||
from osc import oscerr
|
||||
import osclib
|
||||
from osclib.core import source_file_ensure
|
||||
|
||||
SUPPORTED_ARCHS = ['x86_64', 'i586', 'aarch64', 'ppc64le', 's390x']
|
||||
DEFAULT_REPOSITORY = 'standard'
|
||||
|
||||
META_PACKAGE = '000package-groups'
|
||||
|
||||
class SkippkgFinder(object):
|
||||
def __init__(self, opensuse_project, sle_project, print_only, verbose):
|
||||
self.opensuse_project = opensuse_project
|
||||
self.sle_project = sle_project
|
||||
self.print_only = print_only
|
||||
self.verbose = verbose
|
||||
self.apiurl = osc.conf.config['apiurl']
|
||||
self.debug = osc.conf.config['debug']
|
||||
|
||||
def is_sle_specific(self, package):
|
||||
"""
|
||||
Return True if package is provided for SLE only or a SLE forking.
|
||||
Add new condition here if you do not want package being added to
|
||||
selected_binarylist[].
|
||||
"""
|
||||
pkg = package.lower()
|
||||
prefixes = (
|
||||
'desktop-data',
|
||||
'libyui-bindings',
|
||||
'libyui-doc',
|
||||
'libyui-ncurses',
|
||||
'libyui-qt',
|
||||
'libyui-rest',
|
||||
'lifecycle-data-sle',
|
||||
'kernel-livepatch',
|
||||
'kiwi-template',
|
||||
'mgr-',
|
||||
'migrate',
|
||||
'patterns',
|
||||
'release-notes',
|
||||
'sap',
|
||||
'sca-',
|
||||
'skelcd',
|
||||
'sle-',
|
||||
'sle_',
|
||||
'sle15',
|
||||
'sles15',
|
||||
'spacewalk',
|
||||
'supportutils-plugin',
|
||||
'suse-migration',
|
||||
'susemanager-',
|
||||
'yast2-hana'
|
||||
)
|
||||
suffixes = (
|
||||
'-caasp',
|
||||
'-sle',
|
||||
'bootstrap'
|
||||
)
|
||||
matches = (
|
||||
'gtk-vnc2',
|
||||
'ibus-googlepinyin',
|
||||
'infiniband-diags',
|
||||
'llvm',
|
||||
'lua51-luajit',
|
||||
'lvm2-clvm',
|
||||
'osad',
|
||||
'rhncfg',
|
||||
'python-ibus',
|
||||
'python-pymemcache',
|
||||
'suse-build-key',
|
||||
'suse-hpc',
|
||||
'txt2tags',
|
||||
'zypp-plugin-spacewalk',
|
||||
'zypper-search-packages-plugin'
|
||||
)
|
||||
if pkg.startswith(prefixes) or pkg.endswith(suffixes) or pkg in matches:
|
||||
return True
|
||||
if 'sles' in pkg or\
|
||||
'sled' in pkg or\
|
||||
'sap-' in pkg or\
|
||||
'-sap' in pkg or\
|
||||
'eula' in pkg or\
|
||||
'branding' in pkg:
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_packagelist(self, project, by_project=True):
|
||||
"""
|
||||
Return the list of package's info of a project.
|
||||
If the latest package is from an incident then returns incident
|
||||
package.
|
||||
"""
|
||||
|
||||
pkglist = {}
|
||||
packageinfo = {}
|
||||
query = {'expand': 1}
|
||||
root = ET.parse(http_GET(makeurl(self.apiurl, ['source', project],
|
||||
query=query))).getroot()
|
||||
for i in root.findall('entry'):
|
||||
pkgname = i.get('name')
|
||||
orig_project = i.get('originproject')
|
||||
is_incidentpkg = False
|
||||
# Metapackage should not be selected
|
||||
if pkgname.startswith('000') or\
|
||||
pkgname.startswith('_') or\
|
||||
pkgname.startswith('patchinfo.') or\
|
||||
pkgname.startswith('skelcd-') or\
|
||||
pkgname.startswith('installation-images') or\
|
||||
pkgname.endswith('-mini'):
|
||||
continue
|
||||
# Ugly hack for package has dot in source package name
|
||||
# eg. go1.x incidents as the name would be go1.x.xxx
|
||||
if '.' in pkgname and re.match(r'[0-9]+$', pkgname.split('.')[-1]) and \
|
||||
orig_project.startswith('SUSE:') and orig_project.endswith(':Update'):
|
||||
is_incidentpkg = True
|
||||
if pkgname.startswith('go1') or\
|
||||
pkgname.startswith('bazel0') or\
|
||||
pkgname.startswith('dotnet') or\
|
||||
pkgname.startswith('rust1') or\
|
||||
pkgname.startswith('ruby2'):
|
||||
if not (pkgname.count('.') > 1):
|
||||
is_incidentpkg = False
|
||||
|
||||
# If an incident found then update the package origin info
|
||||
if is_incidentpkg:
|
||||
orig_name = re.sub(r'\.[0-9]+$', '', pkgname)
|
||||
incident_number = int(pkgname.split('.')[-1])
|
||||
if orig_name in pkglist and pkglist[orig_name]['Project'] == orig_project:
|
||||
if re.match(r'[0-9]+$', pkglist[orig_name]['Package'].split('.')[-1]):
|
||||
old_incident_number = int(pkglist[orig_name]['Package'].split('.')[-1])
|
||||
if incident_number > old_incident_number:
|
||||
pkglist[orig_name]['Package'] = pkgname
|
||||
else:
|
||||
pkglist[orig_name]['Package'] = pkgname
|
||||
else:
|
||||
pkglist[pkgname] = {'Project': orig_project, 'Package': pkgname}
|
||||
|
||||
if by_project:
|
||||
for pkg in pkglist.keys():
|
||||
if pkglist[pkg]['Project'].startswith('SUSE:') and self.is_sle_specific(pkg):
|
||||
continue
|
||||
if pkglist[pkg]['Project'] not in packageinfo:
|
||||
packageinfo[pkglist[pkg]['Project']] = []
|
||||
if pkglist[pkg]['Package'] not in packageinfo[pkglist[pkg]['Project']]:
|
||||
packageinfo[pkglist[pkg]['Project']].append(pkglist[pkg]['Package'])
|
||||
return packageinfo
|
||||
|
||||
return pkglist
|
||||
|
||||
def get_project_binary_list(self, project, repository, arch, package_binaries={}):
|
||||
"""
|
||||
Returns binarylist of a project
|
||||
"""
|
||||
|
||||
# Use pool repository for SUSE namespace project.
|
||||
# Because RPMs were injected to pool repository on OBS rather than
|
||||
# standard repository.
|
||||
if project.startswith('SUSE:'):
|
||||
repository = 'pool'
|
||||
|
||||
path = ['build', project, repository, arch]
|
||||
url = makeurl(self.apiurl, path, {'view': 'binaryversions'})
|
||||
root = ET.parse(http_GET(url)).getroot()
|
||||
|
||||
for binary_list in root:
|
||||
package = binary_list.get('package')
|
||||
package = package.split(':', 1)[0]
|
||||
index = project + "_" + package
|
||||
|
||||
if index not in package_binaries:
|
||||
package_binaries[index] = []
|
||||
for binary in binary_list:
|
||||
filename = binary.get('name')
|
||||
result = re.match(osclib.core.RPM_REGEX, filename)
|
||||
if not result:
|
||||
continue
|
||||
|
||||
if result.group('arch') == 'src' or result.group('arch') == 'nosrc':
|
||||
continue
|
||||
if result.group('name').endswith('-debuginfo') or result.group('name').endswith('-debuginfo-32bit'):
|
||||
continue
|
||||
if result.group('name').endswith('-debugsource'):
|
||||
continue
|
||||
|
||||
if result.group('name') not in package_binaries[index]:
|
||||
package_binaries[index].append(result.group('name'))
|
||||
|
||||
return package_binaries
|
||||
|
||||
def exception_package(self, package):
|
||||
"""
|
||||
Do not skip the package if matches the condition.
|
||||
package parameter is source package name.
|
||||
"""
|
||||
|
||||
if '-bootstrap' in package or\
|
||||
'Tumbleweed' in package or\
|
||||
'metis' in package:
|
||||
return True
|
||||
# These packages must have a good reason not to be single-speced
|
||||
# from one source.
|
||||
if package.startswith('python2-') or\
|
||||
package.startswith('python3'):
|
||||
return True
|
||||
return False
|
||||
|
||||
def exception_binary(self, package):
|
||||
"""
|
||||
Do not skip the binary if matches the condition
|
||||
package parameter is RPM filename.
|
||||
"""
|
||||
|
||||
if package == 'openSUSE-release' or\
|
||||
package == 'openSUSE-release-ftp' or\
|
||||
package == 'openSUSE-Addon-NonOss-release':
|
||||
return True
|
||||
return False
|
||||
|
||||
def crawl(self):
|
||||
"""Main method"""
|
||||
|
||||
leap_pkglist = self.get_packagelist(self.opensuse_project)
|
||||
sle_pkglist = self.get_packagelist(self.sle_project, by_project=False)
|
||||
# The selected_binarylist[] includes the latest sourcepackage list
|
||||
# binary RPMs from the latest sources need to be presented in ftp eventually
|
||||
selected_binarylist = []
|
||||
# Any existed binary RPMs from any SPx/Leap/Backports
|
||||
fullbinarylist = []
|
||||
# package_binaries[] is a pre-formated binarylist per each package
|
||||
# access to the conotent uses package_binaries['SUSE:SLE-15:Update_libcdio.12032']
|
||||
package_binaries = {}
|
||||
|
||||
# Inject binarylist to a list per package name no matter what archtectures was
|
||||
for arch in SUPPORTED_ARCHS:
|
||||
for prj in leap_pkglist.keys():
|
||||
package_binaries = self.get_project_binary_list(prj, DEFAULT_REPOSITORY, arch, package_binaries)
|
||||
|
||||
for pkg in package_binaries.keys():
|
||||
if not self.exception_package(pkg):
|
||||
fullbinarylist += package_binaries[pkg]
|
||||
|
||||
for prj in leap_pkglist.keys():
|
||||
for pkg in leap_pkglist[prj]:
|
||||
cands = [prj + "_" + pkg]
|
||||
# Handling for SLE forks, or package has different multibuild bits
|
||||
# enablility between SLE and openSUSE
|
||||
if prj.startswith('openSUSE:') and pkg in sle_pkglist and\
|
||||
not self.is_sle_specific(pkg):
|
||||
cands.append(sle_pkglist[pkg]['Project'] + "_" + sle_pkglist[pkg]['Package'])
|
||||
logging.debug(cands)
|
||||
for index in cands:
|
||||
if index in package_binaries:
|
||||
selected_binarylist += package_binaries[index]
|
||||
else:
|
||||
logging.info("Can not find binary of %s" % index)
|
||||
|
||||
# Some packages has been obsoleted by new updated package, however
|
||||
# there are application still depend on old library when it builds
|
||||
# eg. SUSE:SLE-15-SP3:GA has qpdf/libqpdf28 but cups-filter was build
|
||||
# in/when SLE15 SP2 which requiring qpdf/libqpdf6, therefore old
|
||||
# qpdf/libqpdf6 from SLE15 SP2 should not to be missed.
|
||||
extra_packagelist = [
|
||||
# gnome-software requirement
|
||||
'SUSE:SLE-15-SP2:Update_libxmlb.15999',
|
||||
# cups-filter requirement
|
||||
'SUSE:SLE-15-SP2:GA_qpdf',
|
||||
# libcdio_paranoia2 requirement
|
||||
'SUSE:SLE-15:Update_libcdio.12032',
|
||||
# libstoken1 requirement
|
||||
'SUSE:SLE-15:Update_libnettle.19992',
|
||||
# python2-Pillow requirement
|
||||
'SUSE:SLE-15:Update_libwebp.19719',
|
||||
# amarok requirement
|
||||
'SUSE:SLE-15:Update_mariadb.20531',
|
||||
# bogofilter requirement
|
||||
'SUSE:SLE-15:GA_gsl',
|
||||
# gnome-builder requirement
|
||||
'SUSE:SLE-15-SP2:GA_vala',
|
||||
# hfst-ospell requirement
|
||||
'SUSE:SLE-15:Update_icu.14528'
|
||||
]
|
||||
for pkg in extra_packagelist:
|
||||
selected_binarylist += package_binaries[pkg]
|
||||
|
||||
# Preparing a packagelist for the skipping candidate
|
||||
obsoleted = []
|
||||
for pkg in fullbinarylist:
|
||||
if pkg not in selected_binarylist and pkg not in obsoleted:
|
||||
if not self.exception_binary(pkg):
|
||||
obsoleted.append(pkg)
|
||||
|
||||
# Post processing of obsoleted packagelist
|
||||
tmp_obsoleted = obsoleted.copy()
|
||||
for pkg in tmp_obsoleted:
|
||||
# Respect to single-speced python package, when a python2 RPM is
|
||||
# considered then a python3 flavor should also be selected to be
|
||||
# skipped, if not, don't add it.
|
||||
if pkg.startswith('python2-') and re.sub(r'^python2', 'python3', pkg) not in obsoleted:
|
||||
obsoleted.remove(pkg)
|
||||
# Main RPM must to be skipped if -32 bit RPM or -64bit RPM is
|
||||
# considered.
|
||||
if pkg.endswith('-32bit') or pkg.endswith('-64bit'):
|
||||
main_filename = re.sub('-[36][24]bit', '', pkg)
|
||||
if main_filename not in obsoleted:
|
||||
obsoleted.remove(pkg)
|
||||
|
||||
skip_list = ET.Element('group', {'name': 'NON_FTP_PACKAGES'})
|
||||
ET.SubElement(skip_list, 'conditional', {'name': 'drop_from_ftp'})
|
||||
packagelist = ET.SubElement(skip_list, 'packagelist', {'relationship': 'requires'})
|
||||
for pkg in sorted(obsoleted):
|
||||
if not self.print_only and self.verbose:
|
||||
print(pkg)
|
||||
attr = {'name': pkg}
|
||||
ET.SubElement(packagelist, 'package', attr)
|
||||
if not self.print_only:
|
||||
source_file_ensure(self.apiurl, self.opensuse_project, META_PACKAGE, 'NON_FTP_PACKAGES.group',
|
||||
ET.tostring(skip_list, pretty_print=True, encoding='unicode'),
|
||||
'Update the skip list')
|
||||
else:
|
||||
print(ET.tostring(skip_list, pretty_print=True,
|
||||
encoding='unicode'))
|
||||
|
||||
|
||||
def main(args):
|
||||
osc.conf.get_config(override_apiurl=args.apiurl)
|
||||
osc.conf.config['debug'] = args.debug
|
||||
|
||||
if args.opensuse_project is None or args.sle_project is None:
|
||||
print("Please pass --opensuse-project and --sle-project argument. See usage with --help.")
|
||||
quit()
|
||||
|
||||
uc = SkippkgFinder(args.opensuse_project, args.sle_project, args.print_only, args.verbose)
|
||||
uc.crawl()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
description = 'Overwrites NON_FTP_PACKAGES.group according to the latest sources. '\
|
||||
'This tool only works for Leap after CtLG implemented.'
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL')
|
||||
parser.add_argument('-d', '--debug', action='store_true',
|
||||
help='print info useful for debuging')
|
||||
parser.add_argument('-o', '--opensuse-project', dest='opensuse_project', metavar='OPENSUSE_PROJECT',
|
||||
help='openSUSE project on buildservice')
|
||||
parser.add_argument('-s', '--sle-project', dest='sle_project', metavar='SLE_PROJECT',
|
||||
help='SLE project on buildservice')
|
||||
parser.add_argument('-p', '--print-only', action='store_true',
|
||||
help='show the result instead of the uploading')
|
||||
parser.add_argument('-v', '--verbose', action='store_true',
|
||||
help='show the diff')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG if args.debug
|
||||
else logging.INFO)
|
||||
|
||||
sys.exit(main(args))
|
||||
Loading…
x
Reference in New Issue
Block a user