#!/usr/bin/python3 import argparse import logging import sys from urllib.error import HTTPError import re from lxml import etree as ET from collections import namedtuple import osc.conf import osc.core from osc.core import http_GET from osc.core import makeurl from osc import oscerr import osclib from osclib.core import source_file_ensure from osclib.conf import Config SUPPORTED_ARCHS = ['x86_64', 'i586', 'aarch64', 'ppc64le', 's390x'] DEFAULT_REPOSITORY = 'standard' META_PACKAGE = '000package-groups' class SkippkgFinder(object): def __init__(self, opensuse_project, sle_project, alternative_project, print_only, verbose): self.upload_project = opensuse_project self.opensuse_project = opensuse_project if alternative_project: self.opensuse_project = alternative_project self.sle_project = sle_project self.print_only = print_only self.verbose = verbose self.apiurl = osc.conf.config['apiurl'] self.debug = osc.conf.config['debug'] config = Config.get(self.apiurl, opensuse_project) # binary rpms of packages from `skippkg-finder-skiplist-ignores` # be found in the `package_binaries` thus format must to be like # SUSE:SLE-15:Update_libcdio.12032, PROJECT-NAME_PACKAGE-NAME self.skiplist_ignored = set(config.get('skippkg-finder-skiplist-ignores', '').split(' ')) # supplement RPMs for skipping from the ftp-tree self.skiplist_supplement_regex = set(config.get('skippkg-finder-skiplist-supplement-regex', '').split(' ')) # drops off RPM from a list of the supplement RPMs due to regex self.skiplist_supplement_ignores = set(config.get('skippkg-finder-skiplist-supplement-ignores', '').split(' ')) def is_sle_specific(self, package): """ Return True if package is provided for SLE only or a SLE forking. Add new condition here if you do not want package being added to selected_binarylist[]. """ pkg = package.lower() prefixes = ( 'desktop-data', 'libyui-bindings', 'libyui-doc', 'libyui-ncurses', 'libyui-qt', 'libyui-rest', 'lifecycle-data-sle', 'kernel-livepatch', 'kiwi-template', 'mgr-', 'migrate', 'patterns', 'release-notes', 'sap', 'sca-', 'skelcd', 'sle-', 'sle_', 'sle15', 'sles15', 'spacewalk', 'supportutils-plugin', 'suse-migration', 'susemanager-', 'yast2-hana' ) suffixes = ( '-caasp', '-sle', 'bootstrap' ) matches = ( 'gtk-vnc2', 'ibus-googlepinyin', 'infiniband-diags', 'llvm', 'lua51-luajit', 'lvm2-clvm', 'osad', 'rhncfg', 'python-ibus', 'python-pymemcache', 'suse-build-key', 'suse-hpc', 'txt2tags', 'zypp-plugin-spacewalk', 'zypper-search-packages-plugin' ) if pkg.startswith(prefixes) or pkg.endswith(suffixes) or pkg in matches: return True if 'sles' in pkg or\ 'sled' in pkg or\ 'sap-' in pkg or\ '-sap' in pkg or\ 'eula' in pkg or\ 'branding' in pkg: return True return False def get_packagelist(self, project, by_project=True): """ Return the list of package's info of a project. If the latest package is from an incident then returns incident package. """ pkglist = {} packageinfo = {} query = {'expand': 1} root = ET.parse(http_GET(makeurl(self.apiurl, ['source', project], query=query))).getroot() for i in root.findall('entry'): pkgname = i.get('name') orig_project = i.get('originproject') is_incidentpkg = False # Metapackage should not be selected if pkgname.startswith('000') or\ pkgname.startswith('_') or\ pkgname.startswith('patchinfo.') or\ pkgname.startswith('skelcd-') or\ pkgname.startswith('installation-images') or\ pkgname.startswith('Leap-release') or\ pkgname.endswith('-mini') or\ '-mini.' in pkgname: continue # Ugly hack for package has dot in source package name # eg. go1.x incidents as the name would be go1.x.xxx if '.' in pkgname and re.match(r'[0-9]+$', pkgname.split('.')[-1]) and \ orig_project.startswith('SUSE:') and orig_project.endswith(':Update'): is_incidentpkg = True if pkgname.startswith('go1') or\ pkgname.startswith('bazel0') or\ pkgname.startswith('dotnet') or\ pkgname.startswith('rust1') or\ pkgname.startswith('ruby2'): if not (pkgname.count('.') > 1): is_incidentpkg = False # If an incident found then update the package origin info if is_incidentpkg: orig_name = re.sub(r'\.[0-9]+$', '', pkgname) incident_number = int(pkgname.split('.')[-1]) if orig_name in pkglist and pkglist[orig_name]['Project'] == orig_project: if re.match(r'[0-9]+$', pkglist[orig_name]['Package'].split('.')[-1]): old_incident_number = int(pkglist[orig_name]['Package'].split('.')[-1]) if incident_number > old_incident_number: pkglist[orig_name]['Package'] = pkgname else: pkglist[orig_name]['Package'] = pkgname else: pkglist[pkgname] = {'Project': orig_project, 'Package': pkgname} if by_project: for pkg in pkglist.keys(): if pkglist[pkg]['Project'].startswith('SUSE:') and self.is_sle_specific(pkg): continue if pkglist[pkg]['Project'] not in packageinfo: packageinfo[pkglist[pkg]['Project']] = [] if pkglist[pkg]['Package'] not in packageinfo[pkglist[pkg]['Project']]: packageinfo[pkglist[pkg]['Project']].append(pkglist[pkg]['Package']) return packageinfo return pkglist def get_project_binary_list(self, project, repository, arch, package_binaries={}): """ Returns binarylist of a project """ # Use pool repository for SUSE namespace project. # Because RPMs were injected to pool repository on OBS rather than # standard repository. if project.startswith('SUSE:'): repository = 'pool' path = ['build', project, repository, arch] url = makeurl(self.apiurl, path, {'view': 'binaryversions'}) root = ET.parse(http_GET(url)).getroot() for binary_list in root: package = binary_list.get('package') package = package.split(':', 1)[0] index = project + "_" + package if index not in package_binaries: package_binaries[index] = [] for binary in binary_list: filename = binary.get('name') result = re.match(osclib.core.RPM_REGEX, filename) if not result: continue if result.group('arch') == 'src' or result.group('arch') == 'nosrc': continue if result.group('name').endswith('-debuginfo') or result.group('name').endswith('-debuginfo-32bit'): continue if result.group('name').endswith('-debugsource'): continue if result.group('name') not in package_binaries[index]: package_binaries[index].append(result.group('name')) return package_binaries def exception_package(self, package): """ Do not skip the package if matches the condition. package parameter is source package name. """ if '-bootstrap' in package or\ 'Tumbleweed' in package or\ 'metis' in package: return True # These packages must have a good reason not to be single-speced # from one source. if package.startswith('python2-') or\ package.startswith('python3'): return True return False def exception_binary(self, package): """ Do not skip the binary if matches the condition package parameter is RPM filename. """ if package == 'openSUSE-release' or\ package == 'openSUSE-release-ftp' or\ package == 'openSUSE-Addon-NonOss-release': return True return False def crawl(self): """Main method""" leap_pkglist = self.get_packagelist(self.opensuse_project) sle_pkglist = self.get_packagelist(self.sle_project, by_project=False) # The selected_binarylist[] includes the latest sourcepackage list # binary RPMs from the latest sources need to be presented in ftp eventually selected_binarylist = [] # Any existed binary RPMs from any SPx/Leap/Backports fullbinarylist = [] # package_binaries[] is a pre-formated binarylist per each package # access to the conotent uses package_binaries['SUSE:SLE-15:Update_libcdio.12032'] package_binaries = {} # Inject binarylist to a list per package name no matter what archtectures was for arch in SUPPORTED_ARCHS: for prj in leap_pkglist.keys(): package_binaries = self.get_project_binary_list(prj, DEFAULT_REPOSITORY, arch, package_binaries) for pkg in package_binaries.keys(): if not self.exception_package(pkg): fullbinarylist += package_binaries[pkg] for prj in leap_pkglist.keys(): for pkg in leap_pkglist[prj]: cands = [prj + "_" + pkg] # Handling for SLE forks, or package has different multibuild bits # enablility between SLE and openSUSE if prj.startswith('openSUSE:') and pkg in sle_pkglist and\ not self.is_sle_specific(pkg): cands.append(sle_pkglist[pkg]['Project'] + "_" + sle_pkglist[pkg]['Package']) logging.debug(cands) for index in cands: if index in package_binaries: selected_binarylist += package_binaries[index] else: logging.info("Can not find binary of %s" % index) # Some packages has been obsoleted by new updated package, however # there are application still depend on old library when it builds # eg. SUSE:SLE-15-SP3:GA has qpdf/libqpdf28 but cups-filter was build # in/when SLE15 SP2 which requiring qpdf/libqpdf6, therefore old # qpdf/libqpdf6 from SLE15 SP2 should not to be missed. for pkg in self.skiplist_ignored: selected_binarylist += package_binaries[pkg] # Preparing a packagelist for the skipping candidate obsoleted = [] for pkg in fullbinarylist: if pkg not in selected_binarylist and pkg not in obsoleted: if not self.exception_binary(pkg): obsoleted.append(pkg) # Post processing of obsoleted packagelist tmp_obsoleted = obsoleted.copy() for pkg in tmp_obsoleted: # Respect to single-speced python package, when a python2 RPM is # considered then a python3 flavor should also be selected to be # skipped, if not, don't add it. if pkg.startswith('python2-') and re.sub(r'^python2', 'python3', pkg) not in obsoleted: obsoleted.remove(pkg) # Main RPM must to be skipped if -32 bit RPM or -64bit RPM is # considered. if pkg.endswith('-32bit') or pkg.endswith('-64bit'): main_filename = re.sub('-[36][24]bit', '', pkg) if main_filename not in obsoleted: obsoleted.remove(pkg) for regex in self.skiplist_supplement_regex: for binary in fullbinarylist: result = re.match(regex, binary) if result and binary not in obsoleted and\ binary not in self.skiplist_supplement_ignores: obsoleted.append(binary) skip_list = ET.Element('group', {'name': 'NON_FTP_PACKAGES'}) ET.SubElement(skip_list, 'conditional', {'name': 'drop_from_ftp'}) packagelist = ET.SubElement(skip_list, 'packagelist', {'relationship': 'requires'}) for pkg in sorted(obsoleted): if not self.print_only and self.verbose: print(pkg) attr = {'name': pkg} ET.SubElement(packagelist, 'package', attr) if not self.print_only: source_file_ensure(self.apiurl, self.upload_project, META_PACKAGE, 'NON_FTP_PACKAGES.group', ET.tostring(skip_list, pretty_print=True, encoding='unicode'), 'Update the skip list') else: print(ET.tostring(skip_list, pretty_print=True, encoding='unicode')) def main(args): osc.conf.get_config(override_apiurl=args.apiurl) osc.conf.config['debug'] = args.debug if args.opensuse_project is None or args.sle_project is None: print("Please pass --opensuse-project and --sle-project argument. See usage with --help.") quit() uc = SkippkgFinder(args.opensuse_project, args.sle_project, args.alternative_project, args.print_only, args.verbose) uc.crawl() if __name__ == '__main__': description = 'Overwrites NON_FTP_PACKAGES.group according to the latest sources. '\ 'This tool only works for Leap after CtLG implemented.' parser = argparse.ArgumentParser(description=description) parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL') parser.add_argument('-d', '--debug', action='store_true', help='print info useful for debuging') parser.add_argument('-o', '--opensuse-project', dest='opensuse_project', metavar='OPENSUSE_PROJECT', help='openSUSE project on buildservice') parser.add_argument('-s', '--sle-project', dest='sle_project', metavar='SLE_PROJECT', help='SLE project on buildservice') parser.add_argument('-t', '--alternative-project', dest='alternative_project', metavar='ALTERNATIVE_PROJECT', help='check the given project instead of OPENSUSE_PROJECT') parser.add_argument('-p', '--print-only', action='store_true', help='show the result instead of the uploading') parser.add_argument('-v', '--verbose', action='store_true', help='show the diff') args = parser.parse_args() logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) sys.exit(main(args))