diff --git a/osclib/repochecks.py b/osclib/repochecks.py index c4b6330d..031cbf4c 100644 --- a/osclib/repochecks.py +++ b/osclib/repochecks.py @@ -1,9 +1,12 @@ import logging import os import re +import requests import subprocess import tempfile from fnmatch import fnmatch +from lxml import etree as ETL +from osc.core import http_GET import yaml @@ -87,7 +90,7 @@ def filter_release(line): line = re.sub(r'(provided by [^ ]*\-[^-]*)\-[^-]*(\.\w+)$', r'\1\2', line) return line -def parsed_installcheck(pfile, arch, target_packages, whitelist): +def parsed_installcheck(repos, arch, target_packages, whitelist): reported_problems = dict() if not len(target_packages): @@ -100,7 +103,10 @@ def parsed_installcheck(pfile, arch, target_packages, whitelist): return _mapping[arch] return arch - p = subprocess.run(['/usr/bin/installcheck', maparch2installarch(arch), pfile], + if not isinstance(repos, list): + repos = [repos] + + p = subprocess.run(['/usr/bin/installcheck', maparch2installarch(arch)] + repos, stdout=subprocess.PIPE, errors='backslashreplace', text=True) if p.returncode: in_problem = False @@ -162,15 +168,37 @@ def installcheck(directories, arch, whitelist, ignore_conflicts): return parts - def mirror(apiurl, project, repository, arch): """Call bs_mirrorfull script to mirror packages.""" directory = os.path.join(CACHEDIR, project, repository, arch) - # return directory if not os.path.exists(directory): os.makedirs(directory) + meta = ETL.parse(http_GET('{}/public/source/{}/_meta'.format(apiurl, project))).getroot() + repotag = meta.xpath("/project/repository[@name='{}']".format(repository))[0] + if arch not in repotag.xpath("./arch/text()"): + # Arch not in this project, skip mirroring + return directory + + download = repotag.xpath("./download[@arch='{}']".format(arch)) + if download is not None and len(download) > 0: + if len(download) > 1: + raise Exception('Multiple download urls unsupported') + repotype = download[0].get('repotype') + if repotype != 'rpmmd': + raise Exception('repotype {} not supported'.format(repotype)) + url = download[0].get('url') + repoindex = ETL.fromstring(requests.get('{}/repodata/repomd.xml'.format(url)).content) + primarypath = repoindex.xpath("string(./repo:data[@type='primary']/repo:location/@href)", namespaces={'repo': 'http://linux.duke.edu/metadata/repo'}) + primaryname = os.path.basename(primarypath) + primarydest = os.path.join(directory, os.path.basename(primarypath)) + if not os.path.exists(primarydest): + with tempfile.NamedTemporaryFile(dir=directory) as primarytemp: + primarytemp.write(requests.get(url + '/' + primarypath).content) + os.link(primarytemp.name, primarydest) + return primarydest + script = os.path.join(SCRIPT_PATH, '..', 'bs_mirrorfull') path = '/'.join((project, repository, arch)) logger.info('mirroring {}'.format(path)) diff --git a/project-installcheck.py b/project-installcheck.py index d6618d6a..945ea627 100755 --- a/project-installcheck.py +++ b/project-installcheck.py @@ -126,8 +126,13 @@ class RepoChecker(): repository_pairs = repository_path_expand(self.apiurl, project, repository) directories = [] + primaryxmls = [] for pair_project, pair_repository in repository_pairs: - directories.append(mirror(self.apiurl, pair_project, pair_repository, arch)) + mirrored = mirror(self.apiurl, pair_project, pair_repository, arch) + if os.path.isdir(mirrored): + directories.append(mirrored) + else: + primaryxmls.append(mirrored) parsed = dict() with tempfile.TemporaryDirectory(prefix='repochecker') as dir: @@ -148,7 +153,7 @@ class RepoChecker(): catalog = yaml.safe_load(file) target_packages = catalog.get(directories[0], []) - parsed = parsed_installcheck(pfile, arch, target_packages, []) + parsed = parsed_installcheck([pfile] + primaryxmls, arch, target_packages, []) for package in parsed: parsed[package]['output'] = "\n".join(parsed[package]['output'])