Add support for DoD repos to installcheck

osclib/repochecks.py

@@ -1,9 +1,12 @@
 import logging
 import os
 import re
+import requests
 import subprocess
 import tempfile
 from fnmatch import fnmatch
+from lxml import etree as ETL
+from osc.core import http_GET
 
 import yaml
 
@@ -87,7 +90,7 @@ def filter_release(line):
     line = re.sub(r'(provided by [^ ]*\-[^-]*)\-[^-]*(\.\w+)$', r'\1\2', line)
     return line
 
-def parsed_installcheck(pfile, arch, target_packages, whitelist):
+def parsed_installcheck(repos, arch, target_packages, whitelist):
     reported_problems = dict()
 
     if not len(target_packages):
@@ -100,7 +103,10 @@ def parsed_installcheck(pfile, arch, target_packages, whitelist):
             return _mapping[arch]
         return arch
 
-    p = subprocess.run(['/usr/bin/installcheck', maparch2installarch(arch), pfile],
+    if not isinstance(repos, list):
+        repos = [repos]
+
+    p = subprocess.run(['/usr/bin/installcheck', maparch2installarch(arch)] + repos,
                        stdout=subprocess.PIPE, errors='backslashreplace', text=True)
     if p.returncode:
         in_problem = False
@@ -162,15 +168,37 @@ def installcheck(directories, arch, whitelist, ignore_conflicts):
 
         return parts
 
-
 def mirror(apiurl, project, repository, arch):
     """Call bs_mirrorfull script to mirror packages."""
     directory = os.path.join(CACHEDIR, project, repository, arch)
-    # return directory
 
     if not os.path.exists(directory):
         os.makedirs(directory)
 
+    meta = ETL.parse(http_GET('{}/public/source/{}/_meta'.format(apiurl, project))).getroot()
+    repotag = meta.xpath("/project/repository[@name='{}']".format(repository))[0]
+    if arch not in repotag.xpath("./arch/text()"):
+        # Arch not in this project, skip mirroring
+        return directory
+
+    download = repotag.xpath("./download[@arch='{}']".format(arch))
+    if download is not None and len(download) > 0:
+        if len(download) > 1:
+            raise Exception('Multiple download urls unsupported')
+        repotype = download[0].get('repotype')
+        if repotype != 'rpmmd':
+            raise Exception('repotype {} not supported'.format(repotype))
+        url = download[0].get('url')
+        repoindex = ETL.fromstring(requests.get('{}/repodata/repomd.xml'.format(url)).content)
+        primarypath = repoindex.xpath("string(./repo:data[@type='primary']/repo:location/@href)", namespaces={'repo': 'http://linux.duke.edu/metadata/repo'})
+        primaryname = os.path.basename(primarypath)
+        primarydest = os.path.join(directory, os.path.basename(primarypath))
+        if not os.path.exists(primarydest):
+            with tempfile.NamedTemporaryFile(dir=directory) as primarytemp:
+                primarytemp.write(requests.get(url + '/' + primarypath).content)
+                os.link(primarytemp.name, primarydest)
+        return primarydest
+
     script = os.path.join(SCRIPT_PATH, '..', 'bs_mirrorfull')
     path = '/'.join((project, repository, arch))
     logger.info('mirroring {}'.format(path))

project-installcheck.py

@@ -126,8 +126,13 @@ class RepoChecker():
 
         repository_pairs = repository_path_expand(self.apiurl, project, repository)
         directories = []
+        primaryxmls = []
         for pair_project, pair_repository in repository_pairs:
-            directories.append(mirror(self.apiurl, pair_project, pair_repository, arch))
+            mirrored = mirror(self.apiurl, pair_project, pair_repository, arch)
+            if os.path.isdir(mirrored):
+                directories.append(mirrored)
+            else:
+                primaryxmls.append(mirrored)
 
         parsed = dict()
         with tempfile.TemporaryDirectory(prefix='repochecker') as dir:
@@ -148,7 +153,7 @@ class RepoChecker():
                 catalog = yaml.safe_load(file)
                 target_packages = catalog.get(directories[0], [])
 
-            parsed = parsed_installcheck(pfile, arch, target_packages, [])
+            parsed = parsed_installcheck([pfile] + primaryxmls, arch, target_packages, [])
             for package in parsed:
                 parsed[package]['output'] = "\n".join(parsed[package]['output'])