Retire repo_checker.pl

Extract the writing of the susetags content into a new script that
also creates a yaml file which package comes from what directory.
This information is used in repochecks.py to run findfileconflicts
and installcheck as repo_checker.pl used to

findfileconflicts

@@ -63,7 +63,7 @@ sub beautify_mode {
   return "$rts$ft".sprintf("%03o", $fm)." $m[2]";
 }
 
-print "scanning file list\n";
+print STDERR "scanning file list\n";
 if ($ARGV[0] =~ /\.gz$/) {
   open(FL, "-|", 'gunzip', '-dc', $ARGV[0]) || die("open $ARGV[0]: $!\n");
 } else {
@@ -166,10 +166,10 @@ while(<FL>) {
 }
 close(FL) || die("close failed\n");
 
-print "currently have ".@dirs." dirs and ".@modes." modes\n";
+print STDERR "currently have ".@dirs." dirs and ".@modes." modes\n";
 
 # connect dirs and add all dirs as files
-print "connecting ".@dirs." directories\n";
+print STDERR "connecting ".@dirs." directories\n";
 my @implicit_conflicts;
 for (@dirs) {
   next unless /^(.*\/)(.*?)\/$/;
@@ -193,7 +193,7 @@ for (@dirs) {
   next if $have_dir;
   push @implicit_conflicts, $f;
 }
-print "now ".@dirs." directories\n";
+print STDERR "now ".@dirs." directories\n";
 
 # the old and fast way
 #
@@ -203,7 +203,7 @@ print "now ".@dirs." directories\n";
 #}
 
 if (@implicit_conflicts) {
-  print "have implicit conflicts, calculating dir owners\n";
+  print STDERR "have implicit conflicts, calculating dir owners\n";
   my @pdirs;	# parent dirs
   for (@dirs) {
     next unless /^(.*\/)(.*?)\/$/;
@@ -247,7 +247,7 @@ if (@implicit_conflicts) {
 %files = ();	# free mem
 
 # reduce all-dir conflicts and trivial multiarch conflicts
-print "reducing trivial conflicts\n";
+print STDERR "reducing trivial conflicts\n";
 for my $f (sort keys %filesc) {
   my $allm;
   my $allc = 1;
@@ -273,7 +273,7 @@ for my $f (sort keys %filesc) {
   }
 }
 
-print "checking conflicts\n";
+print STDERR "checking conflicts\n";
 my %pkgneeded;
 my %tocheck;
 my %tocheck_files;
@@ -323,8 +323,8 @@ for my $pkg (sort keys %pkgneeded) {
   }
 }
 
-print "found ".(keys %tocheck)." conflict candidates\n";
+print STDERR "found ".(keys %tocheck)." conflict candidates\n";
-print "checking...\n";
+print STDERR "checking...\n";
 # now check each package combination for all candidates
 for my $tc (sort keys %tocheck) {
   my @p = @{$tocheck{$tc}};
@@ -360,9 +360,11 @@ for my $tc (sort keys %tocheck) {
       next unless @con;
       my @sp1 = split(' ', $p1);
       my @sp2 = split(' ', $p2);
-      print "found conflict of $sp1[0]-$sp1[1]-$sp1[2].$sp1[3] with $sp2[0]-$sp2[1]-$sp2[2].$sp2[3]:\n";
+      print "- between:\n";
-      print "  - $_\n" for @con;
+      print "  - [$sp1[0], $sp1[1], $sp1[2], $sp1[3]]\n";
+      print "  - [$sp2[0], $sp2[1], $sp2[2], $sp2[3]]\n";
+      print "  conflicts: |-\n";
+      print "    $_\n" for (@con);
     }
   }
 }
-

osclib/repochecks.py

@@ -0,0 +1,118 @@
+import logging
+import tempfile
+import os
+import re
+import yaml
+import subprocess
+from fnmatch import fnmatch
+
+logger = logging.getLogger('InstallChecker')
+
+SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
+
+class CorruptRepos(Exception):
+    pass
+
+# the content of sp is name, version, release, arch
+def _format_pkg(sp):
+    return "{}-{}-{}.{}".format(sp[0], sp[1], sp[2], sp[3])
+
+def _check_exists_in_whitelist(sp, whitelist):
+    if sp[0] in whitelist:
+        logger.debug("Found %s in whitelist, ignoring", sp[0])
+        return True
+    # check with version
+    long_name = "{}-{}".format(sp[0], sp[1])
+    if long_name in whitelist:
+        logger.debug("Found %s in whitelist, ignoring", long_name)
+        return True
+    for entry in whitelist:
+        if fnmatch(sp[0], entry):
+            logger.debug("Found %s matching whitelist entry %s, ignoring", sp[0], entry)
+            return True
+
+def _check_colon_format(sp1, sp2, whitelist):
+    if "{}:{}".format(sp1, sp2) in whitelist:
+        logger.debug("Found %s:%s in whitelist, ignoring", sp1, sp2)
+        return True
+
+def _check_conflicts_whitelist(sp1, sp2, whitelist):
+    if _check_exists_in_whitelist(sp1, whitelist):
+        return True
+    if _check_exists_in_whitelist(sp2, whitelist):
+        return True
+    if _check_colon_format(sp1[0], sp2[0], whitelist):
+        return True
+    if _check_colon_format(sp2[0], sp1[0], whitelist):
+        return True
+
+def _fileconflicts(pfile, target_packages, whitelist):
+    script = os.path.join(SCRIPT_PATH, '..', 'findfileconflicts')
+    p = subprocess.run(['perl', script, pfile], stdout=subprocess.PIPE)
+    if p.returncode or len(p.stdout):
+        output = ''
+        conflicts = yaml.safe_load(p.stdout)
+        for conflict in conflicts:
+            sp1 = conflict['between'][0]
+            sp2 = conflict['between'][1]
+
+            if not sp1[0] in target_packages and not sp2[0] in target_packages:
+                continue
+
+            if _check_conflicts_whitelist(sp1, sp2, whitelist):
+                continue
+
+            output += "found conflict of {} with {}\n".format(_format_pkg(sp1), _format_pkg(sp2))
+            for file in conflict['conflicts'].split('\n'):
+                output += "  {}\n".format(file)
+            output += "\n"
+
+        if len(output):
+            return output
+
+def _installcheck(pfile, arch, target_packages, whitelist):
+    p = subprocess.run(['/usr/bin/installcheck', arch, pfile], stdout=subprocess.PIPE, text=True)
+    if p.returncode:
+        output = ''
+        in_problem = False
+        install_re = re.compile(r"^can't install (.*)-[^-]+-[^-]+:$")
+        for line in p.stdout.split('\n'):
+            if not line.startswith(' '):
+                in_problem = False
+            match = install_re.match(line)
+            if match:
+                in_problem = match.group(1) in target_packages
+            if in_problem:
+                output += line + "\n"
+        return output
+
+def installcheck(directories, arch, whitelist, ignore_conflicts):
+
+    with tempfile.TemporaryDirectory(prefix='repochecker') as dir:
+        #dir = '.'
+        pfile = os.path.join(dir, 'packages')
+
+        script = os.path.join(SCRIPT_PATH, '..', 'write_repo_susetags_file.pl')
+        parts = ['perl', script, dir] + directories
+
+        p = subprocess.run(parts)
+        if p.returncode:
+            # technically only 126, but there is no other value atm -
+            # so if some other perl error happens, we don't continue
+            raise CorruptRepos
+
+        target_packages = []
+        with open(os.path.join(dir, 'catalog.yml')) as file:
+            catalog = yaml.safe_load(file)
+            target_packages = catalog[directories[0]]
+
+        parts = []
+        output = _fileconflicts(pfile, target_packages, ignore_conflicts)
+        if output:
+            parts.append(output)
+
+        output = _installcheck(pfile, arch, target_packages, whitelist)
+        if output:
+            parts.append(output)
+
+        return parts

staging-installcheck.py

@@ -72,6 +72,7 @@ class InstallChecker(object):
 
         self.existing_problems = self.binary_list_existing_problem(api.project, api.cmain_repo)
         self.ignore_duplicated = set(self.config.get('installcheck-ignore-duplicated-binaries', '').split(' '))
+        self.ignore_conflicts = set(self.config.get('installcheck-ignore-conflicts', '').split(' '))
 
     def check_required_by(self, fileinfo, provides, requiredby, built_binaries, comments):
         if requiredby.get('name') in built_binaries:
@@ -141,7 +142,7 @@ class InstallChecker(object):
             args = match.group('args').strip()
             # allow space and comma to seperate
             args = args.replace(',', ' ').split(' ')
-        return args
+        return set(args)
 
     def staging(self, project, force=False):
         api = self.api
@@ -219,7 +220,8 @@ class InstallChecker(object):
             else:
                 whitelist = self.existing_problems
 
-            whitelist |= set(to_ignore)
+            whitelist |= to_ignore
+            ignore_conflicts = self.ignore_conflicts | to_ignore
 
             check = self.cycle_check(project, repository, arch)
             if not check.success:
@@ -227,7 +229,7 @@ class InstallChecker(object):
                 result_comment.append(check.comment)
                 result = False
 
-            check = self.install_check(target_pair, arch, directories, None, whitelist)
+            check = self.install_check(directories, arch, whitelist, ignore_conflicts)
             if not check.success:
                 self.logger.warning('Install check failed')
                 result_comment.append(check.comment)
@@ -332,6 +334,8 @@ class InstallChecker(object):
     def mirror(self, project, repository, arch):
         """Call bs_mirrorfull script to mirror packages."""
         directory = os.path.join(CACHEDIR, project, repository, arch)
+        #return directory
+
         if not os.path.exists(directory):
             os.makedirs(directory)
 
@@ -367,72 +371,17 @@ class InstallChecker(object):
 
         return binaries
 
-    def install_check(self, target_project_pair, arch, directories,
+    def install_check(self, directories, arch, whitelist, ignored_conflicts):
-                      ignore=None, whitelist=[], parse=False, no_filter=False):
+        self.logger.info('install check: start (whitelist:{})'.format(','.join(whitelist)))
-        self.logger.info('install check: start (ignore:{}, whitelist:{}, parse:{}, no_filter:{})'.format(
+        import osclib.repochecks
-            bool(ignore), len(whitelist), parse, no_filter))
+        parts = osclib.repochecks.installcheck(directories, arch, whitelist, ignored_conflicts)
-
+        if len(parts):
-        with tempfile.NamedTemporaryFile() as ignore_file:
-            # Print ignored rpms on separate lines in ignore file.
-            if ignore:
-                for item in ignore:
-                    ignore_file.write(item + '\n')
-                ignore_file.flush()
-
-            # Invoke repo_checker.pl to perform an install check.
-            script = os.path.join(SCRIPT_PATH, 'repo_checker.pl')
-            parts = ['LC_ALL=C', 'perl', script, arch, ','.join(directories),
-                     '-f', ignore_file.name, '-w', ','.join(whitelist)]
-            if no_filter:
-                parts.append('--no-filter')
-
-            parts = [pipes.quote(part) for part in parts]
-            p = subprocess.Popen(' '.join(parts), shell=True,
-                                 stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE, close_fds=True)
-            stdout, stderr = p.communicate()
-
-        if p.returncode:
-            self.logger.info('install check: failed')
-            if p.returncode == 126:
-                self.logger.warning('mirror cache reset due to corruption')
-                self._invalidate_all()
-            elif parse:
-                # Parse output for later consumption for posting comments.
-                sections = self.install_check_parse(stdout)
-                self.install_check_sections_group(
-                    target_project_pair[0], target_project_pair[1], arch, sections)
-
-            # Format output as markdown comment.
-            parts = []
-
-            stdout = stdout.decode('utf-8').strip()
-            if stdout:
-                parts.append(stdout + '\n')
-            stderr = stderr.strip()
-            if stderr:
-                parts.append(stderr + '\n')
-
             header = '### [install check & file conflicts for {}]'.format(arch)
             return CheckResult(False, header + '\n\n' + ('\n' + ('-' * 80) + '\n\n').join(parts))
 
         self.logger.info('install check: passed')
         return CheckResult(True, None)
 
-    def install_check_sections_group(self, project, repository, arch, sections):
-        _, binary_map = package_binary_list(self.api.apiurl, project, repository, arch)
-
-        for section in sections:
-            # If switch to creating bugs likely makes sense to join packages to
-            # form grouping key and create shared bugs for conflicts.
-            # Added check for b in binary_map after encountering:
-            # https://lists.opensuse.org/opensuse-buildservice/2017-08/msg00035.html
-            # Under normal circumstances this should never occur.
-            packages = set([binary_map[b] for b in section.binaries if b in binary_map])
-            for package in packages:
-                self.package_results.setdefault(package, [])
-                self.package_results[package].append(section)
-
     def install_check_parse(self, output):
         section = None
         text = None

write_repo_susetags_file.pl

@@ -0,0 +1,75 @@
+#! /usr/bin/perl -w
+
+use File::Basename;
+use File::Temp qw/ tempdir  /;
+use XML::Simple;
+use Data::Dumper;
+use Cwd;
+
+use strict;
+
+BEGIN {
+  my ($wd) = $0 =~ m-(.*)/- ;
+  $wd ||= '.';
+  unshift @INC, $wd;
+}
+
+require CreatePackageDescr;
+
+die "Usage: $0 <output directory> <input directories...>" if scalar(@ARGV) < 2;
+
+my $output_directory = shift @ARGV;
+my @directories = @ARGV;
+
+sub write_package {
+    my ($package, $packages_fd, $directory, $written_names) = @_;
+
+    my $name = basename($package);
+    if ($name =~ m/^[a-z0-9]{32}-/) { # repo cache
+       $name =~ s,^[^-]+-(.*)\.rpm,$1,;
+    } else {
+       $name =~ s,^(.*)-[^-]+-[^-]+.rpm,$1,;
+    }
+
+    if ( defined $written_names->{$name} ) {
+        return;
+    }
+    $written_names->{$name} = $directory;
+
+    my $out = CreatePackageDescr::package_snippet($package);
+    if ($out eq "" || $out =~ m/=Pkg:    /) {
+        print STDERR "ERROR: empty package snippet for: $name\n";
+        exit(126);
+    }
+    print $packages_fd $out;
+    return $name;
+}
+
+open( my $packages_fd, ">", "$output_directory/packages" ) || die 'can not open';
+print $packages_fd "=Ver: 2.0\n";
+
+my %written_names;
+
+for my $directory (@directories) {
+    my @rpms = glob("$directory/*.rpm");
+    write_package( $_, $packages_fd, $directory, \%written_names ) for @rpms;
+}
+
+close($packages_fd);
+
+# turn around key->value
+my %per_directory;
+for my $name (keys %written_names) {
+   $per_directory{$written_names{$name}} ||= [];
+   push(@{$per_directory{$written_names{$name}}}, $name);
+}
+
+open( my $yaml_fd, ">", "$output_directory/catalog.yml" ) || die 'can not open';
+for my $directory (@directories) {
+   next unless defined($per_directory{$directory});
+   print $yaml_fd "$directory:\n";
+   for my $name (@{$per_directory{$directory}}) {
+        print $yaml_fd "  - '$name'\n";
+   }
+}
+close($yaml_fd);