diff --git a/findfileconflicts b/findfileconflicts index 0eecd3d9..42ae9cac 100755 --- a/findfileconflicts +++ b/findfileconflicts @@ -63,7 +63,7 @@ sub beautify_mode { return "$rts$ft".sprintf("%03o", $fm)." $m[2]"; } -print "scanning file list\n"; +print STDERR "scanning file list\n"; if ($ARGV[0] =~ /\.gz$/) { open(FL, "-|", 'gunzip', '-dc', $ARGV[0]) || die("open $ARGV[0]: $!\n"); } else { @@ -166,10 +166,10 @@ while() { } close(FL) || die("close failed\n"); -print "currently have ".@dirs." dirs and ".@modes." modes\n"; +print STDERR "currently have ".@dirs." dirs and ".@modes." modes\n"; # connect dirs and add all dirs as files -print "connecting ".@dirs." directories\n"; +print STDERR "connecting ".@dirs." directories\n"; my @implicit_conflicts; for (@dirs) { next unless /^(.*\/)(.*?)\/$/; @@ -193,7 +193,7 @@ for (@dirs) { next if $have_dir; push @implicit_conflicts, $f; } -print "now ".@dirs." directories\n"; +print STDERR "now ".@dirs." directories\n"; # the old and fast way # @@ -203,7 +203,7 @@ print "now ".@dirs." directories\n"; #} if (@implicit_conflicts) { - print "have implicit conflicts, calculating dir owners\n"; + print STDERR "have implicit conflicts, calculating dir owners\n"; my @pdirs; # parent dirs for (@dirs) { next unless /^(.*\/)(.*?)\/$/; @@ -247,7 +247,7 @@ if (@implicit_conflicts) { %files = (); # free mem # reduce all-dir conflicts and trivial multiarch conflicts -print "reducing trivial conflicts\n"; +print STDERR "reducing trivial conflicts\n"; for my $f (sort keys %filesc) { my $allm; my $allc = 1; @@ -273,7 +273,7 @@ for my $f (sort keys %filesc) { } } -print "checking conflicts\n"; +print STDERR "checking conflicts\n"; my %pkgneeded; my %tocheck; my %tocheck_files; @@ -323,8 +323,8 @@ for my $pkg (sort keys %pkgneeded) { } } -print "found ".(keys %tocheck)." conflict candidates\n"; -print "checking...\n"; +print STDERR "found ".(keys %tocheck)." conflict candidates\n"; +print STDERR "checking...\n"; # now check each package combination for all candidates for my $tc (sort keys %tocheck) { my @p = @{$tocheck{$tc}}; @@ -360,9 +360,11 @@ for my $tc (sort keys %tocheck) { next unless @con; my @sp1 = split(' ', $p1); my @sp2 = split(' ', $p2); - print "found conflict of $sp1[0]-$sp1[1]-$sp1[2].$sp1[3] with $sp2[0]-$sp2[1]-$sp2[2].$sp2[3]:\n"; - print " - $_\n" for @con; + print "- between:\n"; + print " - [$sp1[0], $sp1[1], $sp1[2], $sp1[3]]\n"; + print " - [$sp2[0], $sp2[1], $sp2[2], $sp2[3]]\n"; + print " conflicts: |-\n"; + print " $_\n" for (@con); } } } - diff --git a/osclib/repochecks.py b/osclib/repochecks.py new file mode 100644 index 00000000..67eebaa6 --- /dev/null +++ b/osclib/repochecks.py @@ -0,0 +1,118 @@ +import logging +import tempfile +import os +import re +import yaml +import subprocess +from fnmatch import fnmatch + +logger = logging.getLogger('InstallChecker') + +SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) + +class CorruptRepos(Exception): + pass + +# the content of sp is name, version, release, arch +def _format_pkg(sp): + return "{}-{}-{}.{}".format(sp[0], sp[1], sp[2], sp[3]) + +def _check_exists_in_whitelist(sp, whitelist): + if sp[0] in whitelist: + logger.debug("Found %s in whitelist, ignoring", sp[0]) + return True + # check with version + long_name = "{}-{}".format(sp[0], sp[1]) + if long_name in whitelist: + logger.debug("Found %s in whitelist, ignoring", long_name) + return True + for entry in whitelist: + if fnmatch(sp[0], entry): + logger.debug("Found %s matching whitelist entry %s, ignoring", sp[0], entry) + return True + +def _check_colon_format(sp1, sp2, whitelist): + if "{}:{}".format(sp1, sp2) in whitelist: + logger.debug("Found %s:%s in whitelist, ignoring", sp1, sp2) + return True + +def _check_conflicts_whitelist(sp1, sp2, whitelist): + if _check_exists_in_whitelist(sp1, whitelist): + return True + if _check_exists_in_whitelist(sp2, whitelist): + return True + if _check_colon_format(sp1[0], sp2[0], whitelist): + return True + if _check_colon_format(sp2[0], sp1[0], whitelist): + return True + +def _fileconflicts(pfile, target_packages, whitelist): + script = os.path.join(SCRIPT_PATH, '..', 'findfileconflicts') + p = subprocess.run(['perl', script, pfile], stdout=subprocess.PIPE) + if p.returncode or len(p.stdout): + output = '' + conflicts = yaml.safe_load(p.stdout) + for conflict in conflicts: + sp1 = conflict['between'][0] + sp2 = conflict['between'][1] + + if not sp1[0] in target_packages and not sp2[0] in target_packages: + continue + + if _check_conflicts_whitelist(sp1, sp2, whitelist): + continue + + output += "found conflict of {} with {}\n".format(_format_pkg(sp1), _format_pkg(sp2)) + for file in conflict['conflicts'].split('\n'): + output += " {}\n".format(file) + output += "\n" + + if len(output): + return output + +def _installcheck(pfile, arch, target_packages, whitelist): + p = subprocess.run(['/usr/bin/installcheck', arch, pfile], stdout=subprocess.PIPE, text=True) + if p.returncode: + output = '' + in_problem = False + install_re = re.compile(r"^can't install (.*)-[^-]+-[^-]+:$") + for line in p.stdout.split('\n'): + if not line.startswith(' '): + in_problem = False + match = install_re.match(line) + if match: + in_problem = match.group(1) in target_packages + if in_problem: + output += line + "\n" + return output + +def installcheck(directories, arch, whitelist, ignore_conflicts): + + with tempfile.TemporaryDirectory(prefix='repochecker') as dir: + #dir = '.' + pfile = os.path.join(dir, 'packages') + + script = os.path.join(SCRIPT_PATH, '..', 'write_repo_susetags_file.pl') + parts = ['perl', script, dir] + directories + + p = subprocess.run(parts) + if p.returncode: + # technically only 126, but there is no other value atm - + # so if some other perl error happens, we don't continue + raise CorruptRepos + + target_packages = [] + with open(os.path.join(dir, 'catalog.yml')) as file: + catalog = yaml.safe_load(file) + target_packages = catalog[directories[0]] + + parts = [] + output = _fileconflicts(pfile, target_packages, ignore_conflicts) + if output: + parts.append(output) + + output = _installcheck(pfile, arch, target_packages, whitelist) + if output: + parts.append(output) + + return parts diff --git a/staging-installcheck.py b/staging-installcheck.py index e5fb0f5d..b98a3e9b 100755 --- a/staging-installcheck.py +++ b/staging-installcheck.py @@ -72,6 +72,7 @@ class InstallChecker(object): self.existing_problems = self.binary_list_existing_problem(api.project, api.cmain_repo) self.ignore_duplicated = set(self.config.get('installcheck-ignore-duplicated-binaries', '').split(' ')) + self.ignore_conflicts = set(self.config.get('installcheck-ignore-conflicts', '').split(' ')) def check_required_by(self, fileinfo, provides, requiredby, built_binaries, comments): if requiredby.get('name') in built_binaries: @@ -141,7 +142,7 @@ class InstallChecker(object): args = match.group('args').strip() # allow space and comma to seperate args = args.replace(',', ' ').split(' ') - return args + return set(args) def staging(self, project, force=False): api = self.api @@ -219,7 +220,8 @@ class InstallChecker(object): else: whitelist = self.existing_problems - whitelist |= set(to_ignore) + whitelist |= to_ignore + ignore_conflicts = self.ignore_conflicts | to_ignore check = self.cycle_check(project, repository, arch) if not check.success: @@ -227,7 +229,7 @@ class InstallChecker(object): result_comment.append(check.comment) result = False - check = self.install_check(target_pair, arch, directories, None, whitelist) + check = self.install_check(directories, arch, whitelist, ignore_conflicts) if not check.success: self.logger.warning('Install check failed') result_comment.append(check.comment) @@ -332,6 +334,8 @@ class InstallChecker(object): def mirror(self, project, repository, arch): """Call bs_mirrorfull script to mirror packages.""" directory = os.path.join(CACHEDIR, project, repository, arch) + #return directory + if not os.path.exists(directory): os.makedirs(directory) @@ -367,72 +371,17 @@ class InstallChecker(object): return binaries - def install_check(self, target_project_pair, arch, directories, - ignore=None, whitelist=[], parse=False, no_filter=False): - self.logger.info('install check: start (ignore:{}, whitelist:{}, parse:{}, no_filter:{})'.format( - bool(ignore), len(whitelist), parse, no_filter)) - - with tempfile.NamedTemporaryFile() as ignore_file: - # Print ignored rpms on separate lines in ignore file. - if ignore: - for item in ignore: - ignore_file.write(item + '\n') - ignore_file.flush() - - # Invoke repo_checker.pl to perform an install check. - script = os.path.join(SCRIPT_PATH, 'repo_checker.pl') - parts = ['LC_ALL=C', 'perl', script, arch, ','.join(directories), - '-f', ignore_file.name, '-w', ','.join(whitelist)] - if no_filter: - parts.append('--no-filter') - - parts = [pipes.quote(part) for part in parts] - p = subprocess.Popen(' '.join(parts), shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, close_fds=True) - stdout, stderr = p.communicate() - - if p.returncode: - self.logger.info('install check: failed') - if p.returncode == 126: - self.logger.warning('mirror cache reset due to corruption') - self._invalidate_all() - elif parse: - # Parse output for later consumption for posting comments. - sections = self.install_check_parse(stdout) - self.install_check_sections_group( - target_project_pair[0], target_project_pair[1], arch, sections) - - # Format output as markdown comment. - parts = [] - - stdout = stdout.decode('utf-8').strip() - if stdout: - parts.append(stdout + '\n') - stderr = stderr.strip() - if stderr: - parts.append(stderr + '\n') - + def install_check(self, directories, arch, whitelist, ignored_conflicts): + self.logger.info('install check: start (whitelist:{})'.format(','.join(whitelist))) + import osclib.repochecks + parts = osclib.repochecks.installcheck(directories, arch, whitelist, ignored_conflicts) + if len(parts): header = '### [install check & file conflicts for {}]'.format(arch) return CheckResult(False, header + '\n\n' + ('\n' + ('-' * 80) + '\n\n').join(parts)) self.logger.info('install check: passed') return CheckResult(True, None) - def install_check_sections_group(self, project, repository, arch, sections): - _, binary_map = package_binary_list(self.api.apiurl, project, repository, arch) - - for section in sections: - # If switch to creating bugs likely makes sense to join packages to - # form grouping key and create shared bugs for conflicts. - # Added check for b in binary_map after encountering: - # https://lists.opensuse.org/opensuse-buildservice/2017-08/msg00035.html - # Under normal circumstances this should never occur. - packages = set([binary_map[b] for b in section.binaries if b in binary_map]) - for package in packages: - self.package_results.setdefault(package, []) - self.package_results[package].append(section) - def install_check_parse(self, output): section = None text = None diff --git a/write_repo_susetags_file.pl b/write_repo_susetags_file.pl new file mode 100755 index 00000000..c58fd740 --- /dev/null +++ b/write_repo_susetags_file.pl @@ -0,0 +1,75 @@ +#! /usr/bin/perl -w + +use File::Basename; +use File::Temp qw/ tempdir /; +use XML::Simple; +use Data::Dumper; +use Cwd; + +use strict; + +BEGIN { + my ($wd) = $0 =~ m-(.*)/- ; + $wd ||= '.'; + unshift @INC, $wd; +} + +require CreatePackageDescr; + +die "Usage: $0 " if scalar(@ARGV) < 2; + +my $output_directory = shift @ARGV; +my @directories = @ARGV; + +sub write_package { + my ($package, $packages_fd, $directory, $written_names) = @_; + + my $name = basename($package); + if ($name =~ m/^[a-z0-9]{32}-/) { # repo cache + $name =~ s,^[^-]+-(.*)\.rpm,$1,; + } else { + $name =~ s,^(.*)-[^-]+-[^-]+.rpm,$1,; + } + + if ( defined $written_names->{$name} ) { + return; + } + $written_names->{$name} = $directory; + + my $out = CreatePackageDescr::package_snippet($package); + if ($out eq "" || $out =~ m/=Pkg: /) { + print STDERR "ERROR: empty package snippet for: $name\n"; + exit(126); + } + print $packages_fd $out; + return $name; +} + +open( my $packages_fd, ">", "$output_directory/packages" ) || die 'can not open'; +print $packages_fd "=Ver: 2.0\n"; + +my %written_names; + +for my $directory (@directories) { + my @rpms = glob("$directory/*.rpm"); + write_package( $_, $packages_fd, $directory, \%written_names ) for @rpms; +} + +close($packages_fd); + +# turn around key->value +my %per_directory; +for my $name (keys %written_names) { + $per_directory{$written_names{$name}} ||= []; + push(@{$per_directory{$written_names{$name}}}, $name); +} + +open( my $yaml_fd, ">", "$output_directory/catalog.yml" ) || die 'can not open'; +for my $directory (@directories) { + next unless defined($per_directory{$directory}); + print $yaml_fd "$directory:\n"; + for my $name (@{$per_directory{$directory}}) { + print $yaml_fd " - '$name'\n"; + } +} +close($yaml_fd);