From 79ba192353c55989319681e028c21591ce46d1e4 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Wed, 9 Apr 2014 17:32:45 +0200 Subject: [PATCH] add a small tool that rebuilds broken packages replacing rebuildpacs from package-lists git --- CreatePackageDescr.pm | 128 ++++++++++++++++++++ bs_mirrorfull | 6 +- rebuildpacs.pl | 136 +++++++++++++++++++++ repo-checker.pl | 272 +++++++++++++----------------------------- 4 files changed, 353 insertions(+), 189 deletions(-) create mode 100644 CreatePackageDescr.pm create mode 100755 rebuildpacs.pl diff --git a/CreatePackageDescr.pm b/CreatePackageDescr.pm new file mode 100644 index 00000000..3c5b7d1e --- /dev/null +++ b/CreatePackageDescr.pm @@ -0,0 +1,128 @@ +package CreatePackageDescr; + +BEGIN { + unshift @INC, "/usr/lib/build/Build"; +} + +use File::Basename; + +use Rpm; +use Fcntl; + +sub package_snippet($) { + + my $package = shift; + + my $cachedir = dirname($package) . "/.cache/"; + my $cachefile = $cachedir . basename($package); + + my $out = ''; + if ( -f $cachefile ) { + open( C, $cachefile ) || die "no cache for $package"; + while () { + $out .= $_; + } + close(C); + return $out; + } + + # RPMTAG_FILEMODES = 1030, /* h[] */ + # RPMTAG_FILEFLAGS = 1037, /* i[] */ + # RPMTAG_FILEUSERNAME = 1039, /* s[] */ + # RPMTAG_FILEGROUPNAME = 1040, /* s[] */ + + my %qq = Build::Rpm::rpmq( + $package, + qw{NAME VERSION RELEASE ARCH OLDFILENAMES DIRNAMES BASENAMES DIRINDEXES 1030 1037 1039 1040 + 1047 1112 1113 1049 1048 1050 1090 1114 1115 1054 1053 1055 1036 + } + ); + + my $name = $qq{'NAME'}[0]; + + Build::Rpm::add_flagsvers( \%qq, 1049, 1048, 1050 ); # requires + Build::Rpm::add_flagsvers( \%qq, 1047, 1112, 1113 ); # provides + Build::Rpm::add_flagsvers( \%qq, 1090, 1114, 1115 ); # obsoletes + Build::Rpm::add_flagsvers( \%qq, 1054, 1053, 1055 ); # conflicts + + $out .= sprintf( "=Pkg: %s %s %s %s\n", + $name, $qq{'VERSION'}[0], $qq{'RELEASE'}[0], $qq{'ARCH'}[0] ); + $out .= "+Flx:\n"; + my @modes = @{ $qq{1030} || [] }; + my @basenames = @{ $qq{BASENAMES} || [] }; + my @dirs = @{ $qq{DIRNAMES} || [] }; + my @dirindexes = @{ $qq{DIRINDEXES} || [] }; + my @users = @{ $qq{1039} || [] }; + my @groups = @{ $qq{1040} || [] }; + my @flags = @{ $qq{1037} || [] }; + my @linktos = @{ $qq{1036} || [] }; + + my @xprvs; + + foreach my $bname (@basenames) { + my $mode = shift @modes; + my $di = shift @dirindexes; + my $user = shift @users; + my $group = shift @groups; + my $flag = shift @flags; + my $linkto = shift @linktos; + + my $filename = $dirs[$di] . $bname; + my $fs = $filename; + if ( Fcntl::S_ISLNK($mode) ) { + $fs = "$filename -> $linkto"; + } + $out .= sprintf "%o %o %s:%s %s\n", $mode, $flag, $user, $group, $fs; + if ( $filename =~ /^\/etc\// + || $filename =~ /bin\// + || $filename eq "/usr/lib/sendmail" ) + { + push @xprvs, $filename; + } + + } + $out .= "-Flx:\n"; + $out .= "+Prv:\n"; + foreach my $prv ( @{ $qq{1047} || [] } ) { + $out .= "$prv\n"; + } + foreach my $prv (@xprvs) { + $out .= "$prv\n"; + } + $out .= "-Prv:\n"; + $out .= "+Con:\n"; + foreach my $prv ( @{ $qq{1054} || [] } ) { + $out .= "$prv\n"; + } + $out .= "-Con:\n"; + $out .= "+Req:\n"; + foreach my $prv ( @{ $qq{1049} || [] } ) { + next if ( $prv =~ m/^rpmlib/ ); + next + if ( $name eq "libqmmp0-plugin-mplayer" + && $prv eq "/usr/bin/mplayer" ); + next if ( $prv eq "this-is-only-for-build-envs" ); + next + if ( $name eq "installation-images-debuginfodeps" + && $prv =~ m/debuginfo.build/ ); + next + if ( $name eq "installation-images-debuginfodeps-openSUSE" + && $prv =~ m/debuginfo.build/ ); + $out .= "$prv\n"; + } + $out .= "-Req:\n"; + $out .= "+Obs:\n"; + foreach my $prv ( @{ $qq{1090} || [] } ) { + $out .= "$prv\n"; + } + $out .= "-Obs:\n"; + + mkdir($cachedir); + open( C, '>', $cachefile ) || die "can't write $cachefile"; + print C $out; + close(C); + + return $out; +} + +1; diff --git a/bs_mirrorfull b/bs_mirrorfull index 88ba141c..f2fd07d0 100755 --- a/bs_mirrorfull +++ b/bs_mirrorfull @@ -34,7 +34,7 @@ die("uasge: bs_mirrorfull url dir\n") unless @ARGV == 2; my ($url, $dir) = @ARGV; $url =~ s/\/$//; -print "receiving tree state\n"; +#print "receiving tree state\n"; my $bvl = BSRPC::rpc("$url/_repository", $BSXML::binaryversionlist, "view=binaryversions", "nometa=1"); my @localbins = grep {/^[0-9a-f]{32}-.+\.rpm$/} ls($dir); my %localbins = map {$_ => 1} @localbins; @@ -75,7 +75,7 @@ if (@todownload) { }; BSRPC::rpc($param, undef, 'view=cpioheaders', @args); $did += @fetch; - print "$did/$todo\n"; + #print "$did/$todo\n"; } } -print "done, we now have ".(keys %remotebins)." packages.\n"; +#print "done, we now have ".(keys %remotebins)." packages.\n"; diff --git a/rebuildpacs.pl b/rebuildpacs.pl new file mode 100755 index 00000000..4fb1fdd1 --- /dev/null +++ b/rebuildpacs.pl @@ -0,0 +1,136 @@ +#! /usr/bin/perl + +use Data::Dumper; +use XML::Simple; +use URI::Escape; +use File::Basename; +use File::Temp qw/tempdir/; + +require CreatePackageDescr; + +my $repodir; + +sub find_source_container($) { + my $pkg = shift; + + my @rpms = glob("$repodir/*-$pkg.rpm"); + + for my $rpm (@rpms) { + + # 1123 == Disturl + my %qq = Build::Rpm::rpmq( $rpm, qw{NAME 1123} ); + next if ( $qq{NAME}[0] ne $pkg ); + my $distfile = basename( $qq{1123}[0] ); + $distfile =~ s,^[^-]*-,,; + + return $distfile; + } +} + +my $followup = 0; +my $cproblem = ''; +my %problems; + +my $project = $ARGV[0] || "openSUSE:Factory"; +my $repo = $ARGV[1] || "standard"; +my $arch = $ARGV[2] || "x86_64"; + +$repodir = "/var/cache/repo-checker/repo-openSUSE:Factory-$repo-$arch"; +mkdir($repodir); +my $pfile = tempdir() . "/packages"; # the filename is important ;( + +system( +"./bs_mirrorfull --nodebug https://build.opensuse.org/build/$project/$repo/$arch/ $repodir" +); + +my @rpms = glob("$repodir/*.rpm"); + +open( PACKAGES, ">", $pfile ) || die "can not open $pfile"; +print PACKAGES "=Ver: 2.0\n"; + +foreach my $package (@rpms) { + my $out = CreatePackageDescr::package_snippet($package); + print PACKAGES CreatePackageDescr::package_snippet($package); +} +close(PACKAGES); + +# read the problems out of installcheck +open( INSTALLCHECK, "installcheck $arch $pfile|" ); +while () { + chomp; + + if (m/^can't install (.*)\-[^-]*\-[^-]*\.($arch|noarch):/) { + $cproblem = $1; + $cproblem =~ s/kmp-([^-]*)/kmp-default/; + $cproblem = find_source_container($cproblem); + $followup = 0; + next; + } + + $followup = 1 if ( $_ =~ m/none of the providers can be installed/ ); + + # not interesting for me + next if (m/ \(we have /); + next if ($followup); + + # very thin ice here + s,\(\)\(64bit\),,; + + s,(needed by [^ ]*)\-[^-]*\-[^-]*\.($arch|noarch)$,$1,; + + s,^\s*,,; + $problems{$cproblem}->{$_} = 1; + +} +close(INSTALLCHECK); + +my %nproblems; +for my $package ( sort keys %problems ) { + $problems{$package} = join( ', ', sort( keys %{ $problems{$package} } ) ); +} + +open( PROBLEMS, "problems" ); +while () { + chomp; + if (m,^$project/$repo/$arch/([^:]*):\s*(.*)$,) { + my $package = $1; + my $oproblem = $2; + my $nproblem = $problems{$package}; + if ( $oproblem && $nproblem && $oproblem eq $nproblem ) + { # one more rebuild won't help + delete $problems{$package}; + } + } +} +close(PROBLEMS); + +exit(0) if ( !%problems ); + +# check for succeeded packages +my $api = "/build/$project/_result?repository=$repo&arch=$arch&code=succeeded"; +for my $problem ( sort keys %problems ) { + $api .= "&package=" . uri_escape($problem); +} + +open( RESULT, "osc api '$api'|" ); +@result = ; +my $results = XMLin( join( '', @result ), ForceArray => ['status'] ); +close(RESULT); + +my @packages = @{ $results->{result}->{status} }; +exit(0) if ( !@packages ); + +open( PROBLEMS, ">>problems" ); +$api = "/build/$project?cmd=rebuild&repository=$repo&arch=$arch"; +for my $package (@packages) { + print "rebuild ", $package->{package}, ": ", + $problems{ $package->{package} }, "\n"; + $api .= "&package=" . uri_escape( $package->{package} ); + print PROBLEMS "$project/$repo/$arch/" + . $package->{package} . ": " + . $problems{ $package->{package} }, "\n"; +} + +system("osc api -X POST '$api'"); +close(PROBLEMS); + diff --git a/repo-checker.pl b/repo-checker.pl index bd2ecb46..dbb1800d 100644 --- a/repo-checker.pl +++ b/repo-checker.pl @@ -1,245 +1,145 @@ #! /usr/bin/perl -w -BEGIN { - unshift @INC, "/usr/lib/build/Build"; -} use File::Basename; use File::Temp qw/ tempdir /; use XML::Simple; use Data::Dumper; use Cwd; -use Rpm; -use Fcntl; use strict; +require CreatePackageDescr; + my $ret = 0; my $dir = shift @ARGV; my %toignore; my $repodir; while (@ARGV) { - my $switch = shift @ARGV; - if ($switch eq "-f") { - my $toignore = shift @ARGV; - open(TOIGNORE, $toignore) || die "can't open $toignore"; - while ( ) { - chomp; - $toignore{$_} = 1; + my $switch = shift @ARGV; + if ( $switch eq "-f" ) { + my $toignore = shift @ARGV; + open( TOIGNORE, $toignore ) || die "can't open $toignore"; + while () { + chomp; + $toignore{$_} = 1; + } + close(TOIGNORE); + } + elsif ( $switch eq "-r" ) { + $repodir = shift @ARGV; + } + else { + print "read the source luke: $switch ? \n"; + exit(1); } - close(TOIGNORE); - } elsif ($switch eq "-r") { - $repodir = shift @ARGV; - } else { - print "read the source luke: $switch ? \n"; - exit(1); - } } -for my $pdir (glob("$dir/*")) { - if (! -f "$pdir/rpmlint.log") { - print "Couldn't find a rpmlint.log in the build results in $pdir. This is mandatory\n"; - my $name = basename($pdir); - if ($name eq "rpm" || $name eq "rpm-python" || $name eq "popt" || $name eq "rpmlint" || $name eq "rpmlint-Factory" ) { - print "ignoring - whitelist\n"; - } else { - $ret = 1; +for my $pdir ( glob("$dir/*") ) { + if ( !-f "$pdir/rpmlint.log" ) { + print + "Couldn't find a rpmlint.log in the build results in $pdir. This is mandatory\n"; + my $name = basename($pdir); + if ( $name eq "rpm" + || $name eq "rpm-python" + || $name eq "popt" + || $name eq "rpmlint" + || $name eq "rpmlint-Factory" ) + { + print "ignoring - whitelist\n"; + } + else { + $ret = 1; + } } - } else { - open(GREP, "grep 'W:.*invalid-license ' $pdir/rpmlint.log |"); - while ( ) { - print "Found rpmlint warning: "; - print $_; - $ret = 1; + else { + open( GREP, "grep 'W:.*invalid-license ' $pdir/rpmlint.log |" ); + while () { + print "Found rpmlint warning: "; + print $_; + $ret = 1; + } } - } } my %targets; -my %cache; -foreach my $file (glob("$repodir/.cache/*")) { - if ($file =~ m,/(\d+)\.(\d+)-([^/]*)$,) { - $cache{"$1.$2"} = $3; - } +sub write_package($$) { + my $ignore = shift; + my $package = shift; + + my $out = CreatePackageDescr::package_snippet($package); + + my $name = basename($package); + $name =~ s,^[^-]*-(.*)\.rpm,$1,; + + if ( $ignore == 1 && defined $toignore{$name} ) { + return; + } + + print PACKAGES $out; + return $name; } -sub write_package($$) -{ - my $ignore = shift; - my $package = shift; - - # RPMTAG_FILEMODES = 1030, /* h[] */ - # RPMTAG_FILEFLAGS = 1037, /* i[] */ - # RPMTAG_FILEUSERNAME = 1039, /* s[] */ - # RPMTAG_FILEGROUPNAME = 1040, /* s[] */ - - my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, - $atime,$mtime,$ctime,$blksize,$blocks); - - # use cache - if ($ignore == 1) { - ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, - $atime,$mtime,$ctime,$blksize,$blocks) = stat($package); - if ($cache{"$mtime.$ino"}) { - my $name = $cache{"$mtime.$ino"}; - if (defined $toignore{$name}) { - return; - } - open(C, "$repodir/.cache/$mtime.$ino-$name") || die "no cache for $package"; - while ( ) { - print PACKAGES $_; - } - close(C); - return; - } - } - - my %qq = Build::Rpm::rpmq("$package", qw{NAME VERSION RELEASE ARCH OLDFILENAMES DIRNAMES BASENAMES DIRINDEXES 1030 1037 1039 1040 - 1047 1112 1113 1049 1048 1050 1090 1114 1115 1054 1053 1055 1036 - }); - - my $name = $qq{'NAME'}[0]; - if ($ignore == 1 && defined $toignore{$name}) { - return; - } - - if ($ignore == 0) { - $targets{$name} = 1; - } - - Build::Rpm::add_flagsvers(\%qq, 1049, 1048, 1050); # requires - Build::Rpm::add_flagsvers(\%qq, 1047, 1112, 1113); # provides - Build::Rpm::add_flagsvers(\%qq, 1090, 1114, 1115); # obsoletes - Build::Rpm::add_flagsvers(\%qq, 1054, 1053, 1055); # conflicts - - my $out = ''; - $out .= sprintf("=Pkg: %s %s %s %s\n", $name, $qq{'VERSION'}[0], $qq{'RELEASE'}[0], $qq{'ARCH'}[0]); - $out .= "+Flx:\n"; - my @modes = @{$qq{1030} || []}; - my @basenames = @{$qq{BASENAMES} || []}; - my @dirs = @{$qq{DIRNAMES} || []}; - my @dirindexes = @{$qq{DIRINDEXES} || []}; - my @users = @{$qq{1039} || []}; - my @groups = @{$qq{1040} || []}; - my @flags = @{$qq{1037} || []}; - my @linktos = @{$qq{1036} || []}; - - my @xprvs; - - foreach my $bname (@basenames) { - my $mode = shift @modes; - my $di = shift @dirindexes; - my $user = shift @users; - my $group = shift @groups; - my $flag = shift @flags; - my $linkto = shift @linktos; - - my $filename = $dirs[$di] . $bname; - my $fs = $filename; - if (Fcntl::S_ISLNK($mode)) { - $fs = "$filename -> $linkto"; - } - $out .= sprintf "%o %o %s:%s %s\n", $mode, $flag, $user, $group, $fs; - if ( $filename =~ /^\/etc\// || $filename =~ /bin\// || $filename eq "/usr/lib/sendmail" ) { - push @xprvs, $filename; - } - - } - $out .= "-Flx:\n"; - $out .= "+Prv:\n"; - foreach my $prv (@{$qq{1047} || []}) { - $out .= "$prv\n"; - } - foreach my $prv (@xprvs) { - $out .= "$prv\n"; - } - $out .= "-Prv:\n"; - $out .= "+Con:\n"; - foreach my $prv (@{$qq{1054} || []}) { - $out .= "$prv\n"; - } - $out .= "-Con:\n"; - $out .= "+Req:\n"; - foreach my $prv (@{$qq{1049} || []}) { - next if ($prv =~ m/^rpmlib/); - next if ($name eq "libqmmp0-plugin-mplayer" && $prv eq "/usr/bin/mplayer"); - next if ($prv eq "this-is-only-for-build-envs"); - next if ($name eq "installation-images-debuginfodeps" && $prv =~ m/debuginfo.build/); - next if ($name eq "installation-images-debuginfodeps-openSUSE" && $prv =~ m/debuginfo.build/); - $out .= "$prv\n"; - } - $out .= "-Req:\n"; - $out .= "+Obs:\n"; - foreach my $prv (@{$qq{1090} || []}) { - $out .= "$prv\n"; - } - $out .= "-Obs:\n"; - - if ($ignore == 1) { - mkdir("$repodir/.cache"); - open(C, '>', "$repodir/.cache/$mtime.$ino-$name") || die "no writeable cache for $package"; - print C $out; - close(C); - } - - print PACKAGES $out; -} - -my @rpms = glob("$repodir/*.rpm"); +my @rpms = glob("$repodir/*.rpm"); my $pfile = $ENV{'HOME'} . "/packages"; -open(PACKAGES, ">", $pfile) || die 'can not open'; +open( PACKAGES, ">", $pfile ) || die 'can not open'; print PACKAGES "=Ver: 2.0\n"; foreach my $package (@rpms) { - write_package(1, $package); + write_package( 1, $package ); } @rpms = glob("$dir/*/*.rpm"); foreach my $package (@rpms) { - write_package(0, $package); + my $name = write_package( 0, $package ); + $targets{$name} = 1; } close(PACKAGES); #print "calling installcheck\n"; #print Dumper(%targets); -open(INSTALL, "/usr/bin/installcheck x86_64 $pfile 2>&1|") || die 'exec installcheck'; -while ( ) { +open( INSTALL, "/usr/bin/installcheck x86_64 $pfile 2>&1|" ) + || die 'exec installcheck'; +while () { chomp; next if (m/unknown line:.*Flx/); - if ($_ =~ m/can't install (.*)-([^-]*)-[^-\.]/) { -# print "CI $1\n"; - if (defined $targets{$1}) { - print "$_\n"; - while ( ) { - last if (m/^can't install /); - print "$_"; - } - $ret = 1; - last; + if ( $_ =~ m/can't install (.*)-([^-]*)-[^-\.]/ ) { + + # print "CI $1\n"; + if ( defined $targets{$1} ) { + print "$_\n"; + while () { + last if (m/^can't install /); + print "$_"; + } + $ret = 1; + last; } } } close(INSTALL); #print "checking file conflicts\n"; -my $cmd = sprintf("perl %s/findfileconflicts $pfile", dirname($0)); -open(INSTALL, "$cmd |") || die 'exec fileconflicts'; +my $cmd = sprintf( "perl %s/findfileconflicts $pfile", dirname($0) ); +open( INSTALL, "$cmd |" ) || die 'exec fileconflicts'; my $inc = 0; -while ( ) { +while () { chomp; + #print STDERR "$_\n"; - if ($_ =~ m/found conflict of (.*)-[^-]*-[^-]* with (.*)-[^-]*-[^-]*:/) { + if ( $_ =~ m/found conflict of (.*)-[^-]*-[^-]* with (.*)-[^-]*-[^-]*:/ ) { $inc = 0; - #print STDERR "F $1 $2\n"; - if (defined $targets{$1} || defined $targets{$2}) { - $inc = 1; - $ret = 1; + + #print STDERR "F $1 $2\n"; + if ( defined $targets{$1} || defined $targets{$2} ) { + $inc = 1; + $ret = 1; } } if ($inc) { - print "$_\n"; + print "$_\n"; } } close(INSTALL);