Merge pull request #128 from openSUSE/coolo_rebuildpacs

add a small tool that rebuilds broken packages
This commit is contained in:
Tomáš Chvátal 2014-04-10 07:34:34 +02:00
commit e17f5283c6
4 changed files with 353 additions and 189 deletions

128
CreatePackageDescr.pm Normal file
View File

@ -0,0 +1,128 @@
package CreatePackageDescr;
BEGIN {
unshift @INC, "/usr/lib/build/Build";
}
use File::Basename;
use Rpm;
use Fcntl;
sub package_snippet($) {
my $package = shift;
my $cachedir = dirname($package) . "/.cache/";
my $cachefile = $cachedir . basename($package);
my $out = '';
if ( -f $cachefile ) {
open( C, $cachefile ) || die "no cache for $package";
while (<C>) {
$out .= $_;
}
close(C);
return $out;
}
# RPMTAG_FILEMODES = 1030, /* h[] */
# RPMTAG_FILEFLAGS = 1037, /* i[] */
# RPMTAG_FILEUSERNAME = 1039, /* s[] */
# RPMTAG_FILEGROUPNAME = 1040, /* s[] */
my %qq = Build::Rpm::rpmq(
$package,
qw{NAME VERSION RELEASE ARCH OLDFILENAMES DIRNAMES BASENAMES DIRINDEXES 1030 1037 1039 1040
1047 1112 1113 1049 1048 1050 1090 1114 1115 1054 1053 1055 1036
}
);
my $name = $qq{'NAME'}[0];
Build::Rpm::add_flagsvers( \%qq, 1049, 1048, 1050 ); # requires
Build::Rpm::add_flagsvers( \%qq, 1047, 1112, 1113 ); # provides
Build::Rpm::add_flagsvers( \%qq, 1090, 1114, 1115 ); # obsoletes
Build::Rpm::add_flagsvers( \%qq, 1054, 1053, 1055 ); # conflicts
$out .= sprintf( "=Pkg: %s %s %s %s\n",
$name, $qq{'VERSION'}[0], $qq{'RELEASE'}[0], $qq{'ARCH'}[0] );
$out .= "+Flx:\n";
my @modes = @{ $qq{1030} || [] };
my @basenames = @{ $qq{BASENAMES} || [] };
my @dirs = @{ $qq{DIRNAMES} || [] };
my @dirindexes = @{ $qq{DIRINDEXES} || [] };
my @users = @{ $qq{1039} || [] };
my @groups = @{ $qq{1040} || [] };
my @flags = @{ $qq{1037} || [] };
my @linktos = @{ $qq{1036} || [] };
my @xprvs;
foreach my $bname (@basenames) {
my $mode = shift @modes;
my $di = shift @dirindexes;
my $user = shift @users;
my $group = shift @groups;
my $flag = shift @flags;
my $linkto = shift @linktos;
my $filename = $dirs[$di] . $bname;
my $fs = $filename;
if ( Fcntl::S_ISLNK($mode) ) {
$fs = "$filename -> $linkto";
}
$out .= sprintf "%o %o %s:%s %s\n", $mode, $flag, $user, $group, $fs;
if ( $filename =~ /^\/etc\//
|| $filename =~ /bin\//
|| $filename eq "/usr/lib/sendmail" )
{
push @xprvs, $filename;
}
}
$out .= "-Flx:\n";
$out .= "+Prv:\n";
foreach my $prv ( @{ $qq{1047} || [] } ) {
$out .= "$prv\n";
}
foreach my $prv (@xprvs) {
$out .= "$prv\n";
}
$out .= "-Prv:\n";
$out .= "+Con:\n";
foreach my $prv ( @{ $qq{1054} || [] } ) {
$out .= "$prv\n";
}
$out .= "-Con:\n";
$out .= "+Req:\n";
foreach my $prv ( @{ $qq{1049} || [] } ) {
next if ( $prv =~ m/^rpmlib/ );
next
if ( $name eq "libqmmp0-plugin-mplayer"
&& $prv eq "/usr/bin/mplayer" );
next if ( $prv eq "this-is-only-for-build-envs" );
next
if ( $name eq "installation-images-debuginfodeps"
&& $prv =~ m/debuginfo.build/ );
next
if ( $name eq "installation-images-debuginfodeps-openSUSE"
&& $prv =~ m/debuginfo.build/ );
$out .= "$prv\n";
}
$out .= "-Req:\n";
$out .= "+Obs:\n";
foreach my $prv ( @{ $qq{1090} || [] } ) {
$out .= "$prv\n";
}
$out .= "-Obs:\n";
mkdir($cachedir);
open( C, '>', $cachefile ) || die "can't write $cachefile";
print C $out;
close(C);
return $out;
}
1;

View File

@ -34,7 +34,7 @@ die("uasge: bs_mirrorfull url dir\n") unless @ARGV == 2;
my ($url, $dir) = @ARGV;
$url =~ s/\/$//;
print "receiving tree state\n";
#print "receiving tree state\n";
my $bvl = BSRPC::rpc("$url/_repository", $BSXML::binaryversionlist, "view=binaryversions", "nometa=1");
my @localbins = grep {/^[0-9a-f]{32}-.+\.rpm$/} ls($dir);
my %localbins = map {$_ => 1} @localbins;
@ -75,7 +75,7 @@ if (@todownload) {
};
BSRPC::rpc($param, undef, 'view=cpioheaders', @args);
$did += @fetch;
print "$did/$todo\n";
#print "$did/$todo\n";
}
}
print "done, we now have ".(keys %remotebins)." packages.\n";
#print "done, we now have ".(keys %remotebins)." packages.\n";

136
rebuildpacs.pl Executable file
View File

@ -0,0 +1,136 @@
#! /usr/bin/perl
use Data::Dumper;
use XML::Simple;
use URI::Escape;
use File::Basename;
use File::Temp qw/tempdir/;
require CreatePackageDescr;
my $repodir;
sub find_source_container($) {
my $pkg = shift;
my @rpms = glob("$repodir/*-$pkg.rpm");
for my $rpm (@rpms) {
# 1123 == Disturl
my %qq = Build::Rpm::rpmq( $rpm, qw{NAME 1123} );
next if ( $qq{NAME}[0] ne $pkg );
my $distfile = basename( $qq{1123}[0] );
$distfile =~ s,^[^-]*-,,;
return $distfile;
}
}
my $followup = 0;
my $cproblem = '';
my %problems;
my $project = $ARGV[0] || "openSUSE:Factory";
my $repo = $ARGV[1] || "standard";
my $arch = $ARGV[2] || "x86_64";
$repodir = "/var/cache/repo-checker/repo-openSUSE:Factory-$repo-$arch";
mkdir($repodir);
my $pfile = tempdir() . "/packages"; # the filename is important ;(
system(
"./bs_mirrorfull --nodebug https://build.opensuse.org/build/$project/$repo/$arch/ $repodir"
);
my @rpms = glob("$repodir/*.rpm");
open( PACKAGES, ">", $pfile ) || die "can not open $pfile";
print PACKAGES "=Ver: 2.0\n";
foreach my $package (@rpms) {
my $out = CreatePackageDescr::package_snippet($package);
print PACKAGES CreatePackageDescr::package_snippet($package);
}
close(PACKAGES);
# read the problems out of installcheck
open( INSTALLCHECK, "installcheck $arch $pfile|" );
while (<INSTALLCHECK>) {
chomp;
if (m/^can't install (.*)\-[^-]*\-[^-]*\.($arch|noarch):/) {
$cproblem = $1;
$cproblem =~ s/kmp-([^-]*)/kmp-default/;
$cproblem = find_source_container($cproblem);
$followup = 0;
next;
}
$followup = 1 if ( $_ =~ m/none of the providers can be installed/ );
# not interesting for me
next if (m/ \(we have /);
next if ($followup);
# very thin ice here
s,\(\)\(64bit\),,;
s,(needed by [^ ]*)\-[^-]*\-[^-]*\.($arch|noarch)$,$1,;
s,^\s*,,;
$problems{$cproblem}->{$_} = 1;
}
close(INSTALLCHECK);
my %nproblems;
for my $package ( sort keys %problems ) {
$problems{$package} = join( ', ', sort( keys %{ $problems{$package} } ) );
}
open( PROBLEMS, "problems" );
while (<PROBLEMS>) {
chomp;
if (m,^$project/$repo/$arch/([^:]*):\s*(.*)$,) {
my $package = $1;
my $oproblem = $2;
my $nproblem = $problems{$package};
if ( $oproblem && $nproblem && $oproblem eq $nproblem )
{ # one more rebuild won't help
delete $problems{$package};
}
}
}
close(PROBLEMS);
exit(0) if ( !%problems );
# check for succeeded packages
my $api = "/build/$project/_result?repository=$repo&arch=$arch&code=succeeded";
for my $problem ( sort keys %problems ) {
$api .= "&package=" . uri_escape($problem);
}
open( RESULT, "osc api '$api'|" );
@result = <RESULT>;
my $results = XMLin( join( '', @result ), ForceArray => ['status'] );
close(RESULT);
my @packages = @{ $results->{result}->{status} };
exit(0) if ( !@packages );
open( PROBLEMS, ">>problems" );
$api = "/build/$project?cmd=rebuild&repository=$repo&arch=$arch";
for my $package (@packages) {
print "rebuild ", $package->{package}, ": ",
$problems{ $package->{package} }, "\n";
$api .= "&package=" . uri_escape( $package->{package} );
print PROBLEMS "$project/$repo/$arch/"
. $package->{package} . ": "
. $problems{ $package->{package} }, "\n";
}
system("osc api -X POST '$api'");
close(PROBLEMS);

View File

@ -1,245 +1,145 @@
#! /usr/bin/perl -w
BEGIN {
unshift @INC, "/usr/lib/build/Build";
}
use File::Basename;
use File::Temp qw/ tempdir /;
use XML::Simple;
use Data::Dumper;
use Cwd;
use Rpm;
use Fcntl;
use strict;
require CreatePackageDescr;
my $ret = 0;
my $dir = shift @ARGV;
my %toignore;
my $repodir;
while (@ARGV) {
my $switch = shift @ARGV;
if ($switch eq "-f") {
my $toignore = shift @ARGV;
open(TOIGNORE, $toignore) || die "can't open $toignore";
while ( <TOIGNORE> ) {
chomp;
$toignore{$_} = 1;
my $switch = shift @ARGV;
if ( $switch eq "-f" ) {
my $toignore = shift @ARGV;
open( TOIGNORE, $toignore ) || die "can't open $toignore";
while (<TOIGNORE>) {
chomp;
$toignore{$_} = 1;
}
close(TOIGNORE);
}
elsif ( $switch eq "-r" ) {
$repodir = shift @ARGV;
}
else {
print "read the source luke: $switch ? \n";
exit(1);
}
close(TOIGNORE);
} elsif ($switch eq "-r") {
$repodir = shift @ARGV;
} else {
print "read the source luke: $switch ? \n";
exit(1);
}
}
for my $pdir (glob("$dir/*")) {
if (! -f "$pdir/rpmlint.log") {
print "Couldn't find a rpmlint.log in the build results in $pdir. This is mandatory\n";
my $name = basename($pdir);
if ($name eq "rpm" || $name eq "rpm-python" || $name eq "popt" || $name eq "rpmlint" || $name eq "rpmlint-Factory" ) {
print "ignoring - whitelist\n";
} else {
$ret = 1;
for my $pdir ( glob("$dir/*") ) {
if ( !-f "$pdir/rpmlint.log" ) {
print
"Couldn't find a rpmlint.log in the build results in $pdir. This is mandatory\n";
my $name = basename($pdir);
if ( $name eq "rpm"
|| $name eq "rpm-python"
|| $name eq "popt"
|| $name eq "rpmlint"
|| $name eq "rpmlint-Factory" )
{
print "ignoring - whitelist\n";
}
else {
$ret = 1;
}
}
} else {
open(GREP, "grep 'W:.*invalid-license ' $pdir/rpmlint.log |");
while ( <GREP> ) {
print "Found rpmlint warning: ";
print $_;
$ret = 1;
else {
open( GREP, "grep 'W:.*invalid-license ' $pdir/rpmlint.log |" );
while (<GREP>) {
print "Found rpmlint warning: ";
print $_;
$ret = 1;
}
}
}
}
my %targets;
my %cache;
foreach my $file (glob("$repodir/.cache/*")) {
if ($file =~ m,/(\d+)\.(\d+)-([^/]*)$,) {
$cache{"$1.$2"} = $3;
}
sub write_package($$) {
my $ignore = shift;
my $package = shift;
my $out = CreatePackageDescr::package_snippet($package);
my $name = basename($package);
$name =~ s,^[^-]*-(.*)\.rpm,$1,;
if ( $ignore == 1 && defined $toignore{$name} ) {
return;
}
print PACKAGES $out;
return $name;
}
sub write_package($$)
{
my $ignore = shift;
my $package = shift;
# RPMTAG_FILEMODES = 1030, /* h[] */
# RPMTAG_FILEFLAGS = 1037, /* i[] */
# RPMTAG_FILEUSERNAME = 1039, /* s[] */
# RPMTAG_FILEGROUPNAME = 1040, /* s[] */
my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
$atime,$mtime,$ctime,$blksize,$blocks);
# use cache
if ($ignore == 1) {
($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
$atime,$mtime,$ctime,$blksize,$blocks) = stat($package);
if ($cache{"$mtime.$ino"}) {
my $name = $cache{"$mtime.$ino"};
if (defined $toignore{$name}) {
return;
}
open(C, "$repodir/.cache/$mtime.$ino-$name") || die "no cache for $package";
while ( <C> ) {
print PACKAGES $_;
}
close(C);
return;
}
}
my %qq = Build::Rpm::rpmq("$package", qw{NAME VERSION RELEASE ARCH OLDFILENAMES DIRNAMES BASENAMES DIRINDEXES 1030 1037 1039 1040
1047 1112 1113 1049 1048 1050 1090 1114 1115 1054 1053 1055 1036
});
my $name = $qq{'NAME'}[0];
if ($ignore == 1 && defined $toignore{$name}) {
return;
}
if ($ignore == 0) {
$targets{$name} = 1;
}
Build::Rpm::add_flagsvers(\%qq, 1049, 1048, 1050); # requires
Build::Rpm::add_flagsvers(\%qq, 1047, 1112, 1113); # provides
Build::Rpm::add_flagsvers(\%qq, 1090, 1114, 1115); # obsoletes
Build::Rpm::add_flagsvers(\%qq, 1054, 1053, 1055); # conflicts
my $out = '';
$out .= sprintf("=Pkg: %s %s %s %s\n", $name, $qq{'VERSION'}[0], $qq{'RELEASE'}[0], $qq{'ARCH'}[0]);
$out .= "+Flx:\n";
my @modes = @{$qq{1030} || []};
my @basenames = @{$qq{BASENAMES} || []};
my @dirs = @{$qq{DIRNAMES} || []};
my @dirindexes = @{$qq{DIRINDEXES} || []};
my @users = @{$qq{1039} || []};
my @groups = @{$qq{1040} || []};
my @flags = @{$qq{1037} || []};
my @linktos = @{$qq{1036} || []};
my @xprvs;
foreach my $bname (@basenames) {
my $mode = shift @modes;
my $di = shift @dirindexes;
my $user = shift @users;
my $group = shift @groups;
my $flag = shift @flags;
my $linkto = shift @linktos;
my $filename = $dirs[$di] . $bname;
my $fs = $filename;
if (Fcntl::S_ISLNK($mode)) {
$fs = "$filename -> $linkto";
}
$out .= sprintf "%o %o %s:%s %s\n", $mode, $flag, $user, $group, $fs;
if ( $filename =~ /^\/etc\// || $filename =~ /bin\// || $filename eq "/usr/lib/sendmail" ) {
push @xprvs, $filename;
}
}
$out .= "-Flx:\n";
$out .= "+Prv:\n";
foreach my $prv (@{$qq{1047} || []}) {
$out .= "$prv\n";
}
foreach my $prv (@xprvs) {
$out .= "$prv\n";
}
$out .= "-Prv:\n";
$out .= "+Con:\n";
foreach my $prv (@{$qq{1054} || []}) {
$out .= "$prv\n";
}
$out .= "-Con:\n";
$out .= "+Req:\n";
foreach my $prv (@{$qq{1049} || []}) {
next if ($prv =~ m/^rpmlib/);
next if ($name eq "libqmmp0-plugin-mplayer" && $prv eq "/usr/bin/mplayer");
next if ($prv eq "this-is-only-for-build-envs");
next if ($name eq "installation-images-debuginfodeps" && $prv =~ m/debuginfo.build/);
next if ($name eq "installation-images-debuginfodeps-openSUSE" && $prv =~ m/debuginfo.build/);
$out .= "$prv\n";
}
$out .= "-Req:\n";
$out .= "+Obs:\n";
foreach my $prv (@{$qq{1090} || []}) {
$out .= "$prv\n";
}
$out .= "-Obs:\n";
if ($ignore == 1) {
mkdir("$repodir/.cache");
open(C, '>', "$repodir/.cache/$mtime.$ino-$name") || die "no writeable cache for $package";
print C $out;
close(C);
}
print PACKAGES $out;
}
my @rpms = glob("$repodir/*.rpm");
my @rpms = glob("$repodir/*.rpm");
my $pfile = $ENV{'HOME'} . "/packages";
open(PACKAGES, ">", $pfile) || die 'can not open';
open( PACKAGES, ">", $pfile ) || die 'can not open';
print PACKAGES "=Ver: 2.0\n";
foreach my $package (@rpms) {
write_package(1, $package);
write_package( 1, $package );
}
@rpms = glob("$dir/*/*.rpm");
foreach my $package (@rpms) {
write_package(0, $package);
my $name = write_package( 0, $package );
$targets{$name} = 1;
}
close(PACKAGES);
#print "calling installcheck\n";
#print Dumper(%targets);
open(INSTALL, "/usr/bin/installcheck x86_64 $pfile 2>&1|") || die 'exec installcheck';
while ( <INSTALL> ) {
open( INSTALL, "/usr/bin/installcheck x86_64 $pfile 2>&1|" )
|| die 'exec installcheck';
while (<INSTALL>) {
chomp;
next if (m/unknown line:.*Flx/);
if ($_ =~ m/can't install (.*)-([^-]*)-[^-\.]/) {
# print "CI $1\n";
if (defined $targets{$1}) {
print "$_\n";
while ( <INSTALL> ) {
last if (m/^can't install /);
print "$_";
}
$ret = 1;
last;
if ( $_ =~ m/can't install (.*)-([^-]*)-[^-\.]/ ) {
# print "CI $1\n";
if ( defined $targets{$1} ) {
print "$_\n";
while (<INSTALL>) {
last if (m/^can't install /);
print "$_";
}
$ret = 1;
last;
}
}
}
close(INSTALL);
#print "checking file conflicts\n";
my $cmd = sprintf("perl %s/findfileconflicts $pfile", dirname($0));
open(INSTALL, "$cmd |") || die 'exec fileconflicts';
my $cmd = sprintf( "perl %s/findfileconflicts $pfile", dirname($0) );
open( INSTALL, "$cmd |" ) || die 'exec fileconflicts';
my $inc = 0;
while ( <INSTALL> ) {
while (<INSTALL>) {
chomp;
#print STDERR "$_\n";
if ($_ =~ m/found conflict of (.*)-[^-]*-[^-]* with (.*)-[^-]*-[^-]*:/) {
if ( $_ =~ m/found conflict of (.*)-[^-]*-[^-]* with (.*)-[^-]*-[^-]*:/ ) {
$inc = 0;
#print STDERR "F $1 $2\n";
if (defined $targets{$1} || defined $targets{$2}) {
$inc = 1;
$ret = 1;
#print STDERR "F $1 $2\n";
if ( defined $targets{$1} || defined $targets{$2} ) {
$inc = 1;
$ret = 1;
}
}
if ($inc) {
print "$_\n";
print "$_\n";
}
}
close(INSTALL);