yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1855 from coolo/more_rebuildpacs

Browse Source 
Fix rebuildpacs to trigger rpmlint-mini on polkit changes
This commit is contained in:
Stephan Kulow 2019-02-07 11:29:40 +01:00 committed by GitHub
commit 6d706a6a4b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 174 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

310
rebuildpacs.pl
View File

@ -23,16 +23,17 @@ sub find_source_container($) {
my $pkg = shift; my $pkg = shift;
for my $repodir (@repodirs) { for my $repodir (@repodirs) {
my @rpms = glob("$repodir/*-$pkg.rpm"); my @rpms = glob("$repodir/*-$pkg.rpm");
for my $rpm (@rpms) { for my $rpm (@rpms) {
# 1123 == Disturl
my %qq = Build::Rpm::rpmq( $rpm, qw{NAME 1123} );
next if ( $qq{NAME}[0] ne $pkg );
my $distfile = basename( $qq{1123}[0] );
$distfile =~ s,^[^-]*-,,;
return $distfile; # 1123 == Disturl
} my %qq = Build::Rpm::rpmq( $rpm, qw{NAME 1123} );
next if ( $qq{NAME}[0] ne $pkg );
my $distfile = basename( $qq{1123}[0] );
$distfile =~ s,^[^-]*-,,;
return $distfile;
}
} }
} }
@ -47,29 +48,29 @@ my $arch = $ARGV[2] || "x86_64";
my %leafed; my %leafed;
sub read_plain_index($) { sub read_plain_index($) {
my $file = shift; my $file = shift;
my %ret; my %ret;
open(FILE, $file) || return \%ret; open( FILE, $file ) || return \%ret;
while ( <FILE> ) { while (<FILE>) {
if (m/^(.*):(.*)/) { if (m/^(.*):(.*)/) {
$ret{$1} = $2; $ret{$1} = $2;
}
} }
} close(FILE);
close(FILE); return \%ret;
return \%ret;
} }
sub write_plain_index($$) { sub write_plain_index($$) {
my $file = shift; my $file = shift;
my $hash = shift; my $hash = shift;
open(FILE, ">$file") || die "can't write to $file"; open( FILE, ">$file" ) || die "can't write to $file";
for my $key (sort keys %{$hash}) { for my $key ( sort keys %{$hash} ) {
print FILE "$key:" . $hash->{$key} . "\n"; print FILE "$key:" . $hash->{$key} . "\n";
} }
close(FILE); close(FILE);
} }
# defines packages that need to be triggered too # defines packages that need to be triggered too
@ -82,25 +83,27 @@ my %parents = (
kdebase4-openSUSE kde-branding-openSUSE kdebase4-openSUSE kde-branding-openSUSE
bundle-lang-kde bundle-lang-common bundle-lang-kde bundle-lang-common
openSUSE-images installation-images-openSUSE) openSUSE-images installation-images-openSUSE)
], ],
"kdebase4-openSUSE" => [qw(bundle-lang-kde)], "kdebase4-openSUSE" => [qw(bundle-lang-kde)],
"kernel-source" => [qw(perf)], "kernel-source" => [qw(perf)],
); );
# for subsets (staging projects) we need to remember which are ignored # for subsets (staging projects) we need to remember which are ignored
my %ignored; my %ignored;
sub check_leaf_package($$) { sub check_leaf_package($$) {
my $package = shift; my $package = shift;
my $rebuildhash = shift; my $rebuildhash = shift;
if (system("osc api /source/$project/$package/_meta > /dev/null 2>&1 ")) { if ( system("osc api /source/$project/$package/_meta > /dev/null 2>&1 ") ) {
$ignored{$package} = 1; $ignored{$package} = 1;
return; return;
} }
my @lines = (); my @lines = ();
open( OSC, "osc api /build/$project/$repo/$arch/$package/_buildinfo?internal=1|" ); open( OSC,
"osc api /build/$project/$repo/$arch/$package/_buildinfo|" );
while (<OSC>) { while (<OSC>) {
chomp; chomp;
if (m/<subpack>(.*)</) { if (m/<subpack>(.*)</) {
@ -109,11 +112,12 @@ sub check_leaf_package($$) {
if (m/bdep name="([^"]*)"/) { if (m/bdep name="([^"]*)"/) {
my $parent = $leafed{$1}; my $parent = $leafed{$1};
if ( $parent && $parent ne "rpmlint-mini" ) { if ( $parent && $parent ne "rpmlint-mini" ) {
# I dislike grep
unless (grep { $_ eq $package } @{$parents{$parent}}) { # I dislike grep
print "ADD $package to PARENT $parent!!\n"; unless ( grep { $_ eq $package } @{ $parents{$parent} } ) {
} print "ADD $package to PARENT $parent!!\n";
next; }
next;
} }
} }
else { else {
@ -125,111 +129,116 @@ sub check_leaf_package($$) {
close(OSC); close(OSC);
my $ctx = Digest::MD5->new; my $ctx = Digest::MD5->new;
for my $line ( sort @lines ) { for my $line ( sort @lines ) {
$ctx->add($line); $ctx->add($line);
} }
my $rebuilds = read_plain_index("buildinfos"); my $rebuilds = read_plain_index("buildinfos");
my $newmd5 = $ctx->hexdigest; my $newmd5 = $ctx->hexdigest;
if ($rebuilds->{"$project/$repo/$arch/$package"} ne $newmd5) { if ( $rebuilds->{"$project/$repo/$arch/$package"} ne $newmd5 ) {
$rebuildhash->{$package} = 1; $rebuildhash->{$package} = 1;
for my $child (@{$parents{$package}}) { for my $child ( @{ $parents{$package} } ) {
$rebuildhash->{$child} = 1; $rebuildhash->{$child} = 1;
} }
$rebuilds->{"$project/$repo/$arch/$package"} = $newmd5; $rebuilds->{"$project/$repo/$arch/$package"} = $newmd5;
write_plain_index("buildinfos", $rebuilds); write_plain_index( "buildinfos", $rebuilds );
} }
} }
my %torebuild; my %torebuild;
check_leaf_package("rpmlint", \%torebuild); check_leaf_package( "rpmlint-mini", \%torebuild );
check_leaf_package("rpmlint-mini", \%torebuild); check_leaf_package( "rpmlint", \%torebuild );
check_leaf_package("branding-openSUSE", \%torebuild); check_leaf_package( "branding-openSUSE", \%torebuild );
check_leaf_package("glib2-branding-openSUSE", \%torebuild); check_leaf_package( "glib2-branding-openSUSE", \%torebuild );
check_leaf_package("PackageKit-branding-openSUSE", \%torebuild); check_leaf_package( "PackageKit-branding-openSUSE", \%torebuild );
check_leaf_package("kiwi-config-openSUSE", \%torebuild); check_leaf_package( "kiwi-config-openSUSE", \%torebuild );
check_leaf_package("xfce4-branding-openSUSE", \%torebuild); check_leaf_package( "xfce4-branding-openSUSE", \%torebuild );
check_leaf_package("kdebase4-openSUSE", \%torebuild); check_leaf_package( "kdebase4-openSUSE", \%torebuild );
check_leaf_package("kde-branding-openSUSE", \%torebuild); check_leaf_package( "kde-branding-openSUSE", \%torebuild );
check_leaf_package("bundle-lang-common", \%torebuild); check_leaf_package( "bundle-lang-common", \%torebuild );
check_leaf_package("bundle-lang-kde", \%torebuild); check_leaf_package( "bundle-lang-kde", \%torebuild );
check_leaf_package("bundle-lang-gnome", \%torebuild); check_leaf_package( "bundle-lang-gnome", \%torebuild );
check_leaf_package("installation-images-openSUSE", \%torebuild); check_leaf_package( "installation-images-openSUSE", \%torebuild );
check_leaf_package("openSUSE-images", \%torebuild);
if (%torebuild) { if (%torebuild) {
my $api = "/build/$project?cmd=rebuild&repository=$repo&arch=$arch"; my $api = "/build/$project?cmd=rebuild&repository=$repo&arch=$arch";
for my $package (sort keys %torebuild) { for my $package ( sort keys %torebuild ) {
next if (defined $ignored{$package}); next if ( defined $ignored{$package} );
last if (length($api) > 32767); last if ( length($api) > 32767 );
$api .= "&package=" . uri_escape( $package ); $api .= "&package=" . uri_escape($package);
} }
system("osc api -X POST '$api'"); system("osc api -X POST '$api'");
} }
my $pfile = tempdir(CLEANUP => 1) . "/packages"; # the filename is important ;( my $pfile =
tempdir( CLEANUP => 1 ) . "/packages"; # the filename is important ;(
sub mirror_repo($$$) { sub mirror_repo($$$) {
my $project = shift; my $project = shift;
my $repo = shift; my $repo = shift;
my $arch = shift; my $arch = shift;
# Old and new in single directory, but never deployed together. # Old and new in single directory, but never deployed together.
my $repodir = ($ENV{XDG_CACHE_HOME}||$ENV{HOME}."/.cache")."/openSUSE-release-tools/repository-meta/repo-$project-$repo-$arch"; my $repodir = ( $ENV{XDG_CACHE_HOME} || $ENV{HOME} . "/.cache" )
mkdir($repodir); . "/openSUSE-release-tools/repository-meta/repo-$project-$repo-$arch";
mkdir($repodir);
system( system(
"$script_dir/bs_mirrorfull --nodebug https://api.opensuse.org/public/build/$project/$repo/$arch/ $repodir" "$script_dir/bs_mirrorfull --nodebug https://api.opensuse.org/public/build/$project/$repo/$arch/ $repodir"
); );
return $repodir; return $repodir;
} }
sub find_package_in_project($) { sub find_package_in_project($) {
my $project = shift; my $project = shift;
open(OSC, "osc api /source/$project?expand=1 |"); open( OSC, "osc api /source/$project?expand=1 |" );
my $xml = XMLin(join('', <OSC>), ForceArray => 1); my $xml = XMLin( join( '', <OSC> ), ForceArray => 1 );
close(OSC); close(OSC);
my @packs = keys %{$xml->{entry}}; my @packs = keys %{ $xml->{entry} };
return shift @packs; return shift @packs;
} }
# find a random package # find a random package
sub get_paths($$$) { sub get_paths($$$) {
my $project = shift; my $project = shift;
my $repo = shift; my $repo = shift;
my $arch = shift; my $arch = shift;
my $package = find_package_in_project($project); my $package = find_package_in_project($project);
open(OSC, "osc api /build/$project/$repo/$arch/$package/_buildinfo|"); open( OSC, "osc api /build/$project/$repo/$arch/$package/_buildinfo|" );
my $xml = join('', <OSC>); my $xml = join( '', <OSC> );
if ($xml !~ m/^</) { if ( $xml !~ m/^</ ) {
die "failed to open /build/$project/$repo/$arch/$package/_buildinfo"; die "failed to open /build/$project/$repo/$arch/$package/_buildinfo";
} }
$xml = XMLin($xml, ForceArray => 1); $xml = XMLin( $xml, ForceArray => 1 );
close(OSC); close(OSC);
return $xml->{path}; return $xml->{path};
} }
my $paths = get_paths($project, $repo, $arch); my $paths = get_paths( $project, $repo, $arch );
my @rpms; my @rpms;
for my $path (@$paths) { for my $path (@$paths) {
# openSUSE:Factory/ports is in the paths, but not a repo # openSUSE:Factory/ports is in the paths, but not a repo
if (system("osc api /build/$path->{'project'}/$path->{'repository'}/$arch > /dev/null 2>&1 ")) { if (
next; system(
} "osc api /build/$path->{'project'}/$path->{'repository'}/$arch > /dev/null 2>&1 "
)
)
{
next;
}
my $repodir = mirror_repo($path->{'project'}, $path->{'repository'}, $arch); my $repodir =
push(@repodirs, $repodir); mirror_repo( $path->{'project'}, $path->{'repository'}, $arch );
push(@rpms, glob("$repodir/*.rpm")); push( @repodirs, $repodir );
push( @rpms, glob("$repodir/*.rpm") );
} }
open( PACKAGES, ">", $pfile ) || die "can not open $pfile"; open( PACKAGES, ">", $pfile ) || die "can not open $pfile";
@ -247,8 +256,8 @@ close(PACKAGES);
# read the problems out of installcheck # read the problems out of installcheck
my $rpmarch = $arch; my $rpmarch = $arch;
$rpmarch = "armv7hl" if ($arch eq "armv7l"); $rpmarch = "armv7hl" if ( $arch eq "armv7l" );
$rpmarch = "armv6hl" if ($arch eq "armv6l"); $rpmarch = "armv6hl" if ( $arch eq "armv6l" );
open( INSTALLCHECK, "/usr/bin/installcheck $rpmarch $pfile|" ); open( INSTALLCHECK, "/usr/bin/installcheck $rpmarch $pfile|" );
while (<INSTALLCHECK>) { while (<INSTALLCHECK>) {
@ -274,21 +283,49 @@ while (<INSTALLCHECK>) {
s,(needed by [^ ]*)\-[^-]*\-[^-]*\.($rpmarch|noarch)$,$1,; s,(needed by [^ ]*)\-[^-]*\-[^-]*\.($rpmarch|noarch)$,$1,;
s,^\s*,,; s,^\s*,,;
# patterns are too spammy and rebuilding doesn't help # patterns are too spammy and rebuilding doesn't help
next if (grep { $_ eq $cproblem } qw( next
patterns-openSUSE patterns-base patterns-haskell if (
patterns-mate patterns-media patterns-yast grep { $_ eq $cproblem }
installation-images:Kubic fftw3:gnu-openmpi-hpc hdf5:mvapich2 qw(
hdf5:openmpi hdf5:serial scalapack:gnu-mvapich2-hpc fftw3:gnu-openmpi-hpc
scalapack:gnu-openmpi-hpc python-numpy:gnu-hpc hdf5:gnu-hpc
petsc:serial netcdf:serial netcdf:openmpi netcdf:gnu-hpc hdf5:gnu-mpich-hpc
netcdf:gnu-openmpi-hpc netcdf:gnu-mvapich2-hpc)); hdf5:gnu-mvapich2-hpc
hdf5:gnu-openmpi-hpc
hdf5:gnu-openmpi2-hpc
hdf5:gnu-openmpi3-hpc
hdf5:mvapich2
hdf5:openmpi
hdf5:serial
installation-images:Kubic
metis:gnu-hpc
netcdf:gnu-hpc
netcdf:gnu-mvapich2-hpc
netcdf:gnu-openmpi-hpc
netcdf:openmpi
netcdf:serial
patterns-base
patterns-haskell
patterns-mate
patterns-media
patterns-openSUSE
patterns-yast
petsc:serial
python-numpy:gnu-hpc
scalapack:gnu-mvapich2-hpc
scalapack:gnu-openmpi-hpc
warewulf:modules
python-scipy:gnu-hpc
)
);
$problems{$cproblem}->{$_} = 1; $problems{$cproblem}->{$_} = 1;
} }
close(INSTALLCHECK); close(INSTALLCHECK);
unlink($pfile); unlink($pfile);
rmdir(dirname($pfile)); rmdir( dirname($pfile) );
for my $package ( sort keys %problems ) { for my $package ( sort keys %problems ) {
$problems{$package} = join( ', ', sort( keys %{ $problems{$package} } ) ); $problems{$package} = join( ', ', sort( keys %{ $problems{$package} } ) );
@ -301,8 +338,9 @@ open( PROBLEMS, "problems" );
while (<PROBLEMS>) { while (<PROBLEMS>) {
chomp; chomp;
if (m,^$project/$repo/$arch/([^:]*):\s*(.*)$,) { if (m,^$project/$repo/$arch/([^:]*):\s*(.*)$,) {
my $package = $1; my $package = $1;
my $oproblem = $2; my $oproblem = $2;
# remember old problems for current project/repo # remember old problems for current project/repo
$oproblems{$package} = $oproblem; $oproblems{$package} = $oproblem;
} }
@ -323,46 +361,46 @@ open( RESULT, "osc api '$api'|" );
my $results = XMLin( join( '', @result ), ForceArray => ['status'] ); my $results = XMLin( join( '', @result ), ForceArray => ['status'] );
close(RESULT); close(RESULT);
my @packages = @{ $results->{result}->{status} }; my @packages = @{ $results->{result}->{status} };
my $rebuildit = 0; my $rebuildit = 0;
$api = "/build/$project?cmd=rebuild&repository=$repo&arch=$arch"; $api = "/build/$project?cmd=rebuild&repository=$repo&arch=$arch";
for my $package (@packages) { for my $package (@packages) {
$package = $package->{package}; $package = $package->{package};
last if (length($api) > 32767); last if ( length($api) > 32767 );
if (!$problems{$package}) { if ( !$problems{$package} ) {
# it can go
delete $oproblems{$package}; # it can go
next; delete $oproblems{$package};
next;
} }
my $oproblem = $oproblems{$package} || ''; my $oproblem = $oproblems{$package} || '';
if ($problems{$package} eq $oproblem) { if ( $problems{$package} eq $oproblem ) {
# rebuild won't help # rebuild won't help
next; next;
} }
$rebuildit = 1; $rebuildit = 1;
print "rebuild ", $package, ": ", print "rebuild ", $package, ": ", $problems{$package}, "\n";
$problems{ $package }, "\n"; $api .= "&package=" . uri_escape($package);
$api .= "&package=" . uri_escape( $package );
$oproblems{$package} = $problems{$package}; $oproblems{$package} = $problems{$package};
} }
open( PROBLEMS, ">problems" ); open( PROBLEMS, ">problems" );
# write all lines for other projects/repos as they are # write all lines for other projects/repos as they are
foreach (@other_problems) { foreach (@other_problems) {
print PROBLEMS $_, "\n"; print PROBLEMS $_, "\n";
} }
for my $package (keys %oproblems) { for my $package ( keys %oproblems ) {
print PROBLEMS "$project/$repo/$arch/" print PROBLEMS "$project/$repo/$arch/" . $package . ": "
. $package . ": " . $oproblems{$package}, "\n";
. $oproblems{ $package }, "\n";
} }
close(PROBLEMS); close(PROBLEMS);
if ($rebuildit) { if ($rebuildit) {
print "API '$api'\n"; print "API '$api'\n";
system("osc api -X POST '$api'"); system("osc api -X POST '$api'");
} }