yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge from coolo to aplanas

Browse Source
This commit is contained in:
Alberto Planas 2013-08-06 15:54:40 +02:00
commit 810ab399d2
8 changed files with 146 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
findfileconflicts
View File

@ -77,6 +77,11 @@ while(<FL>) {
$fls = 0;
next;
}
my $lnk = '';
if (/^(12.*)( -> .*?)$/) {
$_ = $1;
$lnk = $2;
}
next unless /^(\d+ (\d+) \S+) (.*\/)(.*?)$/;
my $n = $dirs{$3};
if (!defined($n)) {
@ -84,11 +89,13 @@ while(<FL>) {
$dirs{$3} = $n;
$dirs[$n] = $3;
}
my $m = $modes{$1};
# ignore link targets of ghosts
$lnk = '' if $lnk && (oct($2) & 0100) != 0;
my $m = $modes{"$1$lnk"};
if (!defined($m)) {
$m = @modes;
$modes{$1} = $m;
$modes[$m] = $1;
$modes{"$1$lnk"} = $m;
$modes[$m] = "$1$lnk";
$modes_type[$m] = oct($1) & 07770000;
$modes_ghost[$m] = oct($2) & 0100;
}
@ -155,6 +162,7 @@ while(<FL>) {
}
close(FL) || die("close failed\n");
print "currently have ".@dirs." dirs and ".@modes." modes\n";
# connect dirs and add all dirs as files
print "connecting ".@dirs." directories\n";
@ -324,14 +332,14 @@ for my $tc (sort keys %tocheck) {
for my $f (@{$tocheck_files{$tc}}) {
my @pp = grep {s/^(?:\Q$p1\E|\Q$p2\E)\///} map {$_} @{$filesc{$f}};
next unless @pp;
# ignore if (all directories or all ghosts) and all same mode;
# ignore if (all directories or all ghosts or all links) and all same mode;
my %allm = map {$_ => 1} @pp;
my $info = '';
if (keys(%allm) == 1) {
my $m = (keys(%allm))[0];
# all modes/flags are the same
# no conflict if all dirs or all ghosts
next if $modes_type[$m] == 040000 || $modes_ghost[$m] == 0100;
# no conflict if all dirs or all ghosts or all links
next if $modes_type[$m] == 040000 || $modes_type[$m] == 0120000 || $modes_ghost[$m] == 0100;
} else {
# don't report mode mismatches for files/symlinks that are not ghosts
for my $m (keys %allm) {

52
generate-reminder.pl
View File

@ -27,14 +27,15 @@ sub fetch_user_infos($)
}
my $ua = LWP::UserAgent->new;
$ua->timeout(15);
$ua->timeout(180);
$ua->max_size(100000000);
$ua->default_header("Accept" => "application/json");
my $mywork = $ua->get("https://build.opensuse.org/home/requests?user=$user");
my $mywork = $ua->get("https://build.opensuse.org/home/requests.json?user=$user");
unless ($mywork->is_success) { die $mywork->status_line; }
$mywork = from_json( $mywork->decoded_content, { utf8 => 1 });
my $url = "https://build.opensuse.org/stage/project/status?project=$tproject&ignore_pending=0";
my $url = "https://build.opensuse.org/project/status/$tproject?ignore_pending=0";
$url .= "&limit_to_fails=false&limit_to_old=false&include_versions=true&filter_for_user=$user";
my $projstat = $ua->get($url);
die $projstat->status_line unless ($projstat->is_success);
@ -44,7 +45,7 @@ sub fetch_user_infos($)
$st{'mywork'} = $mywork;
$st{'projstat'} = $projstat;
# open(my $fh, '>', "reports/$user");
# print $fh to_json(%st);
# print STDOUT to_json(\%st);
# close $fh;
return ($mywork, $projstat);
}
@ -150,7 +151,8 @@ sub generate_report($)
my $report = '';
for my $request (@{$mywork->{review}}) {
my $reviews = $request->{review};
# as we query the build service as anonymous, we always get the reviews for "the others"
my $reviews = $request->{other_open_reviews};
$reviews = [$reviews] if (ref($reviews) eq "HASH");
for my $review (@{$reviews}) {
next if ($review->{state} ne 'new');
@ -196,10 +198,11 @@ sub generate_report($)
my $comment = $package->{failedcomment} || 'unknown failure';
$comment =~ s,^\s+,,;
$comment =~ s,\s+$,,;
my $url = "$baseurl/package/live_build_log?arch=" . uri_escape($package->{failedarch});
$url .= "&package=" . uri_escape($package->{name});
$url .= "&project=" . uri_escape($tproject);
$url .= "&repository=" . uri_escape($package->{failedrepo});
my $url = "$baseurl/package/live_build_log/";
$url .= uri_escape($tproject) . "/";
$url .= uri_escape($package->{name}) . "/";
$url .= uri_escape($package->{failedrepo}) . "/";
$url .= uri_escape($package->{failedarch});
$url = shorten_url($url, "bf-$package->{name}");
push(@{$lines->{fails}}, " $package->{name} fails for $fail ($comment):");
push(@{$lines->{fails}}, " $url\n");
@ -208,11 +211,11 @@ sub generate_report($)
for my $problem (sort @{$package->{problems}}) {
if ($problem eq 'different_changes') {
my $url = "$baseurl/package/rdiff?";
$url .= "opackage=" . uri_escape($package->{name});
my $url = "$baseurl/package/rdiff/";
$url .= uri_escape($package->{develproject});
$url .= "/" . uri_escape($package->{develpackage});
$url .= "?opackage=" . uri_escape($package->{name});
$url .= "&oproject=" . uri_escape($tproject);
$url .= "&package=" . uri_escape($package->{develpackage});
$url .= "&project=" . uri_escape($package->{develproject});
if ($ignorechanges == 0) {
$url = shorten_url($url, "rd-$package->{name}");
push(@{$lines->{unsubmit}}, " $package->{name} - $url");
@ -327,7 +330,7 @@ END
close(FORTUNE);
$report .= "\n\n-- \nYour fortune cookie:\n" . $fortune;
use Email::Simple;
use MIME::Lite;
use XML::Simple;
my $xml = '';
@ -338,27 +341,22 @@ END
my $info = XMLin($xml);
my $to = $info->{email};
if (ref($info->{realname}) ne "HASH") {
my $octets = decode("iso-8859-1", $info->{realname});
my $name = encode('utf-8', $octets);
$to = "$name <$to>";
my $name = $info->{realname};
$to = encode('MIME-Header', "$name <$to>");
}
my $email =
Email::Simple->create(
header => [
MIME::Lite->new(
From => 'Stephan Kulow <coolo@suse.de>',
To => $to,
Subject => 'Reminder for openSUSE:Factory work',
],
body => $report
Data => $report,
Encoding => '7bit',
);
# update from time to time :)
$email->header_set( 'MIME-Version', '1.0' );
$email->header_set( 'User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20111220 Thunderbird/9.0');
$email->header_set( 'Content-Type', 'text/plain; charset=UTF-8');
$email->header_set( 'X-Mailer', 'https://github.com/coolo/factory-auto/blob/master/generate-reminder.pl');
$email->header_set( 'Content-Transfer-Encoding', '7bit');
$email->add( 'User-Agent' => 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20111220 Thunderbird/9.0');
$email->add( 'X-Mailer' => 'https://github.com/coolo/factory-auto/blob/master/generate-reminder.pl');
print "From - " . Date::Format::time2str("%a %b %d %T %Y\n", time);
print $email->as_string;
$email->print(\*STDOUT);
}

2
osc-check_dups.py
View File

@ -18,7 +18,7 @@ def _checker_check_dups(self, project, opts):
type = a.attrib['type']
assert target != None
if target.attrib['project'] != project: continue
#print id
#print(id)
#ET.dump(target)
if not target.attrib.has_key('package'): continue
package = target.attrib['package']

19
osc-check_repo.py
View File

@ -613,8 +613,9 @@ def _check_repo_repo_list(self, prj, repo, arch, pkg, opts, ignore=False):
continue
files.append((fn, pname, result.group(4)))
except urllib2.HTTPError, e:
if not ignore:
print 'ERROR in URL %s [%s]'%(url, e)
pass
# if not ignore:
# print 'ERROR in URL %s [%s]'%(url, e)
return files
@ -678,7 +679,7 @@ def _get_base_build_bin(self, opts):
"""Get Base:build pagacke list"""
binaries = {}
for arch in ('x86_64', 'i586'):
url = makeurl(opts.apiurl, ['/build/Base:build/standard/%s/_repository'%arch,])
url = makeurl(opts.apiurl, ['/build/openSUSE:Factory:Build/standard/%s/_repository'%arch,])
root = ET.parse(http_GET(url)).getroot()
binaries[arch] = set([e.attrib['filename'][:-4] for e in root.findall('binary')])
return binaries
@ -686,7 +687,7 @@ def _get_base_build_bin(self, opts):
def _get_base_build_src(self, opts):
"""Get Base:build pagacke list"""
url = makeurl(opts.apiurl, ['/source/Base:build',])
url = makeurl(opts.apiurl, ['/source/openSUSE:Factory:Build',])
root = ET.parse(http_GET(url)).getroot()
return set([e.attrib['name'] for e in root.findall('entry')])
@ -799,10 +800,10 @@ def _check_repo_group(self, id_, reqs, opts):
outliers = build_deps - base_build_bin[arch]
if outliers:
print 'OUTLIERS (%s)'%arch, outliers
msg = 'This package is a Base:build and one of the dependencies is outside Base:build (%s)'%(', '.join(outliers))
# msg = 'This package is a Base:build and one of the dependencies is outside Base:build (%s)'%(', '.join(outliers))
# self._check_repo_change_review_state(opts, p.request, 'new', message=msg)
print 'NON-(FIX)-UPDATED', msg
return
# print 'NON-(FIX)-UPDATED', msg
p.updated = True
# Detect cycles - We create the full graph from _builddepinfo.
for arch in ('x86_64',):
@ -855,12 +856,12 @@ def _check_repo_group(self, id_, reqs, opts):
smissing = []
for package in p.missings:
alreadyin = False
print package, packs
# print package, packs
for t in packs:
if package == t.tpackage: alreadyin=True
if alreadyin:
continue
print package, packs, downloads, toignore
#print package, packs, downloads, toignore
request = self._check_repo_find_submit_request(opts, p.tproject, package)
if request:
greqs = opts.groups.get(p.group, [])

23
osc-group.py
View File

@ -6,7 +6,7 @@
# Then try to run 'osc checker --help' to see the usage.
def _group_find_request(self, package, opts):
url = makeurl(opts.apiurl, ['request'], "states=new,review,declined&project=openSUSE:Factory&view=collection&package=%s" % package )
url = makeurl(opts.apiurl, ['request'], "states=new,review&project=openSUSE:Factory&view=collection&package=%s" % package )
f = http_GET(url)
root = ET.parse(f).getroot()
maxid=0
@ -17,6 +17,19 @@ def _group_find_request(self, package, opts):
maxid = id
return maxid
def _group_find_request_project(self, source_project, opts):
url = makeurl(opts.apiurl, ['request'], "states=new,review&project=openSUSE:Factory&view=collection" )
f = http_GET(url)
root = ET.parse(f).getroot()
res = []
for rq in root.findall('request'):
for a in rq.findall('action'):
s = a.find('source')
if s is not None and s.get('project') == source_project:
id = int(rq.attrib['id'])
res.append(id)
return res
def _group_find_group(self, request, opts):
url = makeurl(opts.apiurl, ['search', "request", "id?match=action/grouped/@id=%s" % request] )
f = http_GET(url)
@ -47,8 +60,14 @@ def do_group(self, subcmd, opts, *args):
for p in args[:]:
request = self._group_find_request(p, opts)
if not request:
print("Can't find a request for", p)
prequests = self._group_find_request_project(p, opts)
if not len(prequests):
print("Can't find a request for package", p)
exit(1)
else:
prequests = [ request ]
for request in prequests:
group = self._group_find_group(request, opts)
if group > 0:
if grouptoadd > 0 and grouptoadd != group:

19
queue-all.sh
View File

@ -1,18 +1,17 @@
rm -f missingdeps
wget "http://gitorious.org/opensuse/package-lists/blobs/raw/master/output/opensuse/missingdeps"
list=`osc api /status/project/openSUSE:Factory | grep '<develpack' | sed -e 's,.*proj=",,; s,".*,,' | sort -u`
list=`osc api /search/package/?match='@project="openSUSE:Factory"' | grep "<devel project=" | sed -e 's,.*project=",,; s,".*,,' | sort -u`
users=`mktemp`
mbox=`mktemp`
dir=reports-`date +%F`
mkdir $dir
( for i in $list ; do
osc meta prj $i | grep '<person.*role="maintainer"'
done
osc api /status/project/openSUSE:Factory | grep '<person.*role="maintainer"'
echo "query $i" >&2
osc meta prj $i
osc api /search/package/?match='@project="$i"'
done | grep '<person.*role="maintainer"'
) | sed -e 's,.*userid=",,; s,".*,,' | sort -u > $users
tuser=`mktemp`
#tail -f $mbox &
for i in `cat $users`; do
echo "generate $i"
osc meta user -- $i > $tuser
perl generate-reminder.pl $i >> $mbox
perl generate-reminder.pl $i > $dir/$i.txt
done
echo "DONE $mbox"
echo "DONE $dir"

13
repo-checker.pl
View File

@ -10,6 +10,7 @@ use XML::Simple;
use Data::Dumper;
use Cwd;
use Rpm;
use Fcntl;
use strict;
@ -80,7 +81,7 @@ sub write_package($$)
}
my %qq = Build::Rpm::rpmq("$package", qw{NAME VERSION RELEASE ARCH OLDFILENAMES DIRNAMES BASENAMES DIRINDEXES 1030 1037 1039 1040
1047 1112 1113 1049 1048 1050 1090 1114 1115 1054 1053 1055
1047 1112 1113 1049 1048 1050 1090 1114 1115 1054 1053 1055 1036
});
my $name = $qq{'NAME'}[0];
@ -107,6 +108,7 @@ sub write_package($$)
my @users = @{$qq{1039} || []};
my @groups = @{$qq{1040} || []};
my @flags = @{$qq{1037} || []};
my @linktos = @{$qq{1036} || []};
my @xprvs;
@ -116,12 +118,18 @@ sub write_package($$)
my $user = shift @users;
my $group = shift @groups;
my $flag = shift @flags;
my $linkto = shift @linktos;
my $filename = $dirs[$di] . $bname;
$out .= sprintf "%o %o %s:%s %s\n", $mode, $flag, $user, $group, $filename;
my $fs = $filename;
if (Fcntl::S_ISLNK($mode)) {
$fs = "$filename -> $linkto";
}
$out .= sprintf "%o %o %s:%s %s\n", $mode, $flag, $user, $group, $fs;
if ( $filename =~ /^\/etc\// || $filename =~ /bin\// || $filename eq "/usr/lib/sendmail" ) {
push @xprvs, $filename;
}
}
$out .= "-Flx:\n";
$out .= "+Prv:\n";
@ -143,6 +151,7 @@ sub write_package($$)
next if ($name eq "libqmmp0-plugin-mplayer" && $prv eq "/usr/bin/mplayer");
next if ($name eq "systemd-mini" && $prv eq "this-is-only-for-build-envs");
next if ($name eq "build-config" && $prv eq "this-is-only-for-build-envs");
next if ($name eq "installation-images-debuginfodeps" && $prv =~ m/debuginfo.build/);
$out .= "$prv\n";
}
$out .= "-Req:\n";

34
source-checker.pl
View File

@ -5,6 +5,10 @@ use File::Temp qw/ tempdir /;
use XML::Simple;
use Data::Dumper;
use Cwd;
BEGIN {
unshift @INC, ($::ENV{'BUILD_DIR'} || '/usr/lib/build');
}
use Build;
my $old = $ARGV[0];
my $dir = $ARGV[1];
@ -105,21 +109,26 @@ foreach my $test (glob("/usr/lib/obs/service/source_validators/*")) {
}
}
if (-d "_old") {
chdir("_old") || die "chdir _old failed";
if (-d "$old") {
my $odir = getcwd();
chdir($old) || die "chdir $old failed";
my $cf = Build::read_config("x86_64", "/usr/lib/build/configs/default.conf");
my %thash = ();
my %rhash = ();
for my $spec (glob("*.spec")) {
open(PIPE, "grep '^Source' $spec |");
while (<PIPE>) {
chomp;
s/^Source[0-9]*\s*:\s*//;
$thash{$_} = 1;
my $ps = Build::Rpm::parse($cf, $spec);
while (my ($k, $v) = each %$ps) {
if ($k =~ m/^source/) {
$thash{$v} = 1;
}
close(PIPE);
}
chdir("../$dir") || die "chdir failed";
}
chdir($odir) || die "chdir $odir failed";
chdir($dir) || die "chdir $dir failed";
for my $spec (glob("*.spec")) {
my $ps = Build::Rpm::parse($cf, $spec);
open(OSPEC, "$spec");
open(NSPEC, ">$spec.new");
while (<OSPEC>) {
@ -128,7 +137,8 @@ if (-d "_old") {
my $line = $_;
$line =~ s/^(Source[0-9]*)\s*:\s*//;
my $prefix = $1;
if (defined $thash{$line}) {
my $parsedline = $ps->{lc $prefix};
if (defined $thash{$parsedline}) {
my $file = $line;
my $bname = basename($file);
print NSPEC "$prefix: $bname\n";
@ -141,9 +151,11 @@ if (-d "_old") {
}
close(OSPEC);
close(NSPEC);
#system("diff -u $spec $spec.new");
#exit(0);
rename("$spec.new", "$spec") || die "rename failed";
}
chdir("..");
chdir($odir);
}
my $odir = getcwd;