Merge from coolo to aplanas

findfileconflicts

@@ -77,6 +77,11 @@ while(<FL>) {
       $fls = 0;
       next;
     }
+    my $lnk = '';
+    if (/^(12.*)( -> .*?)$/) {
+      $_ = $1;
+      $lnk = $2;
+    }
     next unless /^(\d+ (\d+) \S+) (.*\/)(.*?)$/;
     my $n = $dirs{$3};
     if (!defined($n)) {
@@ -84,11 +89,13 @@ while(<FL>) {
       $dirs{$3} = $n;
       $dirs[$n] = $3;
     }
-    my $m = $modes{$1};
+    # ignore link targets of ghosts
+    $lnk = '' if $lnk && (oct($2) & 0100) != 0;
+    my $m = $modes{"$1$lnk"};
     if (!defined($m)) {
       $m = @modes;
-      $modes{$1} = $m;
+      $modes{"$1$lnk"} = $m;
-      $modes[$m] = $1;
+      $modes[$m] = "$1$lnk";
       $modes_type[$m] = oct($1) & 07770000;
       $modes_ghost[$m] = oct($2) & 0100;
     }
@@ -155,6 +162,7 @@ while(<FL>) {
 }
 close(FL) || die("close failed\n");
 
+print "currently have ".@dirs." dirs and ".@modes." modes\n";
 
 # connect dirs and add all dirs as files
 print "connecting ".@dirs." directories\n";
@@ -324,14 +332,14 @@ for my $tc (sort keys %tocheck) {
       for my $f (@{$tocheck_files{$tc}}) {
 	my @pp = grep {s/^(?:\Q$p1\E|\Q$p2\E)\///} map {$_} @{$filesc{$f}};
 	next unless @pp;
-	# ignore if (all directories or all ghosts) and all same mode;
+	# ignore if (all directories or all ghosts or all links) and all same mode;
 	my %allm = map {$_ => 1} @pp;
 	my $info = '';
 	if (keys(%allm) == 1) {
 	  my $m = (keys(%allm))[0];
 	  # all modes/flags are the same
-	  # no conflict if all dirs or all ghosts
+	  # no conflict if all dirs or all ghosts or all links
-	  next if $modes_type[$m] == 040000 || $modes_ghost[$m] == 0100;
+	  next if $modes_type[$m] == 040000 || $modes_type[$m] == 0120000 || $modes_ghost[$m] == 0100;
 	} else {
 	  # don't report mode mismatches for files/symlinks that are not ghosts
 	  for my $m (keys %allm) {

generate-reminder.pl

@@ -27,14 +27,15 @@ sub fetch_user_infos($)
     }
 
     my $ua = LWP::UserAgent->new;
-    $ua->timeout(15);
+    $ua->timeout(180);
+    $ua->max_size(100000000);
     $ua->default_header("Accept" => "application/json");
-    my $mywork = $ua->get("https://build.opensuse.org/home/requests?user=$user");
+    my $mywork = $ua->get("https://build.opensuse.org/home/requests.json?user=$user");
     unless ($mywork->is_success) { die $mywork->status_line; }
 
     $mywork = from_json( $mywork->decoded_content, { utf8 => 1 });
 
-    my $url = "https://build.opensuse.org/stage/project/status?project=$tproject&ignore_pending=0";
+    my $url = "https://build.opensuse.org/project/status/$tproject?ignore_pending=0";
     $url .= "&limit_to_fails=false&limit_to_old=false&include_versions=true&filter_for_user=$user";
     my $projstat = $ua->get($url);
     die $projstat->status_line unless ($projstat->is_success);
@@ -44,7 +45,7 @@ sub fetch_user_infos($)
     $st{'mywork'} = $mywork;
     $st{'projstat'} = $projstat;
    # open(my $fh, '>', "reports/$user");
-   # print $fh to_json(%st);
+   # print STDOUT to_json(\%st);
    # close $fh;
     return ($mywork, $projstat);
 }
@@ -150,7 +151,8 @@ sub generate_report($)
     my $report = '';
 
     for my $request (@{$mywork->{review}}) {
-	my $reviews = $request->{review};
+      # as we query the build service as anonymous, we always get the reviews for "the others"
+	my $reviews = $request->{other_open_reviews};
 	$reviews = [$reviews] if (ref($reviews) eq "HASH");
 	for my $review (@{$reviews}) {
 	    next if ($review->{state} ne 'new');
@@ -196,10 +198,11 @@ sub generate_report($)
 		my $comment = $package->{failedcomment} || 'unknown failure';
 		$comment =~ s,^\s+,,;
 		$comment =~ s,\s+$,,;
-		my $url = "$baseurl/package/live_build_log?arch=" . uri_escape($package->{failedarch});
+		my $url = "$baseurl/package/live_build_log/";
-		$url   .= "&package=" . uri_escape($package->{name});
+		$url   .= uri_escape($tproject) . "/";
-		$url   .= "&project=" . uri_escape($tproject);
+		$url   .= uri_escape($package->{name}) . "/";
-		$url   .= "&repository=" . uri_escape($package->{failedrepo});
+		$url   .= uri_escape($package->{failedrepo}) . "/";
+		$url   .= uri_escape($package->{failedarch});
 		$url = shorten_url($url, "bf-$package->{name}");
 		push(@{$lines->{fails}}, "  $package->{name} fails for $fail ($comment):");
 		push(@{$lines->{fails}}, "    $url\n");
@@ -208,11 +211,11 @@ sub generate_report($)
 
 	    for my $problem (sort @{$package->{problems}}) {
 		if ($problem eq 'different_changes') {
-                    my $url = "$baseurl/package/rdiff?";
+                    my $url = "$baseurl/package/rdiff/";
-		    $url .= "opackage=" .  uri_escape($package->{name});
+                    $url .= uri_escape($package->{develproject});
+		    $url .= "/" . uri_escape($package->{develpackage});
+		    $url .= "?opackage=" .  uri_escape($package->{name});
                     $url .= "&oproject=" . uri_escape($tproject);
-                    $url .= "&package=" . uri_escape($package->{develpackage});
-                    $url .= "&project=" . uri_escape($package->{develproject});
 		    if ($ignorechanges == 0) {
 			$url = shorten_url($url, "rd-$package->{name}");
 			push(@{$lines->{unsubmit}}, "    $package->{name} - $url");
@@ -327,7 +330,7 @@ END
     close(FORTUNE);
     $report .= "\n\n-- \nYour fortune cookie:\n" . $fortune;
 
-    use Email::Simple;
+    use MIME::Lite;
     use XML::Simple;
 
     my $xml = '';
@@ -338,27 +341,22 @@ END
     my $info = XMLin($xml);
     my $to = $info->{email};
     if (ref($info->{realname}) ne "HASH") {
-      my $octets = decode("iso-8859-1", $info->{realname});
+      my $name = $info->{realname};
-      my $name = encode('utf-8', $octets);
+      $to = encode('MIME-Header', "$name <$to>");
-      $to = "$name <$to>";
     }
     my $email = 
-	Email::Simple->create(
+	MIME::Lite->new(
-	    header => [
    	   From    => 'Stephan Kulow <coolo@suse.de>',
 	   To      => $to,
 	   Subject => 'Reminder for openSUSE:Factory work',
-	    ],
+	   Data => $report,
-	    body => $report
+	   Encoding => '7bit',
 	);
 
     # update from time to time :)
-    $email->header_set( 'MIME-Version', '1.0' );
+    $email->add( 'User-Agent' => 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20111220 Thunderbird/9.0');
-    $email->header_set( 'User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0) Gecko/20111220 Thunderbird/9.0');
+    $email->add( 'X-Mailer' => 'https://github.com/coolo/factory-auto/blob/master/generate-reminder.pl');
-    $email->header_set( 'Content-Type', 'text/plain; charset=UTF-8');
-    $email->header_set( 'X-Mailer', 'https://github.com/coolo/factory-auto/blob/master/generate-reminder.pl');
-    $email->header_set( 'Content-Transfer-Encoding', '7bit');
 
     print "From - " . Date::Format::time2str("%a %b %d %T %Y\n", time);
-    print $email->as_string;
+    $email->print(\*STDOUT);
 }

osc-check_dups.py

@@ -18,7 +18,7 @@ def _checker_check_dups(self, project, opts):
             type = a.attrib['type']
             assert target != None
             if target.attrib['project'] != project: continue
-	    #print id
+	    #print(id)
             #ET.dump(target)
 	    if not target.attrib.has_key('package'): continue
             package = target.attrib['package']

osc-check_repo.py

@@ -613,8 +613,9 @@ def _check_repo_repo_list(self, prj, repo, arch, pkg, opts, ignore=False):
                 continue
             files.append((fn, pname, result.group(4)))
     except urllib2.HTTPError, e:
-        if not ignore:
+        pass
-            print 'ERROR in URL %s [%s]'%(url, e)
+        # if not ignore:
+        #     print 'ERROR in URL %s [%s]'%(url, e)
     return files
 
 
@@ -678,7 +679,7 @@ def _get_base_build_bin(self, opts):
     """Get Base:build pagacke list"""
     binaries = {}
     for arch in ('x86_64', 'i586'):
-        url = makeurl(opts.apiurl, ['/build/Base:build/standard/%s/_repository'%arch,])
+        url = makeurl(opts.apiurl, ['/build/openSUSE:Factory:Build/standard/%s/_repository'%arch,])
         root = ET.parse(http_GET(url)).getroot()
         binaries[arch] = set([e.attrib['filename'][:-4] for e in root.findall('binary')])
     return binaries
@@ -686,7 +687,7 @@ def _get_base_build_bin(self, opts):
 
 def _get_base_build_src(self, opts):
     """Get Base:build pagacke list"""
-    url = makeurl(opts.apiurl, ['/source/Base:build',])
+    url = makeurl(opts.apiurl, ['/source/openSUSE:Factory:Build',])
     root = ET.parse(http_GET(url)).getroot()
     return set([e.attrib['name'] for e in root.findall('entry')])
 
@@ -799,10 +800,10 @@ def _check_repo_group(self, id_, reqs, opts):
                 outliers = build_deps - base_build_bin[arch]
                 if outliers:
                     print 'OUTLIERS (%s)'%arch, outliers
-                    msg = 'This package is a Base:build and one of the dependencies is outside Base:build (%s)'%(', '.join(outliers))
+                    # msg = 'This package is a Base:build and one of the dependencies is outside Base:build (%s)'%(', '.join(outliers))
                     # self._check_repo_change_review_state(opts, p.request, 'new', message=msg)
-                    print 'NON-(FIX)-UPDATED', msg
+                    # print 'NON-(FIX)-UPDATED', msg
-                    return
+                    p.updated = True
 
     # Detect cycles - We create the full graph from _builddepinfo.
     for arch in ('x86_64',):
@@ -855,12 +856,12 @@ def _check_repo_group(self, id_, reqs, opts):
         smissing = []
         for package in p.missings:
             alreadyin = False
-            print package, packs
+            # print package, packs
             for t in packs:
                 if package == t.tpackage: alreadyin=True
             if alreadyin:
                 continue
-            print package, packs, downloads, toignore
+            #print package, packs, downloads, toignore
             request = self._check_repo_find_submit_request(opts, p.tproject, package)
             if request:
                 greqs = opts.groups.get(p.group, [])

osc-group.py

@@ -6,7 +6,7 @@
 # Then try to run 'osc checker --help' to see the usage.
 
 def _group_find_request(self, package, opts):
-    url = makeurl(opts.apiurl, ['request'], "states=new,review,declined&project=openSUSE:Factory&view=collection&package=%s" % package )
+    url = makeurl(opts.apiurl, ['request'], "states=new,review&project=openSUSE:Factory&view=collection&package=%s" % package )
     f = http_GET(url)
     root = ET.parse(f).getroot()
     maxid=0
@@ -17,6 +17,19 @@ def _group_find_request(self, package, opts):
             maxid = id
     return maxid
 
+def _group_find_request_project(self, source_project, opts):
+    url = makeurl(opts.apiurl, ['request'], "states=new,review&project=openSUSE:Factory&view=collection" )
+    f = http_GET(url)
+    root = ET.parse(f).getroot()
+    res = []
+    for rq in root.findall('request'):
+        for a in rq.findall('action'):
+            s = a.find('source')
+            if s is not None and s.get('project') ==  source_project:
+                id = int(rq.attrib['id'])
+                res.append(id)
+    return res
+
 def _group_find_group(self, request, opts):
     url = makeurl(opts.apiurl, ['search', "request", "id?match=action/grouped/@id=%s" % request] )
     f = http_GET(url)
@@ -47,8 +60,14 @@ def do_group(self, subcmd, opts, *args):
     for p in args[:]:
         request = self._group_find_request(p, opts)
         if not request:
-            print("Can't find a request for", p)
+            prequests = self._group_find_request_project(p, opts)
+            if not len(prequests):
+                print("Can't find a request for package", p)
                 exit(1)
+        else:
+            prequests = [ request ]
+
+        for request in prequests:
             group = self._group_find_group(request, opts)
             if group > 0:
                 if grouptoadd > 0 and grouptoadd != group:

queue-all.sh

@@ -1,18 +1,17 @@
 rm -f missingdeps
 wget "http://gitorious.org/opensuse/package-lists/blobs/raw/master/output/opensuse/missingdeps"
-list=`osc api /status/project/openSUSE:Factory | grep '<develpack' | sed -e 's,.*proj=",,; s,".*,,' | sort -u`
+list=`osc api /search/package/?match='@project="openSUSE:Factory"' | grep "<devel project=" | sed -e 's,.*project=",,; s,".*,,' | sort -u`
 users=`mktemp`
-mbox=`mktemp`
+dir=reports-`date +%F`
+mkdir $dir
 ( for i in $list ; do 
-  osc meta prj $i | grep '<person.*role="maintainer"'
+  echo "query $i" >&2
-done
+  osc meta prj $i 
-osc api /status/project/openSUSE:Factory | grep '<person.*role="maintainer"'
+  osc api /search/package/?match='@project="$i"'
+done | grep '<person.*role="maintainer"'
 ) | sed -e 's,.*userid=",,; s,".*,,'  | sort -u > $users
-tuser=`mktemp`
-#tail -f $mbox &
 for i in `cat $users`; do 
   echo "generate $i"
-  osc meta user -- $i  > $tuser
+  perl generate-reminder.pl $i > $dir/$i.txt
-  perl generate-reminder.pl $i >> $mbox
 done
-echo "DONE $mbox"
+echo "DONE $dir"

repo-checker.pl

@@ -10,6 +10,7 @@ use XML::Simple;
 use Data::Dumper;
 use Cwd;
 use Rpm;
+use Fcntl;
 
 use strict;
 
@@ -80,7 +81,7 @@ sub write_package($$)
   }
 
   my %qq = Build::Rpm::rpmq("$package", qw{NAME VERSION RELEASE ARCH OLDFILENAMES DIRNAMES BASENAMES DIRINDEXES 1030 1037 1039 1040
-					   1047 1112 1113 1049 1048 1050 1090 1114 1115 1054 1053 1055
+					   1047 1112 1113 1049 1048 1050 1090 1114 1115 1054 1053 1055 1036
 					});
 
   my $name = $qq{'NAME'}[0];
@@ -107,6 +108,7 @@ sub write_package($$)
   my @users = @{$qq{1039} || []};
   my @groups = @{$qq{1040} || []};
   my @flags = @{$qq{1037} || []};
+  my @linktos = @{$qq{1036} || []};
 
   my @xprvs;
 
@@ -116,12 +118,18 @@ sub write_package($$)
     my $user = shift @users;
     my $group = shift @groups;
     my $flag = shift @flags;
+    my $linkto = shift @linktos;
 
     my $filename = $dirs[$di] . $bname;
-    $out .= sprintf "%o %o %s:%s %s\n", $mode, $flag, $user, $group, $filename;
+    my $fs = $filename;
+    if (Fcntl::S_ISLNK($mode)) {
+      $fs = "$filename -> $linkto";
+    }
+    $out .= sprintf "%o %o %s:%s %s\n", $mode, $flag, $user, $group, $fs;
     if ( $filename =~ /^\/etc\// || $filename =~ /bin\// || $filename eq "/usr/lib/sendmail" ) {
       push @xprvs, $filename;
     }
+
   }
   $out .= "-Flx:\n";
   $out .= "+Prv:\n";
@@ -143,6 +151,7 @@ sub write_package($$)
     next if ($name eq "libqmmp0-plugin-mplayer" && $prv eq "/usr/bin/mplayer");
     next if ($name eq "systemd-mini" && $prv eq "this-is-only-for-build-envs");
     next if ($name eq "build-config" && $prv eq "this-is-only-for-build-envs");
+    next if ($name eq "installation-images-debuginfodeps" && $prv =~ m/debuginfo.build/);
     $out .= "$prv\n";
   }
   $out .= "-Req:\n";

source-checker.pl

@@ -5,6 +5,10 @@ use File::Temp qw/ tempdir  /;
 use XML::Simple;
 use Data::Dumper;
 use Cwd;
+BEGIN {
+  unshift @INC, ($::ENV{'BUILD_DIR'} || '/usr/lib/build');
+}
+use Build;
 
 my $old = $ARGV[0];
 my $dir = $ARGV[1];
@@ -105,21 +109,26 @@ foreach my $test (glob("/usr/lib/obs/service/source_validators/*")) {
     }
 }
 
-if (-d "_old") {
+if (-d "$old") {
-    chdir("_old") || die "chdir _old failed";
+    my $odir = getcwd();
+    chdir($old) || die "chdir $old failed";
+    my $cf = Build::read_config("x86_64", "/usr/lib/build/configs/default.conf");
+
     my %thash = ();
     my %rhash = ();
     for my $spec (glob("*.spec")) {
-	open(PIPE, "grep '^Source' $spec |");
+	my $ps = Build::Rpm::parse($cf, $spec);
-	while (<PIPE>) {
+
-	    chomp;
+	while (my ($k, $v) = each %$ps) {
-	    s/^Source[0-9]*\s*:\s*//;
+	    if ($k =~ m/^source/) {
-	    $thash{$_} = 1;
+		$thash{$v} = 1;
 	    }
-	close(PIPE);
 	}
-    chdir("../$dir") || die "chdir failed";
+    }
+    chdir($odir) || die "chdir $odir failed";
+    chdir($dir) || die "chdir $dir failed";
     for my $spec (glob("*.spec")) {
+	my $ps = Build::Rpm::parse($cf, $spec);
 	open(OSPEC, "$spec");
 	open(NSPEC, ">$spec.new");
 	while (<OSPEC>) {
@@ -128,7 +137,8 @@ if (-d "_old") {
 		my $line = $_;
 		$line =~ s/^(Source[0-9]*)\s*:\s*//;
 		my $prefix = $1;
-		if (defined $thash{$line}) {
+		my $parsedline = $ps->{lc $prefix};
+		if (defined $thash{$parsedline}) {
 		    my $file = $line;
 		    my $bname = basename($file);
 		    print NSPEC "$prefix: $bname\n";
@@ -141,9 +151,11 @@ if (-d "_old") {
 	}
 	close(OSPEC);
 	close(NSPEC);
+	#system("diff -u $spec $spec.new");
+	#exit(0);
 	rename("$spec.new", "$spec") || die "rename failed";
     }
-    chdir("..");
+    chdir($odir);
 }
 
 my $odir = getcwd;