more url parsing

common/config.go

@@ -54,7 +54,6 @@ type ReviewGroup struct {
 type QAConfig struct {
 	Name   string
 	Origin string
-	BuildDisableRepos []string // which repos to build disable in the new project
 }
 
 type Permissions struct {

devel-importer/find_factory_commit_in_gitpkg.pl

@@ -0,0 +1,181 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+use IPC::Open2;
+use JSON;
+
+sub FindFactoryCommit {
+    my ($package) = @_;
+
+    # Execute osc cat and capture output
+    my $osc_cmd = "osc cat openSUSE:Factory $package $package.changes";
+    open( my $osc_fh, "$osc_cmd |" ) or die "Failed to run osc: $!";
+    my $data = do { local $/; <$osc_fh> };
+    close($osc_fh);
+
+    # Calculate size
+    my $size = length($data);
+
+    # Create blob header
+    my $blob = "blob $size\0$data";
+
+    # Open a pipe to openssl to compute the hash
+    my ( $reader, $writer );
+    my $pid = open2( $reader, $writer, "openssl sha256" );
+
+    # Send blob data
+    print $writer $blob;
+    close $writer;
+
+    # Read the hash result and extract it
+    my $hash_line = <$reader>;
+    waitpid( $pid, 0 );
+    my ($hash) = $hash_line =~ /([a-fA-F0-9]{64})/;
+
+    # Run git search command with the hash
+    print("looking for hash: $hash\n");
+    my @hashes;
+    my $git_cmd =
+"git -C $package rev-list --all pool/HEAD | while read commit; do git -C $package ls-tree \"\$commit\" | grep -q '^100644 blob $hash' && echo \"\$commit\"; done";
+    open( my $git_fh, "$git_cmd |" ) or die "Failed to run git search: $!";
+    while ( my $commit = <$git_fh> ) {
+        chomp $commit;
+        print "Found commit $commit\n";
+        push( @hashes, $commit );
+    }
+    close($git_fh);
+    return @hashes;
+}
+
+sub FactoryMd5 {
+	my ($package) = @_;
+	my $out = "";
+
+	if (system("osc ls openSUSE:Factory $package | grep -q build.specials.obscpio") == 0) {
+		system("mkdir _extract") == 0 || die "_extract exists or can't make it. Aborting.";
+		chdir("_extract") || die;
+		system("osc cat openSUSE:Factory $package build.specials.obscpio | cpio -dium 2> /dev/null") == 0 || die;
+		system("rm .* 2> /dev/null");
+		open( my $fh, "find -type f -exec /usr/bin/basename {} \\; | xargs md5sum | awk '{print \$1 FS \$2}' | grep -v d41d8cd98f00b204e9800998ecf8427e |") or die;
+		while ( my $l = <$fh>) {
+			$out = $out.$l;
+		}
+		close($fh);
+		chdir("..") && system("rm -rf _extract") == 0 || die;
+	}
+	open( my $fh, "osc ls -v openSUSE:Factory $package | awk '{print \$1 FS \$7}' | grep -v -F '_scmsync.obsinfo\nbuild.specials.obscpio' |") or die;
+	while (my $l = <$fh>) {
+		$out = $out.$l;
+	}
+	close($fh);
+	return $out;
+}
+
+# Read project from first argument
+sub Usage {
+    die "Usage: $0 <OBS Project> <package> <repo>";
+}
+
+my $project = shift or Usage();
+my $pkg     = shift;
+my $repo    = shift;
+
+if (not defined($repo)) {
+	Usage();
+}
+
+
+my $meta_url = `osc meta pkg $project $pkg | grep scmsync | sed -e 's,\\s*</\\?scmsync>\\s*,,g'`;
+chomp($meta_url);
+if ($meta_url ne $repo) {
+	die "meta not equal to repo for $pkg: $meta_url != $repo";
+}
+
+my ($org, $repo_path, $branch) = $meta_url =~ m{^https?://src\.opensuse\.org/([^/]+)/([^/#\s]+)(?:.*#(\S+))?};
+die "Only src.opensuse.org is supported" unless $org;
+
+if ($org eq "pool") {
+	print "Already a pool package. We are done.\n";
+	exit(0);
+}
+
+my @packages = ($pkg) if defined $pkg;
+
+if ( ! -e $org ) {
+    mkdir($org);
+}
+chdir($org);
+
+my $super_user = $ENV{SUPER};
+if (defined($super_user)) {
+    $super_user = "-G $super_user";
+} else {
+    $super_user = "";
+}
+
+for my $package ( sort(@packages) ) {
+    print " ----- PROCESSING $package\n";
+    my $url      = "https://src.opensuse.org/$org/$repo_path.git";
+    my $push_url = "gitea\@src.opensuse.org:pool/$package.git";
+    if ( not -e $package ) {
+        print("cloning...\n");
+        system("git clone --origin pool $url $package") == 0
+          or die "Can't clone $org/$repo_path";
+    }
+    else {
+        print("adding remote...\n");
+        system("git -C $package remote rm pool > /dev/null");
+        system("git -C $package remote add pool $url") == 0
+          or die "Can't add pool for $package";
+    }
+    system("git -C $package remote set-url pool --push $push_url") == 0
+      or die "Can't add push remote for $package";
+    print("fetching remote...\n");
+    system("git -C $package fetch pool") == 0 or die "Can't fetch pool for $package";
+
+    my @commits = FindFactoryCommit($package);
+    my $Md5Hashes = FactoryMd5($package);
+    my $c;
+    my $match = 0;
+    for my $commit (@commits) {
+        if ( length($commit) != 64 ) {
+            print("Failed to find factory commit. Aborting.");
+            exit(1);
+        }
+
+        if (
+                system("git -C $package lfs fetch pool $commit") == 0
+            and system("git -C $package checkout -B factory $commit") == 0
+            and system("git -C $package lfs checkout") == 0
+            and chdir($package)) {
+
+	    open(my $fh, "|-", "md5sum -c --quiet") or die $!;
+            print $fh $Md5Hashes;
+            close $fh;
+	    if ($? >> 8 != 0) {
+		    chdir("..") || die;
+		    next;
+	    }
+	    open($fh, "|-", "awk '{print \$2}' | sort | bash -c \"diff <(ls -1 | sort) -\"") or die $!;
+	    print $fh $Md5Hashes;
+            close $fh;
+            my $ec = $? >> 8;
+	    chdir("..") || die;
+
+	    if ($ec == 0) {
+               $c     = $commit;
+               $match = 1;
+               last;
+	    }
+
+        }
+    }
+
+    if ( !$match ) {
+        die "Match not found. Aborting.";
+    }
+
+    system ("git -C $package push -f pool factory");
+    print "$package: $c\n";
+}
+

devel-importer/migrated_projects

@@ -1,3 +1,4 @@
+SystemsManagement
 Java:packages
 Kernel:firmware
 Kernel:kdump

obs-staging-bot/README.md

@@ -34,8 +34,7 @@ It's a JSON file with following syntax:
  "QA": [
    {
      "Name": "SLES",
-     "Origin": "SUSE:SLFO:Products:SLES:16.0",
+     "Origin": "SUSE:SLFO:Products:SLES:16.0"
-     "BuildDisableRepos": ["product"]
    }
  ]
 }
@@ -48,7 +47,6 @@ It's a JSON file with following syntax:
 | *QA* | Crucial for generating a product build (such as an ISO or FTP tree) that incorporates the packages. | no | array of objects | | |
 | *QA > Name* | Suffix for the QA OBS staging project. The project is named *StagingProject:<PR_Number>:Name*. | no | string | | |
 | *QA > Origin* | OBS reference project | no | string | | |
-| *QA > BuildDisableRepos* | The names of OBS repositories to build-disable, if any. | no | array of strings | | [] |
 
 
 Details
@@ -67,6 +65,6 @@ Details
 * **PrjGit PR - QA staging project**
   * The QA staging project is meant for building the product; the relative build config is inherited from the `QA > Origin` project.
   * In this case, the **scmsync** tag is inherited from the `QA > Origin` project.
-  * It is desirable in some cases to avoid building some specific build service repositories when not needed. In this case, `QA > BuildDisableRepos` can be specified.
+
-    These repositories would be disabled in the project meta when generating the QA project.
+
 

obs-staging-bot/main.go

@@ -109,11 +109,6 @@ const (
 	BuildStatusSummaryUnknown  = 4
 )
 
-type DisableFlag struct {
-	XMLName string `xml:"disable"`
-	Name    string `xml:"repository,attr"`
-}
-
 func ProcessBuildStatus(project, refProject *common.BuildResultList) BuildStatusSummary {
 	if _, finished := refProject.BuildResultSummary(); !finished {
 		common.LogDebug("refProject not finished building??")
@@ -382,7 +377,7 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
 // stagingProject:$buildProject
 //  ^-  stagingProject:$buildProject:$subProjectName (based on templateProject)
 
-func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingProject, templateProject, subProjectName string, buildDisableRepos []string) error {
+func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingProject, templateProject, subProjectName string) error {
 	common.LogDebug("Setup QA sub projects")
 	templateMeta, err := ObsClient.GetProjectMeta(templateProject)
 	if err != nil {
@@ -412,21 +407,7 @@ func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, git
 		repository.Fragment  = branch.SHA
 		templateMeta.ScmSync = repository.String()
 		common.LogDebug("Setting scmsync url to ", templateMeta.ScmSync)
-	}
+        }
-	// Build-disable repositories if asked
-	if len(buildDisableRepos) > 0 {
-		toDisable := make([]DisableFlag, len(buildDisableRepos))
-		for idx, repositoryName := range buildDisableRepos {
-			toDisable[idx] = DisableFlag{Name: repositoryName}
-		}
-
-		output, err := xml.Marshal(toDisable)
-		if err != nil {
-			common.LogError("error while marshalling, skipping BuildDisableRepos: ", err)
-		} else {
-			templateMeta.BuildFlags.Contents += string(output)
-		}
-	}
 	// Cleanup ReleaseTarget and modify affected path entries
 	for idx, r := range templateMeta.Repositories {
 		templateMeta.Repositories[idx].ReleaseTargets = nil
@@ -941,8 +922,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 				CreateQASubProject(stagingConfig, git, gitea, pr,
 					stagingProject,
 					setup.Origin,
-					setup.Name,
+					setup.Name)
-					setup.BuildDisableRepos)
 				msg = msg + ObsWebHost + "/project/show/" +
 					stagingProject + ":" + setup.Name + "\n"
 			}