.

Using the BuildDisableRepos configuration, it is now possible to
define which repositories to build-disable in the QA project meta.

This is for example useful for the SLES development workflow, where
the product repository should only be enabled after the packagelist
definitions have been built - so it is not desirable to have them
built as soon as the QA project is created.

Example:

    {
      "ObsProject": "SUSE:SLFO:Main",
      "StagingProject": "SUSE:SLFO:Main:PullRequest",
      "QA": [
        {
          "Name": "SLES",
          "Origin": "SUSE:SLFO:Products:SLES:16.1",
          "BuildDisableRepos": ["product"]
        }
      ]
    }

Signed-off-by: Eugenio Paolantonio <eugenio.paolantonio@suse.com>

common/config.go

@@ -54,6 +54,7 @@ type ReviewGroup struct {
 type QAConfig struct {
 	Name   string
 	Origin string
+	BuildDisableRepos []string // which repos to build disable in the new project
 }
 
 type Permissions struct {

devel-importer/find_factory_commit_in_gitpkg.pl

@@ -0,0 +1,187 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+use IPC::Open2;
+use URI;
+
+sub FindFactoryCommit {
+    my ($package) = @_;
+
+    # Execute osc cat and capture output
+    my $osc_cmd = "osc cat openSUSE:Factory $package $package.changes";
+    open( my $osc_fh, "$osc_cmd |" ) or die "Failed to run osc: $!";
+    my $data = do { local $/; <$osc_fh> };
+    close($osc_fh);
+
+    # Calculate size
+    my $size = length($data);
+
+    # Create blob header
+    my $blob = "blob $size\0$data";
+
+    # Open a pipe to openssl to compute the hash
+    my ( $reader, $writer );
+    my $pid = open2( $reader, $writer, "openssl sha256" );
+
+    # Send blob data
+    print $writer $blob;
+    close $writer;
+
+    # Read the hash result and extract it
+    my $hash_line = <$reader>;
+    waitpid( $pid, 0 );
+    my ($hash) = $hash_line =~ /([a-fA-F0-9]{64})/;
+
+    # Run git search command with the hash
+    print("looking for hash: $hash\n");
+    my @hashes;
+    my $git_cmd =
+"git -C $package rev-list --all pool/HEAD | while read commit; do git -C $package ls-tree \"\$commit\" | grep -q '^100644 blob $hash' && echo \"\$commit\"; done";
+    open( my $git_fh, "$git_cmd |" ) or die "Failed to run git search: $!";
+    while ( my $commit = <$git_fh> ) {
+        chomp $commit;
+        print "Found commit $commit\n";
+        push( @hashes, $commit );
+    }
+    close($git_fh);
+    return @hashes;
+}
+
+sub FactoryMd5 {
+	my ($package) = @_;
+	my $out = "";
+
+	if (system("osc ls openSUSE:Factory $package | grep -q build.specials.obscpio") == 0) {
+		system("mkdir _extract") == 0 || die "_extract exists or can't make it. Aborting.";
+		chdir("_extract") || die;
+		system("osc cat openSUSE:Factory $package build.specials.obscpio | cpio -dium 2> /dev/null") == 0 || die;
+		system("rm .* 2> /dev/null");
+		open( my $fh, "find -type f -exec /usr/bin/basename {} \\; | xargs md5sum | awk '{print \$1 FS \$2}' | grep -v d41d8cd98f00b204e9800998ecf8427e |") or die;
+		while ( my $l = <$fh>) {
+			$out = $out.$l;
+		}
+		close($fh);
+		chdir("..") && system("rm -rf _extract") == 0 || die;
+	}
+	open( my $fh, "osc ls -v openSUSE:Factory $package | awk '{print \$1 FS \$7}' | grep -v -F '_scmsync.obsinfo\nbuild.specials.obscpio' |") or die;
+	while (my $l = <$fh>) {
+		$out = $out.$l;
+	}
+	close($fh);
+	return $out;
+}
+
+# Read project from first argument
+sub Usage {
+    die "Usage: $0 <OBS Project> <package> <repo>";
+}
+
+my $project = shift or Usage();
+my $pkg     = shift;
+my $repo    = shift;
+
+if (not defined($repo)) {
+	Usage();
+}
+
+
+my $meta_url = `osc meta pkg $project $pkg | grep scmsync | sed -e 's,\\s*</\\?scmsync>\\s*,,g'`;
+chomp($meta_url);
+if ($meta_url ne $repo) {
+	die "meta not equal to repo for $pkg: $meta_url != $repo";
+}
+
+my $u = URI->new($meta_url);
+die "Only src.opensuse.org is supported" unless $u->scheme =~ /^https?$/ && $u->host eq 'src.opensuse.org';
+my (undef, $org, $repo_path) = split('/', $u->path);
+my $branch = $u->fragment;
+die "Only src.opensuse.org is supported" unless $org;
+if ($org eq "pool") {
+	print "Already a pool package. We are done.\n";
+	exit(0);
+}
+
+my %params = $u->query_form;
+delete $params{trackingbranch};
+die "Unsupported query parameters: " . join(', ', keys %params) if keys %params;
+
+my @packages = ($pkg) if defined $pkg;
+
+if ( ! -e $org ) {
+    mkdir($org);
+}
+chdir($org);
+
+my $super_user = $ENV{SUPER};
+if (defined($super_user)) {
+    $super_user = "-G $super_user";
+} else {
+    $super_user = "";
+}
+
+for my $package ( sort(@packages) ) {
+    print " ----- PROCESSING $package\n";
+    my $url      = "https://src.opensuse.org/$org/$repo_path.git";
+    my $push_url = "gitea\@src.opensuse.org:pool/$package.git";
+    if ( not -e $package ) {
+        print("cloning...\n");
+        system("git clone --origin pool $url $package") == 0
+          or die "Can't clone $org/$repo_path";
+    }
+    else {
+        print("adding remote...\n");
+        system("git -C $package remote rm pool > /dev/null");
+        system("git -C $package remote add pool $url") == 0
+          or die "Can't add pool for $package";
+    }
+    system("git -C $package remote set-url pool --push $push_url") == 0
+      or die "Can't add push remote for $package";
+    print("fetching remote...\n");
+    system("git -C $package fetch pool") == 0 or die "Can't fetch pool for $package";
+
+    my @commits = FindFactoryCommit($package);
+    my $Md5Hashes = FactoryMd5($package);
+    my $c;
+    my $match = 0;
+    for my $commit (@commits) {
+        if ( length($commit) != 64 ) {
+            print("Failed to find factory commit. Aborting.");
+            exit(1);
+        }
+
+        if (
+                system("git -C $package lfs fetch pool $commit") == 0
+            and system("git -C $package checkout -B factory $commit") == 0
+            and system("git -C $package lfs checkout") == 0
+            and chdir($package)) {
+
+	    open(my $fh, "|-", "md5sum -c --quiet") or die $!;
+            print $fh $Md5Hashes;
+            close $fh;
+	    if ($? >> 8 != 0) {
+		    chdir("..") || die;
+		    next;
+	    }
+	    open($fh, "|-", "awk '{print \$2}' | sort | bash -c \"diff <(ls -1 | sort) -\"") or die $!;
+	    print $fh $Md5Hashes;
+            close $fh;
+            my $ec = $? >> 8;
+	    chdir("..") || die;
+
+	    if ($ec == 0) {
+               $c     = $commit;
+               $match = 1;
+               last;
+	    }
+
+        }
+    }
+
+    if ( !$match ) {
+        die "Match not found. Aborting.";
+    }
+
+    system ("git -C $package push -f pool factory");
+    print "$package: $c\n";
+}
+

devel-importer/migrated_projects

@@ -1,3 +1,4 @@
+SystemsManagement
 Java:packages
 Kernel:firmware
 Kernel:kdump

devel-importer/update_git_packages.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/bash
+
+osc api '/search/package?match=scmsync' | ../xml_package_parse | ../find_factory_commit_in_gitpkg.pl
+

devel-importer/xml_package_parse

@@ -0,0 +1,83 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+use XML::Parser;
+
+my $parser = XML::Parser->new(Handlers => {
+    Start => \&handle_start,
+    End   => \&handle_end,
+    Char  => \&handle_char,
+});
+
+my $current_element = '';
+my $current_package_attrs = {};
+my $scmsync_content = '';
+
+my %devel_pkgs;
+open(my $dfh, "curl -s https://src.opensuse.org/openSUSE/Factory/raw/branch/main/pkgs/_meta/devel_packages |") or die $!;
+while(<$dfh>) {
+    chomp;
+    $devel_pkgs{$_} = 1;
+}
+close($dfh);
+
+my $xml_content = do { local $/; <STDIN> };
+$parser->parse($xml_content);
+
+sub handle_start {
+    my ($expat, $element, %attrs) = @_;
+    $current_element = $element;
+    
+    if ($element eq 'package') {
+        $current_package_attrs = \%attrs;
+    }
+    if ($element eq 'scmsync') {
+        $scmsync_content = '';
+    }
+}
+
+sub handle_char {
+    my ($expat, $string) = @_;
+    if ($current_element eq 'scmsync') {
+        $scmsync_content .= $string;
+    }
+}
+
+sub handle_end {
+    my ($expat, $element) = @_;
+    
+    if ($element eq 'scmsync') {
+        my $project = $current_package_attrs->{project};
+        my $name = $current_package_attrs->{name};
+        my $scmsync = $scmsync_content;
+
+        # Use checks
+        $project = '' unless defined $project;
+        $name = '' unless defined $name;
+        $scmsync = '' unless defined $scmsync;
+
+        # Trim
+        $project =~ s/^\s+|\s+$//g;
+        $name =~ s/^\s+|\s+$//g;
+        $scmsync =~ s/^\s+|\s+$//g;
+
+        my $has_error = 0;
+        foreach my $val ($project, $name, $scmsync) {
+            if ($val =~ /\s/) {
+                print STDERR "Error: Value '$val' contains whitespace.\n";
+                $has_error = 1;
+            }
+        }
+
+        unless ($has_error) {
+            if ($devel_pkgs{"$name $project"}) {
+                print "$name $project $scmsync\n";
+            }
+        }
+    }
+    
+    # Reset current element if we are closing it
+    if ($current_element eq $element) {
+        $current_element = '';
+    }
+}

obs-staging-bot/README.md

@@ -34,7 +34,8 @@ It's a JSON file with following syntax:
  "QA": [
    {
      "Name": "SLES",
-     "Origin": "SUSE:SLFO:Products:SLES:16.0"
+     "Origin": "SUSE:SLFO:Products:SLES:16.0",
+     "BuildDisableRepos": ["product"]
    }
  ]
 }
@@ -47,6 +48,7 @@ It's a JSON file with following syntax:
 | *QA* | Crucial for generating a product build (such as an ISO or FTP tree) that incorporates the packages. | no | array of objects | | |
 | *QA > Name* | Suffix for the QA OBS staging project. The project is named *StagingProject:<PR_Number>:Name*. | no | string | | |
 | *QA > Origin* | OBS reference project | no | string | | |
+| *QA > BuildDisableRepos* | The names of OBS repositories to build-disable, if any. | no | array of strings | | [] |
 
 
 Details
@@ -65,6 +67,6 @@ Details
 * **PrjGit PR - QA staging project**
   * The QA staging project is meant for building the product; the relative build config is inherited from the `QA > Origin` project.
   * In this case, the **scmsync** tag is inherited from the `QA > Origin` project.
-
-
+  * It is desirable in some cases to avoid building some specific build service repositories when not needed. In this case, `QA > BuildDisableRepos` can be specified.
+    These repositories would be disabled in the project meta when generating the QA project.
 

obs-staging-bot/main.go

@@ -109,6 +109,11 @@ const (
 	BuildStatusSummaryUnknown  = 4
 )
 
+type DisableFlag struct {
+	XMLName string `xml:"disable"`
+	Name    string `xml:"repository,attr"`
+}
+
 func ProcessBuildStatus(project, refProject *common.BuildResultList) BuildStatusSummary {
 	if _, finished := refProject.BuildResultSummary(); !finished {
 		common.LogDebug("refProject not finished building??")
@@ -377,7 +382,7 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
 // stagingProject:$buildProject
 //  ^-  stagingProject:$buildProject:$subProjectName (based on templateProject)
 
-func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingProject, templateProject, subProjectName string) error {
+func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingProject, templateProject, subProjectName string, buildDisableRepos []string) error {
 	common.LogDebug("Setup QA sub projects")
 	templateMeta, err := ObsClient.GetProjectMeta(templateProject)
 	if err != nil {
@@ -407,7 +412,21 @@ func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, git
 		repository.Fragment  = branch.SHA
 		templateMeta.ScmSync = repository.String()
 		common.LogDebug("Setting scmsync url to ", templateMeta.ScmSync)
-        }
+	}
+	// Build-disable repositories if asked
+	if len(buildDisableRepos) > 0 {
+		toDisable := make([]DisableFlag, len(buildDisableRepos))
+		for idx, repositoryName := range buildDisableRepos {
+			toDisable[idx] = DisableFlag{Name: repositoryName}
+		}
+
+		output, err := xml.Marshal(toDisable)
+		if err != nil {
+			common.LogError("error while marshalling, skipping BuildDisableRepos: ", err)
+		} else {
+			templateMeta.BuildFlags.Contents += string(output)
+		}
+	}
 	// Cleanup ReleaseTarget and modify affected path entries
 	for idx, r := range templateMeta.Repositories {
 		templateMeta.Repositories[idx].ReleaseTargets = nil
@@ -922,7 +941,8 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 				CreateQASubProject(stagingConfig, git, gitea, pr,
 					stagingProject,
 					setup.Origin,
-					setup.Name)
+					setup.Name,
+					setup.BuildDisableRepos)
 				msg = msg + ObsWebHost + "/project/show/" +
 					stagingProject + ":" + setup.Name + "\n"
 			}