common/config.go

@@ -54,6 +54,7 @@ type ReviewGroup struct {
 type QAConfig struct {
 	Name   string
 	Origin string
+	Label  string // requires this gitea lable to be set or skipped
 	BuildDisableRepos []string // which repos to build disable in the new project
 }
 
@@ -63,9 +64,9 @@ type Permissions struct {
 }
 
 const (
-	Label_StagingAuto   = "staging/Auto"
-	Label_ReviewPending = "review/Pending"
-	Label_ReviewDone    = "review/Done"
+	Label_StagingAuto    = "staging/Auto"
+	Label_ReviewPending  = "review/Pending"
+	Label_ReviewDone     = "review/Done"
 )
 
 func LabelKey(tag_value string) string {

common/consts.go

@@ -20,10 +20,13 @@ package common
 
 const (
 	GiteaTokenEnv    = "GITEA_TOKEN"
+	GiteaHostEnv     = "GITEA_HOST"
 	ObsUserEnv       = "OBS_USER"
 	ObsPasswordEnv   = "OBS_PASSWORD"
 	ObsSshkeyEnv     = "OBS_SSHKEY"
 	ObsSshkeyFileEnv = "OBS_SSHKEYFILE"
+	ObsApiEnv        = "OBS_API"
+	ObsWebEnv        = "OBS_WEB"
 
 	DefaultGitPrj      = "_ObsPrj"
 	PrjLinksFile       = "links.json"

common/git_utils.go

@@ -283,7 +283,7 @@ func (e *GitHandlerImpl) GitClone(repo, branch, remoteUrl string) (string, error
 func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error) {
 	id, err := e.GitExecWithOutput(gitDir, "show-ref", "--heads", "--hash", branchName)
 	if err != nil {
-		return "", fmt.Errorf("Can't find default branch: %s", branchName)
+		return "", fmt.Errorf("Can't find default branch: %s in %s", branchName, gitDir)
 	}
 
 	id = strings.TrimSpace(SplitLines(id)[0])
@@ -297,7 +297,7 @@ func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error
 func (e *GitHandlerImpl) GitRemoteHead(gitDir, remote, branchName string) (string, error) {
 	id, err := e.GitExecWithOutput(gitDir, "show-ref", "--hash", "--verify", "refs/remotes/"+remote+"/"+branchName)
 	if err != nil {
-		return "", fmt.Errorf("Can't find default branch: %s", branchName)
+		return "", fmt.Errorf("Can't find default branch: %s in %s", branchName, gitDir)
 	}
 
 	return strings.TrimSpace(id), nil
@@ -904,6 +904,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
 				for _, te := range tree.items {
 					if te.isTree() {
 						trees[p+te.name+"/"] = te.hash
+						submoduleList[p+te.name] = te.hash
 					} else if te.isSubmodule() {
 						submoduleList[p+te.name] = te.hash
 					}

common/obs_utils.go

@@ -116,13 +116,18 @@ type Flags struct {
 	Contents string `xml:",innerxml"`
 }
 
+type ProjectLinkMeta struct {
+	Project string `xml:"project,attr"`
+}
+
 type ProjectMeta struct {
 	XMLName      xml.Name         `xml:"project"`
 	Name         string           `xml:"name,attr"`
 	Title        string           `xml:"title"`
 	Description  string           `xml:"description"`
 	Url          string           `xml:"url,omitempty"`
-	ScmSync      string           `xml:"scmsync"`
+	ScmSync      string           `xml:"scmsync,omitempty"`
+	Link         []ProjectLinkMeta `xml:"link"`
 	Persons      []PersonRepoMeta `xml:"person"`
 	Groups       []GroupRepoMeta  `xml:"group"`
 	Repositories []RepositoryMeta `xml:"repository"`
@@ -138,8 +143,8 @@ type ProjectMeta struct {
 type PackageMeta struct {
 	XMLName xml.Name         `xml:"package"`
 	Name    string           `xml:"name,attr"`
-	Project string           `xml:"project,attr"`
-	ScmSync string           `xml:"scmsync"`
+	Project string           `xml:"project,attr,omitempty"`
+	ScmSync string           `xml:"scmsync,omitempty"`
 	Persons []PersonRepoMeta `xml:"person"`
 	Groups  []GroupRepoMeta  `xml:"group"`
 
@@ -689,13 +694,15 @@ func (r *BuildResultList) BuildResultSummary() (success, finished bool) {
 			if !ok {
 				panic("Unknown result code: " + result.Code)
 			}
-			if r.isLastBuild && result.Code == "unknown" {
-				// it means the package has never build yet,
-				// but we don't know the reason
-				detail.Finished = true
+			if r.isLastBuild {
+				// we are always finished, since it is the last result
+				// also when there is "unknown" state, it just means it
+				// it was never done
+				finished = true
+			} else {
+			        finished = finished && detail.Finished
 			}
 
-			finished = finished && detail.Finished
 			success = success && detail.Success
 
 			if !finished {

obs-staging-bot/README.md

@@ -35,6 +35,7 @@ It's a JSON file with following syntax:
    {
      "Name": "SLES",
      "Origin": "SUSE:SLFO:Products:SLES:16.0",
+     "Label": "BootstrapRing",
      "BuildDisableRepos": ["product"]
    }
  ]
@@ -48,6 +49,7 @@ It's a JSON file with following syntax:
 | *QA* | Crucial for generating a product build (such as an ISO or FTP tree) that incorporates the packages. | no | array of objects | | |
 | *QA > Name* | Suffix for the QA OBS staging project. The project is named *StagingProject:<PR_Number>:Name*. | no | string | | |
 | *QA > Origin* | OBS reference project | no | string | | |
+| *QA > Label* | Setup the project only when the given gitea label is set on pull request | no | string | | |
 | *QA > BuildDisableRepos* | The names of OBS repositories to build-disable, if any. | no | array of strings | | [] |
 
 
@@ -69,4 +71,10 @@ Details
   * In this case, the **scmsync** tag is inherited from the `QA > Origin` project.
   * It is desirable in some cases to avoid building some specific build service repositories when not needed. In this case, `QA > BuildDisableRepos` can be specified.
     These repositories would be disabled in the project meta when generating the QA project.
-
+  * QA projects can build on each other. In this case it is important that the order to setup is correct
+    in the staging.config file.
+  * Based on Label settings QA projects can get created or removed. The staging bot is also checking that these
+    projects build successfully.
+  * It is possible to include the sources from the staging project also in the QA project. Define a template using
+    a project link pointing to the project defined as "StagingProject". You must *not* use scmsync directly in the
+    same project then, but you can use it indirectly via a second project link

obs-staging-bot/main.go

@@ -19,6 +19,7 @@ package main
  */
 
 import (
+	"bufio"
 	"encoding/xml"
 	"errors"
 	"flag"
@@ -26,6 +27,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"runtime/debug"
 	"slices"
@@ -114,161 +116,105 @@ type DisableFlag struct {
 	Name    string `xml:"repository,attr"`
 }
 
-func ProcessBuildStatus(project, refProject *common.BuildResultList) BuildStatusSummary {
-	if _, finished := refProject.BuildResultSummary(); !finished {
-		common.LogDebug("refProject not finished building??")
-		return BuildStatusSummaryUnknown
-	}
-
+func ProcessBuildStatus(project *common.BuildResultList) BuildStatusSummary {
 	if _, finished := project.BuildResultSummary(); !finished {
 		common.LogDebug("Still building...")
 		return BuildStatusSummaryBuilding
 	}
 
-	// the repositories should be setup equally between the projects. We
-	// need to verify that packages that are building in `refProject` are not
-	// failing in the `project`
-	BuildResultSorter := func(a, b *common.BuildResult) int {
-		if c := strings.Compare(a.Repository, b.Repository); c != 0 {
-			return c
-		}
-		if c := strings.Compare(a.Arch, b.Arch); c != 0 {
-			return c
-		}
-
-		panic("Should not happen -- BuiltResultSorter equal repos?")
-	}
-	slices.SortFunc(project.Result, BuildResultSorter)
-	if refProject == nil {
-		// just return if buid finished and have some successes, since new package
-		common.LogInfo("New package. Only need some success...")
-		SomeSuccess := false
-		for i := 0; i < len(project.Result); i++ {
-			repoRes := project.Result[i]
-			repoResStatus, ok := common.ObsRepoStatusDetails[repoRes.Code]
-			if !ok {
-				common.LogDebug("cannot find code:", repoRes.Code)
-				return BuildStatusSummaryUnknown
-			}
-			if !repoResStatus.Finished {
-				return BuildStatusSummaryBuilding
-			}
-
-			for _, pkg := range repoRes.Status {
-				pkgStatus, ok := common.ObsBuildStatusDetails[pkg.Code]
-				if !ok {
-					common.LogInfo("Unknown package build status:", pkg.Code, "for", pkg.Package)
-					common.LogDebug("Details:", pkg.Details)
-				}
-
-				if pkgStatus.Success {
-					SomeSuccess = true
-				}
-			}
-		}
-
-		if SomeSuccess {
-			return BuildStatusSummarySuccess
-		}
-		return BuildStatusSummaryFailed
-	}
-
-	slices.SortFunc(refProject.Result, BuildResultSorter)
-
-	common.LogDebug("comparing results", len(project.Result), "vs. ref", len(refProject.Result))
-	SomeSuccess := false
+	common.LogDebug("build results", len(project.Result))
 	for i := 0; i < len(project.Result); i++ {
 		common.LogDebug("searching for", project.Result[i].Repository, "/", project.Result[i].Arch)
-		j := 0
 	found:
-		for ; j < len(refProject.Result); j++ {
-			if project.Result[i].Repository != refProject.Result[j].Repository ||
-				project.Result[i].Arch != refProject.Result[j].Arch {
-				continue
-			}
-
+		for j := 0; j < len(project.Result); j++ {
 			common.LogDebug("  found match for @ idx:", j)
-			res, success := ProcessRepoBuildStatus(project.Result[i].Status, refProject.Result[j].Status)
+			res := ProcessRepoBuildStatus(project.Result[i].Status)
 			switch res {
 			case BuildStatusSummarySuccess:
-				SomeSuccess = SomeSuccess || success
 				break found
+			case BuildStatusSummaryFailed:
+				return BuildStatusSummaryFailed
 			default:
 				return res
 			}
 		}
-
-		if j >= len(refProject.Result) {
-			common.LogDebug("Cannot find results...")
-			common.LogDebug(project.Result[i])
-			common.LogDebug(refProject.Result)
-			return BuildStatusSummaryUnknown
-		}
 	}
 
-	if SomeSuccess {
-		return BuildStatusSummarySuccess
-	}
-
-	return BuildStatusSummaryFailed
+	return BuildStatusSummarySuccess
 }
 
-func ProcessRepoBuildStatus(results, ref []*common.PackageBuildStatus) (status BuildStatusSummary, SomeSuccess bool) {
-	PackageBuildStatusSorter := func(a, b *common.PackageBuildStatus) int {
-		return strings.Compare(a.Package, b.Package)
-	}
+func ProcessRepoBuildStatus(results []*common.PackageBuildStatus) (status BuildStatusSummary) {
+
+ 	PackageBuildStatusSorter := func(a, b *common.PackageBuildStatus) int {
+ 	        return strings.Compare(a.Package, b.Package)
+ 	}
 
-	common.LogDebug("********  REF: ")
-	data, _ := xml.MarshalIndent(ref, "", "  ")
-	common.LogDebug(string(data))
 	common.LogDebug("******* RESULTS: ")
-	data, _ = xml.MarshalIndent(results, "", "  ")
+	data, _ := xml.MarshalIndent(results, "", "  ")
 	common.LogDebug(string(data))
 	common.LogDebug("*******")
 
 	// compare build result
 	slices.SortFunc(results, PackageBuildStatusSorter)
-	slices.SortFunc(ref, PackageBuildStatusSorter)
 
-	j := 0
-	SomeSuccess = false
 	for i := 0; i < len(results); i++ {
 		res, ok := common.ObsBuildStatusDetails[results[i].Code]
 		if !ok {
 			common.LogInfo("unknown package result code:", results[i].Code, "for package:", results[i].Package)
-			return BuildStatusSummaryUnknown, SomeSuccess
+			return BuildStatusSummaryUnknown
 		}
 
 		if !res.Finished {
-			return BuildStatusSummaryBuilding, SomeSuccess
+			return BuildStatusSummaryBuilding
 		}
 
 		if !res.Success {
-			// not failed if reference project also failed for same package here
-			for ; j < len(results) && strings.Compare(results[i].Package, ref[j].Package) < 0; j++ {
-			}
-
-			if j < len(results) && results[i].Package == ref[j].Package {
-				refRes, ok := common.ObsBuildStatusDetails[ref[j].Code]
-				if !ok {
-					common.LogInfo("unknown ref package result code:", ref[j].Code, "package:", ref[j].Package)
-					return BuildStatusSummaryUnknown, SomeSuccess
-				}
-
-				if !refRes.Finished {
-					common.LogDebug("Not finished building in reference project?")
-				}
-
-				if refRes.Success {
-					return BuildStatusSummaryFailed, SomeSuccess
-				}
-			}
-		} else {
-			SomeSuccess = true
+			return BuildStatusSummaryFailed
 		}
 	}
 
-	return BuildStatusSummarySuccess, SomeSuccess
+	return BuildStatusSummarySuccess
+}
+
+func GetPackageBuildStatus(project *common.BuildResultList, packageName string) (bool, BuildStatusSummary) {
+	var packageStatuses []*common.PackageBuildStatus
+
+	// Collect all statuses for the package
+	for _, result := range project.Result {
+		for _, pkgStatus := range result.Status {
+			if pkgStatus.Package == packageName {
+				packageStatuses = append(packageStatuses, pkgStatus)
+			}
+		}
+	}
+
+	if len(packageStatuses) == 0 {
+		return true, BuildStatusSummaryUnknown // true for 'missing'
+	}
+
+	// Check for any failures
+	for _, pkgStatus := range packageStatuses {
+		res, ok := common.ObsBuildStatusDetails[pkgStatus.Code]
+		if !ok {
+			common.LogInfo("unknown package result code:", pkgStatus.Code, "for package:", pkgStatus.Package)
+			return false, BuildStatusSummaryUnknown
+		}
+		if !res.Success {
+			return false, BuildStatusSummaryFailed
+		}
+	}
+
+	// Check for any unfinished builds
+	for _, pkgStatus := range packageStatuses {
+		res, _ := common.ObsBuildStatusDetails[pkgStatus.Code]
+		// 'ok' is already checked in the loop above
+		if !res.Finished {
+			return false, BuildStatusSummaryBuilding
+		}
+	}
+
+	// If we got here, all are finished and successful
+	return false, BuildStatusSummarySuccess
 }
 
 func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingPrj, buildPrj string, stagingMasterPrj string) (*common.ProjectMeta, error) {
@@ -327,9 +273,9 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
 		common.LogError("error fetching project meta for", buildPrj, ". Err:", err)
 		return nil, err
 	}
+	common.LogInfo("Meta: ", meta)
 
 	// generate new project with paths pointinig back to original repos
-	// disable publishing
 
 	meta.Name = stagingPrj
 	meta.Description = fmt.Sprintf(`Pull request build job PR#%d to branch %s of %s/%s`,
@@ -344,7 +290,10 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
 
 	urlPkg := make([]string, 0, len(modifiedOrNew))
 	for _, pkg := range modifiedOrNew {
-		urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(pkg))
+		// FIXME: skip manifest subdirectories itself
+		// strip any leading directory name and just hand over last directory as package name
+		onlybuilds := strings.Split(pkg, "/")
+		urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(onlybuilds[len(onlybuilds)-1]))
 	}
 	meta.ScmSync = pr.Head.Repo.CloneURL + "?" + strings.Join(urlPkg, "&") + "#" + pr.Head.Sha
 	if len(meta.ScmSync) >= 65535 {
@@ -384,12 +333,14 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
 
 func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingProject, templateProject, subProjectName string, buildDisableRepos []string) error {
 	common.LogDebug("Setup QA sub projects")
+	common.LogDebug("reading templateProject ", templateProject)
 	templateMeta, err := ObsClient.GetProjectMeta(templateProject)
 	if err != nil {
 		common.LogError("error fetching template project meta for", templateProject, ":", err)
 		return err
 	}
 	// patch baseMeta to become the new project
+	common.LogDebug("upcoming project name ", stagingProject, ":", subProjectName)
 	templateMeta.Name = stagingProject + ":" + subProjectName
 	// freeze tag for now
 	if len(templateMeta.ScmSync) > 0 {
@@ -427,19 +378,50 @@ func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, git
 			templateMeta.BuildFlags.Contents += string(output)
 		}
 	}
+
+	// include sources from submission project when link points to staging project
+	for idx, l := range templateMeta.Link {
+		if l.Project == stagingConfig.StagingProject {
+			templateMeta.Link[idx].Project = stagingProject
+		}
+	}
+
 	// Cleanup ReleaseTarget and modify affected path entries
 	for idx, r := range templateMeta.Repositories {
 		templateMeta.Repositories[idx].ReleaseTargets = nil
 
 		for pidx, path := range r.Paths {
 			// Check for path building against code stream
+			common.LogDebug("   checking in ", templateMeta.Name)
+			common.LogDebug("      stagingProject ", stagingProject)
+			common.LogDebug("      checking for ", templateMeta.Repositories[idx].Paths[pidx].Project)
+			common.LogDebug("         path.Project ", path.Project)
+			common.LogDebug("         stagingConfig.ObsProject ", stagingConfig.ObsProject)
+			common.LogDebug("         stagingConfig.StagingProject ", stagingConfig.StagingProject)
+			common.LogDebug("         templateProject ", templateProject)
 			if path.Project == stagingConfig.ObsProject {
 				templateMeta.Repositories[idx].Paths[pidx].Project = stagingProject
-			}
+			} else
 			// Check for path building against a repo in template project itself
 			if path.Project == templateProject {
 				templateMeta.Repositories[idx].Paths[pidx].Project = templateMeta.Name
+			} else
+			// Check for path prefixes against a template project inside of template project area
+			if strings.HasPrefix(path.Project, stagingConfig.StagingProject + ":") {
+				newProjectName := stagingProject
+				// find project name
+				for _, setup := range stagingConfig.QA {
+					if setup.Origin == path.Project  {
+						common.LogDebug("   Match:", setup.Origin)
+						newProjectName = newProjectName + ":" + setup.Name
+						common.LogDebug("   New:", newProjectName)
+						break
+					}
+				}
+				templateMeta.Repositories[idx].Paths[pidx].Project = newProjectName
+			        common.LogDebug("   Matched prefix")
 			}
+			common.LogDebug("   Path using project ", templateMeta.Repositories[idx].Paths[pidx].Project)
 		}
 	}
 
@@ -447,6 +429,8 @@ func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, git
 		err = ObsClient.SetProjectMeta(templateMeta)
 		if err != nil {
 			common.LogError("cannot create project:", templateMeta.Name, err)
+			x, _ := xml.MarshalIndent(templateMeta, "", "   ")
+			common.LogError(string(x))
 			return err
 		}
 	} else {
@@ -553,7 +537,7 @@ func FetchOurLatestActionableReview(gitea common.Gitea, org, repo string, id int
 }
 
 func ParseNotificationToPR(thread *models.NotificationThread) (org string, repo string, num int64, err error) {
-	rx := regexp.MustCompile(`^https://src\.(?:open)?suse\.(?:org|de)/api/v\d+/repos/(?<org>[-_a-zA-Z0-9]+)/(?<project>[-_a-zA-Z0-9]+)/issues/(?<num>[0-9]+)$`)
+	rx := regexp.MustCompile(`^.*/api/v\d+/repos/(?<org>[-_a-zA-Z0-9]+)/(?<project>[-_a-zA-Z0-9]+)/issues/(?<num>[0-9]+)$`)
 	notification := thread.Subject
 	match := rx.FindStringSubmatch(notification.URL)
 	if match == nil {
@@ -702,6 +686,64 @@ func SetStatus(gitea common.Gitea, org, repo, hash string, status *models.Commit
 	return err
 }
 
+func commentOnPackagePR(gitea common.Gitea, org string, repo string, prNum int64, msg string) {
+	if IsDryRun {
+		common.LogInfo("Would comment on package PR %s/%s#%d: %s", org, repo, prNum, msg)
+		return
+	}
+
+	pr, err := gitea.GetPullRequest(org, repo, prNum)
+	if err != nil {
+		common.LogError("Failed to get package PR %s/%s#%d: %v", org, repo, prNum, err)
+		return
+	}
+
+	err = gitea.AddComment(pr, msg)
+	if err != nil {
+		common.LogError("Failed to comment on package PR %s/%s#%d: %v", org, repo, prNum, err)
+	}
+}
+
+// Create and remove QA projects
+func ProcessQaProjects(stagingConfig *common.StagingConfig, git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingProject string) []string {
+	usedQAprojects := make([]string, 0)
+	prLabelNames := make(map[string]int)
+	for _, label := range pr.Labels {
+		prLabelNames[label.Name] = 1
+	}
+	msg := ""
+	for _, setup := range stagingConfig.QA {
+		QAproject := stagingProject + ":" + setup.Name
+		if len(setup.Label) > 0 {
+			if _, ok := prLabelNames[setup.Label]; !ok {
+				if !IsDryRun {
+					// blindly remove, will fail when not existing
+			 		ObsClient.DeleteProject(QAproject)
+				}
+				common.LogInfo("QA project ", setup.Name, "has no matching Label")
+				continue
+			}
+		}
+
+		usedQAprojects = append(usedQAprojects, QAproject)
+		// check for existens first, no error, but no meta is a 404
+		if meta, err := ObsClient.GetProjectMeta(QAproject); meta == nil && err == nil {
+			common.LogInfo("Create QA project ", QAproject)
+			CreateQASubProject(stagingConfig, git, gitea, pr,
+				stagingProject,
+				setup.Origin,
+				setup.Name,
+				setup.BuildDisableRepos)
+				msg = msg + "QA Project added: " + ObsWebHost + "/project/show/" +
+        		              QAproject + "\n"
+		}
+	}
+	if len(msg) > 1 {
+		gitea.AddComment(pr, msg)
+	}
+	return usedQAprojects
+}
+
 func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, error) {
 	dir, err := os.MkdirTemp(os.TempDir(), BotName)
 	common.PanicOnError(err)
@@ -779,11 +821,12 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 		meta, err := ObsClient.GetProjectMeta(stagingConfig.ObsProject)
 		if err != nil || meta == nil {
 			common.LogError("Cannot find reference project meta:", stagingConfig.ObsProject, err)
-			if !IsDryRun {
+			if !IsDryRun && err == nil {
+				common.LogError("Reference project is absent:", stagingConfig.ObsProject, err)
 				_, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot fetch reference project meta")
 				return true, err
 			}
-			return true, nil
+			return true, err
 		}
 
 		if metaUrl, err := url.Parse(meta.ScmSync); err != nil {
@@ -843,10 +886,13 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 		if !stagingConfig.RebuildAll {
 			for pkg, headOid := range headSubmodules {
 				if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid {
+					if pkg != "rpms" && pkg != "dependencies" {
+					_, spkg := filepath.Split(pkg)
 					if exists {
-						modifiedPackages = append(modifiedPackages, pkg)
+						modifiedPackages = append(modifiedPackages, spkg)
 					} else {
-						newPackages = append(newPackages, pkg)
+						newPackages = append(newPackages, spkg)
+					}
 					}
 					common.LogDebug(pkg, ":", baseOid, "->", headOid)
 				}
@@ -933,36 +979,49 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 			msg = "Build is started in " + ObsWebHost + "/project/show/" +
 				stagingProject + " .\n"
 
-			if len(stagingConfig.QA) > 0 {
-				msg = msg + "\nAdditional QA builds: \n"
-			}
 			SetStatus(gitea, org, repo, pr.Head.Sha, status)
-			for _, setup := range stagingConfig.QA {
-				CreateQASubProject(stagingConfig, git, gitea, pr,
-					stagingProject,
-					setup.Origin,
-					setup.Name,
-					setup.BuildDisableRepos)
-				msg = msg + ObsWebHost + "/project/show/" +
-					stagingProject + ":" + setup.Name + "\n"
-			}
 		}
 		if change != RequestModificationNoChange && !IsDryRun {
 			gitea.AddComment(pr, msg)
 		}
 
-		baseResult, err := ObsClient.LastBuildResults(stagingConfig.ObsProject, modifiedPackages...)
-		if err != nil {
-			common.LogError("failed fetching ref project status for", stagingConfig.ObsProject, ":", err)
-		}
 		stagingResult, err := ObsClient.BuildStatus(stagingProject)
 		if err != nil {
 			common.LogError("failed fetching stage project status for", stagingProject, ":", err)
 		}
-		buildStatus := ProcessBuildStatus(stagingResult, baseResult)
+
+		_, packagePRs := common.ExtractDescriptionAndPRs(bufio.NewScanner(strings.NewReader(pr.Body)))
+
+		// always update QA projects because Labels can change
+		qaProjects := ProcessQaProjects(stagingConfig, git, gitea, pr, stagingProject)
 
 		done := false
-		switch buildStatus {
+		overallBuildStatus := ProcessBuildStatus(stagingResult)
+		commentSuffix := ""
+		if len(qaProjects) > 0 && overallBuildStatus == BuildStatusSummarySuccess {
+			seperator := " in "
+			for _, qaProject := range qaProjects {
+				qaResult, err := ObsClient.BuildStatus(qaProject)
+				if err != nil {
+					common.LogError("failed fetching stage project status for", qaProject, ":", err)
+				}
+				qaBuildStatus := ProcessBuildStatus(qaResult)
+				if qaBuildStatus != BuildStatusSummarySuccess {
+					// either still building or in failed state
+					overallBuildStatus = qaBuildStatus
+					commentSuffix = commentSuffix + seperator + qaProject
+					seperator = ", "
+				}
+				if qaBuildStatus == BuildStatusSummaryFailed {
+					// main project was successful, but QA project, adapt the link to QA project
+					// and change commit state to fail
+					status.Status = common.CommitStatus_Fail
+					status.TargetURL = ObsWebHost + "/project/show/" + qaProject
+					SetStatus(gitea, org, repo, pr.Head.Sha, status)
+				}
+			}
+		}
+		switch overallBuildStatus {
 		case BuildStatusSummarySuccess:
 			status.Status = common.CommitStatus_Success
 			done = true
@@ -982,7 +1041,44 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 				}
 			}
 		}
-		common.LogInfo("Build status:", buildStatus)
+
+		if overallBuildStatus == BuildStatusSummarySuccess || overallBuildStatus == BuildStatusSummaryFailed {
+			// avoid commenting while build is in progress
+			missingPkgs := []string{}
+
+			for _, packagePR := range packagePRs {
+				missing, packageBuildStatus := GetPackageBuildStatus(stagingResult, packagePR.Repo)
+				if missing {
+					missingPkgs = append(missingPkgs, packagePR.Repo)
+					continue
+				}
+				var msg string
+				switch packageBuildStatus {
+				case BuildStatusSummarySuccess:
+					msg = fmt.Sprintf("Build successful, for more information go in %s/project/show/%s.\n", ObsWebHost, stagingProject)
+				case BuildStatusSummaryFailed:
+					msg = fmt.Sprintf("Build failed, for more information go in %s/project/show/%s.\n", ObsWebHost, stagingProject)
+				default:
+					continue
+				}
+				commentOnPackagePR(gitea, packagePR.Org, packagePR.Repo, packagePR.Num, msg)
+			}
+
+			if len(missingPkgs) > 0 {
+				overallBuildStatus = BuildStatusSummaryFailed
+				msg := "The following packages were not found in the staging project:\n"
+				for _, pkg := range missingPkgs {
+					msg = msg + " - " + pkg + "\n"
+				}
+				common.LogInfo(msg)
+				err := gitea.AddComment(pr, msg)
+				if err != nil {
+					common.LogError(err)
+				}
+			}
+		}
+
+		common.LogInfo("Build status:", overallBuildStatus)
 		if !IsDryRun {
 			if err = SetStatus(gitea, org, repo, pr.Head.Sha, status); err != nil {
 				return false, err
@@ -1047,6 +1143,7 @@ func PollWorkNotifications(giteaUrl string) {
 
 var ListPullNotificationsOnly bool
 var GiteaUrl string
+var ObsApiHost string
 var ObsWebHost string
 var IsDryRun bool
 var ProcessPROnly string
@@ -1069,8 +1166,8 @@ func main() {
 	flag.BoolVar(&ListPullNotificationsOnly, "list-notifications-only", false, "Only lists notifications without acting on them")
 	ProcessPROnly := flag.String("pr", "", "Process only specific PR and ignore the rest. Use for debugging")
 	buildRoot := flag.String("build-root", "", "Default build location for staging projects. Default is bot's home project")
-	flag.StringVar(&GiteaUrl, "gitea-url", "https://src.opensuse.org", "Gitea instance")
-	obsApiHost := flag.String("obs", "https://api.opensuse.org", "API for OBS instance")
+	flag.StringVar(&GiteaUrl, "gitea-url", "", "Gitea instance")
+	flag.StringVar(&ObsApiHost, "obs", "", "API for OBS instance")
 	flag.StringVar(&ObsWebHost, "obs-web", "", "Web OBS instance, if not derived from the obs config")
 	flag.BoolVar(&IsDryRun, "dry", false, "Dry-run, don't actually create any build projects or review changes")
 	debug := flag.Bool("debug", false, "Turns on debug logging")
@@ -1082,19 +1179,34 @@ func main() {
 		common.SetLoggingLevel(common.LogLevelInfo)
 	}
 
+	if len(GiteaUrl) == 0 {
+		GiteaUrl = os.Getenv(common.GiteaHostEnv)
+	}
+	if len(GiteaUrl) == 0 {
+		GiteaUrl = "https://src.opensuse.org"
+	}
+	if len(ObsApiHost) == 0 {
+		ObsApiHost = os.Getenv(common.ObsApiEnv)
+	}
+	if len(ObsApiHost) == 0 {
+		ObsApiHost = "https://api.opensuse.org"
+	}
 	if len(ObsWebHost) == 0 {
-		ObsWebHost = ObsWebHostFromApiHost(*obsApiHost)
+		ObsWebHost = os.Getenv(common.ObsWebEnv)
+	}
+	if len(ObsWebHost) == 0 {
+		ObsWebHost = "https://build.opensuse.org"
 	}
 
 	common.LogDebug("OBS Gitea Host:", GiteaUrl)
 	common.LogDebug("OBS Web Host:", ObsWebHost)
-	common.LogDebug("OBS API Host:", *obsApiHost)
+	common.LogDebug("OBS API Host:", ObsApiHost)
 
 	common.PanicOnErrorWithMsg(common.RequireGiteaSecretToken(), "Cannot find GITEA_TOKEN")
 	common.PanicOnErrorWithMsg(common.RequireObsSecretToken(), "Cannot find OBS_USER and OBS_PASSWORD")
 
 	var err error
-	if ObsClient, err = common.NewObsClient(*obsApiHost); err != nil {
+	if ObsClient, err = common.NewObsClient(ObsApiHost); err != nil {
 		log.Error(err)
 		return
 	}

systemd/obs-staging-bot.service

@@ -6,9 +6,13 @@ After=network-online.target
 Type=exec
 ExecStart=/usr/bin/obs-staging-bot
 EnvironmentFile=-/etc/default/obs-staging-bot.env
-DynamicUser=yes
-NoNewPrivileges=yes
-ProtectSystem=strict
+User=obs_staging_bot
+Group=users
+
+# This may work when not using ssh api connections:
+#DynamicUser=yes
+#NoNewPrivileges=yes
+#ProtectSystem=strict
 
 [Install]
 WantedBy=multi-user.target

workflow-direct/main.go

@@ -123,7 +123,7 @@ func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, co
 			common.LogError("   - ", action.Repository.Name, "repo is not sha256. Ignoring.")
 			return
 		}
-		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
+		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", "../" + action.Repository.Name, action.Repository.Name))
 		defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all", "-f")
 
 		branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current"))
@@ -215,7 +215,7 @@ func processConfiguredPushAction(action *common.PushWebhookEvent, config *common
 	}
 
 	if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil {
-		git.GitExecOrPanic(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name)
+		git.GitExecOrPanic(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", "../" + action.Repository.Name, action.Repository.Name)
 		common.LogDebug("Pushed to package that is not part of the project. Re-adding...", err)
 	} else if !stat.IsDir() {
 		common.LogError("Pushed to a package that is not a submodule but exists in the project. Ignoring.")
@@ -420,7 +420,7 @@ next_repo:
 		}
 
 		// add repository to git project
-		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", r.CloneURL, r.Name))
+		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", "../" + r.Name, r.Name))
 
 		curBranch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current"))
 		if branch != curBranch {

workflow-pr/pr_processor.go

@@ -180,6 +180,33 @@ func (pr *PRProcessor) SetSubmodulesToMatchPRSet(prset *common.PRSet) error {
 					}
 
 					updateSubmoduleInPR(submodulePath, prHead, git)
+					err := git.GitExec(path.Join(common.DefaultGitPrj, submodulePath), "lfs", "fetch")
+					common.LogError("lfs fetch err: ", err)
+					if err = git.GitExec(path.Join(common.DefaultGitPrj, submodulePath), "lfs", "fsck"); err != nil {
+
+						found_comment := false
+						timeline, terr := common.FetchTimelineSinceLastPush(Gitea, prHead, org, repo, idx)
+						if terr != nil {
+							common.LogError("lfs fsck error, but timeline fetch failed")
+							break
+						}
+						msgPrefix := "The LFS objects are broken!"
+						for _, t := range timeline {
+						        if t.Type == common.TimelineCommentType_Comment && strings.HasPrefix(t.Body, msgPrefix) {               
+						                found_comment = true
+								common.LogError("lfs fsck Comment already found")
+
+						                break
+						        }
+						}
+						        
+						if !found_comment && !common.IsDryRun {
+							Gitea.AddComment(pr.PR, msgPrefix + " Please verify with 'git lfs fsck'")
+						}
+						common.LogError("lfs fsck failed with: ", err.Error())
+						return err
+					}
+
 					status, err := git.GitStatus(common.DefaultGitPrj)
 					common.LogDebug("status:", status)
 					common.LogDebug("submodule", repo, " hash:", id, " -> ", prHead)
@@ -302,14 +329,16 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
 	PrjGit := PrjGitPR.PR.Base.Repo
 	prjGitPRbranch := PrjGitPR.PR.Head.Name
 	if PrjGitPR.PR.Base.RepoID != PrjGitPR.PR.Head.RepoID {
-		PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, "", PrjGit.SSHURL)
-		git.GitExecOrPanic(common.DefaultGitPrj, "fetch", PrjGitPR.RemoteName, PrjGitPR.PR.Head.Sha)
-		git.GitExecOrPanic(common.DefaultGitPrj, "checkout", PrjGitPR.PR.Head.Sha)
-		common.LogInfo("Cannot update this PR as it's on another remote, not branch:", prjGitPRbranch, "Assuming this is by-design. (eg. project git PR only)")
-		return nil
+		// permission check, if submission comes from foreign repo
+		if !PrjGitPR.PR.AllowMaintainerEdit {
+                	common.LogError("Warning: source and target branch are in different repositories. We may not have the right permissions...")
+	//		Gitea.AddComment(PrjGitPR.PR, "This PR does not allow maintainer changes, but referenced package branch has changed!")
+			return nil
+		}
 	}
 
-	PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, prjGitPRbranch, PrjGit.SSHURL)
+//	PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, prjGitPRbranch, PrjGit.SSHURL)
+	PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, PrjGitPR.PR.Head.Ref, PrjGitPR.PR.Head.Repo.SSHURL)
 	common.PanicOnError(err)
 	git.GitExecOrPanic(common.DefaultGitPrj, "fetch", PrjGitPR.RemoteName, PrjGitBranch)
 
@@ -357,6 +386,7 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
 			}
 			common.PanicOnError(git.GitExec(common.DefaultGitPrj, params...))
 			PrjGitPR.PR.Head.Sha = newHeadCommit
+			Gitea.SetLabels(PrjGit.Owner.UserName, PrjGit.Name, PrjGitPR.PR.Index, []string{prset.Config.Label("PR/updated")})
 		}
 
 		// update PR
@@ -373,7 +403,7 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
 			}
 			return CurrentTitle == NewTitle
 		}
-		if PrjGitPR.PR.User.UserName == CurrentUser.UserName && (PrjGitPR.PR.Body != PrjGitBody || !isPrTitleSame(PrjGitPR.PR.Title, PrjGitTitle)) {
+		if !pr.config.NoProjectGitPR && PrjGitPR.PR.User.UserName == CurrentUser.UserName && (PrjGitPR.PR.Body != PrjGitBody || !isPrTitleSame(PrjGitPR.PR.Title, PrjGitTitle)) {
 			Gitea.UpdatePullRequest(PrjGit.Owner.UserName, PrjGit.Name, PrjGitPR.PR.Index, &models.EditPullRequestOption{
 				RemoveDeadline: true,
 				Title:          PrjGitTitle,
@@ -464,11 +494,11 @@ func (pr *PRProcessor) Process(req *models.PullRequest) error {
 						if _, ok := err.(*repository.RepoMergePullRequestConflict); !ok {
 							common.PanicOnError(err)
 						}
-					} else {
-						Gitea.AddComment(pr.PR, "Closing here because the associated Project PR has been closed.")
-						Gitea.UpdatePullRequest(org, repo, idx, &models.EditPullRequestOption{
-							State: "closed",
-						})
+//					} else {
+//						Gitea.AddComment(pr.PR, "Closing here because the associated Project PR has been closed.")
+//						Gitea.UpdatePullRequest(org, repo, idx, &models.EditPullRequestOption{
+//							State: "closed",
+//						})
 					}
 				}
 			}
@@ -481,10 +511,10 @@ func (pr *PRProcessor) Process(req *models.PullRequest) error {
 					continue
 				}
 			}
-		}
 
-		if err = pr.UpdatePrjGitPR(prset); err != nil {
-			return err
+			if err = pr.UpdatePrjGitPR(prset); err != nil {
+				return err
+			}
 		}
 	}