Add an "lfs fsck" check after submodule update

to avoid merging pull requests where lfs objects are not correctly
registered. Can happen when user has not installed lfs for example.

common/config.go

@@ -62,9 +62,9 @@ type Permissions struct {
 }
 
 const (
-	Label_StagingAuto   = "staging/Auto"
-	Label_ReviewPending = "review/Pending"
-	Label_ReviewDone    = "review/Done"
+	Label_StagingAuto    = "staging/Auto"
+	Label_ReviewPending  = "review/Pending"
+	Label_ReviewDone     = "review/Done"
 )
 
 func LabelKey(tag_value string) string {

common/git_utils.go

@@ -283,7 +283,7 @@ func (e *GitHandlerImpl) GitClone(repo, branch, remoteUrl string) (string, error
 func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error) {
 	id, err := e.GitExecWithOutput(gitDir, "show-ref", "--heads", "--hash", branchName)
 	if err != nil {
-		return "", fmt.Errorf("Can't find default branch: %s", branchName)
+		return "", fmt.Errorf("Can't find default branch: %s in %s", branchName, gitDir)
 	}
 
 	id = strings.TrimSpace(SplitLines(id)[0])
@@ -297,7 +297,7 @@ func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error
 func (e *GitHandlerImpl) GitRemoteHead(gitDir, remote, branchName string) (string, error) {
 	id, err := e.GitExecWithOutput(gitDir, "show-ref", "--hash", "--verify", "refs/remotes/"+remote+"/"+branchName)
 	if err != nil {
-		return "", fmt.Errorf("Can't find default branch: %s", branchName)
+		return "", fmt.Errorf("Can't find default branch: %s in %s", branchName, gitDir)
 	}
 
 	return strings.TrimSpace(id), nil
@@ -904,6 +904,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
 				for _, te := range tree.items {
 					if te.isTree() {
 						trees[p+te.name+"/"] = te.hash
+						submoduleList[p+te.name] = te.hash
 					} else if te.isSubmodule() {
 						submoduleList[p+te.name] = te.hash
 					}

common/obs_utils.go

@@ -689,13 +689,15 @@ func (r *BuildResultList) BuildResultSummary() (success, finished bool) {
 			if !ok {
 				panic("Unknown result code: " + result.Code)
 			}
-			if r.isLastBuild && result.Code == "unknown" {
-				// it means the package has never build yet,
-				// but we don't know the reason
-				detail.Finished = true
+			if r.isLastBuild {
+				// we are always finished, since it is the last result
+				// also when there is "unknown" state, it just means it
+				// it was never done
+				finished = true
+			} else {
+			        finished = finished && detail.Finished
 			}
 
-			finished = finished && detail.Finished
 			success = success && detail.Success
 
 			if !finished {

obs-staging-bot/main.go

@@ -26,6 +26,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"runtime/debug"
 	"slices"
@@ -109,161 +110,67 @@ const (
 	BuildStatusSummaryUnknown  = 4
 )
 
-func ProcessBuildStatus(project, refProject *common.BuildResultList) BuildStatusSummary {
-	if _, finished := refProject.BuildResultSummary(); !finished {
-		common.LogDebug("refProject not finished building??")
-		return BuildStatusSummaryUnknown
-	}
-
+func ProcessBuildStatus(project *common.BuildResultList) BuildStatusSummary {
 	if _, finished := project.BuildResultSummary(); !finished {
 		common.LogDebug("Still building...")
 		return BuildStatusSummaryBuilding
 	}
 
-	// the repositories should be setup equally between the projects. We
-	// need to verify that packages that are building in `refProject` are not
-	// failing in the `project`
-	BuildResultSorter := func(a, b *common.BuildResult) int {
-		if c := strings.Compare(a.Repository, b.Repository); c != 0 {
-			return c
-		}
-		if c := strings.Compare(a.Arch, b.Arch); c != 0 {
-			return c
-		}
+	//slices.SortFunc(project.Result, BuildResultSorter)
 
-		panic("Should not happen -- BuiltResultSorter equal repos?")
-	}
-	slices.SortFunc(project.Result, BuildResultSorter)
-	if refProject == nil {
-		// just return if buid finished and have some successes, since new package
-		common.LogInfo("New package. Only need some success...")
-		SomeSuccess := false
-		for i := 0; i < len(project.Result); i++ {
-			repoRes := project.Result[i]
-			repoResStatus, ok := common.ObsRepoStatusDetails[repoRes.Code]
-			if !ok {
-				common.LogDebug("cannot find code:", repoRes.Code)
-				return BuildStatusSummaryUnknown
-			}
-			if !repoResStatus.Finished {
-				return BuildStatusSummaryBuilding
-			}
-
-			for _, pkg := range repoRes.Status {
-				pkgStatus, ok := common.ObsBuildStatusDetails[pkg.Code]
-				if !ok {
-					common.LogInfo("Unknown package build status:", pkg.Code, "for", pkg.Package)
-					common.LogDebug("Details:", pkg.Details)
-				}
-
-				if pkgStatus.Success {
-					SomeSuccess = true
-				}
-			}
-		}
-
-		if SomeSuccess {
-			return BuildStatusSummarySuccess
-		}
-		return BuildStatusSummaryFailed
-	}
-
-	slices.SortFunc(refProject.Result, BuildResultSorter)
-
-	common.LogDebug("comparing results", len(project.Result), "vs. ref", len(refProject.Result))
-	SomeSuccess := false
+	common.LogDebug("build results", len(project.Result))
 	for i := 0; i < len(project.Result); i++ {
 		common.LogDebug("searching for", project.Result[i].Repository, "/", project.Result[i].Arch)
 		j := 0
 	found:
-		for ; j < len(refProject.Result); j++ {
-			if project.Result[i].Repository != refProject.Result[j].Repository ||
-				project.Result[i].Arch != refProject.Result[j].Arch {
-				continue
-			}
-
+		for j = 0; j < len(project.Result); j++ {
 			common.LogDebug("  found match for @ idx:", j)
-			res, success := ProcessRepoBuildStatus(project.Result[i].Status, refProject.Result[j].Status)
+			res := ProcessRepoBuildStatus(project.Result[i].Status)
 			switch res {
 			case BuildStatusSummarySuccess:
-				SomeSuccess = SomeSuccess || success
 				break found
+			case BuildStatusSummaryFailed:
+				return BuildStatusSummaryFailed
 			default:
 				return res
 			}
 		}
-
-		if j >= len(refProject.Result) {
-			common.LogDebug("Cannot find results...")
-			common.LogDebug(project.Result[i])
-			common.LogDebug(refProject.Result)
-			return BuildStatusSummaryUnknown
-		}
 	}
 
-	if SomeSuccess {
-		return BuildStatusSummarySuccess
-	}
-
-	return BuildStatusSummaryFailed
+	return BuildStatusSummarySuccess
 }
 
-func ProcessRepoBuildStatus(results, ref []*common.PackageBuildStatus) (status BuildStatusSummary, SomeSuccess bool) {
-	PackageBuildStatusSorter := func(a, b *common.PackageBuildStatus) int {
-		return strings.Compare(a.Package, b.Package)
-	}
+func ProcessRepoBuildStatus(results []*common.PackageBuildStatus) (status BuildStatusSummary) {
+
+ 	PackageBuildStatusSorter := func(a, b *common.PackageBuildStatus) int {
+ 	        return strings.Compare(a.Package, b.Package)
+ 	}
 
-	common.LogDebug("********  REF: ")
-	data, _ := xml.MarshalIndent(ref, "", "  ")
-	common.LogDebug(string(data))
 	common.LogDebug("******* RESULTS: ")
-	data, _ = xml.MarshalIndent(results, "", "  ")
+	data, _ := xml.MarshalIndent(results, "", "  ")
 	common.LogDebug(string(data))
 	common.LogDebug("*******")
 
 	// compare build result
 	slices.SortFunc(results, PackageBuildStatusSorter)
-	slices.SortFunc(ref, PackageBuildStatusSorter)
 
-	j := 0
-	SomeSuccess = false
 	for i := 0; i < len(results); i++ {
 		res, ok := common.ObsBuildStatusDetails[results[i].Code]
 		if !ok {
 			common.LogInfo("unknown package result code:", results[i].Code, "for package:", results[i].Package)
-			return BuildStatusSummaryUnknown, SomeSuccess
+			return BuildStatusSummaryUnknown
 		}
 
 		if !res.Finished {
-			return BuildStatusSummaryBuilding, SomeSuccess
+			return BuildStatusSummaryBuilding
 		}
 
 		if !res.Success {
-			// not failed if reference project also failed for same package here
-			for ; j < len(results) && strings.Compare(results[i].Package, ref[j].Package) < 0; j++ {
-			}
-
-			if j < len(results) && results[i].Package == ref[j].Package {
-				refRes, ok := common.ObsBuildStatusDetails[ref[j].Code]
-				if !ok {
-					common.LogInfo("unknown ref package result code:", ref[j].Code, "package:", ref[j].Package)
-					return BuildStatusSummaryUnknown, SomeSuccess
-				}
-
-				if !refRes.Finished {
-					common.LogDebug("Not finished building in reference project?")
-				}
-
-				if refRes.Success {
-					return BuildStatusSummaryFailed, SomeSuccess
-				}
-			}
-		} else {
-			SomeSuccess = true
+			return BuildStatusSummaryFailed
 		}
 	}
 
-	return BuildStatusSummarySuccess, SomeSuccess
+	return BuildStatusSummarySuccess
 }
 
 func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingPrj, buildPrj string, stagingMasterPrj string) (*common.ProjectMeta, error) {
@@ -322,9 +229,9 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
 		common.LogError("error fetching project meta for", buildPrj, ". Err:", err)
 		return nil, err
 	}
+	common.LogInfo("Meta: ", meta)
 
 	// generate new project with paths pointinig back to original repos
-	// disable publishing
 
 	meta.Name = stagingPrj
 	meta.Description = fmt.Sprintf(`Pull request build job PR#%d to branch %s of %s/%s`,
@@ -339,7 +246,10 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
 
 	urlPkg := make([]string, 0, len(modifiedOrNew))
 	for _, pkg := range modifiedOrNew {
-		urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(pkg))
+		// FIXME: skip manifest subdirectories itself
+		// strip any leading directory name and just hand over last directory as package name
+		onlybuilds := strings.Split(pkg, "/")
+		urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(onlybuilds[len(onlybuilds)-1]))
 	}
 	meta.ScmSync = pr.Head.Repo.CloneURL + "?" + strings.Join(urlPkg, "&") + "#" + pr.Head.Sha
 	if len(meta.ScmSync) >= 65535 {
@@ -824,10 +734,13 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 		if !stagingConfig.RebuildAll {
 			for pkg, headOid := range headSubmodules {
 				if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid {
+					if pkg != "rpms" && pkg != "dependencies" {
+					_, spkg := filepath.Split(pkg)
 					if exists {
-						modifiedPackages = append(modifiedPackages, pkg)
+						modifiedPackages = append(modifiedPackages, spkg)
 					} else {
-						newPackages = append(newPackages, pkg)
+						newPackages = append(newPackages, spkg)
+					}
 					}
 					common.LogDebug(pkg, ":", baseOid, "->", headOid)
 				}
@@ -931,15 +844,11 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 			gitea.AddComment(pr, msg)
 		}
 
-		baseResult, err := ObsClient.LastBuildResults(stagingConfig.ObsProject, modifiedPackages...)
-		if err != nil {
-			common.LogError("failed fetching ref project status for", stagingConfig.ObsProject, ":", err)
-		}
 		stagingResult, err := ObsClient.BuildStatus(stagingProject)
 		if err != nil {
 			common.LogError("failed fetching stage project status for", stagingProject, ":", err)
 		}
-		buildStatus := ProcessBuildStatus(stagingResult, baseResult)
+		buildStatus := ProcessBuildStatus(stagingResult)
 
 		done := false
 		switch buildStatus {

workflow-direct/main.go

@@ -123,7 +123,7 @@ func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, co
 			common.LogError("   - ", action.Repository.Name, "repo is not sha256. Ignoring.")
 			return
 		}
-		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
+		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", "../" + action.Repository.Name, action.Repository.Name))
 		defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all", "-f")
 
 		branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current"))
@@ -215,7 +215,7 @@ func processConfiguredPushAction(action *common.PushWebhookEvent, config *common
 	}
 
 	if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil {
-		git.GitExecOrPanic(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name)
+		git.GitExecOrPanic(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", "../" + action.Repository.Name, action.Repository.Name)
 		common.LogDebug("Pushed to package that is not part of the project. Re-adding...", err)
 	} else if !stat.IsDir() {
 		common.LogError("Pushed to a package that is not a submodule but exists in the project. Ignoring.")
@@ -420,7 +420,7 @@ next_repo:
 		}
 
 		// add repository to git project
-		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", r.CloneURL, r.Name))
+		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", "../" + r.Name, r.Name))
 
 		curBranch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current"))
 		if branch != curBranch {

workflow-pr/README.md

@@ -85,6 +85,7 @@ The following labels are used, when defined in Repo/Org.
 | StagingAuto        | staging/Auto   | Assigned to Project Git PRs when first staged
 | ReviewPending      | review/Pending | Assigned to PR when reviews are still pending
 | ReviewDone         | review/Done    | Assigned to PR when reviews are complete on this particular PR
+| ProjectUpdated     |                | Assigned if defined and git project got modified
 
 
 Maintainership

workflow-pr/pr_processor.go

@@ -26,6 +26,7 @@ func PrjGitDescription(prset *common.PRSet) (title string, desc string) {
 	title_refs := make([]string, 0, len(prset.PRs)-1)
 	refs := make([]string, 0, len(prset.PRs)-1)
 
+	prefix := ""
 	for _, pr := range prset.PRs {
 		if prset.IsPrjGitPR(pr.PR) {
 			continue
@@ -34,6 +35,9 @@ func PrjGitDescription(prset *common.PRSet) (title string, desc string) {
 			// remove PRs that are not open from description
 			continue
 		}
+		if strings.HasPrefix(pr.PR.Title, "WIP:") {
+			prefix = "WIP: "
+		}
 		org, repo, idx := pr.PRComponents()
 
 		title_refs = append(title_refs, repo)
@@ -44,7 +48,7 @@ func PrjGitDescription(prset *common.PRSet) (title string, desc string) {
 	slices.Sort(title_refs)
 	slices.Sort(refs)
 
-	title = "Forwarded PRs: " + strings.Join(title_refs, ", ")
+	title = prefix + "Forwarded PRs: " + strings.Join(title_refs, ", ")
 	desc = fmt.Sprintf("This is a forwarded pull request by %s\nreferencing the following pull request(s):\n\n", GitAuthor) + strings.Join(refs, "\n") + "\n"
 
 	if prset.Config.ManualMergeOnly {
@@ -80,6 +84,7 @@ func updateSubmoduleInPR(submodule, headSha string, git common.Git) {
 	git.GitExec(common.DefaultGitPrj, "submodule", "update", "--init", "--checkout", "--depth", "1", submodule)
 	common.PanicOnError(git.GitExec(path.Join(common.DefaultGitPrj, submodule), "fetch", "--depth", "1", "origin", headSha))
 	common.PanicOnError(git.GitExec(path.Join(common.DefaultGitPrj, submodule), "checkout", "-f", headSha))
+	common.PanicOnError(git.GitExec(path.Join(common.DefaultGitPrj, submodule), "lfs", "fsck"))
 }
 
 type PRProcessor struct {
@@ -297,14 +302,16 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
 	PrjGit := PrjGitPR.PR.Base.Repo
 	prjGitPRbranch := PrjGitPR.PR.Head.Name
 	if PrjGitPR.PR.Base.RepoID != PrjGitPR.PR.Head.RepoID {
-		PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, "", PrjGit.SSHURL)
-		git.GitExecOrPanic(common.DefaultGitPrj, "fetch", PrjGitPR.RemoteName, PrjGitPR.PR.Head.Sha)
-		git.GitExecOrPanic(common.DefaultGitPrj, "checkout", PrjGitPR.PR.Head.Sha)
-		common.LogInfo("Cannot update this PR as it's on another remote, not branch:", prjGitPRbranch, "Assuming this is by-design. (eg. project git PR only)")
-		return nil
+		// permission check, if submission comes from foreign repo
+		if !PrjGitPR.PR.AllowMaintainerEdit {
+                	common.LogError("Warning: source and target branch are in different repositories. We may not have the right permissions...")
+	//		Gitea.AddComment(PrjGitPR.PR, "This PR does not allow maintainer changes, but referenced package branch has changed!")
+			return nil
+		}
 	}
 
-	PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, prjGitPRbranch, PrjGit.SSHURL)
+//	PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, prjGitPRbranch, PrjGit.SSHURL)
+	PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, PrjGitPR.PR.Head.Ref, PrjGitPR.PR.Head.Repo.SSHURL)
 	common.PanicOnError(err)
 	git.GitExecOrPanic(common.DefaultGitPrj, "fetch", PrjGitPR.RemoteName, PrjGitBranch)
 
@@ -352,6 +359,7 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
 			}
 			common.PanicOnError(git.GitExec(common.DefaultGitPrj, params...))
 			PrjGitPR.PR.Head.Sha = newHeadCommit
+			Gitea.SetLabels(PrjGit.Owner.UserName, PrjGit.Name, PrjGitPR.PR.Index, []string{prset.Config.Label("PR/updated")})
 		}
 
 		// update PR
@@ -368,7 +376,7 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
 			}
 			return CurrentTitle == NewTitle
 		}
-		if PrjGitPR.PR.User.UserName == CurrentUser.UserName && (PrjGitPR.PR.Body != PrjGitBody || !isPrTitleSame(PrjGitPR.PR.Title, PrjGitTitle)) {
+		if !pr.config.NoProjectGitPR && PrjGitPR.PR.User.UserName == CurrentUser.UserName && (PrjGitPR.PR.Body != PrjGitBody || !isPrTitleSame(PrjGitPR.PR.Title, PrjGitTitle)) {
 			Gitea.UpdatePullRequest(PrjGit.Owner.UserName, PrjGit.Name, PrjGitPR.PR.Index, &models.EditPullRequestOption{
 				RemoveDeadline: true,
 				Title:          PrjGitTitle,
@@ -459,11 +467,11 @@ func (pr *PRProcessor) Process(req *models.PullRequest) error {
 						if _, ok := err.(*repository.RepoMergePullRequestConflict); !ok {
 							common.PanicOnError(err)
 						}
-					} else {
-						Gitea.AddComment(pr.PR, "Closing here because the associated Project PR has been closed.")
-						Gitea.UpdatePullRequest(org, repo, idx, &models.EditPullRequestOption{
-							State: "closed",
-						})
+//					} else {
+//						Gitea.AddComment(pr.PR, "Closing here because the associated Project PR has been closed.")
+//						Gitea.UpdatePullRequest(org, repo, idx, &models.EditPullRequestOption{
+//							State: "closed",
+//						})
 					}
 				}
 			}
@@ -476,10 +484,10 @@ func (pr *PRProcessor) Process(req *models.PullRequest) error {
 					continue
 				}
 			}
-		}
 
-		if err = pr.UpdatePrjGitPR(prset); err != nil {
-			return err
+			if err = pr.UpdatePrjGitPR(prset); err != nil {
+				return err
+			}
 		}
 	}