From 0bee48472d8ac7316816b52a1f6326f541df5e6bcc045e47211a8ae6dab5794a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Wed, 7 May 2025 14:15:57 +0200 Subject: [PATCH 01/10] Implementing cleanup of closed requests --- common/obs_utils.go | 1 - obs-staging-bot/main.go | 87 ++++++++++++++++++++++++----------------- 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/common/obs_utils.go b/common/obs_utils.go index c611ec8..6f12388 100644 --- a/common/obs_utils.go +++ b/common/obs_utils.go @@ -551,7 +551,6 @@ func (c *ObsClient) DeleteProject(project string) error { query.Add("force", "1") url.RawQuery = query.Encode() res, err := c.ObsRequestRaw("DELETE", url.String(), nil) - if err != nil { return err } diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index 6822875..c43374d 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -641,7 +641,6 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e common.LogError("No PR associated with review:", org, "/", repo, "#", id, "Error:", err) return true, err } - common.LogDebug("PR state:", pr.State) if pr.State == "closed" { // dismiss the review @@ -658,40 +657,68 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e } } - if review, err := FetchOurLatestActionableReview(gitea, org, repo, id); err == nil { - common.LogInfo("processing review", review.HTMLURL, "state", review.State) + // Fetching data + review, review_error := FetchOurLatestActionableReview(gitea, org, repo, id) + if pr.State != "closed" && review_error != nil { + // Nothing to do + return true, nil + } - err = FetchPrGit(git, pr) - if err != nil { - common.LogError("Cannot fetch PR git:", pr.URL) - return false, err + err = FetchPrGit(git, pr) + if err != nil { + common.LogError("Cannot fetch PR git:", pr.URL) + return false, err + } + + // we want the possibly pending modification here, in case stagings are added, etc. + // jobs of review team to deal with issues + common.LogDebug("QA configuration fetching ...", common.StagingConfigFile) + data, err := git.GitCatFile(pr.Head.Sha, pr.Head.Sha, common.StagingConfigFile) + if err != nil { + common.LogError("Staging config", common.StagingConfigFile, "not found in PR to the project. Aborting.") + if !IsDryRun { + _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.ProjectConfigFile) } + return true, err + } - // we want the possibly pending modification here, in case stagings are added, etc. - // jobs of review team to deal with issues - common.LogDebug("QA configuration fetching ...", common.StagingConfigFile) - data, err := git.GitCatFile(pr.Head.Sha, pr.Head.Sha, common.StagingConfigFile) - if err != nil { - common.LogError("Staging config", common.StagingConfigFile, "not found in PR to the project. Aborting.") - if !IsDryRun { - _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.ProjectConfigFile) - } + stagingConfig, err := common.ParseStagingConfig(data) + if err != nil { + common.LogError("Error parsing config file", common.StagingConfigFile, err) + } + + if stagingConfig.ObsProject == "" { + common.LogError("Cannot find reference project for PR#", pr.Index) + if !IsDryRun && review_error == nil { + _, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project") return true, err } + return true, nil + } - stagingConfig, err := common.ParseStagingConfig(data) - if err != nil { - common.LogError("Error parsing config file", common.StagingConfigFile, err) - } + common.LogDebug("ObsProject:", stagingConfig.ObsProject) + stagingProject := GetObsProjectAssociatedWithPr(stagingConfig, obsClient.HomeProject, pr) - if stagingConfig.ObsProject == "" { - common.LogError("Cannot find reference project for PR#", pr.Index) + // Cleanup projects + if pr.State == "closed" { + // review is done, cleanup + common.LogInfo(" -- closed request, cleanup staging projects") + for _, setup := range stagingConfig.QA { if !IsDryRun { - _, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project") - return true, err + obsClient.DeleteProject(stagingProject + ":" + setup.Name) } - return true, nil } + if stagingProject != "" { + if !IsDryRun { + obsClient.DeleteProject(stagingProject) + } + } + return true, nil + } + + // Process review aka setup projects + if review_error == nil { + common.LogInfo("processing review", review.HTMLURL, "state", review.State) meta, err := ObsClient.GetProjectMeta(stagingConfig.ObsProject) if err != nil { @@ -725,16 +752,6 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e } } - if stagingConfig.StagingProject != "" { - // staging project must either be nothing or be *under* the target project. - // other setups are currently not supported - // NOTE: this is user input, so we need some limits here - l := len(stagingConfig.ObsProject) - if l >= len(stagingConfig.StagingProject) || stagingConfig.ObsProject != stagingConfig.StagingProject[0:l] { - common.LogError("StagingProject (", stagingConfig.StagingProject, ") is not child of target project", stagingConfig.ObsProject) - } - } - if meta.Name != stagingConfig.ObsProject { common.LogError("staging.config . ObsProject:", stagingConfig.ObsProject, " is not target project name", meta.Name) if !IsDryRun { -- 2.51.1 From c11def600520ea365c895bb3a4444e434e041d3e109d6d3fe9e3d27a11d02517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Wed, 7 May 2025 16:52:50 +0200 Subject: [PATCH 02/10] handle build results different when request with lastbuild=1 In that case we need to * ignore repo state as it is the current one. There is no last state * handle "unkown" state as finished as the package was never attempted, but we don't know the reason (eg. broken source or unresolvable) -- 2.51.1 From 2f8b6b4ade43de15a2dd84ed97db7761f5075d370d7437ad396f20ffc5763f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Thu, 8 May 2025 10:57:17 +0200 Subject: [PATCH 03/10] Temporary hack to include also changed directories Need to be clean'd up via proper subdir handling --- common/git_utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/common/git_utils.go b/common/git_utils.go index a86e335..6707113 100644 --- a/common/git_utils.go +++ b/common/git_utils.go @@ -802,6 +802,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi for _, te := range tree.items { if te.isTree() { trees[p+te.name+"/"] = te.hash + submoduleList[p+te.name] = te.hash } else if te.isSubmodule() { submoduleList[p+te.name] = te.hash } -- 2.51.1 From cc9ad1703d1df51b999b5563ac1c2fb4f60291cd36a5a2ae079a7d29565c8805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Thu, 8 May 2025 10:58:10 +0200 Subject: [PATCH 04/10] Don't crash when new packages got added The build result request of the base project is failing in this situation, since the requested package does not exist. Therefore we need to have seperate lists for proper handling. --- obs-staging-bot/main.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index c43374d..090bacb 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -697,7 +697,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e } common.LogDebug("ObsProject:", stagingConfig.ObsProject) - stagingProject := GetObsProjectAssociatedWithPr(stagingConfig, obsClient.HomeProject, pr) + stagingProject := GetObsProjectAssociatedWithPr(stagingConfig, ObsClient.HomeProject, pr) // Cleanup projects if pr.State == "closed" { @@ -705,12 +705,12 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e common.LogInfo(" -- closed request, cleanup staging projects") for _, setup := range stagingConfig.QA { if !IsDryRun { - obsClient.DeleteProject(stagingProject + ":" + setup.Name) + ObsClient.DeleteProject(stagingProject + ":" + setup.Name) } } if stagingProject != "" { if !IsDryRun { - obsClient.DeleteProject(stagingProject) + ObsClient.DeleteProject(stagingProject) } } return true, nil @@ -772,17 +772,22 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e common.LogDebug(" # head submodules:", len(headSubmodules)) common.LogDebug(" # base submodules:", len(baseSubmodules)) - modifiedOrNew := make([]string, 0, 16) + modifiedPackages := make([]string, 0, 16) + newPackages := make([]string, 0, 16) if !stagingConfig.RebuildAll { for pkg, headOid := range headSubmodules { if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid { - modifiedOrNew = append(modifiedOrNew, pkg) + if len(baseOid) > 0 { + modifiedPackages = append(modifiedPackages, pkg) + } else { + newPackages = append(newPackages, pkg) + } common.LogDebug(pkg, ":", baseOid, "->", headOid) } } } - if len(modifiedOrNew) == 0 { + if len(modifiedPackages) == 0 && len(newPackages) == 0 { rebuild_all := false || stagingConfig.RebuildAll reviews, err := gitea.GetPullRequestReviews(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Index) @@ -859,13 +864,13 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e gitea.AddComment(pr, msg) } - baseResult, err := ObsClient.LastBuildResults(stagingConfig.ObsProject, modifiedOrNew...) + baseResult, err := ObsClient.LastBuildResults(stagingConfig.ObsProject, modifiedPackages...) if err != nil { common.LogError("failed fetching ref project status for", stagingConfig.ObsProject, ":", err) } stagingResult, err := ObsClient.BuildStatus(stagingProject) if err != nil { - common.LogError("failed fetching ref project status for", stagingProject, ":", err) + common.LogError("failed fetching stage project status for", stagingProject, ":", err) } buildStatus := ProcessBuildStatus(stagingResult, baseResult) -- 2.51.1 From c286e12b675beaeb4f439d0112f144645003a33f765a05f1178ddb8b3f270810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Mon, 12 May 2025 16:22:06 +0200 Subject: [PATCH 05/10] Try to use Staging Master Project as default build target if available This allows us to set custom build configuration or repository sets for pull request projects. --- obs-staging-bot/main.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index 090bacb..92390a4 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -263,7 +263,7 @@ func ProcessRepoBuildStatus(results, ref []common.PackageBuildStatus) (status Bu return BuildStatusSummarySuccess, SomeSuccess } -func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingPrj, buildPrj string) (*common.ProjectMeta, error) { +func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingPrj, buildPrj string, stagingMasterPrj string) (*common.ProjectMeta, error) { common.LogDebug("repo content fetching ...") err := FetchPrGit(git, pr) if err != nil { @@ -289,7 +289,15 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque } } - meta, err := ObsClient.GetProjectMeta(buildPrj) + common.LogDebug("Trying first staging master project: ", stagingMasterPrj) + meta, err := ObsClient.GetProjectMeta(stagingMasterPrj) + if err == nil { + // success, so we use that staging master project as our build project + buildPrj = stagingMasterPrj + } else { + common.LogInfo("error fetching project meta for ", stagingMasterPrj, ". Fall Back to ", buildPrj) + meta, err = ObsClient.GetProjectMeta(buildPrj) + } if err != nil { common.LogError("error fetching project meta for", buildPrj, ". Err:", err) return nil, err @@ -412,7 +420,8 @@ func StartOrUpdateBuild(config *common.StagingConfig, git common.Git, gitea comm var state RequestModification = RequestModificationSourceChanged if meta == nil { // new build - meta, err = GenerateObsPrjMeta(git, gitea, pr, obsPrProject, config.ObsProject) + common.LogDebug(" Staging master:", config.StagingProject) + meta, err = GenerateObsPrjMeta(git, gitea, pr, obsPrProject, config.ObsProject, config.StagingProject) if err != nil { return RequestModificationNoChange, err } @@ -426,6 +435,8 @@ func StartOrUpdateBuild(config *common.StagingConfig, git common.Git, gitea comm } else { err = ObsClient.SetProjectMeta(meta) if err != nil { + x, _ := xml.MarshalIndent(meta, "", " ") + common.LogDebug(" meta:", string(x)) common.LogError("cannot create meta project:", err) return RequestModificationNoChange, err } -- 2.51.1 From 4378568953b99c10774f4dd200eadadb757b6ee2e140cbb7f406f1ff7cd5222e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Tue, 13 May 2025 07:48:45 +0200 Subject: [PATCH 06/10] Fix logic in crash protection We must not access review.User object if it is nil --- obs-staging-bot/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index 92390a4..a358a27 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -470,7 +470,7 @@ func FetchOurLatestActionableReview(gitea common.Gitea, org, repo string, id int for idx := len(reviews) - 1; idx >= 0; idx-- { review := reviews[idx] - if review.User != nil || review.User.UserName == Username { + if review.User == nil || review.User.UserName == Username { if IsDryRun { // for purposes of moving forward a no-op check return review, nil -- 2.51.1 From cde46e85f365320ee7655b63f4699e5656d573fde738e3f5471a26ec7e595e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Wed, 9 Jul 2025 08:44:08 +0200 Subject: [PATCH 07/10] Enable code stream publishing --- obs-staging-bot/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index a358a27..0ee5f41 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -323,7 +323,9 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque } meta.ScmSync = pr.Head.Repo.CloneURL + "?" + strings.Join(urlPkg, "&") + "#" + pr.Head.Sha meta.Title = fmt.Sprintf("PR#%d to %s", pr.Index, pr.Base.Name) - meta.PublicFlags = common.Flags{Contents: ""} +// QE wants it published ... also we should not hardcode it here, since +// it is configurable via the :PullRequest project +// meta.PublicFlags = common.Flags{Contents: ""} meta.Groups = nil meta.Persons = nil -- 2.51.1 From 74f40f536ad48c4ee260e50a254f667b33d922f8a5c487929f685cb2be8c65b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Wed, 9 Jul 2025 08:45:22 +0200 Subject: [PATCH 08/10] message typo --- obs-staging-bot/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index 0ee5f41..6714166 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -560,7 +560,7 @@ func CleanupPullNotification(gitea common.Gitea, thread *models.NotificationThre } if pr.State != "closed" { - common.LogInfo(" ignoring peding PR", thread.Subject.HTMLURL, " state:", pr.State) + common.LogInfo(" ignoring pending PR", thread.Subject.HTMLURL, " state:", pr.State) return false } -- 2.51.1 From d7132727a77530990e4b13b66ac11687389acd29333306637c8ac4b15aae1115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Fri, 11 Jul 2025 10:52:00 +0200 Subject: [PATCH 09/10] Create Pull Requests to specified branches instead of always using DefaultBranch. This means that target needs always gets specified now. --- common/gitea_utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/gitea_utils.go b/common/gitea_utils.go index c06a8c6..a44b031 100644 --- a/common/gitea_utils.go +++ b/common/gitea_utils.go @@ -603,14 +603,14 @@ func (gitea *GiteaTransport) CreateRepositoryIfNotExist(git Git, org, repoName s func (gitea *GiteaTransport) CreatePullRequestIfNotExist(repo *models.Repository, srcId, targetId, title, body string) (*models.PullRequest, error) { prOptions := models.CreatePullRequestOption{ - Base: repo.DefaultBranch, + Base: targetId, Head: srcId, Title: title, Body: body, } if pr, err := gitea.client.Repository.RepoGetPullRequestByBaseHead( - repository.NewRepoGetPullRequestByBaseHeadParams().WithOwner(repo.Owner.UserName).WithRepo(repo.Name).WithBase(repo.DefaultBranch).WithHead(srcId), + repository.NewRepoGetPullRequestByBaseHeadParams().WithOwner(repo.Owner.UserName).WithRepo(repo.Name).WithBase(targetId).WithHead(srcId), gitea.transport.DefaultAuthentication, ); err == nil { return pr.Payload, nil -- 2.51.1 From ec5ac4fca3e8c8b5aba891c6c6bd3d59e5cf76b54ecafb4c49eee1e8608642f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Fri, 11 Jul 2025 10:53:38 +0200 Subject: [PATCH 10/10] Don't fail on project git pull request creation. --- workflow-pr/pr_processor.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/workflow-pr/pr_processor.go b/workflow-pr/pr_processor.go index cde94d1..8569482 100644 --- a/workflow-pr/pr_processor.go +++ b/workflow-pr/pr_processor.go @@ -87,11 +87,6 @@ func AllocatePRProcessor(req *common.PullRequestWebhookEvent, configs common.Aut } func (pr *PRProcessor) SetSubmodulesToMatchPRSet(prset *common.PRSet) ([]string, []string, error) { - prjGitPR, err := prset.GetPrjGitPR() - if err != nil { - return nil, nil, err - } - git := pr.git subList, err := git.GitSubmoduleList(common.DefaultGitPrj, "HEAD") if err != nil { @@ -113,13 +108,15 @@ func (pr *PRProcessor) SetSubmodulesToMatchPRSet(prset *common.PRSet) ([]string, revert := false if pr.PR.State != "open" { - // remove PR from PrjGit - var valid bool - if prHead, valid = git.GitSubmoduleCommitId(common.DefaultGitPrj, repo, prjGitPR.PR.MergeBase); !valid { - common.LogError("Failed fetching original submodule commit id for repo") - return nil, nil, err + prjGitPR, err := prset.GetPrjGitPR() + if prjGitPR != nil { + // remove PR from PrjGit + var valid bool + if prHead, valid = git.GitSubmoduleCommitId(common.DefaultGitPrj, repo, prjGitPR.PR.MergeBase); !valid { + common.LogError("Failed fetching original submodule commit id for repo") + return nil, nil, err + } } - revert = true } -- 2.51.1