From fa61af0db6bd4a5a7af2b24a8fd623c93b035548cf1429e898d8740e43b90e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Mon, 5 May 2025 11:26:07 +0200 Subject: [PATCH 1/6] Implement detection for local repositories Repositories which build against another repo in the same project need to do so also in the forked project. This is eg for consuming rpms from one repo in an image build from same project. --- obs-staging-bot/main.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index ea184f1..8085da4 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -300,10 +300,20 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque // set paths to parent project for idx, r := range meta.Repositories { - meta.Repositories[idx].Paths = []common.RepositoryPathMeta{{ - Project: buildPrj, - Repository: r.Name, - }} + localRepository := false + for pidx, path := range r.Paths { + // Check for path building against a repo in template project itself + if path.Project == buildPrj { + meta.Repositories[idx].Paths[pidx].Project = meta.Name + localRepository = true + } + } + if localRepository != true { + meta.Repositories[idx].Paths = []common.RepositoryPathMeta{{ + Project: buildPrj, + Repository: r.Name, + }} + } } return meta, nil } -- 2.51.1 From 1661dc451b6db8fbe87cd74486cd82896fe9028800e712eb7ce0145fc3f6fb0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Mon, 5 May 2025 13:02:09 +0200 Subject: [PATCH 2/6] Fix QA project setup handling --- obs-staging-bot/main.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index 8085da4..4c97bb3 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -544,7 +544,6 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { // we want the possibly pending modification here, in case stagings are added, etc. // jobs of review team to deal with issues common.LogDebug("QA configuration fetching ...", common.StagingConfigFile) - QA := []common.QAConfig{} data, err := git.GitCatFile(pr.Head.Sha, pr.Head.Sha, common.StagingConfigFile) if err != nil { common.LogError("Staging config", common.StagingConfigFile, "not found in PR to the project. Aborting.") @@ -687,25 +686,26 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { common.LogDebug("ObsProject:", stagingConfig.ObsProject) stagingProject := GetObsProjectAssociatedWithPr(stagingConfig, obsClient.HomeProject, pr) change, err := StartOrUpdateBuild(stagingConfig, git, gitea, pr, obsClient) - if change != RequestModificationNoChange { - msg := "Changed source updated for build" - if change == RequestModificationProjectCreated { - msg = "Build is started in https://" + ObsWebHost + "/project/show/" + - stagingProject - } - if !IsDryRun { - gitea.AddComment(pr, msg) - } - } - + msg := "Changed source updated for build" if change == RequestModificationProjectCreated { - for _, setup := range QA { + msg = "Build is started in https://" + ObsWebHost + "/project/show/" + + stagingProject + " ." + + if len(stagingConfig.QA) > 0 { + msg = msg + "Additional QA builds: " + } + for _, setup := range stagingConfig.QA { CreateQASubProject(stagingConfig, git, gitea, pr, obsClient, stagingProject, setup.Origin, setup.Name) + msg = msg + "https://" + ObsWebHost + "/project/show/" + + stagingProject + ":" + setup.Name + " " } } + if change != RequestModificationNoChange && !IsDryRun { + gitea.AddComment(pr, msg) + } baseResult, err := obsClient.LastBuildResults(stagingConfig.ObsProject, modifiedOrNew...) if err != nil { -- 2.51.1 From c359827361d64a635a5a2269c9714798825fad2ba09a39f9b4aaa51a6ada2227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Mon, 5 May 2025 14:14:32 +0200 Subject: [PATCH 3/6] Drop release targets in pull request projects --- obs-staging-bot/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index 4c97bb3..a57b598 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -300,6 +300,7 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque // set paths to parent project for idx, r := range meta.Repositories { + meta.Repositories[idx].ReleaseTargets = nil localRepository := false for pidx, path := range r.Paths { // Check for path building against a repo in template project itself -- 2.51.1 From 657b36e07d10eaeafc49b805398f2729396893ceb5f98f514063471d07b99813 Mon Sep 17 00:00:00 2001 From: Adam Majer Date: Wed, 7 May 2025 12:31:11 +0200 Subject: [PATCH 4/6] common: fix parsing commit messages --- common/git_utils.go | 22 ++++++++++------ common/git_utils_test.go | 54 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/common/git_utils.go b/common/git_utils.go index dcfdd56..745ab52 100644 --- a/common/git_utils.go +++ b/common/git_utils.go @@ -446,21 +446,29 @@ func parseGitMsg(data <-chan byte) (GitMsg, error) { }, nil } -func parseGitCommitHdr(data <-chan byte) ([2]string, error) { +func parseGitCommitHdr(oldHdr [2]string, data <-chan byte) ([2]string, int, error) { hdr := make([]byte, 0, 60) val := make([]byte, 0, 1000) c := <-data + size := 1 if c != '\n' { // end of header marker for ; c != ' '; c = <-data { hdr = append(hdr, c) + size++ + } + if size == 1 { // continuation header here + hdr = []byte(oldHdr[0]) + val = append([]byte(oldHdr[1]), '\n') } for c := <-data; c != '\n'; c = <-data { val = append(val, c) + size++ } + size++ } - return [2]string{string(hdr), string(val)}, nil + return [2]string{string(hdr), string(val)}, size, nil } func parseGitCommitMsg(data <-chan byte, l int) (string, error) { @@ -470,7 +478,6 @@ func parseGitCommitMsg(data <-chan byte, l int) (string, error) { msg = append(msg, c) l-- } - // l-- if l != 0 { return "", fmt.Errorf("Unexpected data in the git commit msg: l=%d", l) @@ -490,12 +497,14 @@ func parseGitCommit(data <-chan byte) (GitCommit, error) { var c GitCommit l := hdr.size for { - hdr, err := parseGitCommitHdr(data) + var hdr [2]string + hdr, size, err := parseGitCommitHdr(hdr, data) if err != nil { return GitCommit{}, nil } + l -= size - if len(hdr[0])+len(hdr[1]) == 0 { // hdr end marker + if size == 1 { break } @@ -503,10 +512,7 @@ func parseGitCommit(data <-chan byte) (GitCommit, error) { case "tree": c.Tree = hdr[1] } - - l -= len(hdr[0]) + len(hdr[1]) + 2 } - l-- c.Msg, err = parseGitCommitMsg(data, l) return c, err diff --git a/common/git_utils_test.go b/common/git_utils_test.go index 88d6fae..ff135b9 100644 --- a/common/git_utils_test.go +++ b/common/git_utils_test.go @@ -29,6 +29,15 @@ import ( "testing" ) +func TestGitSLFO(t *testing.T) { + SetLoggingLevel(LogLevelDebug) + a, _ := AllocateGitWorkTree("/tmp", "test", "test") + git, _ := a.ReadExistingPath("SLFO") + data, err := git.GitCatFile("", "c07c52c57a10fb355956df3caad2986613838f149274fbe312ad76560764829d", "staging.config") + t.Log(err) + t.Fatal(string(data)) +} + func TestGitClone(t *testing.T) { tests := []struct { name string @@ -251,6 +260,51 @@ Reviewed-By: Marco Ippolito ` + "\x00" } }) + t.Run("parse multiline headers", func(t *testing.T) { + const commitData = "c07c52c57a10fb355956df3caad2986613838f149274fbe312ad76560764829d commit 1150\000" + `tree 3e06b280ea056141ed5e8af9794a41ae5281930c45321803eab53a240cb60044 +parent 19362a2cecb1fd25a89e03611d08ac68dcb1732f9dc0a68a40926356787fa4ca +author Adrian Schröter 1746600403 +0200 +committer Adrian Schröter 1746600403 +0200 +gpgsig-sha256 -----BEGIN PGP SIGNATURE----- + + iQIzBAABCgAdFiEE1QF1zm/pNbvyhgLFkY2MlUwI22cFAmgbAd0ACgkQkY2MlUwI + 22dxtA//eUCzIqxVdaEnOrFeTyxKig/mCOjaAyctmwr0vXUyElRtjXe4TzVG3QtR + uDfhIrKYLZ2tU/0TewTW/4XopWxLuqEzVQLrjuYl7K5P3GoYk52W1yGT0szzm7/i + 87j4UdRL9YGU/gYO7nSzstcfTP6AcmYzVUoOnwYR0K2vyOVjO4niL3mFXxLkIgIt + jd82xcE4JpQz9Yjyq2nDdz4A55kLAwsqY+dOct4oC6bZmj1/JeoGQfPvUsvsQgcI + syCHVh0GBxjvSv50V/VPzxQTFMal/TdtvAD4kmP/9RDi/5THzus8Peam8pV0gEIC + Q15ZcuLwIsC9i7ifUDYgzLgBBRdpSI0qji4Y6clWULPVjsyghgyfQw1trBSySpC8 + O1XfajUM+rXyrBLP6kzY+zl/zyzRdJ8JhljmC+SmNuyyEB77Hkn83k0f+aBhhqC2 + 4b3fIsKtwJZ1w6gr6SSz1BottiT9ShQzRaL8iRoF/2l5MkHPR+QFg2J7EIBqCbCQ + hFUjdvWAXQBWkkTQlJmLmJBXDOLQg3o6xCbnZM0gPFjZWE7e3Mpky7H0+xPnoeg9 + ukuvkexXQ6yrdiekA7HRLc76Te/I0m7KDOOWZ3rbJV6uH/3ps4FbLQTZO12AtZ6J + n8hYdYfw9yjCxiKUjnEtXtDRe8DJpqv+hO0Wj4MI5gIA2JE2lzY= + =Keg5 + -----END PGP SIGNATURE----- + +dummy change, don't merge +` + "\000" + ch := make(chan byte) + go func() { + for _, b := range []byte(commitData) { + ch <- b + } + }() + commit, err := parseGitCommit(ch) + + if err != nil { + t.Error(err) + } + + if commit.Tree != "3e06b280ea056141ed5e8af9794a41ae5281930c45321803eab53a240cb60044" { + t.Errorf("Invalid commit object: %#v", commit) + } + + if commit.Msg != "dummy change, don't merge\n" { + t.Errorf("Invalid commit msg: '%s'", commit.Msg) + } + }) + t.Run("parse tree object", func(t *testing.T) { const treeData = "\x31\x61\x30\x35\x64\x62\x37\x33\x36\x39\x33\x37\x34\x33\x30\x65\x31\x38\x64\x66\x34\x33\x61\x32\x37\x61\x39\x38\x30\x30\x31\x30\x31\x32\x65\x31\x65\x64\x32\x30\x34\x38\x32\x39\x38\x36\x37\x31\x32\x38\x66\x32\x63\x65\x38\x34\x30\x36\x62\x35\x63\x66\x63\x39\x20\x74\x72\x65\x65\x20\x32\x30\x35\x00\x34\x30\x30\x30\x30\x20\x62\x6f\x74\x73\x2d\x63\x6f\x6d\x6d\x6f\x6e\x00\x93\x17\xaa\x47\xf6\xea\x37\xe8\xbc\xe2\x80\x77\x57\x90\xf4\xa8\x01\xd7\xe3\x70\x2f\x84\xfb\xe1\xb0\x0e\x4a\x2c\x1c\x75\x2c\x2b\x34\x30\x30\x30\x30\x20\x6f\x62\x73\x2d\x73\x74\x61\x67\x69\x6e\x67\x2d\x62\x6f\x74\x00\x79\x77\x8b\x28\x7d\x37\x10\x59\xb9\x71\x28\x36\xed\x20\x31\x5f\xfb\xe1\xed\xb5\xba\x4f\x5e\xbb\x65\x65\x68\x23\x77\x32\x58\xfe\x34\x30\x30\x30\x30\x20\x70\x72\x2d\x72\x65\x76\x69\x65\x77\x00\x36\x0d\x45\xcb\x76\xb8\x93\xb3\x21\xba\xfa\xd5\x00\x9d\xfc\x59\xab\x88\xc1\x3c\x81\xcb\x48\x5a\xe0\x29\x29\x0f\xe3\x6b\x3c\x5e\x34\x30\x30\x30\x30\x20\x70\x72\x6a\x67\x69\x74\x2d\x75\x70\x64\x61\x74\x65\x72\x00\xb4\x0b\x1c\xf5\xfb\xec\x9a\xb2\x9f\x48\x3e\x21\x18\x0d\x51\xb7\x98\x6e\x21\x99\x74\x84\x67\x71\x41\x24\x42\xfc\xc9\x04\x12\x99\x00" -- 2.51.1 From 2f3f5aa56ca9cb5c51312c343242aad07269caae856a71b5378ce85cb950eb86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Wed, 7 May 2025 14:15:57 +0200 Subject: [PATCH 5/6] Implementing cleanup of closed requests --- common/obs_utils.go | 8 ++- obs-staging-bot/main.go | 115 +++++++++++++++++++++++----------------- 2 files changed, 73 insertions(+), 50 deletions(-) diff --git a/common/obs_utils.go b/common/obs_utils.go index 5f8a598..caae9c6 100644 --- a/common/obs_utils.go +++ b/common/obs_utils.go @@ -424,7 +424,11 @@ func (c *ObsClient) SetProjectMeta(meta *ProjectMeta) error { } func (c *ObsClient) DeleteProject(project string) error { - res, err := c.ObsRequest("DELETE", c.baseUrl.JoinPath("source", project).String(), nil) + u := c.baseUrl.JoinPath("source", project) + query := u.Query() + query.Add("force", "1") + u.RawQuery = query.Encode() + res, err := c.ObsRequest("DELETE", u.String(), nil) if err != nil { return err } @@ -614,7 +618,7 @@ var ObsBuildStatusDetails map[string]ObsBuildStatusDetail = map[string]ObsBuildS "unknown": ObsBuildStatusDetail{ Code: "unknown", Description: "The scheduler has not yet evaluated this package. Should be a short intermediate state for new packages.", - Finished: false, + Finished: true, // for lastbuild it is true, otherwise not... }, "error": ObsBuildStatusDetail{ diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index a57b598..8af5116 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -507,13 +507,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { common.LogError("No PR associated with review:", org, "/", repo, "#", id, "Error:", err) return err } - common.LogDebug("PR state:", pr.State) - if pr.State == "closed" { - // dismiss the review - common.LogInfo(" -- closed request, so nothing to review") - return nil - } obsClient, err := common.NewObsClient(ObsApiHost) if err != nil { @@ -533,40 +527,79 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { } } - if review, err := FetchOurLatestActionableReview(gitea, org, repo, id); err == nil { - common.LogInfo("processing review", review.HTMLURL, "state", review.State) + // Fetching data + review, review_error := FetchOurLatestActionableReview(gitea, org, repo, id) + if pr.State != "closed" && review_error != nil { + // Nothing to do + return nil + } - err = FetchPrGit(git, pr) - if err != nil { - common.LogError("Cannot fetch PR git:", pr.URL) + err = FetchPrGit(git, pr) + if err != nil { + common.LogError("Cannot fetch PR git:", pr.URL) + return err + } + + // we want the possibly pending modification here, in case stagings are added, etc. + // jobs of review team to deal with issues + common.LogDebug("QA configuration fetching ...", common.StagingConfigFile) + data, err := git.GitCatFile(pr.Head.Sha, pr.Head.Sha, common.StagingConfigFile) + if err != nil { + common.LogError("Staging config", common.StagingConfigFile, "not found in PR to the project. Aborting.") + if !IsDryRun { + _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.ProjectConfigFile) + } + return err + } + + stagingConfig, err := common.ParseStagingConfig(data) + if err != nil { + common.LogError("Error parsing config file", common.StagingConfigFile, err) + } + + if stagingConfig.ObsProject == "" { + common.LogError("Cannot find reference project for PR#", pr.Index) + if !IsDryRun && review_error == nil { + _, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project") return err } + return nil + } - // we want the possibly pending modification here, in case stagings are added, etc. - // jobs of review team to deal with issues - common.LogDebug("QA configuration fetching ...", common.StagingConfigFile) - data, err := git.GitCatFile(pr.Head.Sha, pr.Head.Sha, common.StagingConfigFile) - if err != nil { - common.LogError("Staging config", common.StagingConfigFile, "not found in PR to the project. Aborting.") - if !IsDryRun { - _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.ProjectConfigFile) - } - return err - } - - stagingConfig, err := common.ParseStagingConfig(data) - if err != nil { - common.LogError("Error parsing config file", common.StagingConfigFile, err) - } - - if stagingConfig.ObsProject == "" { - common.LogError("Cannot find reference project for PR#", pr.Index) - if !IsDryRun { - _, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project") - return err - } + if stagingConfig.StagingProject != "" { + // staging project must either be nothing or be *under* the target project. + // other setups are currently not supported + // NOTE: this is user input, so we need some limits here + l := len(stagingConfig.ObsProject) + if l >= len(stagingConfig.StagingProject) || stagingConfig.ObsProject != stagingConfig.StagingProject[0:l] { + common.LogError("StagingProject (", stagingConfig.StagingProject, ") is not child of target project", stagingConfig.ObsProject) return nil } + } + + common.LogDebug("ObsProject:", stagingConfig.ObsProject) + stagingProject := GetObsProjectAssociatedWithPr(stagingConfig, obsClient.HomeProject, pr) + + // Cleanup projects + if pr.State == "closed" { + // review is done, cleanup + common.LogInfo(" -- closed request, cleanup staging projects") + for _, setup := range stagingConfig.QA { + if !IsDryRun { + obsClient.DeleteProject(stagingProject + ":" + setup.Name) + } + } + if stagingProject != "" { + if !IsDryRun { + obsClient.DeleteProject(stagingProject) + } + } + return nil + } + + // Process review aka setup projects + if review_error == nil { + common.LogInfo("processing review", review.HTMLURL, "state", review.State) meta, err := obsClient.GetProjectMeta(stagingConfig.ObsProject) if err != nil { @@ -600,16 +633,6 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { } } - if stagingConfig.StagingProject != "" { - // staging project must either be nothing or be *under* the target project. - // other setups are currently not supported - // NOTE: this is user input, so we need some limits here - l := len(stagingConfig.ObsProject) - if l >= len(stagingConfig.StagingProject) || stagingConfig.ObsProject != stagingConfig.StagingProject[0:l] { - common.LogError("StagingProject (", stagingConfig.StagingProject, ") is not child of target project", stagingConfig.ObsProject) - } - } - if meta.Name != stagingConfig.ObsProject { common.LogError("staging.config . ObsProject:", stagingConfig.ObsProject, " is not target project name", meta.Name) if !IsDryRun { @@ -684,8 +707,6 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { } } - common.LogDebug("ObsProject:", stagingConfig.ObsProject) - stagingProject := GetObsProjectAssociatedWithPr(stagingConfig, obsClient.HomeProject, pr) change, err := StartOrUpdateBuild(stagingConfig, git, gitea, pr, obsClient) msg := "Changed source updated for build" if change == RequestModificationProjectCreated { @@ -735,8 +756,6 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { common.LogInfo("Build status:", buildStatus) // waiting for build results -- nothing to do - } else if err == NonActionableReviewError || err == NoReviewsFoundError { - return nil } return nil -- 2.51.1 From 909df35141a280d39114723fe4232e6510d297aa4dfa87bb5f34d7ca49f9fb68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Wed, 7 May 2025 16:52:50 +0200 Subject: [PATCH 6/6] handle build results different when request with lastbuild=1 In that case we need to * ignore repo state as it is the current one. There is no last state * handle "unkown" state as finished as the package was never attempted, but we don't know the reason (eg. broken source or unresolvable) --- common/obs_utils.go | 13 +++++++++---- obs-staging-bot/main.go | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/common/obs_utils.go b/common/obs_utils.go index caae9c6..7c8e743 100644 --- a/common/obs_utils.go +++ b/common/obs_utils.go @@ -496,7 +496,7 @@ func (r *BuildResultList) GetPackageList() []string { return pkgList } -func (r *BuildResultList) BuildResultSummary() (success, finished bool) { +func (r *BuildResultList) BuildResultSummary(LastBuild bool) (success, finished bool) { if r == nil { return true, true } @@ -511,7 +511,7 @@ func (r *BuildResultList) BuildResultSummary() (success, finished bool) { panic("Unknown repo result code: " + resultSet.Code) } - finished = repoDetail.Finished + finished = LastBuild || repoDetail.Finished if !finished || resultSet.Dirty { return } @@ -522,6 +522,11 @@ func (r *BuildResultList) BuildResultSummary() (success, finished bool) { if !ok { panic("Unknown result code: " + result.Code) } + if LastBuild && result.Code == "unknown" { + // it means the package has never build yet, + // but we don't know the reason + detail.Finished = true + } finished = finished && detail.Finished success = success && detail.Success @@ -617,8 +622,8 @@ var ObsBuildStatusDetails map[string]ObsBuildStatusDetail = map[string]ObsBuildS }, "unknown": ObsBuildStatusDetail{ Code: "unknown", - Description: "The scheduler has not yet evaluated this package. Should be a short intermediate state for new packages.", - Finished: true, // for lastbuild it is true, otherwise not... + Description: "The scheduler has not yet evaluated this package. Should be a short intermediate state for new packages. When used for lastbuild state it means it was never possible to attempt a build", + Finished: false, }, "error": ObsBuildStatusDetail{ diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index 8af5116..7a463a8 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -105,11 +105,13 @@ const ( ) func ProcessBuildStatus(project, refProject *common.BuildResultList) BuildStatusSummary { - if _, finished := project.BuildResultSummary(); !finished { + // interpret current build results here + if _, finished := project.BuildResultSummary(false); !finished { return BuildStatusSummaryBuilding } - if _, finished := refProject.BuildResultSummary(); !finished { + // interpret lastbuild results here, so we need to give a hint for unknown + if _, finished := refProject.BuildResultSummary(true); !finished { common.LogDebug("refProject not finished building??") return BuildStatusSummaryUnknown } -- 2.51.1