diff --git a/common/git_utils.go b/common/git_utils.go index dcfdd56..745ab52 100644 --- a/common/git_utils.go +++ b/common/git_utils.go @@ -446,21 +446,29 @@ func parseGitMsg(data <-chan byte) (GitMsg, error) { }, nil } -func parseGitCommitHdr(data <-chan byte) ([2]string, error) { +func parseGitCommitHdr(oldHdr [2]string, data <-chan byte) ([2]string, int, error) { hdr := make([]byte, 0, 60) val := make([]byte, 0, 1000) c := <-data + size := 1 if c != '\n' { // end of header marker for ; c != ' '; c = <-data { hdr = append(hdr, c) + size++ + } + if size == 1 { // continuation header here + hdr = []byte(oldHdr[0]) + val = append([]byte(oldHdr[1]), '\n') } for c := <-data; c != '\n'; c = <-data { val = append(val, c) + size++ } + size++ } - return [2]string{string(hdr), string(val)}, nil + return [2]string{string(hdr), string(val)}, size, nil } func parseGitCommitMsg(data <-chan byte, l int) (string, error) { @@ -470,7 +478,6 @@ func parseGitCommitMsg(data <-chan byte, l int) (string, error) { msg = append(msg, c) l-- } - // l-- if l != 0 { return "", fmt.Errorf("Unexpected data in the git commit msg: l=%d", l) @@ -490,12 +497,14 @@ func parseGitCommit(data <-chan byte) (GitCommit, error) { var c GitCommit l := hdr.size for { - hdr, err := parseGitCommitHdr(data) + var hdr [2]string + hdr, size, err := parseGitCommitHdr(hdr, data) if err != nil { return GitCommit{}, nil } + l -= size - if len(hdr[0])+len(hdr[1]) == 0 { // hdr end marker + if size == 1 { break } @@ -503,10 +512,7 @@ func parseGitCommit(data <-chan byte) (GitCommit, error) { case "tree": c.Tree = hdr[1] } - - l -= len(hdr[0]) + len(hdr[1]) + 2 } - l-- c.Msg, err = parseGitCommitMsg(data, l) return c, err diff --git a/common/git_utils_test.go b/common/git_utils_test.go index 88d6fae..ff135b9 100644 --- a/common/git_utils_test.go +++ b/common/git_utils_test.go @@ -29,6 +29,15 @@ import ( "testing" ) +func TestGitSLFO(t *testing.T) { + SetLoggingLevel(LogLevelDebug) + a, _ := AllocateGitWorkTree("/tmp", "test", "test") + git, _ := a.ReadExistingPath("SLFO") + data, err := git.GitCatFile("", "c07c52c57a10fb355956df3caad2986613838f149274fbe312ad76560764829d", "staging.config") + t.Log(err) + t.Fatal(string(data)) +} + func TestGitClone(t *testing.T) { tests := []struct { name string @@ -251,6 +260,51 @@ Reviewed-By: Marco Ippolito ` + "\x00" } }) + t.Run("parse multiline headers", func(t *testing.T) { + const commitData = "c07c52c57a10fb355956df3caad2986613838f149274fbe312ad76560764829d commit 1150\000" + `tree 3e06b280ea056141ed5e8af9794a41ae5281930c45321803eab53a240cb60044 +parent 19362a2cecb1fd25a89e03611d08ac68dcb1732f9dc0a68a40926356787fa4ca +author Adrian Schröter 1746600403 +0200 +committer Adrian Schröter 1746600403 +0200 +gpgsig-sha256 -----BEGIN PGP SIGNATURE----- + + iQIzBAABCgAdFiEE1QF1zm/pNbvyhgLFkY2MlUwI22cFAmgbAd0ACgkQkY2MlUwI + 22dxtA//eUCzIqxVdaEnOrFeTyxKig/mCOjaAyctmwr0vXUyElRtjXe4TzVG3QtR + uDfhIrKYLZ2tU/0TewTW/4XopWxLuqEzVQLrjuYl7K5P3GoYk52W1yGT0szzm7/i + 87j4UdRL9YGU/gYO7nSzstcfTP6AcmYzVUoOnwYR0K2vyOVjO4niL3mFXxLkIgIt + jd82xcE4JpQz9Yjyq2nDdz4A55kLAwsqY+dOct4oC6bZmj1/JeoGQfPvUsvsQgcI + syCHVh0GBxjvSv50V/VPzxQTFMal/TdtvAD4kmP/9RDi/5THzus8Peam8pV0gEIC + Q15ZcuLwIsC9i7ifUDYgzLgBBRdpSI0qji4Y6clWULPVjsyghgyfQw1trBSySpC8 + O1XfajUM+rXyrBLP6kzY+zl/zyzRdJ8JhljmC+SmNuyyEB77Hkn83k0f+aBhhqC2 + 4b3fIsKtwJZ1w6gr6SSz1BottiT9ShQzRaL8iRoF/2l5MkHPR+QFg2J7EIBqCbCQ + hFUjdvWAXQBWkkTQlJmLmJBXDOLQg3o6xCbnZM0gPFjZWE7e3Mpky7H0+xPnoeg9 + ukuvkexXQ6yrdiekA7HRLc76Te/I0m7KDOOWZ3rbJV6uH/3ps4FbLQTZO12AtZ6J + n8hYdYfw9yjCxiKUjnEtXtDRe8DJpqv+hO0Wj4MI5gIA2JE2lzY= + =Keg5 + -----END PGP SIGNATURE----- + +dummy change, don't merge +` + "\000" + ch := make(chan byte) + go func() { + for _, b := range []byte(commitData) { + ch <- b + } + }() + commit, err := parseGitCommit(ch) + + if err != nil { + t.Error(err) + } + + if commit.Tree != "3e06b280ea056141ed5e8af9794a41ae5281930c45321803eab53a240cb60044" { + t.Errorf("Invalid commit object: %#v", commit) + } + + if commit.Msg != "dummy change, don't merge\n" { + t.Errorf("Invalid commit msg: '%s'", commit.Msg) + } + }) + t.Run("parse tree object", func(t *testing.T) { const treeData = "\x31\x61\x30\x35\x64\x62\x37\x33\x36\x39\x33\x37\x34\x33\x30\x65\x31\x38\x64\x66\x34\x33\x61\x32\x37\x61\x39\x38\x30\x30\x31\x30\x31\x32\x65\x31\x65\x64\x32\x30\x34\x38\x32\x39\x38\x36\x37\x31\x32\x38\x66\x32\x63\x65\x38\x34\x30\x36\x62\x35\x63\x66\x63\x39\x20\x74\x72\x65\x65\x20\x32\x30\x35\x00\x34\x30\x30\x30\x30\x20\x62\x6f\x74\x73\x2d\x63\x6f\x6d\x6d\x6f\x6e\x00\x93\x17\xaa\x47\xf6\xea\x37\xe8\xbc\xe2\x80\x77\x57\x90\xf4\xa8\x01\xd7\xe3\x70\x2f\x84\xfb\xe1\xb0\x0e\x4a\x2c\x1c\x75\x2c\x2b\x34\x30\x30\x30\x30\x20\x6f\x62\x73\x2d\x73\x74\x61\x67\x69\x6e\x67\x2d\x62\x6f\x74\x00\x79\x77\x8b\x28\x7d\x37\x10\x59\xb9\x71\x28\x36\xed\x20\x31\x5f\xfb\xe1\xed\xb5\xba\x4f\x5e\xbb\x65\x65\x68\x23\x77\x32\x58\xfe\x34\x30\x30\x30\x30\x20\x70\x72\x2d\x72\x65\x76\x69\x65\x77\x00\x36\x0d\x45\xcb\x76\xb8\x93\xb3\x21\xba\xfa\xd5\x00\x9d\xfc\x59\xab\x88\xc1\x3c\x81\xcb\x48\x5a\xe0\x29\x29\x0f\xe3\x6b\x3c\x5e\x34\x30\x30\x30\x30\x20\x70\x72\x6a\x67\x69\x74\x2d\x75\x70\x64\x61\x74\x65\x72\x00\xb4\x0b\x1c\xf5\xfb\xec\x9a\xb2\x9f\x48\x3e\x21\x18\x0d\x51\xb7\x98\x6e\x21\x99\x74\x84\x67\x71\x41\x24\x42\xfc\xc9\x04\x12\x99\x00" diff --git a/common/obs_utils.go b/common/obs_utils.go index 5f8a598..7c8e743 100644 --- a/common/obs_utils.go +++ b/common/obs_utils.go @@ -424,7 +424,11 @@ func (c *ObsClient) SetProjectMeta(meta *ProjectMeta) error { } func (c *ObsClient) DeleteProject(project string) error { - res, err := c.ObsRequest("DELETE", c.baseUrl.JoinPath("source", project).String(), nil) + u := c.baseUrl.JoinPath("source", project) + query := u.Query() + query.Add("force", "1") + u.RawQuery = query.Encode() + res, err := c.ObsRequest("DELETE", u.String(), nil) if err != nil { return err } @@ -492,7 +496,7 @@ func (r *BuildResultList) GetPackageList() []string { return pkgList } -func (r *BuildResultList) BuildResultSummary() (success, finished bool) { +func (r *BuildResultList) BuildResultSummary(LastBuild bool) (success, finished bool) { if r == nil { return true, true } @@ -507,7 +511,7 @@ func (r *BuildResultList) BuildResultSummary() (success, finished bool) { panic("Unknown repo result code: " + resultSet.Code) } - finished = repoDetail.Finished + finished = LastBuild || repoDetail.Finished if !finished || resultSet.Dirty { return } @@ -518,6 +522,11 @@ func (r *BuildResultList) BuildResultSummary() (success, finished bool) { if !ok { panic("Unknown result code: " + result.Code) } + if LastBuild && result.Code == "unknown" { + // it means the package has never build yet, + // but we don't know the reason + detail.Finished = true + } finished = finished && detail.Finished success = success && detail.Success @@ -613,7 +622,7 @@ var ObsBuildStatusDetails map[string]ObsBuildStatusDetail = map[string]ObsBuildS }, "unknown": ObsBuildStatusDetail{ Code: "unknown", - Description: "The scheduler has not yet evaluated this package. Should be a short intermediate state for new packages.", + Description: "The scheduler has not yet evaluated this package. Should be a short intermediate state for new packages. When used for lastbuild state it means it was never possible to attempt a build", Finished: false, }, diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index ea184f1..7a463a8 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -105,11 +105,13 @@ const ( ) func ProcessBuildStatus(project, refProject *common.BuildResultList) BuildStatusSummary { - if _, finished := project.BuildResultSummary(); !finished { + // interpret current build results here + if _, finished := project.BuildResultSummary(false); !finished { return BuildStatusSummaryBuilding } - if _, finished := refProject.BuildResultSummary(); !finished { + // interpret lastbuild results here, so we need to give a hint for unknown + if _, finished := refProject.BuildResultSummary(true); !finished { common.LogDebug("refProject not finished building??") return BuildStatusSummaryUnknown } @@ -300,10 +302,21 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque // set paths to parent project for idx, r := range meta.Repositories { - meta.Repositories[idx].Paths = []common.RepositoryPathMeta{{ - Project: buildPrj, - Repository: r.Name, - }} + meta.Repositories[idx].ReleaseTargets = nil + localRepository := false + for pidx, path := range r.Paths { + // Check for path building against a repo in template project itself + if path.Project == buildPrj { + meta.Repositories[idx].Paths[pidx].Project = meta.Name + localRepository = true + } + } + if localRepository != true { + meta.Repositories[idx].Paths = []common.RepositoryPathMeta{{ + Project: buildPrj, + Repository: r.Name, + }} + } } return meta, nil } @@ -496,13 +509,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { common.LogError("No PR associated with review:", org, "/", repo, "#", id, "Error:", err) return err } - common.LogDebug("PR state:", pr.State) - if pr.State == "closed" { - // dismiss the review - common.LogInfo(" -- closed request, so nothing to review") - return nil - } obsClient, err := common.NewObsClient(ObsApiHost) if err != nil { @@ -522,41 +529,79 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { } } - if review, err := FetchOurLatestActionableReview(gitea, org, repo, id); err == nil { - common.LogInfo("processing review", review.HTMLURL, "state", review.State) + // Fetching data + review, review_error := FetchOurLatestActionableReview(gitea, org, repo, id) + if pr.State != "closed" && review_error != nil { + // Nothing to do + return nil + } - err = FetchPrGit(git, pr) - if err != nil { - common.LogError("Cannot fetch PR git:", pr.URL) + err = FetchPrGit(git, pr) + if err != nil { + common.LogError("Cannot fetch PR git:", pr.URL) + return err + } + + // we want the possibly pending modification here, in case stagings are added, etc. + // jobs of review team to deal with issues + common.LogDebug("QA configuration fetching ...", common.StagingConfigFile) + data, err := git.GitCatFile(pr.Head.Sha, pr.Head.Sha, common.StagingConfigFile) + if err != nil { + common.LogError("Staging config", common.StagingConfigFile, "not found in PR to the project. Aborting.") + if !IsDryRun { + _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.ProjectConfigFile) + } + return err + } + + stagingConfig, err := common.ParseStagingConfig(data) + if err != nil { + common.LogError("Error parsing config file", common.StagingConfigFile, err) + } + + if stagingConfig.ObsProject == "" { + common.LogError("Cannot find reference project for PR#", pr.Index) + if !IsDryRun && review_error == nil { + _, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project") return err } + return nil + } - // we want the possibly pending modification here, in case stagings are added, etc. - // jobs of review team to deal with issues - common.LogDebug("QA configuration fetching ...", common.StagingConfigFile) - QA := []common.QAConfig{} - data, err := git.GitCatFile(pr.Head.Sha, pr.Head.Sha, common.StagingConfigFile) - if err != nil { - common.LogError("Staging config", common.StagingConfigFile, "not found in PR to the project. Aborting.") - if !IsDryRun { - _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.ProjectConfigFile) - } - return err - } - - stagingConfig, err := common.ParseStagingConfig(data) - if err != nil { - common.LogError("Error parsing config file", common.StagingConfigFile, err) - } - - if stagingConfig.ObsProject == "" { - common.LogError("Cannot find reference project for PR#", pr.Index) - if !IsDryRun { - _, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project") - return err - } + if stagingConfig.StagingProject != "" { + // staging project must either be nothing or be *under* the target project. + // other setups are currently not supported + // NOTE: this is user input, so we need some limits here + l := len(stagingConfig.ObsProject) + if l >= len(stagingConfig.StagingProject) || stagingConfig.ObsProject != stagingConfig.StagingProject[0:l] { + common.LogError("StagingProject (", stagingConfig.StagingProject, ") is not child of target project", stagingConfig.ObsProject) return nil } + } + + common.LogDebug("ObsProject:", stagingConfig.ObsProject) + stagingProject := GetObsProjectAssociatedWithPr(stagingConfig, obsClient.HomeProject, pr) + + // Cleanup projects + if pr.State == "closed" { + // review is done, cleanup + common.LogInfo(" -- closed request, cleanup staging projects") + for _, setup := range stagingConfig.QA { + if !IsDryRun { + obsClient.DeleteProject(stagingProject + ":" + setup.Name) + } + } + if stagingProject != "" { + if !IsDryRun { + obsClient.DeleteProject(stagingProject) + } + } + return nil + } + + // Process review aka setup projects + if review_error == nil { + common.LogInfo("processing review", review.HTMLURL, "state", review.State) meta, err := obsClient.GetProjectMeta(stagingConfig.ObsProject) if err != nil { @@ -590,16 +635,6 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { } } - if stagingConfig.StagingProject != "" { - // staging project must either be nothing or be *under* the target project. - // other setups are currently not supported - // NOTE: this is user input, so we need some limits here - l := len(stagingConfig.ObsProject) - if l >= len(stagingConfig.StagingProject) || stagingConfig.ObsProject != stagingConfig.StagingProject[0:l] { - common.LogError("StagingProject (", stagingConfig.StagingProject, ") is not child of target project", stagingConfig.ObsProject) - } - } - if meta.Name != stagingConfig.ObsProject { common.LogError("staging.config . ObsProject:", stagingConfig.ObsProject, " is not target project name", meta.Name) if !IsDryRun { @@ -674,28 +709,27 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { } } - common.LogDebug("ObsProject:", stagingConfig.ObsProject) - stagingProject := GetObsProjectAssociatedWithPr(stagingConfig, obsClient.HomeProject, pr) change, err := StartOrUpdateBuild(stagingConfig, git, gitea, pr, obsClient) - if change != RequestModificationNoChange { - msg := "Changed source updated for build" - if change == RequestModificationProjectCreated { - msg = "Build is started in https://" + ObsWebHost + "/project/show/" + - stagingProject - } - if !IsDryRun { - gitea.AddComment(pr, msg) - } - } - + msg := "Changed source updated for build" if change == RequestModificationProjectCreated { - for _, setup := range QA { + msg = "Build is started in https://" + ObsWebHost + "/project/show/" + + stagingProject + " ." + + if len(stagingConfig.QA) > 0 { + msg = msg + "Additional QA builds: " + } + for _, setup := range stagingConfig.QA { CreateQASubProject(stagingConfig, git, gitea, pr, obsClient, stagingProject, setup.Origin, setup.Name) + msg = msg + "https://" + ObsWebHost + "/project/show/" + + stagingProject + ":" + setup.Name + " " } } + if change != RequestModificationNoChange && !IsDryRun { + gitea.AddComment(pr, msg) + } baseResult, err := obsClient.LastBuildResults(stagingConfig.ObsProject, modifiedOrNew...) if err != nil { @@ -724,8 +758,6 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) error { common.LogInfo("Build status:", buildStatus) // waiting for build results -- nothing to do - } else if err == NonActionableReviewError || err == NoReviewsFoundError { - return nil } return nil