diff --git a/autogits.spec b/autogits.spec index 3e95bc2..d2c010a 100644 --- a/autogits.spec +++ b/autogits.spec @@ -22,6 +22,8 @@ Release: 0 Summary: GitWorkflow utilities License: GPL-2.0-or-later URL: https://src.opensuse.org/adamm/autogits +#!RemoteAsset: git+https://src.suse.de/adrianSuSE/autogits#ibs_state +Source0: %name-%version.tar.xz BuildRequires: git BuildRequires: systemd-rpm-macros BuildRequires: go diff --git a/common/config.go b/common/config.go index 602d916..05b3118 100644 --- a/common/config.go +++ b/common/config.go @@ -55,6 +55,7 @@ type QAConfig struct { Name string Origin string Label string // requires this gitea lable to be set or skipped + Labels []string // requires any of the lables to be set BuildDisableRepos []string // which repos to build disable in the new project } @@ -326,6 +327,14 @@ func ParseStagingConfig(data []byte) (*StagingConfig, error) { if err := json.Unmarshal(data, &staging); err != nil { return nil, err } + // backward compability, transfer Label to new Labels array + // should we fail when both are set? + for i := range staging.QA { + if len(staging.QA[i].Labels) == 0 && len(staging.QA[i].Label) > 0 { + staging.QA[i].Labels = []string{staging.QA[i].Label} + staging.QA[i].Label = "" + } + } return &staging, nil } diff --git a/common/git_utils.go b/common/git_utils.go index 92b9e4b..3a05cc6 100644 --- a/common/git_utils.go +++ b/common/git_utils.go @@ -288,7 +288,7 @@ func (e *GitHandlerImpl) GitClone(repo, branch, remoteUrl string) (string, error func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error) { id, err := e.GitExecWithOutput(gitDir, "show-ref", "--heads", "--hash", branchName) if err != nil { - return "", fmt.Errorf("Can't find default branch: %s", branchName) + return "", fmt.Errorf("Can't find default branch: %s in %s", branchName, gitDir) } id = strings.TrimSpace(SplitLines(id)[0]) @@ -302,7 +302,7 @@ func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error func (e *GitHandlerImpl) GitRemoteHead(gitDir, remote, branchName string) (string, error) { id, err := e.GitExecWithOutput(gitDir, "show-ref", "--hash", "--verify", "refs/remotes/"+remote+"/"+branchName) if err != nil { - return "", fmt.Errorf("Can't find default branch: %s", branchName) + return "", fmt.Errorf("Can't find default branch: %s in %s", branchName, gitDir) } return strings.TrimSpace(id), nil diff --git a/common/obs_utils.go b/common/obs_utils.go index 0417968..f567336 100644 --- a/common/obs_utils.go +++ b/common/obs_utils.go @@ -520,7 +520,7 @@ func ObsSafeProjectName(prjname string) string { } var ValidBlockModes []string = []string{"all", "local", "never"} -var ValidPrjLinkModes []string = []string{"off", "localdep", "alldirect", "all"} +var ValidPrjLinkModes []string = []string{"off", "localdep", "alldirect", "alldirect_or_localdep", "all"} var ValidTriggerModes []string = []string{"transitive", "direct", "local"} func (c *ObsClient) SetProjectMeta(meta *ProjectMeta) error { @@ -694,13 +694,15 @@ func (r *BuildResultList) BuildResultSummary() (success, finished bool) { if !ok { panic("Unknown result code: " + result.Code) } - if r.isLastBuild && result.Code == "unknown" { - // it means the package has never build yet, - // but we don't know the reason - detail.Finished = true + if r.isLastBuild { + // we are always finished, since it is the last result + // also when there is "unknown" state, it just means it + // it was never done + finished = true + } else { + finished = finished && detail.Finished } - finished = finished && detail.Finished success = success && detail.Success if !finished { diff --git a/obs-staging-bot/README.md b/obs-staging-bot/README.md index ffffee5..d042165 100644 --- a/obs-staging-bot/README.md +++ b/obs-staging-bot/README.md @@ -50,6 +50,7 @@ It's a JSON file with following syntax: | *QA > Name* | Suffix for the QA OBS staging project. The project is named *StagingProject::Name*. | no | string | | | | *QA > Origin* | OBS reference project | no | string | | | | *QA > Label* | Setup the project only when the given gitea label is set on pull request | no | string | | | +| *QA > Labels* | Setup the project only when the one of gitea labels is set on pull request. Do not combine with Label. | no | array of strings | | [] | | *QA > BuildDisableRepos* | The names of OBS repositories to build-disable, if any. | no | array of strings | | [] | diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index dfe836f..fef232f 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -27,6 +27,7 @@ import ( "net/url" "os" "path" + "path/filepath" "regexp" "runtime/debug" "slices" @@ -127,6 +128,10 @@ func ProcessBuildStatus(project *common.BuildResultList) BuildStatusSummary { found: for j := 0; j < len(project.Result); j++ { common.LogDebug(" found match for @ idx:", j) + if project.Result[i].Dirty { + // ignore possible temporary failures and wait for settling + return BuildStatusSummaryBuilding + } res := ProcessRepoBuildStatus(project.Result[i].Status) switch res { case BuildStatusSummarySuccess: @@ -713,8 +718,16 @@ func ProcessQaProjects(stagingConfig *common.StagingConfig, git common.Git, gite msg := "" for _, setup := range stagingConfig.QA { QAproject := stagingProject + ":" + setup.Name - if len(setup.Label) > 0 { - if _, ok := prLabelNames[setup.Label]; !ok { + if len(setup.Labels) > 0 { + matchedLabel := false + for labelName := range prLabelNames { + if slices.Contains(setup.Labels, labelName) { + matchedLabel = true + break + } + } + + if ! matchedLabel { if !IsDryRun { // blindly remove, will fail when not existing ObsClient.DeleteProject(QAproject) @@ -797,7 +810,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e if err != nil { common.LogError("Staging config", common.StagingConfigFile, "not found in PR to the project. Aborting.") if !IsDryRun { - _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.ProjectConfigFile) + _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find staging config in PR ("+common.StagingConfigFile+")") } return true, err } @@ -856,8 +869,11 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e // NOTE: this is user input, so we need some limits here l := len(stagingConfig.ObsProject) if l >= len(stagingConfig.StagingProject) || stagingConfig.ObsProject != stagingConfig.StagingProject[0:l] { - common.LogError("StagingProject (", stagingConfig.StagingProject, ") is not child of target project", stagingConfig.ObsProject) - return true, nil + // TEMPORARY HACK: We remove this when Factory has switched to git + if ( stagingConfig.ObsProject != "openSUSE:Factory:git" && stagingConfig.StagingProject != "openSUSE:Factory:PullRequest" ) { + common.LogError("StagingProject (", stagingConfig.StagingProject, ") is not child of target project", stagingConfig.ObsProject) + return true, nil + } } } @@ -886,10 +902,13 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e if !stagingConfig.RebuildAll { for pkg, headOid := range headSubmodules { if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid { + if pkg != "rpms" && pkg != "dependencies" { + _, spkg := filepath.Split(pkg) if exists { - modifiedPackages = append(modifiedPackages, pkg) + modifiedPackages = append(modifiedPackages, spkg) } else { - newPackages = append(newPackages, pkg) + newPackages = append(newPackages, spkg) + } } common.LogDebug(pkg, ":", baseOid, "->", headOid) } @@ -995,9 +1014,11 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e done := false overallBuildStatus := ProcessBuildStatus(stagingResult) commentSuffix := "" + common.LogDebug("Number of QA Projects: ", len(qaProjects)) if len(qaProjects) > 0 && overallBuildStatus == BuildStatusSummarySuccess { seperator := " in " for _, qaProject := range qaProjects { + common.LogDebug(" QA Project: ", qaProject) qaResult, err := ObsClient.BuildStatus(qaProject) if err != nil { common.LogError("failed fetching stage project status for", qaProject, ":", err) @@ -1018,6 +1039,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e } } } + switch overallBuildStatus { case BuildStatusSummarySuccess: status.Status = common.CommitStatus_Success diff --git a/systemd/obs-staging-bot.service b/systemd/obs-staging-bot.service index d3dce81..cce596a 100644 --- a/systemd/obs-staging-bot.service +++ b/systemd/obs-staging-bot.service @@ -6,9 +6,13 @@ After=network-online.target Type=exec ExecStart=/usr/bin/obs-staging-bot EnvironmentFile=-/etc/default/obs-staging-bot.env -DynamicUser=yes -NoNewPrivileges=yes -ProtectSystem=strict +User=autogits_obs_staging_bot +Group=users + +# This may work when not using ssh api connections: +#DynamicUser=yes +#NoNewPrivileges=yes +#ProtectSystem=strict [Install] WantedBy=multi-user.target diff --git a/workflow-direct/main.go b/workflow-direct/main.go index b6f1dfc..416f9ec 100644 --- a/workflow-direct/main.go +++ b/workflow-direct/main.go @@ -123,7 +123,7 @@ func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, co common.LogError(" - ", action.Repository.Name, "repo is not sha256. Ignoring.") return } - common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name)) + common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", "../" + action.Repository.Name, action.Repository.Name)) defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all", "-f") branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current")) @@ -215,7 +215,7 @@ func processConfiguredPushAction(action *common.PushWebhookEvent, config *common } if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil { - git.GitExecOrPanic(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name) + git.GitExecOrPanic(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", "../" + action.Repository.Name, action.Repository.Name) common.LogDebug("Pushed to package that is not part of the project. Re-adding...", err) } else if !stat.IsDir() { common.LogError("Pushed to a package that is not a submodule but exists in the project. Ignoring.") @@ -420,7 +420,7 @@ next_repo: } // add repository to git project - common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", r.CloneURL, r.Name)) + common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", "../" + r.Name, r.Name)) curBranch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current")) if branch != curBranch { diff --git a/workflow-pr/pr_processor.go b/workflow-pr/pr_processor.go index a460a5f..35199ba 100644 --- a/workflow-pr/pr_processor.go +++ b/workflow-pr/pr_processor.go @@ -180,6 +180,33 @@ func (pr *PRProcessor) SetSubmodulesToMatchPRSet(prset *common.PRSet) error { } updateSubmoduleInPR(submodulePath, prHead, git) + err := git.GitExec(path.Join(common.DefaultGitPrj, submodulePath), "lfs", "fetch") + common.LogError("lfs fetch err: ", err) + if err = git.GitExec(path.Join(common.DefaultGitPrj, submodulePath), "lfs", "fsck"); err != nil { + + found_comment := false + timeline, terr := common.FetchTimelineSinceLastPush(Gitea, prHead, org, repo, idx) + if terr != nil { + common.LogError("lfs fsck error, but timeline fetch failed") + break + } + msgPrefix := "The LFS objects are broken!" + for _, t := range timeline { + if t.Type == common.TimelineCommentType_Comment && strings.HasPrefix(t.Body, msgPrefix) { + found_comment = true + common.LogError("lfs fsck Comment already found") + + break + } + } + + if !found_comment && !common.IsDryRun { + Gitea.AddComment(pr.PR, msgPrefix + " Please verify with 'git lfs fsck'") + } + common.LogError("lfs fsck failed with: ", err.Error()) + return err + } + status, err := git.GitStatus(common.DefaultGitPrj) common.LogDebug("status:", status) common.LogDebug("submodule", repo, " hash:", id, " -> ", prHead) @@ -309,14 +336,16 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error { PrjGit := PrjGitPR.PR.Base.Repo prjGitPRbranch := PrjGitPR.PR.Head.Name if PrjGitPR.PR.Base.RepoID != PrjGitPR.PR.Head.RepoID { - PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, "", PrjGit.SSHURL) - git.GitExecOrPanic(common.DefaultGitPrj, "fetch", PrjGitPR.RemoteName, PrjGitPR.PR.Head.Sha) - git.GitExecOrPanic(common.DefaultGitPrj, "checkout", PrjGitPR.PR.Head.Sha) - common.LogInfo("Cannot update this PR as it's on another remote, not branch:", prjGitPRbranch, "Assuming this is by-design. (eg. project git PR only)") - return nil + // permission check, if submission comes from foreign repo + if !PrjGitPR.PR.AllowMaintainerEdit { + common.LogError("Warning: source and target branch are in different repositories. We may not have the right permissions...") + // Gitea.AddComment(PrjGitPR.PR, "This PR does not allow maintainer changes, but referenced package branch has changed!") + return nil + } } - PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, prjGitPRbranch, PrjGit.SSHURL) +// PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, prjGitPRbranch, PrjGit.SSHURL) + PrjGitPR.RemoteName, err = git.GitClone(common.DefaultGitPrj, PrjGitPR.PR.Head.Ref, PrjGitPR.PR.Head.Repo.SSHURL) common.PanicOnError(err) git.GitExecOrPanic(common.DefaultGitPrj, "fetch", PrjGitPR.RemoteName, PrjGitBranch) @@ -364,6 +393,7 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error { } common.PanicOnError(git.GitExec(common.DefaultGitPrj, params...)) PrjGitPR.PR.Head.Sha = newHeadCommit + Gitea.SetLabels(PrjGit.Owner.UserName, PrjGit.Name, PrjGitPR.PR.Index, []string{prset.Config.Label("PR/updated")}) } // update PR @@ -380,7 +410,7 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error { } return CurrentTitle == NewTitle } - if PrjGitPR.PR.User.UserName == CurrentUser.UserName && (PrjGitPR.PR.Body != PrjGitBody || !isPrTitleSame(PrjGitPR.PR.Title, PrjGitTitle)) { + if !pr.config.NoProjectGitPR && PrjGitPR.PR.User.UserName == CurrentUser.UserName && (PrjGitPR.PR.Body != PrjGitBody || !isPrTitleSame(PrjGitPR.PR.Title, PrjGitTitle)) { Gitea.UpdatePullRequest(PrjGit.Owner.UserName, PrjGit.Name, PrjGitPR.PR.Index, &models.EditPullRequestOption{ RemoveDeadline: true, Title: PrjGitTitle,