package main import ( "bytes" "errors" "fmt" "log" "net/url" "os" "path" "regexp" "slices" "strconv" "strings" "time" "src.opensuse.org/autogits/common" "src.opensuse.org/autogits/common/gitea-generated/models" ) const ( GitAuthor = "GiteaBot - Obs Staging" BotName = "ObsStaging" ObsBuildBot = "/obsbuild" Username = "autogits_obs_staging_bot" ) var GiteaToken string var runId uint func failOnError(err error, msg string) { if err != nil { log.Panicf("%s: %s", err, msg) } } func fetchPrGit(h *common.RequestHandler, pr *models.PullRequest) error { // clone PR head and base and return path if h.HasError() { return h.Error } if _, err := os.Stat(path.Join(h.GitPath, pr.Head.Sha)); os.IsNotExist(err) { h.GitExec("", "clone", "--depth", "1", pr.Head.Repo.CloneURL, pr.Head.Sha) h.GitExec(pr.Head.Sha, "fetch", "--depth", "1", "origin", pr.Head.Sha, pr.Base.Sha) } else if err != nil { h.Error = err } return h.Error } func getObsProjectAssociatedWithPr(baseProject string, pr *models.PullRequest) string { return fmt.Sprintf( "%s:%s:%s:PR:%d", baseProject, common.ObsSafeProjectName(pr.Base.Repo.Owner.UserName), common.ObsSafeProjectName(pr.Base.Repo.Name), pr.Index, ) } func processBuildStatusUpdate() { } type BuildStatusSummary int const ( BuildStatusSummarySuccess = 1 BuildStatusSummaryFailed = 2 BuildStatusSummaryBuilding = 3 BuildStatusSummaryUnknown = 4 ) func processBuildStatus(h *common.RequestHandler, project, refProject *common.BuildResultList) BuildStatusSummary { if _, finished := project.BuildResultSummary(); !finished { return BuildStatusSummaryBuilding } if _, finished := refProject.BuildResultSummary(); !finished { h.LogError("refProject not finished building??") return BuildStatusSummaryUnknown } // the repositories should be setup equally between the projects. We // need to verify that packages that are building in `refProject` are not // failing in the `project` BuildResultSorter := func(a, b common.BuildResult) int { if c := strings.Compare(a.Repository, b.Repository); c != 0 { return c } if c := strings.Compare(a.Arch, b.Arch); c != 0 { return c } panic("Should not happen -- BuiltResultSorter equal repos?") } slices.SortFunc(project.Result, BuildResultSorter) if refProject == nil { // just return if buid finished and have some successes, since new package SomeSuccess := false for i := 0; i < len(project.Result); i++ { repoRes := &project.Result[i] repoResStatus, ok := common.ObsRepoStatusDetails[repoRes.Code] if !ok { h.LogError("cannot find code: %s", repoRes.Code) return BuildStatusSummaryUnknown } if !repoResStatus.Finished { return BuildStatusSummaryBuilding } for _, pkg := range repoRes.Status { pkgStatus, ok := common.ObsBuildStatusDetails[pkg.Code] if !ok { h.LogError("Unknown package build status: %s for %s", pkg.Code, pkg.Package) h.LogError("Details: %s", pkg.Details) return BuildStatusSummaryUnknown } if pkgStatus.Success { SomeSuccess = true } } } if SomeSuccess { return BuildStatusSummarySuccess } return BuildStatusSummaryFailed } slices.SortFunc(refProject.Result, BuildResultSorter) h.Log("comparing results %d vs. ref %d", len(project.Result), len(refProject.Result)) for i := 0; i < len(project.Result); i++ { h.Log("searching for %s/%s", project.Result[i].Repository, project.Result[i].Arch) j := 0 found: for ; j < len(refProject.Result); j++ { if project.Result[i].Repository != refProject.Result[j].Repository || project.Result[i].Arch != refProject.Result[j].Arch { continue } h.Log("found match for %s/%s @ %d", project.Result[i].Repository, project.Result[i].Arch, j) res := processRepoBuildStatus(h, project.Result[i].Status, refProject.Result[j].Status) switch res { case BuildStatusSummarySuccess: break found default: return res } } h.Log("%d", j) if j >= len(refProject.Result) { h.LogError("Cannot find results... %#v \n %#v\n", project.Result[i], refProject.Result) return BuildStatusSummaryUnknown } } return BuildStatusSummarySuccess } func processRepoBuildStatus(h *common.RequestHandler, results, ref []common.PackageBuildStatus) BuildStatusSummary { PackageBuildStatusSorter := func(a, b common.PackageBuildStatus) int { return strings.Compare(a.Package, b.Package) } // compare build result slices.SortFunc(results, PackageBuildStatusSorter) slices.SortFunc(ref, PackageBuildStatusSorter) j := 0 for i := 0; i < len(results); i++ { res, ok := common.ObsBuildStatusDetails[results[i].Code] if !ok { h.LogError("unknown package result code: %s for package %s", results[i].Code, results[i].Package) return BuildStatusSummaryUnknown } if !res.Finished { return BuildStatusSummaryBuilding } if !res.Success { // not failed if reference project also failed for same package here for ; j < len(results) && strings.Compare(results[i].Package, ref[j].Package) < 0; j++ { } if j < len(results) && results[i].Package == ref[j].Package { refRes, ok := common.ObsBuildStatusDetails[ref[j].Code] if !ok { h.LogError("unknown package result code: %s for package %s", ref[j].Code, ref[j].Package) return BuildStatusSummaryUnknown } if !refRes.Finished { h.LogError("not finished building in reference project?!") } if refRes.Success { return BuildStatusSummaryFailed } } } } return BuildStatusSummarySuccess } func generateObsPrjMeta(h *common.RequestHandler, pr *models.PullRequest, obsClient *common.ObsClient) (*common.ProjectMeta, error) { h.Log("repo content fetching ...") err := fetchPrGit(h, pr) if err != nil { h.LogError("Cannot fetch PR git: %s", pr.URL) return nil, err } // find modified submodules and new submodules -- build them dir := pr.Head.Sha headSubmodules := h.GitSubmoduleList(dir, pr.Head.Sha) baseSubmodules := h.GitSubmoduleList(dir, pr.Base.Sha) modifiedOrNew := make([]string, 0, 16) for pkg, headOid := range headSubmodules { if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid { modifiedOrNew = append(modifiedOrNew, pkg) } } buildPrj := string(bytes.TrimSpace(h.GitCatFile(dir, pr.Head.Sha, "project.build"))) if len(buildPrj) < 1 { _, err := h.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project") if err != nil { h.LogPlainError(err) return nil, h.Error } return nil, fmt.Errorf("Cannot find reference project for %s PR#%d", pr.Base.Name, pr.Index) } if h.HasError() { h.LogPlainError(h.Error) return nil, h.Error } meta, err := obsClient.GetProjectMeta(buildPrj) if err != nil { h.Log("error fetching project meta for %s: %v", buildPrj, err) return nil, err } // generate new project with paths pointinig back to original repos // disable publishing meta.Name = getObsProjectAssociatedWithPr(obsClient.HomeProject, pr) meta.Description = fmt.Sprintf(`Pull request build job PR#%d to branch %s of %s/%s`, pr.Index, pr.Base.Name, pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name) meta.Url = fmt.Sprintf( "https://src.opensuse.org/%s/%s/pulls/%d", url.PathEscape(pr.Base.Repo.Owner.UserName), url.PathEscape(pr.Base.Repo.Name), pr.Index, ) urlPkg := make([]string, 0, len(modifiedOrNew)) for _, pkg := range modifiedOrNew { urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(pkg)) } meta.ScmSync = pr.Head.Repo.CloneURL + "?" + strings.Join(urlPkg, "&") + "#" + pr.Head.Sha meta.Title = fmt.Sprintf("PR#%d to %s", pr.Index, pr.Base.Name) meta.PublicFlags = common.Flags{Contents: ""} // set paths to parent project for idx, r := range meta.Repositories { meta.Repositories[idx].Paths = []common.RepositoryPathMeta{{ Project: buildPrj, Repository: r.Name, }} } h.Log("%#v", meta) return meta, nil } func startOrUpdateBuild(h *common.RequestHandler, pr *models.PullRequest, obsClient *common.ObsClient) error { h.Log("fetching OBS project Meta") obsPrProject := getObsProjectAssociatedWithPr(obsClient.HomeProject, pr) meta, err := obsClient.GetProjectMeta(obsPrProject) if err != nil { h.Log("error fetching project meta for %s: %v", obsPrProject, err) return err } if meta != nil { path, err := url.Parse(meta.ScmSync) if err != nil { h.Log("Cannot parse SCMSYNC url: '%s' .. regenerating build", meta.ScmSync) meta = nil } else { if path.Fragment == pr.Head.Sha { // build in progress return nil } // build needs update h.Log("Detected Head update... regenerating build...") meta = nil } } if meta == nil { // new build meta, err = generateObsPrjMeta(h, pr, obsClient) if err != nil { return err } } err = obsClient.SetProjectMeta(meta) if err != nil { h.Error = err h.LogError("cannot create meta project: %#v", err) return h.Error } return nil } func processPullNotification(h *common.RequestHandler, thread *models.NotificationThread) { rx := regexp.MustCompile(`^https://src\.(?:open)?suse\.(?:org|de)/api/v\d+/repos/(?[a-zA-Z0-9]+)/(?[_a-zA-Z0-9]+)/issues/(?[0-9]+)$`) notification := thread.Subject match := rx.FindStringSubmatch(notification.URL) if match == nil { log.Panicf("Unexpected format of notification: %s", notification.URL) } h.Log("processing") h.Log("project: %s", match[2]) h.Log("org: %s", match[1]) h.Log("number: %s", match[3]) org := match[1] repo := match[2] id, _ := strconv.ParseInt(match[3], 10, 64) pr, reviews, err := h.GetPullRequestAndReviews(org, repo, id) if err != nil { h.LogError("No PR associated with review: %s. Error: %v", notification.URL, err) return } obsClient, err := common.NewObsClient("api.opensuse.org") if err != nil { h.LogPlainError(err) return } reviewRequested := false for _, reviewer := range pr.RequestedReviewers { if reviewer.UserName == Username { reviewRequested = true break } } if !reviewRequested { h.Log("Review not requested in notification. Setting to status 'read'") h.SetNotificationRead(thread.ID) return } newReviews := make([]*models.PullReview, 0, len(reviews)) for _, review := range reviews { if review.User.UserName == Username { newReviews = append(newReviews, review) } } reviews = newReviews slices.SortFunc(reviews, func(a, b *models.PullReview) int { return time.Time(a.Submitted).Compare(time.Time(b.Submitted)) }) for idx := len(reviews) - 1; idx >= 0; idx-- { review := reviews[idx] h.Log("state: %s, body: %s, id:%d\n", string(review.State), review.Body, review.ID) if review.User.UserName != "autogits_obs_staging_bot" { continue } h.Log("processing state...") switch review.State { // create build project, if doesn't exist, and add it to pending requests case common.ReviewStateUnknown, common.ReviewStateRequestReview: if err := startOrUpdateBuild(h, pr, obsClient); err != nil { return } msg := "Build is started in https://build.opensuse.org/project/show/" + getObsProjectAssociatedWithPr(obsClient.HomeProject, pr) h.AddReviewComment(pr, common.ReviewStatePending, msg) case common.ReviewStatePending: err := fetchPrGit(h, pr) if err != nil { h.LogError("Cannot fetch PR git: %s", pr.URL) return } // find modified submodules and new submodules -- build them dir := pr.Head.Sha headSubmodules := h.GitSubmoduleList(dir, pr.Head.Sha) baseSubmodules := h.GitSubmoduleList(dir, pr.Base.Sha) modifiedOrNew := make([]string, 0, 16) for pkg, headOid := range headSubmodules { if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid { modifiedOrNew = append(modifiedOrNew, pkg) } } h.Log("repo content fetching ...") refPrj := string(bytes.TrimSpace(h.GitCatFile(dir, pr.Head.Sha, "project.build"))) if len(refPrj) < 1 { _, err := h.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project") if err != nil { h.LogPlainError(err) return } h.LogError("Cannot find reference project for %s PR#%d", pr.Base.Name, pr.Index) return } if h.HasError() { h.LogPlainError(h.Error) return } obsProject := getObsProjectAssociatedWithPr(obsClient.HomeProject, pr) prjResult, err := obsClient.BuildStatus(obsProject) if err != nil { if errors.Is(err, common.ObsProjectNotFound{Project: obsProject}) { // recreate missing project h.LogError("missing OBS project ... recreating '%s': %v", obsProject, err) startOrUpdateBuild(h, pr, obsClient) return } h.LogError("failed fetching build status for '%s': %v", obsProject, err) return } refProjectResult, err := obsClient.BuildStatus(refPrj, prjResult.GetPackageList()...) if err != nil { h.LogError("failed fetching ref project status for '%s': %v", refPrj, err) } buildStatus := processBuildStatus(h, prjResult, refProjectResult) switch buildStatus { case BuildStatusSummarySuccess: _, err := h.AddReviewComment(pr, common.ReviewStateApproved, "Build successful") if err != nil { h.LogPlainError(err) } case BuildStatusSummaryFailed: _, err := h.AddReviewComment(pr, common.ReviewStateRequestChanges, "Build failed") if err != nil { h.LogPlainError(err) } } h.Log("Build status waiting: %d", buildStatus) // waiting for build results -- nothing to do case common.ReviewStateApproved: // done, mark notification as read h.Log("processing request for success build ...") h.SetNotificationRead(thread.ID) case common.ReviewStateRequestChanges: // build failures, nothing to do here, mark notification as read h.Log("processing request for failed request changes...") h.SetNotificationRead(thread.ID) } break } } func pollWorkNotifications() { h := common.CreateRequestHandler(GitAuthor, BotName) data, err := h.GetPullNotifications(nil) if err != nil { h.LogPlainError(err) return } if data != nil { for _, notification := range data { switch notification.Subject.Type { case "Pull": processPullNotification(h, notification) default: h.SetNotificationRead(notification.ID) } } } } func main() { failOnError(common.RequireGiteaSecretToken(), "Cannot find GITEA_TOKEN") failOnError(common.RequireObsSecretToken(), "Cannot find OBS_USER and OBS_PASSWORD") // go ProcessingObsMessages("rabbit.opensuse.org", "opensuse", "opensuse", "") for { pollWorkNotifications() time.Sleep(10 * time.Minute) } stuck := make(chan int) <-stuck }