autogits/obs-staging-bot/main.go

package main

/*
 * This file is part of Autogits.
 *
 * Copyright © 2024 SUSE LLC
 *
 * Autogits is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License as published by the Free Software
 * Foundation, either version 2 of the License, or (at your option) any later
 * version.
 *
 * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 * PARTICULAR PURPOSE. See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * Foobar. If not, see <https://www.gnu.org/licenses/>.
 */

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"net/url"
	"os"
	"path"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"time"

	"src.opensuse.org/autogits/common"
	"src.opensuse.org/autogits/common/gitea-generated/models"
)

const (
	GitAuthor   = "GiteaBot - Obs Staging"
	BotName     = "ObsStaging"
	ObsBuildBot = "/obsbuild"

	Username = "autogits_obs_staging_bot"
)

var GiteaToken string
var runId uint

func failOnError(err error, msg string) {
	if err != nil {
		log.Panicf("%s: %s", err, msg)
	}
}

func fetchPrGit(h *common.RequestHandler, pr *models.PullRequest) error {
	// clone PR head and base and return path
	if h.HasError() {
		return h.Error
	}
	if _, err := os.Stat(path.Join(h.GitPath, pr.Head.Sha)); os.IsNotExist(err) {
		h.GitExec("", "clone", "--depth", "1", pr.Head.Repo.CloneURL, pr.Head.Sha)
		h.GitExec(pr.Head.Sha, "fetch", "--depth", "1", "origin", pr.Head.Sha, pr.Base.Sha)
	} else if err != nil {
		h.Error = err
	}

	return h.Error
}

func getObsProjectAssociatedWithPr(baseProject string, pr *models.PullRequest) string {
	if pr.Base.Repo.Name == common.DefaultGitPrj {
		// if default project git, we don't need it in project name to be unique
		return fmt.Sprintf(
			"%s:%s:PR:%d",
			baseProject,
			common.ObsSafeProjectName(pr.Base.Repo.Owner.UserName),
			pr.Index,
		)
	}
	return fmt.Sprintf(
		"%s:%s:%s:PR:%d",
		baseProject,
		common.ObsSafeProjectName(pr.Base.Repo.Owner.UserName),
		common.ObsSafeProjectName(pr.Base.Repo.Name),
		pr.Index,
	)
}

func processBuildStatusUpdate() {
}

type BuildStatusSummary int

const (
	BuildStatusSummarySuccess  = 1
	BuildStatusSummaryFailed   = 2
	BuildStatusSummaryBuilding = 3
	BuildStatusSummaryUnknown  = 4
)

func processBuildStatus(h *common.RequestHandler, project, refProject *common.BuildResultList) BuildStatusSummary {
	if _, finished := project.BuildResultSummary(); !finished {
		return BuildStatusSummaryBuilding
	}

	if _, finished := refProject.BuildResultSummary(); !finished {
		h.LogError("refProject not finished building??")
		return BuildStatusSummaryUnknown
	}

	// the repositories should be setup equally between the projects. We
	// need to verify that packages that are building in `refProject` are not
	// failing in the `project`
	BuildResultSorter := func(a, b common.BuildResult) int {
		if c := strings.Compare(a.Repository, b.Repository); c != 0 {
			return c
		}
		if c := strings.Compare(a.Arch, b.Arch); c != 0 {
			return c
		}

		panic("Should not happen -- BuiltResultSorter equal repos?")
	}
	slices.SortFunc(project.Result, BuildResultSorter)
	if refProject == nil {
		// just return if buid finished and have some successes, since new package
		SomeSuccess := false
		for i := 0; i < len(project.Result); i++ {
			repoRes := &project.Result[i]
			repoResStatus, ok := common.ObsRepoStatusDetails[repoRes.Code]
			if !ok {
				h.LogError("cannot find code: %s", repoRes.Code)
				return BuildStatusSummaryUnknown
			}
			if !repoResStatus.Finished {
				return BuildStatusSummaryBuilding
			}

			for _, pkg := range repoRes.Status {
				pkgStatus, ok := common.ObsBuildStatusDetails[pkg.Code]
				if !ok {
					h.LogError("Unknown package build status: %s for %s", pkg.Code, pkg.Package)
					h.LogError("Details: %s", pkg.Details)
					return BuildStatusSummaryUnknown
				}

				if pkgStatus.Success {
					SomeSuccess = true
				}
			}
		}

		if SomeSuccess {
			return BuildStatusSummarySuccess
		}
		return BuildStatusSummaryFailed
	}

	slices.SortFunc(refProject.Result, BuildResultSorter)

	h.Log("comparing results %d vs. ref %d", len(project.Result), len(refProject.Result))
	for i := 0; i < len(project.Result); i++ {
		h.Log("searching for %s/%s", project.Result[i].Repository, project.Result[i].Arch)
		j := 0
	found:
		for ; j < len(refProject.Result); j++ {
			if project.Result[i].Repository != refProject.Result[j].Repository ||
				project.Result[i].Arch != refProject.Result[j].Arch {
				continue
			}

			h.Log("found match for %s/%s @ %d", project.Result[i].Repository, project.Result[i].Arch, j)
			res := processRepoBuildStatus(h, project.Result[i].Status, refProject.Result[j].Status)
			switch res {
			case BuildStatusSummarySuccess:
				break found
			default:
				return res
			}
		}

		h.Log("%d", j)
		if j >= len(refProject.Result) {
			h.LogError("Cannot find results... %#v \n %#v\n", project.Result[i], refProject.Result)
			return BuildStatusSummaryUnknown
		}
	}
	return BuildStatusSummarySuccess
}

func processRepoBuildStatus(h *common.RequestHandler, results, ref []common.PackageBuildStatus) BuildStatusSummary {
	PackageBuildStatusSorter := func(a, b common.PackageBuildStatus) int {
		return strings.Compare(a.Package, b.Package)
	}

	// compare build result
	slices.SortFunc(results, PackageBuildStatusSorter)
	slices.SortFunc(ref, PackageBuildStatusSorter)

	j := 0
	for i := 0; i < len(results); i++ {
		res, ok := common.ObsBuildStatusDetails[results[i].Code]
		if !ok {
			h.LogError("unknown package result code: %s for package %s", results[i].Code, results[i].Package)
			return BuildStatusSummaryUnknown
		}

		if !res.Finished {
			return BuildStatusSummaryBuilding
		}

		if !res.Success {
			// not failed if reference project also failed for same package here
			for ; j < len(results) && strings.Compare(results[i].Package, ref[j].Package) < 0; j++ {
			}

			if j < len(results) && results[i].Package == ref[j].Package {
				refRes, ok := common.ObsBuildStatusDetails[ref[j].Code]
				if !ok {
					h.LogError("unknown package result code: %s for package %s", ref[j].Code, ref[j].Package)
					return BuildStatusSummaryUnknown
				}

				if !refRes.Finished {
					h.LogError("not finished building in reference project?!")
				}

				if refRes.Success {
					return BuildStatusSummaryFailed
				}
			}
		}
	}

	return BuildStatusSummarySuccess
}

func generateObsPrjMeta(h *common.RequestHandler, pr *models.PullRequest, obsClient *common.ObsClient) (*common.ProjectMeta, error) {
	h.Log("repo content fetching ...")
	err := fetchPrGit(h, pr)
	if err != nil {
		h.LogError("Cannot fetch PR git: %s", pr.URL)
		return nil, err
	}

	// find modified submodules and new submodules -- build them
	dir := pr.Head.Sha
	headSubmodules := h.GitSubmoduleList(dir, pr.Head.Sha)
	baseSubmodules := h.GitSubmoduleList(dir, pr.Base.Sha)

	modifiedOrNew := make([]string, 0, 16)
	for pkg, headOid := range headSubmodules {
		if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid {
			modifiedOrNew = append(modifiedOrNew, pkg)
		}
	}
	buildPrj := string(bytes.TrimSpace(h.GitCatFile(dir, pr.Head.Sha, "project.build")))

	if len(buildPrj) < 1 {
		_, err := h.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project")
		if err != nil {
			h.LogPlainError(err)
			return nil, h.Error
		}
		return nil, fmt.Errorf("Cannot find reference project for %s PR#%d", pr.Base.Name, pr.Index)
	}
	if h.HasError() {
		h.LogPlainError(h.Error)
		return nil, h.Error
	}

	meta, err := obsClient.GetProjectMeta(buildPrj)
	if err != nil {
		h.Log("error fetching project meta for %s: %v", buildPrj, err)
		return nil, err
	}

	// generate new project with paths pointinig back to original repos
	// disable publishing

	meta.Name = getObsProjectAssociatedWithPr(obsClient.HomeProject, pr)
	meta.Description = fmt.Sprintf(`Pull request build job PR#%d to branch %s of %s/%s`,
		pr.Index, pr.Base.Name, pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name)
	meta.Url = fmt.Sprintf(
		"https://src.opensuse.org/%s/%s/pulls/%d",
		url.PathEscape(pr.Base.Repo.Owner.UserName),
		url.PathEscape(pr.Base.Repo.Name),
		pr.Index,
	)

	urlPkg := make([]string, 0, len(modifiedOrNew))
	for _, pkg := range modifiedOrNew {
		urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(pkg))
	}
	meta.ScmSync = pr.Head.Repo.CloneURL + "?" + strings.Join(urlPkg, "&") + "#" + pr.Head.Sha
	meta.Title = fmt.Sprintf("PR#%d to %s", pr.Index, pr.Base.Name)
	meta.PublicFlags = common.Flags{Contents: "<disable/>"}

	// set paths to parent project
	for idx, r := range meta.Repositories {
		meta.Repositories[idx].Paths = []common.RepositoryPathMeta{{
			Project:    buildPrj,
			Repository: r.Name,
		}}
	}
	h.Log("%#v", meta)
	return meta, nil
}

func startOrUpdateBuild(h *common.RequestHandler, pr *models.PullRequest, obsClient *common.ObsClient) error {
	h.Log("fetching OBS project Meta")
	obsPrProject := getObsProjectAssociatedWithPr(obsClient.HomeProject, pr)
	meta, err := obsClient.GetProjectMeta(obsPrProject)
	if err != nil {
		h.Log("error fetching project meta for %s: %v", obsPrProject, err)
		return err
	}

	if meta != nil {
		path, err := url.Parse(meta.ScmSync)
		if err != nil {
			h.Log("Cannot parse SCMSYNC url: '%s' .. regenerating build", meta.ScmSync)
			meta = nil
		} else {
			if path.Fragment == pr.Head.Sha {
				// build in progress
				return nil
			}
			// build needs update
			h.Log("Detected Head update... regenerating build...")
			meta = nil
		}
	}
	if meta == nil {
		// new build
		meta, err = generateObsPrjMeta(h, pr, obsClient)
		if err != nil {
			return err
		}
	}

	err = obsClient.SetProjectMeta(meta)
	if err != nil {
		h.Error = err
		h.LogError("cannot create meta project: %#v", err)
		return h.Error
	}

	return nil
}

func processPullNotification(h *common.RequestHandler, thread *models.NotificationThread) {
	rx := regexp.MustCompile(`^https://src\.(?:open)?suse\.(?:org|de)/api/v\d+/repos/(?<org>[a-zA-Z0-9]+)/(?<project>[_a-zA-Z0-9]+)/issues/(?<num>[0-9]+)$`)
	notification := thread.Subject
	match := rx.FindStringSubmatch(notification.URL)
	if match == nil {
		log.Panicf("Unexpected format of notification: %s", notification.URL)
	}

	h.Log("processing")
	h.Log("project: %s", match[2])
	h.Log("org:     %s", match[1])
	h.Log("number:  %s", match[3])

	org := match[1]
	repo := match[2]
	id, _ := strconv.ParseInt(match[3], 10, 64)

	pr, reviews, err := h.GetPullRequestAndReviews(org, repo, id)
	if err != nil {
		h.LogError("No PR associated with review: %s. Error: %v", notification.URL, err)
		return
	}

	obsClient, err := common.NewObsClient("api.opensuse.org")
	if err != nil {
		h.LogPlainError(err)
		return
	}

	reviewRequested := false
	for _, reviewer := range pr.RequestedReviewers {
		if reviewer.UserName == Username {
			reviewRequested = true
			break
		}
	}

	if !reviewRequested {
		h.Log("Review not requested in notification. Setting to status 'read'")
		h.SetNotificationRead(thread.ID)
		return
	}

	newReviews := make([]*models.PullReview, 0, len(reviews))
	for _, review := range reviews {
		if review.User.UserName == Username {
			newReviews = append(newReviews, review)
		}
	}
	reviews = newReviews

	slices.SortFunc(reviews, func(a, b *models.PullReview) int {
		return time.Time(a.Submitted).Compare(time.Time(b.Submitted))
	})

	for idx := len(reviews) - 1; idx >= 0; idx-- {
		review := reviews[idx]
		h.Log("state: %s,   body: %s,   id:%d\n", string(review.State), review.Body, review.ID)

		if review.User.UserName != "autogits_obs_staging_bot" {
			continue
		}

		h.Log("processing state...")

		switch review.State {

		// create build project, if doesn't exist, and add it to pending requests
		case common.ReviewStateUnknown, common.ReviewStateRequestReview:
			if err := startOrUpdateBuild(h, pr, obsClient); err != nil {
				return
			}

			msg := "Build is started in https://build.opensuse.org/project/show/" +
				getObsProjectAssociatedWithPr(obsClient.HomeProject, pr)
			h.AddReviewComment(pr, common.ReviewStatePending, msg)

		case common.ReviewStatePending:
			err := fetchPrGit(h, pr)
			if err != nil {
				h.LogError("Cannot fetch PR git: %s", pr.URL)
				return
			}

			// find modified submodules and new submodules -- build them
			dir := pr.Head.Sha
			headSubmodules := h.GitSubmoduleList(dir, pr.Head.Sha)
			baseSubmodules := h.GitSubmoduleList(dir, pr.Base.Sha)

			modifiedOrNew := make([]string, 0, 16)
			for pkg, headOid := range headSubmodules {
				if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid {
					modifiedOrNew = append(modifiedOrNew, pkg)
				}
			}

			h.Log("repo content fetching ...")
			refPrj := string(bytes.TrimSpace(h.GitCatFile(dir, pr.Head.Sha, "project.build")))

			if len(refPrj) < 1 {
				_, err := h.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project")
				if err != nil {
					h.LogPlainError(err)
					return
				}
				h.LogError("Cannot find reference project for %s PR#%d", pr.Base.Name, pr.Index)
				return
			}
			if h.HasError() {
				h.LogPlainError(h.Error)
				return
			}

			obsProject := getObsProjectAssociatedWithPr(obsClient.HomeProject, pr)
			prjResult, err := obsClient.BuildStatus(obsProject)
			if err != nil {
				if errors.Is(err, common.ObsProjectNotFound{Project: obsProject}) {
					// recreate missing project
					h.LogError("missing OBS project ... recreating '%s': %v", obsProject, err)
					startOrUpdateBuild(h, pr, obsClient)

					return
				}
				h.LogError("failed fetching build status for '%s': %v", obsProject, err)
				return
			}

			refProjectResult, err := obsClient.BuildStatus(refPrj, prjResult.GetPackageList()...)
			if err != nil {
				h.LogError("failed fetching ref project status for '%s': %v", refPrj, err)
			}
			buildStatus := processBuildStatus(h, prjResult, refProjectResult)

			switch buildStatus {
			case BuildStatusSummarySuccess:
				_, err := h.AddReviewComment(pr, common.ReviewStateApproved, "Build successful")
				if err != nil {
					h.LogPlainError(err)
				}
			case BuildStatusSummaryFailed:
				_, err := h.AddReviewComment(pr, common.ReviewStateRequestChanges, "Build failed")
				if err != nil {
					h.LogPlainError(err)
				}
			}
			h.Log("Build status waiting: %d", buildStatus)
			// waiting for build results -- nothing to do

		case common.ReviewStateApproved:
			// done, mark notification as read
			h.Log("processing request for success build ...")
			h.SetNotificationRead(thread.ID)

		case common.ReviewStateRequestChanges:
			// build failures, nothing to do here, mark notification as read
			h.Log("processing request for failed request changes...")
			h.SetNotificationRead(thread.ID)
		}

		break
	}
}

func pollWorkNotifications() {
	h := common.CreateRequestHandler(GitAuthor, BotName)
	data, err := h.GetPullNotifications(nil)

	if err != nil {
		h.LogPlainError(err)
		return
	}

	if data != nil {
		for _, notification := range data {
			switch notification.Subject.Type {
			case "Pull":
				processPullNotification(h, notification)
			default:
				h.SetNotificationRead(notification.ID)
			}
		}
	}
}

func main() {
	failOnError(common.RequireGiteaSecretToken(), "Cannot find GITEA_TOKEN")
	failOnError(common.RequireObsSecretToken(), "Cannot find OBS_USER and OBS_PASSWORD")

	//	go ProcessingObsMessages("rabbit.opensuse.org", "opensuse", "opensuse", "")

	for {
		pollWorkNotifications()
		time.Sleep(10 * time.Minute)
	}

	stuck := make(chan int)
	<-stuck
}