autogits/obs-staging-bot/main.go

package main

/*
 * This file is part of Autogits.
 *
 * Copyright © 2024 SUSE LLC
 *
 * Autogits is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License as published by the Free Software
 * Foundation, either version 2 of the License, or (at your option) any later
 * version.
 *
 * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 * PARTICULAR PURPOSE. See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * Foobar. If not, see <https://www.gnu.org/licenses/>.
 */

import (
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"net/url"
	"os"
	"path"
	"regexp"
	"runtime/debug"
	"slices"
	"strconv"
	"strings"
	"time"

	"github.com/opentracing/opentracing-go/log"
	"src.opensuse.org/autogits/common"
	"src.opensuse.org/autogits/common/gitea-generated/client/repository"
	"src.opensuse.org/autogits/common/gitea-generated/models"
)

const (
	GitAuthor   = "GiteaBot - Obs Staging"
	BotName     = "ObsStaging"
	ObsBuildBot = "/obsbuild"

	Username = "autogits_obs_staging_bot"
)

var GiteaToken string
var runId uint

func FetchPrGit(git common.Git, pr *models.PullRequest) error {
	// clone PR head and base and return path
	cloneURL := pr.Head.Repo.CloneURL
	if GiteaUseSshClone {
		cloneURL = pr.Head.Repo.SSHURL
	}
	if _, err := os.Stat(path.Join(git.GetPath(), pr.Head.Sha)); os.IsNotExist(err) {
		common.PanicOnError(git.GitExec("", "clone", "--depth", "1", cloneURL, pr.Head.Sha))
		common.PanicOnError(git.GitExec(pr.Head.Sha, "fetch", "--depth", "1", "origin", pr.Head.Sha, pr.MergeBase))
	} else if err != nil {
		return err
	}

	return nil
}

func GetObsProjectAssociatedWithPr(config *common.StagingConfig, baseProject string, pr *models.PullRequest) string {
	if config.StagingProject != "" {
		return fmt.Sprintf("%s:%d", config.StagingProject, pr.Index)
	}

	if pr.Base.Repo.Name == common.DefaultGitPrj {
		// if default project git, we don't need it in project name to be unique
		return fmt.Sprintf(
			"%s:%s:PR:%d",
			baseProject,
			common.ObsSafeProjectName(pr.Base.Repo.Owner.UserName),
			pr.Index,
		)
	}
	return fmt.Sprintf(
		"%s:%s:%s:PR:%d",
		baseProject,
		common.ObsSafeProjectName(pr.Base.Repo.Owner.UserName),
		common.ObsSafeProjectName(pr.Base.Repo.Name),
		pr.Index,
	)
}

type RequestModification int

const (
	RequestModificationNoChange       = 1
	RequestModificationProjectCreated = 2
	RequestModificationSourceChanged  = 3
)

type BuildStatusSummary int

const (
	BuildStatusSummarySuccess  = 1
	BuildStatusSummaryFailed   = 2
	BuildStatusSummaryBuilding = 3
	BuildStatusSummaryUnknown  = 4
)

func ProcessBuildStatus(project, refProject *common.BuildResultList) BuildStatusSummary {
	if _, finished := refProject.BuildResultSummary(); !finished {
		common.LogDebug("refProject not finished building??")
		return BuildStatusSummaryUnknown
	}

	if _, finished := project.BuildResultSummary(); !finished {
		common.LogDebug("Still building...")
		return BuildStatusSummaryBuilding
	}

	// the repositories should be setup equally between the projects. We
	// need to verify that packages that are building in `refProject` are not
	// failing in the `project`
	BuildResultSorter := func(a, b common.BuildResult) int {
		if c := strings.Compare(a.Repository, b.Repository); c != 0 {
			return c
		}
		if c := strings.Compare(a.Arch, b.Arch); c != 0 {
			return c
		}

		panic("Should not happen -- BuiltResultSorter equal repos?")
	}
	slices.SortFunc(project.Result, BuildResultSorter)
	if refProject == nil {
		// just return if buid finished and have some successes, since new package
		common.LogInfo("New package. Only need some success...")
		SomeSuccess := false
		for i := 0; i < len(project.Result); i++ {
			repoRes := &project.Result[i]
			repoResStatus, ok := common.ObsRepoStatusDetails[repoRes.Code]
			if !ok {
				common.LogDebug("cannot find code:", repoRes.Code)
				return BuildStatusSummaryUnknown
			}
			if !repoResStatus.Finished {
				return BuildStatusSummaryBuilding
			}

			for _, pkg := range repoRes.Status {
				pkgStatus, ok := common.ObsBuildStatusDetails[pkg.Code]
				if !ok {
					common.LogInfo("Unknown package build status:", pkg.Code, "for", pkg.Package)
					common.LogDebug("Details:", pkg.Details)
				}

				if pkgStatus.Success {
					SomeSuccess = true
				}
			}
		}

		if SomeSuccess {
			return BuildStatusSummarySuccess
		}
		return BuildStatusSummaryFailed
	}

	slices.SortFunc(refProject.Result, BuildResultSorter)

	common.LogDebug("comparing results", len(project.Result), "vs. ref", len(refProject.Result))
	SomeSuccess := false
	for i := 0; i < len(project.Result); i++ {
		common.LogDebug("searching for", project.Result[i].Repository, "/", project.Result[i].Arch)
		j := 0
	found:
		for ; j < len(refProject.Result); j++ {
			if project.Result[i].Repository != refProject.Result[j].Repository ||
				project.Result[i].Arch != refProject.Result[j].Arch {
				continue
			}

			common.LogDebug("  found match for @ idx:", j)
			res, success := ProcessRepoBuildStatus(project.Result[i].Status, refProject.Result[j].Status)
			switch res {
			case BuildStatusSummarySuccess:
				SomeSuccess = SomeSuccess || success
				break found
			default:
				return res
			}
		}

		if j >= len(refProject.Result) {
			common.LogDebug("Cannot find results...")
			common.LogDebug(project.Result[i])
			common.LogDebug(refProject.Result)
			return BuildStatusSummaryUnknown
		}
	}

	if SomeSuccess {
		return BuildStatusSummarySuccess
	}

	return BuildStatusSummaryFailed
}

func ProcessRepoBuildStatus(results, ref []common.PackageBuildStatus) (status BuildStatusSummary, SomeSuccess bool) {
	PackageBuildStatusSorter := func(a, b common.PackageBuildStatus) int {
		return strings.Compare(a.Package, b.Package)
	}

	common.LogDebug("********  REF: ")
	data, _ := xml.MarshalIndent(ref, "", "  ")
	common.LogDebug(string(data))
	common.LogDebug("******* RESULTS: ")
	data, _ = xml.MarshalIndent(results, "", "  ")
	common.LogDebug(string(data))
	common.LogDebug("*******")

	// compare build result
	slices.SortFunc(results, PackageBuildStatusSorter)
	slices.SortFunc(ref, PackageBuildStatusSorter)

	j := 0
	SomeSuccess = false
	for i := 0; i < len(results); i++ {
		res, ok := common.ObsBuildStatusDetails[results[i].Code]
		if !ok {
			common.LogInfo("unknown package result code:", results[i].Code, "for package:", results[i].Package)
			return BuildStatusSummaryUnknown, SomeSuccess
		}

		if !res.Finished {
			return BuildStatusSummaryBuilding, SomeSuccess
		}

		if !res.Success {
			// not failed if reference project also failed for same package here
			for ; j < len(results) && strings.Compare(results[i].Package, ref[j].Package) < 0; j++ {
			}

			if j < len(results) && results[i].Package == ref[j].Package {
				refRes, ok := common.ObsBuildStatusDetails[ref[j].Code]
				if !ok {
					common.LogInfo("unknown ref package result code:", ref[j].Code, "package:", ref[j].Package)
					return BuildStatusSummaryUnknown, SomeSuccess
				}

				if !refRes.Finished {
					common.LogDebug("Not finished building in reference project?")
				}

				if refRes.Success {
					return BuildStatusSummaryFailed, SomeSuccess
				}
			}
		} else {
			SomeSuccess = true
		}
	}

	return BuildStatusSummarySuccess, SomeSuccess
}

func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingPrj, buildPrj string, stagingMasterPrj string) (*common.ProjectMeta, error) {
	common.LogDebug("repo content fetching ...")
	err := FetchPrGit(git, pr)
	if err != nil {
		common.LogError("Cannot fetch PR git:", pr.URL)
		return nil, err
	}

	// find modified submodules and new submodules -- build them
	dir := pr.Head.Sha
	headSubmodules, err := git.GitSubmoduleList(dir, pr.Head.Sha)
	if err != nil {
		return nil, err
	}
	baseSubmodules, err := git.GitSubmoduleList(dir, pr.MergeBase)
	if err != nil {
		return nil, err
	}

	modifiedOrNew := make([]string, 0, 16)
	for pkg, headOid := range headSubmodules {
		if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid {
			modifiedOrNew = append(modifiedOrNew, pkg)
		}
	}

	common.LogDebug("Trying first staging master project: ", stagingMasterPrj)
	meta, err := ObsClient.GetProjectMeta(stagingMasterPrj)
	if err == nil {
		// success, so we use that staging master project as our build project
		buildPrj = stagingMasterPrj
	} else {
		common.LogInfo("error fetching project meta for ", stagingMasterPrj, ". Fall Back to ", buildPrj)
		meta, err = ObsClient.GetProjectMeta(buildPrj)
	}
	if err != nil {
		common.LogError("error fetching project meta for", buildPrj, ". Err:", err)
		return nil, err
	}

	// generate new project with paths pointinig back to original repos
	// disable publishing

	meta.Name = stagingPrj
	meta.Description = fmt.Sprintf(`Pull request build job PR#%d to branch %s of %s/%s`,
		pr.Index, pr.Base.Name, pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name)
	meta.Url = fmt.Sprintf(
		"%s/%s/%s/pulls/%d",
		GiteaUrl,
		url.PathEscape(pr.Base.Repo.Owner.UserName),
		url.PathEscape(pr.Base.Repo.Name),
		pr.Index,
	)

	urlPkg := make([]string, 0, len(modifiedOrNew))
	for _, pkg := range modifiedOrNew {
		urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(pkg))
	}
	meta.ScmSync = pr.Head.Repo.CloneURL + "?" + strings.Join(urlPkg, "&") + "#" + pr.Head.Sha
	meta.Title = fmt.Sprintf("PR#%d to %s", pr.Index, pr.Base.Name)
//      QE wants it published ... also we should not hardcode it here, since
//      it is configurable via the :PullRequest project
//	meta.PublicFlags = common.Flags{Contents: "<disable/>"}

	meta.Groups = nil
	meta.Persons = nil

	// set paths to parent project
	for idx, r := range meta.Repositories {
		meta.Repositories[idx].ReleaseTargets = nil
		localRepository := false
		for pidx, path := range r.Paths {
			// Check for path building against a repo in template project itself
			if path.Project == buildPrj {
				meta.Repositories[idx].Paths[pidx].Project = meta.Name
				localRepository = true
			}
		}
		if localRepository != true {
			meta.Repositories[idx].Paths = []common.RepositoryPathMeta{{
				Project:    buildPrj,
				Repository: r.Name,
			}}
		}
	}
	return meta, nil
}

// buildProject
//  ^-  templateProject
//
// stagingProject:$buildProject
//  ^-  stagingProject:$buildProject:$subProjectName (based on templateProject)

func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, gitea common.Gitea, pr *models.PullRequest, stagingProject, templateProject, subProjectName string) error {
	common.LogDebug("Setup QA sub projects")
	templateMeta, err := ObsClient.GetProjectMeta(templateProject)
	if err != nil {
		common.LogError("error fetching template project meta for", templateProject, ":", err)
		return err
	}
	// patch baseMeta to become the new project
	templateMeta.Name = stagingProject + ":" + subProjectName
	// Cleanup ReleaseTarget and modify affected path entries
	for idx, r := range templateMeta.Repositories {
		templateMeta.Repositories[idx].ReleaseTargets = nil

		for pidx, path := range r.Paths {
			// Check for path building against code stream
			if path.Project == stagingConfig.ObsProject {
				templateMeta.Repositories[idx].Paths[pidx].Project = stagingProject
			}
			// Check for path building against a repo in template project itself
			if path.Project == templateProject {
				templateMeta.Repositories[idx].Paths[pidx].Project = templateMeta.Name
			}
		}
	}

	if !IsDryRun {
		err = ObsClient.SetProjectMeta(templateMeta)
		if err != nil {
			common.LogError("cannot create project:", templateMeta.Name, err)
			return err
		}
	} else {
		common.LogDebug("Create project:", templateMeta.Name)
		common.LogDebug(templateMeta)
	}
	return nil
}

func StartOrUpdateBuild(config *common.StagingConfig, git common.Git, gitea common.Gitea, pr *models.PullRequest) (RequestModification, error) {
	common.LogDebug("fetching OBS project Meta")
	obsPrProject := GetObsProjectAssociatedWithPr(config, ObsClient.HomeProject, pr)
	meta, err := ObsClient.GetProjectMeta(obsPrProject)
	if err != nil {
		common.LogError("error fetching project meta for", obsPrProject, ":", err)
		return RequestModificationNoChange, err
	}

	if meta != nil {
		path, err := url.Parse(meta.ScmSync)
		if err != nil {
			common.LogError("Cannot parse SCMSYNC url: '", meta.ScmSync, "' .. regenerating build")
			meta = nil
		} else {
			if path.Fragment == pr.Head.Sha {
				// build in progress
				return RequestModificationNoChange, nil
			}
			// build needs update
			common.LogInfo("Detected Head update... regenerating build...")
			meta = nil
		}
	}
	var state RequestModification = RequestModificationSourceChanged
	if meta == nil {
		// new build
		common.LogDebug(" Staging master:", config.StagingProject)
		meta, err = GenerateObsPrjMeta(git, gitea, pr, obsPrProject, config.ObsProject, config.StagingProject)
		if err != nil {
			return RequestModificationNoChange, err
		}
		state = RequestModificationProjectCreated
	}

	if IsDryRun {
		x, _ := xml.MarshalIndent(meta, "", "   ")
		common.LogDebug("Creating build project:")
		common.LogDebug(" meta:", string(x))
	} else {
		err = ObsClient.SetProjectMeta(meta)
		if err != nil {
			x, _ := xml.MarshalIndent(meta, "", "   ")
			common.LogDebug(" meta:", string(x))
			common.LogError("cannot create meta project:", err)
			return RequestModificationNoChange, err
		}
	}

	return state, nil
}

func IsReviewerRequested(pr *models.PullRequest) bool {
	for _, reviewer := range pr.RequestedReviewers {
		if reviewer != nil && reviewer.UserName == Username {
			return true
		}
	}

	return false
}

var NonActionableReviewError = errors.New("Ignoring non-actionable review request. Marking as done.")
var NoReviewsFoundError = errors.New("No review requests found for the review user. Marking as done.")

func FetchOurLatestActionableReview(gitea common.Gitea, org, repo string, id int64) (*models.PullReview, error) {
	reviews, err := gitea.GetPullRequestReviews(org, repo, id)
	if err != nil {
		return nil, err
	}

	slices.SortFunc(reviews, func(a, b *models.PullReview) int {
		return time.Time(a.Submitted).Compare(time.Time(b.Submitted))
	})

	for idx := len(reviews) - 1; idx >= 0; idx-- {
		review := reviews[idx]
		if review.User == nil || review.User.UserName == Username {
			if IsDryRun {
				// for purposes of moving forward a no-op check
				return review, nil
			}

			switch review.State {
			default:
				// non-actionable state, mark as done
				common.LogInfo("Ignoring non-actionable review request. Marking as done.")
				return nil, NonActionableReviewError
			case common.ReviewStatePending, common.ReviewStateRequestReview:
				return review, nil
			}
		}
	}
	return nil, NoReviewsFoundError
}

func ParseNotificationToPR(thread *models.NotificationThread) (org string, repo string, num int64, err error) {
	rx := regexp.MustCompile(`^https://src\.(?:open)?suse\.(?:org|de)/api/v\d+/repos/(?<org>[-_a-zA-Z0-9]+)/(?<project>[-_a-zA-Z0-9]+)/issues/(?<num>[0-9]+)$`)
	notification := thread.Subject
	match := rx.FindStringSubmatch(notification.URL)
	if match == nil {
		err = fmt.Errorf("Unexpected notification format: %s", notification.URL)
		return
	}

	org = match[1]
	repo = match[2]
	num, err = strconv.ParseInt(match[3], 10, 64)
	return
}

func ProcessPullNotification(gitea common.Gitea, thread *models.NotificationThread) {
	defer func() {
		err := recover()
		if err != nil {
			common.LogError(err)
			common.LogError(string(debug.Stack()))
		}
	}()

	org, repo, num, err := ParseNotificationToPR(thread)
	if err != nil {
		common.LogError(err.Error())
		return
	}
	common.LogInfo("processing PR:", org, "/", repo, "#", num)

	done, err := ProcessPullRequest(gitea, org, repo, num)
	if !IsDryRun && err == nil && done {
		gitea.SetNotificationRead(thread.ID)
	} else if err != nil {
		common.LogError(err)
	}
}

var CleanedUpIssues []int64 = []int64{}

func CleanupPullNotification(gitea common.Gitea, thread *models.NotificationThread) (CleanupComplete bool) {
	defer func() {
		err := recover()
		if err != nil {
			common.LogError(err)
			common.LogError(string(debug.Stack()))
		}
	}()

	cleanUpIdx, cleanedUp := slices.BinarySearch(CleanedUpIssues, thread.ID)
	if cleanedUp {
		return true
	}

	common.LogDebug("  processing notification:", thread.Subject.HTMLURL)
	org, repo, num, err := ParseNotificationToPR(thread)
	if err != nil {
		common.LogError(err.Error())
		return false
	}

	pr, err := gitea.GetPullRequest(org, repo, num)
	if err != nil {
		common.LogError("Cannot fetch PR ", org, "/", repo, "#", num, " error:", err)
		return false
	}

	if pr.State != "closed" {
		common.LogInfo("   ignoring pending PR", thread.Subject.HTMLURL, " state:", pr.State)
		return false
	}

	data, _, err := gitea.GetRepositoryFileContent(org, repo, pr.Head.Sha, common.StagingConfigFile)

	// TODO: remove this legacy PR cleanup stuff
	var config *common.StagingConfig = &common.StagingConfig{}
	if err != nil {
		if errors.Is(err, &repository.RepoGetContentsNotFound{}) {
			data, _, err := gitea.GetRepositoryFileContent(org, repo, pr.Head.Sha, "project.build")
			if err == nil {
				common.LogDebug("   --> Legacy PR cleanup")
				config.ObsProject = string(data)
			} else {
				common.LogDebug("assuming cleaned up already...", pr.URL)
				CleanedUpIssues = slices.Insert(CleanedUpIssues, cleanUpIdx, thread.ID)
				return true
			}
		}

		if config.ObsProject == "" {
			common.LogDebug("Cannot fetch config file in the PR. Skipping cleanup.")
			return false
		}
	} else {
		config, err = common.ParseStagingConfig(data)
		if err != nil {
			common.LogDebug("Failed to parse config ... so probably nothing to cleanup anyway. Marked as done")
			CleanedUpIssues = slices.Insert(CleanedUpIssues, cleanUpIdx, thread.ID)
			return true
		}
	}

	if !pr.HasMerged && time.Since(time.Time(pr.Closed)) < time.Duration(config.CleanupDelay)*time.Hour {
		common.LogInfo("Cooldown period for cleanup of", thread.URL)
		return false
	}

	stagingProject := GetObsProjectAssociatedWithPr(config, ObsClient.HomeProject, pr)
	if prj, err := ObsClient.GetProjectMeta(stagingProject); err != nil {
		common.LogError("Failed fetching meta for project:", stagingProject, ". Not cleaning up")
		return false
	} else if prj == nil && err == nil {
		// cleanup already done
		CleanedUpIssues = slices.Insert(CleanedUpIssues, cleanUpIdx, thread.ID)
		return true
	}

	common.LogDebug("Cleaning up", stagingProject)
	for _, qa := range config.QA {
		project := stagingProject + ":" + qa.Name
		common.LogDebug("Cleaning up QA staging", project)
		if !IsDryRun {
			if err := ObsClient.DeleteProject(project); err != nil {
				common.LogError("Failed to cleanup QA staging", project, err)
			}
		}
	}
	if !IsDryRun {
		if err := ObsClient.DeleteProject(stagingProject); err != nil {
			common.LogError("Failed to cleanup staging", stagingProject, err)
		}
	}

	CleanedUpIssues = slices.Insert(CleanedUpIssues, cleanUpIdx, thread.ID)
	if l := len(CleanedUpIssues); l > 100000 {
		common.LogInfo("** Problem? ** Cleaning up massive CleanedUpIssues cache")
		CleanedUpIssues = CleanedUpIssues[l-50000:]
	}
	return false // cleaned up now, but the cleanup was not aleady done
}

func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, error) {
	dir, err := os.MkdirTemp(os.TempDir(), BotName)
	common.PanicOnError(err)
	if IsDryRun {
		common.LogInfo("will keep temp directory:", dir)
	} else {
		defer os.RemoveAll(dir)
	}

	gh, err := common.AllocateGitWorkTree(dir, GitAuthor, "noaddress@suse.de")
	common.PanicOnError(err)

	git, err := gh.CreateGitHandler(org)
	common.PanicOnError(err)

	common.LogDebug("Fetching PR:", org, repo, id)
	pr, err := gitea.GetPullRequest(org, repo, id)
	if err != nil {
		common.LogError("No PR associated with review:", org, "/", repo, "#", id, "Error:", err)
		return true, err
	}
	common.LogDebug("PR state:", pr.State)
	if pr.State == "closed" {
		// dismiss the review
		common.LogInfo(" -- closed request, so nothing to review")
		return true, nil
	}

	if !IsReviewerRequested(pr) {
		common.LogError("Review not requested in notification. Setting to status 'read'")
		if !IsDryRun {
			return true, nil
		} else {
			common.LogDebug(" -- continueing dry mode")
		}
	}

	// Fetching data
	review, review_error := FetchOurLatestActionableReview(gitea, org, repo, id)
	if pr.State != "closed" && review_error != nil {
		// Nothing to do
		return true, nil
	}

	err = FetchPrGit(git, pr)
	if err != nil {
		common.LogError("Cannot fetch PR git:", pr.URL)
		return false, err
	}

	// we want the possibly pending modification here, in case stagings are added, etc.
	// jobs of review team to deal with issues
	common.LogDebug("QA configuration fetching ...", common.StagingConfigFile)
	data, err := git.GitCatFile(pr.Head.Sha, pr.Head.Sha, common.StagingConfigFile)
	if err != nil {
		common.LogError("Staging config", common.StagingConfigFile, "not found in PR to the project. Aborting.")
		if !IsDryRun {
			_, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.ProjectConfigFile)
		}
		return true, err
	}

	stagingConfig, err := common.ParseStagingConfig(data)
	if err != nil {
		common.LogError("Error parsing config file", common.StagingConfigFile, err)
	}

	if stagingConfig.ObsProject == "" {
		common.LogError("Cannot find reference project for PR#", pr.Index)
		if !IsDryRun && review_error == nil {
			_, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find reference project")
			return true, err
		}
		return true, nil
	}

	common.LogDebug("ObsProject:", stagingConfig.ObsProject)
	stagingProject := GetObsProjectAssociatedWithPr(stagingConfig, ObsClient.HomeProject, pr)

	// Cleanup projects
	if pr.State == "closed" {
		// review is done, cleanup
		common.LogInfo(" -- closed request, cleanup staging projects")
		for _, setup := range stagingConfig.QA {
			if !IsDryRun {
				ObsClient.DeleteProject(stagingProject + ":" + setup.Name)
			}
		}
		if stagingProject != "" {
			if !IsDryRun {
				ObsClient.DeleteProject(stagingProject)
			}
		}
		return true, nil
	}

	// Process review aka setup projects
	if review_error == nil {
		common.LogInfo("processing review", review.HTMLURL, "state", review.State)

		meta, err := ObsClient.GetProjectMeta(stagingConfig.ObsProject)
		if err != nil {
			common.LogError("Cannot find reference project meta:", stagingConfig.ObsProject, err)
			if !IsDryRun {
				_, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot fetch reference project meta")
				return true, err
			}
			return true, nil
		}

		if metaUrl, err := url.Parse(meta.ScmSync); err != nil {
			return false, err
		} else if targetPRSyncUrl, err := url.Parse(pr.Base.Repo.CloneURL); err != nil {
			return false, err
		} else {
			metaUrl.RawQuery = ""
			metaUrl.Fragment = ""
			metaUrl.Path = strings.TrimSuffix(metaUrl.Path, ".git")
			targetPRSyncUrl.RawQuery = ""
			targetPRSyncUrl.Fragment = ""
			targetPRSyncUrl.Path = strings.TrimSuffix(targetPRSyncUrl.Path, ".git")

			if metaUrl.String() != targetPRSyncUrl.String() {
				common.LogError("SCMSYNC in meta", meta.ScmSync, "!= PR CloneURL from Gitea", pr.Base.Repo.CloneURL, ". Skipping staging")
				if !IsDryRun {
					_, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "SCMSYNC of target project is not equal to CloneURL of this PR. Can't review.")
					return true, err
				}
				return true, nil
			}
		}

		if meta.Name != stagingConfig.ObsProject {
			common.LogError("staging.config . ObsProject:", stagingConfig.ObsProject, " is not target project name", meta.Name)
			if !IsDryRun {
				_, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "OBS Meta and staging.config are inconsistent")
				return true, err
			}
			return true, nil
		}

		// find modified submodules and new submodules -- build them
		dir := pr.Head.Sha
		common.LogDebug("PR:", pr.MergeBase, "->", pr.Head.Sha)
		headSubmodules, err := git.GitSubmoduleList(dir, pr.Head.Sha)
		common.PanicOnError(err)
		baseSubmodules, err := git.GitSubmoduleList(dir, pr.MergeBase)
		common.PanicOnError(err)

		common.LogDebug(" # head submodules:", len(headSubmodules))
		common.LogDebug(" # base submodules:", len(baseSubmodules))

		modifiedPackages := make([]string, 0, 16)
		newPackages := make([]string, 0, 16)
		if !stagingConfig.RebuildAll {
			for pkg, headOid := range headSubmodules {
				if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid {
					if len(baseOid) > 0 {
						modifiedPackages = append(modifiedPackages, pkg)
					} else {
						newPackages = append(newPackages, pkg)
					}
					common.LogDebug(pkg, ":", baseOid, "->", headOid)
				}
			}
		}

		if len(modifiedPackages) == 0 && len(newPackages) == 0 {
			rebuild_all := false || stagingConfig.RebuildAll

			reviews, err := gitea.GetPullRequestReviews(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Index)
			common.LogDebug("num reviews:", len(reviews))
			if err == nil {
				rebuild_rx := regexp.MustCompile("^@autogits_obs_staging_bot\\s*:\\s*(re)?build\\s*all$")
			done:
				for _, r := range reviews {
					for _, l := range common.SplitLines(r.Body) {
						if rebuild_rx.MatchString(strings.ToLower(l)) {
							rebuild_all = true
							break done
						}
					}
				}

				comments, err := gitea.GetIssueComments(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Index)
				common.LogDebug("num comments:", len(comments))
				if err == nil {
				done2:
					for _, r := range comments {

						for _, l := range common.SplitLines(r.Body) {
							if rebuild_rx.MatchString(strings.ToLower(l)) {
								rebuild_all = true
								break done2
							}
						}
					}
				}
			} else {
				common.LogError(err)
			}

			if !rebuild_all {
				common.LogInfo("No package changes detected. Ignoring")
				if !IsDryRun {
					_, err = gitea.AddReviewComment(pr, common.ReviewStateComment, "No package changes. Not rebuilding project by default")
				}
				return true, err
			}
		}

		common.LogDebug("ObsProject:", stagingConfig.ObsProject)
		stagingProject := GetObsProjectAssociatedWithPr(stagingConfig, ObsClient.HomeProject, pr)
		change, err := StartOrUpdateBuild(stagingConfig, git, gitea, pr)
		status := &models.CommitStatus{
			Context:     BotName,
			Description: "OBS Staging build",
			Status:      common.CommitStatus_Pending,
			TargetURL:   ObsWebHost + "/project/show/" + stagingProject,
		}

		msg := "Changed source updated for build"
		if change == RequestModificationProjectCreated {
			msg = "Build is started in " + ObsWebHost + "/project/show/" +
				stagingProject + " .\n"

			if len(stagingConfig.QA) > 0 {
				msg = msg + "\nAdditional QA builds: \n"
			}
			gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)

			for _, setup := range stagingConfig.QA {
				CreateQASubProject(stagingConfig, git, gitea, pr,
					stagingProject,
					setup.Origin,
					setup.Name)
				msg = msg + ObsWebHost + "/project/show/" +
					stagingProject + ":" + setup.Name + "\n"
			}
		}
		if change != RequestModificationNoChange && !IsDryRun {
			gitea.AddComment(pr, msg)
		}

		baseResult, err := ObsClient.LastBuildResults(stagingConfig.ObsProject, modifiedPackages...)
		if err != nil {
			common.LogError("failed fetching ref project status for", stagingConfig.ObsProject, ":", err)
		}
		stagingResult, err := ObsClient.BuildStatus(stagingProject)
		if err != nil {
			common.LogError("failed fetching stage project status for", stagingProject, ":", err)
		}
		buildStatus := ProcessBuildStatus(stagingResult, baseResult)

		switch buildStatus {
		case BuildStatusSummarySuccess:
			status.Status = common.CommitStatus_Success
			if !IsDryRun {
				_, err := gitea.AddReviewComment(pr, common.ReviewStateApproved, "Build successful")
				if err != nil {
					common.LogError(err)
				} else {
					return true, nil
				}
			}
		case BuildStatusSummaryFailed:
			status.Status = common.CommitStatus_Fail
			if !IsDryRun {
				_, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Build failed")
				if err != nil {
					common.LogError(err)
				} else {
					return true, nil
				}
			}
		}
		common.LogInfo("Build status:", buildStatus)
		gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)

		// waiting for build results -- nothing to do

	} else if err == NonActionableReviewError || err == NoReviewsFoundError {
		return true, nil
	}

	return false, nil
}

func PollWorkNotifications(giteaUrl string) {
	gitea := common.AllocateGiteaTransport(giteaUrl)
	data, err := gitea.GetNotifications(common.GiteaNotificationType_Pull, nil)

	if err != nil {
		common.LogError(err)
		return
	}

	if data != nil {
		common.LogDebug("Processing", len(data), "notifications.")
		for _, notification := range data {
			common.LogInfo("notification", notification.ID, "--", notification.Subject.HTMLURL)

			if !ListPullNotificationsOnly {
				switch notification.Subject.Type {
				case "Pull":
					ProcessPullNotification(gitea, notification)
				default:
					if !IsDryRun {
						gitea.SetNotificationRead(notification.ID)
					}
				}
			}
		}
	}

	// cleanup old projects
	common.LogDebug("Cleaning up old pull requests")
	cleanupFinished := false
	for page := int64(1); !cleanupFinished; page++ {
		cleanupFinished = true
		if data, err := gitea.GetDoneNotifications(common.GiteaNotificationType_Pull, page); data != nil {
			for _, n := range data {
				if n.Unread {
					common.LogError("Done notification is unread or pinned?", *n.Subject)
					cleanupFinished = false
					continue
				}

				cleanupFinished = CleanupPullNotification(gitea, n) && cleanupFinished
			}
		} else if err != nil {
			common.LogError(err)
		}
	}
}

var ListPullNotificationsOnly bool
var GiteaUrl string
var GiteaUseSshClone bool
var ObsWebHost string
var IsDryRun bool
var ProcessPROnly string
var ObsClient *common.ObsClient

func ObsWebHostFromApiHost(apihost string) string {
	u, err := url.Parse(apihost)
	if err != nil {
		common.LogError("Cannot parse OBS API URL")
		panic(err)
	}
	if len(u.Host) > 4 && u.Host[0:4] == "api." {
		u.Host = "build" + u.Host[3:]
	}

	return u.String()
}

func main() {
	flag.BoolVar(&ListPullNotificationsOnly, "list-notifications-only", false, "Only lists notifications without acting on them")
	ProcessPROnly := flag.String("pr", "", "Process only specific PR and ignore the rest. Use for debugging")
	buildRoot := flag.String("build-root", "", "Default build location for staging projects. Default is bot's home project")
	flag.StringVar(&GiteaUrl, "gitea-url", "https://src.opensuse.org", "Gitea instance")
	flag.BoolVar(&GiteaUseSshClone, "use-ssh-clone", false, "enforce cloning via ssh")
	obsApiHost := flag.String("obs", "https://api.opensuse.org", "API for OBS instance")
	flag.StringVar(&ObsWebHost, "obs-web", "", "Web OBS instance, if not derived from the obs config")
	flag.BoolVar(&IsDryRun, "dry", false, "Dry-run, don't actually create any build projects or review changes")
	debug := flag.Bool("debug", false, "Turns on debug logging")
	flag.Parse()

	if *debug {
		common.SetLoggingLevel(common.LogLevelDebug)
	} else {
		common.SetLoggingLevel(common.LogLevelInfo)
	}

	if len(ObsWebHost) == 0 {
		ObsWebHost = ObsWebHostFromApiHost(*obsApiHost)
	}

	common.LogDebug("OBS Web Host:", ObsWebHost)
	common.LogDebug("OBS API Host:", *obsApiHost)

	common.PanicOnErrorWithMsg(common.RequireGiteaSecretToken(), "Cannot find GITEA_TOKEN")
	common.PanicOnErrorWithMsg(common.RequireObsSecretToken(), "Cannot find OBS_USER and OBS_PASSWORD")

	var err error
	if ObsClient, err = common.NewObsClient(*obsApiHost); err != nil {
		log.Error(err)
		return
	}

	if len(*buildRoot) > 0 {
		ObsClient.HomeProject = *buildRoot
	}

	if len(*ProcessPROnly) > 0 {
		rx := regexp.MustCompile("^(\\w+)/(\\w+)#(\\d+)$")
		m := rx.FindStringSubmatch(*ProcessPROnly)
		if m == nil {
			common.LogError("Cannot find any PR matches in", *ProcessPROnly)
			return
		}

		gitea := common.AllocateGiteaTransport(GiteaUrl)
		id, _ := strconv.ParseInt(m[3], 10, 64)

		ProcessPullRequest(gitea, m[1], m[2], id)
		return
	}

	for {
		PollWorkNotifications(GiteaUrl)
		common.LogInfo("Poll cycle finished")
		time.Sleep(5 * time.Minute)
	}
}