obs-staging-bot: Fix setting of commit status

In most cases anyway an error in pull request.

common/git_utils.go

@@ -821,6 +821,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
 				for _, te := range tree.items {
 					if te.isTree() {
 						trees[p+te.name+"/"] = te.hash
+						submoduleList[p+te.name] = te.hash
 					} else if te.isSubmodule() {
 						submoduleList[p+te.name] = te.hash
 					}

common/gitea_utils.go

@@ -24,11 +24,13 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"net/url"
 	"os"
 	"path"
 	"path/filepath"
 	"slices"
+	"strconv"
 	"time"
 
 	transport "github.com/go-openapi/runtime/client"
@@ -182,7 +184,6 @@ type Gitea interface {
 	GiteaCommitStatusGetter
 	GiteaCommitStatusSetter
 	GiteaSetRepoOptions
-	GiteaTimelineFetcher
 
 	GetNotifications(Type string, since *time.Time) ([]*models.NotificationThread, error)
 	GetDoneNotifications(Type string, page int64) ([]*models.NotificationThread, error)
@@ -199,7 +200,32 @@ type Gitea interface {
 	GetCurrentUser() (*models.User, error)
 }
 
+type GiteaHeaderInterceptor struct {
+	Length int
+	http.RoundTripper
+}
+
+func (i *GiteaHeaderInterceptor) RoundTrip(req *http.Request) (*http.Response, error) {
+	resp, err := i.RoundTripper.RoundTrip(req)
+	if err != nil {
+		return nil, err
+	}
+	count_header := resp.Header["X-Total-Count"]
+	if len(count_header) == 1 {
+		i.Length, err = strconv.Atoi(resp.Header["X-Total-Count"][0])
+		if err != nil {
+			LogError("Converting X-Total-Count response header error", err)
+			i.Length = -1
+			return nil, err
+		}
+	} else {
+		i.Length = -1
+	}
+	return resp, nil
+}
+
 type GiteaTransport struct {
+	headers   *GiteaHeaderInterceptor
 	transport *transport.Runtime
 	client    *apiclient.GiteaAPI
 }
@@ -212,7 +238,9 @@ func AllocateGiteaTransport(giteaUrl string) Gitea {
 		log.Panicln("Failed to parse gitea url:", err)
 	}
 
+	r.headers = &GiteaHeaderInterceptor{RoundTripper: http.DefaultTransport}
 	r.transport = transport.New(url.Host, apiclient.DefaultBasePath, [](string){url.Scheme})
+	r.transport.Transport = r.headers
 	r.transport.DefaultAuthentication = transport.BearerToken(giteaToken)
 
 	r.client = apiclient.New(r.transport, nil)
@@ -287,10 +315,9 @@ func (gitea *GiteaTransport) ManualMergePR(org, repo string, num int64, commitid
 }
 
 func (gitea *GiteaTransport) GetPullRequests(org, repo string) ([]*models.PullRequest, error) {
-	var page, limit int64
+	var page int64
 
 	prs := make([]*models.PullRequest, 0)
-	limit = 20
 	state := "open"
 
 	for {
@@ -302,16 +329,18 @@ func (gitea *GiteaTransport) GetPullRequests(org, repo string) ([]*models.PullRe
 				WithOwner(org).
 				WithRepo(repo).
 				WithState(&state).
-				WithPage(&page).
-				WithLimit(&limit),
+				WithPage(&page),
 			gitea.transport.DefaultAuthentication)
 
 		if err != nil {
 			return nil, fmt.Errorf("cannot fetch PR list for %s / %s : %w", org, repo, err)
 		}
 
+		if len(req.Payload) == 0 {
+			break
+		}
 		prs = slices.Concat(prs, req.Payload)
-		if len(req.Payload) < int(limit) {
+		if len(prs) >= gitea.headers.Length {
 			break
 		}
 	}
@@ -320,21 +349,23 @@ func (gitea *GiteaTransport) GetPullRequests(org, repo string) ([]*models.PullRe
 }
 
 func (gitea *GiteaTransport) GetCommitStatus(org, repo, hash string) ([]*models.CommitStatus, error) {
-	page := int64(1)
-	limit := int64(10)
+	var page int64
 	var res []*models.CommitStatus
 
 	for {
+		page++
 		r, err := gitea.client.Repository.RepoListStatuses(
-			repository.NewRepoListStatusesParams().WithDefaults().WithOwner(org).WithRepo(repo).WithSha(hash).WithPage(&page).WithLimit(&limit),
+			repository.NewRepoListStatusesParams().WithDefaults().WithOwner(org).WithRepo(repo).WithSha(hash).WithPage(&page),
 			gitea.transport.DefaultAuthentication)
 
 		if err != nil {
 			return res, err
 		}
-
+		if len(r.Payload) == 0 {
+			break
+		}
 		res = append(res, r.Payload...)
-		if len(r.Payload) < int(limit) {
+		if len(res) >= gitea.headers.Length {
 			break
 		}
 	}
@@ -377,19 +408,18 @@ func (gitea *GiteaTransport) GetRepository(org, pkg string) (*models.Repository,
 }
 
 func (gitea *GiteaTransport) GetPullRequestReviews(org, project string, PRnum int64) ([]*models.PullReview, error) {
-	limit := int64(20)
 	var page int64
 	var allReviews []*models.PullReview
 
 	for {
+		page++
 		reviews, err := gitea.client.Repository.RepoListPullReviews(
 			repository.NewRepoListPullReviewsParams().
 				WithDefaults().
 				WithOwner(org).
 				WithRepo(project).
 				WithIndex(PRnum).
-				WithPage(&page).
-				WithLimit(&limit),
+				WithPage(&page),
 			gitea.transport.DefaultAuthentication,
 		)
 
@@ -397,11 +427,13 @@ func (gitea *GiteaTransport) GetPullRequestReviews(org, project string, PRnum in
 			return nil, err
 		}
 
-		allReviews = slices.Concat(allReviews, reviews.Payload)
-		if len(reviews.Payload) < int(limit) {
+		if len(reviews.Payload) == 0 {
+			break
+		}
+		allReviews = slices.Concat(allReviews, reviews.Payload)
+		if len(allReviews) >= gitea.headers.Length {
 			break
 		}
-		page++
 	}
 
 	return allReviews, nil
@@ -469,7 +501,6 @@ const (
 )
 
 func (gitea *GiteaTransport) GetNotifications(Type string, since *time.Time) ([]*models.NotificationThread, error) {
-	bigLimit := int64(20)
 	ret := make([]*models.NotificationThread, 0, 100)
 
 	for page := int64(1); ; page++ {
@@ -477,7 +508,6 @@ func (gitea *GiteaTransport) GetNotifications(Type string, since *time.Time) ([]
 			WithDefaults().
 			WithSubjectType([]string{Type}).
 			WithStatusTypes([]string{"unread"}).
-			WithLimit(&bigLimit).
 			WithPage(&page)
 
 		if since != nil {
@@ -490,8 +520,11 @@ func (gitea *GiteaTransport) GetNotifications(Type string, since *time.Time) ([]
 			return nil, err
 		}
 
+		if len(list.Payload) == 0 {
+			break
+		}
 		ret = slices.Concat(ret, list.Payload)
-		if len(list.Payload) < int(bigLimit) {
+		if len(ret) >= gitea.headers.Length {
 			break
 		}
 	}
@@ -500,7 +533,6 @@ func (gitea *GiteaTransport) GetNotifications(Type string, since *time.Time) ([]
 }
 
 func (gitea *GiteaTransport) GetDoneNotifications(Type string, page int64) ([]*models.NotificationThread, error) {
-	limit := int64(20)
 	t := true
 
 	if page <= 0 {
@@ -511,7 +543,6 @@ func (gitea *GiteaTransport) GetDoneNotifications(Type string, page int64) ([]*m
 			WithAll(&t).
 			WithSubjectType([]string{Type}).
 			WithStatusTypes([]string{"read"}).
-			WithLimit(&limit).
 			WithPage(&page),
 		gitea.transport.DefaultAuthentication)
 	if err != nil {
@@ -564,9 +595,12 @@ func (gitea *GiteaTransport) GetOrganizationRepositories(orgName string) ([]*mod
 		if len(ret.Payload) == 0 {
 			break
 		}
-
 		repos = append(repos, ret.Payload...)
 		page++
+
+		if len(repos) >= gitea.headers.Length {
+			break
+		}
 	}
 
 	return repos, nil
@@ -780,15 +814,18 @@ func (gitea *GiteaTransport) GetTimeline(org, repo string, idx int64) ([]*models
 
 		resCount = len(res.Payload)
 		LogDebug("page:", page, "len:", resCount)
+		if resCount == 0 {
+			break
+		}
 		page++
 
-		for _, d := range res.Payload {
-			if d != nil {
-				retData = append(retData, d)
-			}
+		retData = append(retData, res.Payload...)
+		if len(retData) >= gitea.headers.Length {
+			break
 		}
 	}
 	LogDebug("total results:", len(retData))
+	retData = slices.DeleteFunc(retData, func(a *models.TimelineComment) bool { return a == nil })
 	slices.SortFunc(retData, func(a, b *models.TimelineComment) int {
 		return time.Time(b.Created).Compare(time.Time(a.Created))
 	})

obs-staging-bot/main.go

@@ -222,7 +222,6 @@ func ProcessRepoBuildStatus(results, ref []*common.PackageBuildStatus) (status B
 	slices.SortFunc(results, PackageBuildStatusSorter)
 	slices.SortFunc(ref, PackageBuildStatusSorter)
 
-	j := 0
 	SomeSuccess = false
 	for i := 0; i < len(results); i++ {
 		res, ok := common.ObsBuildStatusDetails[results[i].Code]
@@ -237,10 +236,11 @@ func ProcessRepoBuildStatus(results, ref []*common.PackageBuildStatus) (status B
 
 		if !res.Success {
 			// not failed if reference project also failed for same package here
-			for ; j < len(results) && strings.Compare(results[i].Package, ref[j].Package) < 0; j++ {
+			j := 0
+			for ; j < len(ref) && strings.Compare(results[i].Package, ref[j].Package) != 0; j++ {
 			}
 
-			if j < len(results) && results[i].Package == ref[j].Package {
+			if j < len(ref) {
 				refRes, ok := common.ObsBuildStatusDetails[ref[j].Code]
 				if !ok {
 					common.LogInfo("unknown ref package result code:", ref[j].Code, "package:", ref[j].Package)
@@ -322,6 +322,9 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
 		urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(pkg))
 	}
 	meta.ScmSync = pr.Head.Repo.CloneURL + "?" + strings.Join(urlPkg, "&") + "#" + pr.Head.Sha
+	if len(meta.ScmSync) >= 65535 {
+		return nil, errors.New("Reached max amount of package changes per request")
+	}
 	meta.Title = fmt.Sprintf("PR#%d to %s", pr.Index, pr.Base.Name)
 //      QE wants it published ... also we should not hardcode it here, since
 //      it is configurable via the :PullRequest project
@@ -837,6 +840,25 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 			TargetURL:   ObsWebHost + "/project/show/" + stagingProject,
 		}
 
+		if err != nil {
+			msg := "Unable to setup stage propject " + stagingConfig.ObsProject
+			status.Status = common.CommitStatus_Fail
+			common.LogError(msg)
+			if !IsDryRun {
+				_, err := gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
+				if err != nil {
+					common.LogError(err)
+				}
+				_, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, msg)
+				if err != nil {
+					common.LogError(err)
+				} else {
+					return true, nil
+				}
+			}
+			return false, nil
+		}
+
 		msg := "Changed source updated for build"
 		if change == RequestModificationProjectCreated {
 			msg = "Build is started in " + ObsWebHost + "/project/show/" +
@@ -845,7 +867,10 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 			if len(stagingConfig.QA) > 0 {
 				msg = msg + "\nAdditional QA builds: \n"
 			}
-			gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
+			_, err := gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
+			if err != nil {
+				common.LogError(err)
+			}
 
 			for _, setup := range stagingConfig.QA {
 				CreateQASubProject(stagingConfig, git, gitea, pr,
@@ -870,32 +895,36 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 		}
 		buildStatus := ProcessBuildStatus(stagingResult, baseResult)
 
+		done := false
 		switch buildStatus {
 		case BuildStatusSummarySuccess:
 			status.Status = common.CommitStatus_Success
+			done = true
 			if !IsDryRun {
 				_, err := gitea.AddReviewComment(pr, common.ReviewStateApproved, "Build successful")
 				if err != nil {
 					common.LogError(err)
-				} else {
-					return true, nil
 				}
 			}
 		case BuildStatusSummaryFailed:
 			status.Status = common.CommitStatus_Fail
+			done = true
 			if !IsDryRun {
 				_, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Build failed")
 				if err != nil {
 					common.LogError(err)
-				} else {
-					return true, nil
 				}
 			}
 		}
 		common.LogInfo("Build status:", buildStatus)
-		gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
-
-		// waiting for build results -- nothing to do
+		if !IsDryRun {
+			_, err = gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
+			if err != nil {
+				common.LogError(err)
+				return false, err
+			}
+		}
+		return done, nil
 
 	} else if err == NonActionableReviewError || err == NoReviewsFoundError {
 		return true, nil

obs-status-service/main.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
@@ -268,6 +269,32 @@ func main() {
 
 		res.Write(BuildStatusSvg(nil, &common.PackageBuildStatus{Package: pkg, Code: "unknown"}))
 	})
+	http.HandleFunc("GET /search", func(res http.ResponseWriter, req *http.Request) {
+		common.LogInfo("GET /serach?" + req.URL.RawQuery)
+		queries := req.URL.Query()
+		if !queries.Has("q") {
+			res.WriteHeader(400)
+			return
+		}
+
+		names := queries["q"]
+		if len(names) < 1 || len(names) > 10 {
+			res.WriteHeader(400)
+			return
+		}
+
+		packages := FindPackages(names)
+		data, err := json.MarshalIndent(packages, "", "   ")
+		if err != nil {
+			res.WriteHeader(500)
+			common.LogError("Error in marshalling data.", err)
+			return
+		}
+
+		res.Write(data)
+		res.WriteHeader(200)
+	})
+
 	http.HandleFunc("GET /buildlog/{Project}/{Package}/{Repository}/{Arch}", func(res http.ResponseWriter, req *http.Request) {
 		prj := req.PathValue("Project")
 		pkg := req.PathValue("Package")

obs-status-service/redis.go

@@ -110,6 +110,33 @@ func FindRepoResults(project, repo string) []*common.BuildResult {
 	return ret
 }
 
+func FindPackages(search_terms []string) []string {
+	RepoStatusLock.RLock()
+	defer RepoStatusLock.RUnlock()
+
+	data := make([]string, 0, 100)
+	for _, repo := range RepoStatus {
+		for _, status := range repo.Status {
+			pkg := status.Package
+			match := true
+			for _, term := range search_terms {
+				match = match && strings.Contains(pkg, term)
+			}
+
+			if match {
+				entry := repo.Project + "/" + repo.Status[0].Package
+				if idx, found := slices.BinarySearch(data, entry); !found {
+					data = slices.Insert(data, idx, entry)
+					if len(data) >= 100 {
+						return data
+					}
+				}
+			}
+		}
+	}
+	return data
+}
+
 func FindAndUpdateProjectResults(project string) []*common.BuildResult {
 	res := FindProjectResults(project)
 	wg := &sync.WaitGroup{}

workflow-pr/pr_processor.go

@@ -317,10 +317,15 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
 	}
 
 	PrjGitTitle, PrjGitBody := PrjGitDescription(prset)
+	if PrjGitPR.PR.User.UserName == CurrentUser.UserName {
 		if PrjGitPR.PR.Title != PrjGitTitle || PrjGitPR.PR.Body != PrjGitBody {
 			common.LogDebug("New title:", PrjGitTitle)
 			common.LogDebug(PrjGitBody)
 		}
+	} else {
+		// TODO: find our first comment in timeline
+
+	}
 
 	if !common.IsDryRun {
 		if headCommit != newHeadCommit {