Don't crash when new packages got added

The build result request of the base project is failing in this
situation, since the requested package does not exist.

Therefore we need to have seperate lists for proper handling.

common/git_utils.go

@@ -821,6 +821,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
 				for _, te := range tree.items {
 					if te.isTree() {
 						trees[p+te.name+"/"] = te.hash
+						submoduleList[p+te.name] = te.hash
 					} else if te.isSubmodule() {
 						submoduleList[p+te.name] = te.hash
 					}

common/gitea_utils.go

@@ -182,7 +182,6 @@ type Gitea interface {
 	GiteaCommitStatusGetter
 	GiteaCommitStatusSetter
 	GiteaSetRepoOptions
-	GiteaTimelineFetcher
 
 	GetNotifications(Type string, since *time.Time) ([]*models.NotificationThread, error)
 	GetDoneNotifications(Type string, page int64) ([]*models.NotificationThread, error)
@@ -310,6 +309,9 @@ func (gitea *GiteaTransport) GetPullRequests(org, repo string) ([]*models.PullRe
 			return nil, fmt.Errorf("cannot fetch PR list for %s / %s : %w", org, repo, err)
 		}
 
+		if len(req.Payload) == 0 {
+			break
+		}
 		prs = slices.Concat(prs, req.Payload)
 		if len(req.Payload) < int(limit) {
 			break
@@ -332,11 +334,11 @@ func (gitea *GiteaTransport) GetCommitStatus(org, repo, hash string) ([]*models.
 		if err != nil {
 			return res, err
 		}
-
-		res = append(res, r.Payload...)
-		if len(r.Payload) < int(limit) {
+		if len(r.Payload) == 0 {
 			break
 		}
+		res = append(res, r.Payload...)
+		page++
 	}
 
 	return res, nil
@@ -397,10 +399,10 @@ func (gitea *GiteaTransport) GetPullRequestReviews(org, project string, PRnum in
 			return nil, err
 		}
 
-		allReviews = slices.Concat(allReviews, reviews.Payload)
-		if len(reviews.Payload) < int(limit) {
+		if len(reviews.Payload) == 0 {
 			break
 		}
+		allReviews = slices.Concat(allReviews, reviews.Payload)
 		page++
 	}
 
@@ -490,6 +492,9 @@ func (gitea *GiteaTransport) GetNotifications(Type string, since *time.Time) ([]
 			return nil, err
 		}
 
+		if len(list.Payload) == 0 {
+			break
+		}
 		ret = slices.Concat(ret, list.Payload)
 		if len(list.Payload) < int(bigLimit) {
 			break
@@ -780,6 +785,9 @@ func (gitea *GiteaTransport) GetTimeline(org, repo string, idx int64) ([]*models
 
 		resCount = len(res.Payload)
 		LogDebug("page:", page, "len:", resCount)
+		if resCount == 0 {
+			break
+		}
 		page++
 
 		for _, d := range res.Payload {

obs-staging-bot/main.go

@@ -222,7 +222,6 @@ func ProcessRepoBuildStatus(results, ref []*common.PackageBuildStatus) (status B
 	slices.SortFunc(results, PackageBuildStatusSorter)
 	slices.SortFunc(ref, PackageBuildStatusSorter)
 
-	j := 0
 	SomeSuccess = false
 	for i := 0; i < len(results); i++ {
 		res, ok := common.ObsBuildStatusDetails[results[i].Code]
@@ -237,10 +236,11 @@ func ProcessRepoBuildStatus(results, ref []*common.PackageBuildStatus) (status B
 
 		if !res.Success {
 			// not failed if reference project also failed for same package here
-			for ; j < len(results) && strings.Compare(results[i].Package, ref[j].Package) < 0; j++ {
+			j := 0
+			for ; j < len(ref) && strings.Compare(results[i].Package, ref[j].Package) != 0; j++ {
 			}
 
-			if j < len(results) && results[i].Package == ref[j].Package {
+			if j < len(ref) {
 				refRes, ok := common.ObsBuildStatusDetails[ref[j].Code]
 				if !ok {
 					common.LogInfo("unknown ref package result code:", ref[j].Code, "package:", ref[j].Package)
@@ -322,10 +322,13 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
 		urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(pkg))
 	}
 	meta.ScmSync = pr.Head.Repo.CloneURL + "?" + strings.Join(urlPkg, "&") + "#" + pr.Head.Sha
+	if len(meta.ScmSync) >= 65535 {
+		return nil, errors.New("Reached max amount of package changes per request")
+	}
 	meta.Title = fmt.Sprintf("PR#%d to %s", pr.Index, pr.Base.Name)
-//      QE wants it published ... also we should not hardcode it here, since
-//      it is configurable via the :PullRequest project
-//	meta.PublicFlags = common.Flags{Contents: "<disable/>"}
+	//      QE wants it published ... also we should not hardcode it here, since
+	//      it is configurable via the :PullRequest project
+	//	meta.PublicFlags = common.Flags{Contents: "<disable/>"}
 
 	meta.Groups = nil
 	meta.Persons = nil
@@ -633,6 +636,14 @@ func CleanupPullNotification(gitea common.Gitea, thread *models.NotificationThre
 	return false // cleaned up now, but the cleanup was not aleady done
 }
 
+func SetStatus(gitea common.Gitea, org, repo, hash string, status *models.CommitStatus) error {
+	_, err := gitea.SetCommitStatus(org, repo, hash, status)
+	if err != nil {
+		common.LogError(err)
+	}
+	return err
+}
+
 func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, error) {
 	dir, err := os.MkdirTemp(os.TempDir(), BotName)
 	common.PanicOnError(err)
@@ -768,17 +779,22 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 		common.LogDebug(" # head submodules:", len(headSubmodules))
 		common.LogDebug(" # base submodules:", len(baseSubmodules))
 
-		modifiedOrNew := make([]string, 0, 16)
+		modifiedPackages := make([]string, 0, 16)
+		newPackages := make([]string, 0, 16)
 		if !stagingConfig.RebuildAll {
 			for pkg, headOid := range headSubmodules {
 				if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid {
-					modifiedOrNew = append(modifiedOrNew, pkg)
+					if len(baseOid) > 0 {
+						modifiedPackages = append(modifiedPackages, pkg)
+					} else {
+						newPackages = append(newPackages, pkg)
+					}
 					common.LogDebug(pkg, ":", baseOid, "->", headOid)
 				}
 			}
 		}
 
-		if len(modifiedOrNew) == 0 {
+		if len(modifiedPackages) == 0 && len(newPackages) == 0 {
 			rebuild_all := false || stagingConfig.RebuildAll
 
 			reviews, err := gitea.GetPullRequestReviews(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Index)
@@ -837,6 +853,22 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 			TargetURL:   ObsWebHost + "/project/show/" + stagingProject,
 		}
 
+		if err != nil {
+			msg := "Unable to setup stage project " + stagingConfig.ObsProject
+			status.Status = common.CommitStatus_Fail
+			common.LogError(msg)
+			if !IsDryRun {
+				SetStatus(gitea, org, repo, pr.Head.Sha, status)
+				_, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, msg)
+				if err != nil {
+					common.LogError(err)
+				} else {
+					return true, nil
+				}
+			}
+			return false, nil
+		}
+
 		msg := "Changed source updated for build"
 		if change == RequestModificationProjectCreated {
 			msg = "Build is started in " + ObsWebHost + "/project/show/" +
@@ -845,8 +877,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 			if len(stagingConfig.QA) > 0 {
 				msg = msg + "\nAdditional QA builds: \n"
 			}
-			gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
-
+			SetStatus(gitea, org, repo, pr.Head.Sha, status)
 			for _, setup := range stagingConfig.QA {
 				CreateQASubProject(stagingConfig, git, gitea, pr,
 					stagingProject,
@@ -860,42 +891,44 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 			gitea.AddComment(pr, msg)
 		}
 
-		baseResult, err := ObsClient.LastBuildResults(stagingConfig.ObsProject, modifiedOrNew...)
+		baseResult, err := ObsClient.LastBuildResults(stagingConfig.ObsProject, modifiedPackages...)
 		if err != nil {
 			common.LogError("failed fetching ref project status for", stagingConfig.ObsProject, ":", err)
 		}
 		stagingResult, err := ObsClient.BuildStatus(stagingProject)
 		if err != nil {
-			common.LogError("failed fetching ref project status for", stagingProject, ":", err)
+			common.LogError("failed fetching stage project status for", stagingProject, ":", err)
 		}
 		buildStatus := ProcessBuildStatus(stagingResult, baseResult)
 
+		done := false
 		switch buildStatus {
 		case BuildStatusSummarySuccess:
 			status.Status = common.CommitStatus_Success
+			done = true
 			if !IsDryRun {
 				_, err := gitea.AddReviewComment(pr, common.ReviewStateApproved, "Build successful")
 				if err != nil {
 					common.LogError(err)
-				} else {
-					return true, nil
 				}
 			}
 		case BuildStatusSummaryFailed:
 			status.Status = common.CommitStatus_Fail
+			done = true
 			if !IsDryRun {
 				_, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Build failed")
 				if err != nil {
 					common.LogError(err)
-				} else {
-					return true, nil
 				}
 			}
 		}
 		common.LogInfo("Build status:", buildStatus)
-		gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
-
-		// waiting for build results -- nothing to do
+		if !IsDryRun {
+			if err = SetStatus(gitea, org, repo, pr.Head.Sha, status); err != nil {
+				return false, err
+			}
+		}
+		return done, nil
 
 	} else if err == NonActionableReviewError || err == NoReviewsFoundError {
 		return true, nil

obs-status-service/main.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
@@ -268,6 +269,33 @@ func main() {
 
 		res.Write(BuildStatusSvg(nil, &common.PackageBuildStatus{Package: pkg, Code: "unknown"}))
 	})
+	http.HandleFunc("GET /search", func(res http.ResponseWriter, req *http.Request) {
+		common.LogInfo("GET /serach?" + req.URL.RawQuery)
+		queries := req.URL.Query()
+		if !queries.Has("q") {
+			res.WriteHeader(400)
+			return
+		}
+
+		names := queries["q"]
+		if len(names) != 1 {
+			res.WriteHeader(400)
+			return
+		}
+
+		packages := FindPackages(names[0])
+		data, err := json.MarshalIndent(packages, "", "   ")
+		if err != nil {
+			res.WriteHeader(500)
+			common.LogError("Error in marshalling data.", err)
+			return
+		}
+
+		res.Write(data)
+		res.Header().Add("content-type", "application/json")
+		res.WriteHeader(200)
+	})
+
 	http.HandleFunc("GET /buildlog/{Project}/{Package}/{Repository}/{Arch}", func(res http.ResponseWriter, req *http.Request) {
 		prj := req.PathValue("Project")
 		pkg := req.PathValue("Package")

obs-status-service/redis.go

@@ -29,13 +29,15 @@ func UpdateResults(r *common.BuildResult) {
 	RepoStatusLock.Lock()
 	defer RepoStatusLock.Unlock()
 
+	updateResultsWithoutLocking(r)
+}
+
+func updateResultsWithoutLocking(r *common.BuildResult) {
 	key := "result." + r.Project + "/" + r.Repository + "/" + r.Arch
-	common.LogDebug(" + Updating", key)
 	data, err := redisClient.HGetAll(context.Background(), key).Result()
 	if err != nil {
 		common.LogError("Failed fetching build results for", key, err)
 	}
-	common.LogDebug(" + Update size", len(data))
 
 	reset_time := time.Date(1000, 1, 1, 1, 1, 1, 1, time.Local)
 	for _, pkg := range r.Status {
@@ -110,6 +112,27 @@ func FindRepoResults(project, repo string) []*common.BuildResult {
 	return ret
 }
 
+func FindPackages(pkg string) []string {
+	RepoStatusLock.RLock()
+	defer RepoStatusLock.RUnlock()
+
+	data := make([]string, 0, 100)
+	for _, repo := range RepoStatus {
+		for _, status := range repo.Status {
+			if pkg == status.Package {
+				entry := repo.Project + "/" + pkg
+				if idx, found := slices.BinarySearch(data, entry); !found {
+					data = slices.Insert(data, idx, entry)
+					if len(data) >= 100 {
+						return data
+					}
+				}
+			}
+		}
+	}
+	return data
+}
+
 func FindAndUpdateProjectResults(project string) []*common.BuildResult {
 	res := FindProjectResults(project)
 	wg := &sync.WaitGroup{}
@@ -161,6 +184,8 @@ func RescanRepositories() error {
 	RepoStatusLock.Unlock()
 	var count int
 
+	projectsLooked := make([]string, 0, 10000)
+
 	for {
 		var data []string
 		data, cursor, err = redisClient.ScanType(ctx, cursor, "", 1000, "hash").Result()
@@ -169,6 +194,7 @@ func RescanRepositories() error {
 			return err
 		}
 
+		wg := &sync.WaitGroup{}
 		RepoStatusLock.Lock()
 		for _, repo := range data {
 			r := strings.Split(repo, "/")
@@ -180,14 +206,28 @@ func RescanRepositories() error {
 				Repository: r[1],
 				Arch:       r[2],
 			}
-			if pos, found := slices.BinarySearchFunc(RepoStatus, d, common.BuildResultComp); found {
+
+			var pos int
+			var found bool
+			if pos, found = slices.BinarySearchFunc(RepoStatus, d, common.BuildResultComp); found {
 				RepoStatus[pos].Dirty = true
 			} else {
 				d.Dirty = true
 				RepoStatus = slices.Insert(RepoStatus, pos, d)
 				count++
 			}
+
+			// fetch all keys, one per non-maintenance/non-home: projects, for package search
+			if idx, found := slices.BinarySearch(projectsLooked, d.Project); !found && !strings.Contains(d.Project, ":Maintenance:") && (len(d.Project) < 5 || d.Project[0:5] != "home:") {
+				projectsLooked = slices.Insert(projectsLooked, idx, d.Project)
+				wg.Add(1)
+				go func(r *common.BuildResult) {
+					updateResultsWithoutLocking(r)
+					wg.Done()
+				}(RepoStatus[pos])
 			}
+		}
+		wg.Wait()
 		RepoStatusLock.Unlock()
 
 		if cursor == 0 {

workflow-pr/pr_processor.go

@@ -317,10 +317,15 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
 	}
 
 	PrjGitTitle, PrjGitBody := PrjGitDescription(prset)
+	if PrjGitPR.PR.User.UserName == CurrentUser.UserName {
 		if PrjGitPR.PR.Title != PrjGitTitle || PrjGitPR.PR.Body != PrjGitBody {
 			common.LogDebug("New title:", PrjGitTitle)
 			common.LogDebug(PrjGitBody)
 		}
+	} else {
+		// TODO: find our first comment in timeline
+
+	}
 
 	if !common.IsDryRun {
 		if headCommit != newHeadCommit {
@@ -436,12 +441,12 @@ func (pr *PRProcessor) Process(req *models.PullRequest) error {
 					continue
 				}
 			}
-		}
 
 			if err = pr.UpdatePrjGitPR(prset); err != nil {
 				return err
 			}
 		}
+	}
 
 	if prjGitPR == nil {
 		prjGitPR, err = prset.GetPrjGitPR()