Don't crash when new packages got added

The build result request of the base project is failing in this
situation, since the requested package does not exist.

Therefore we need to have seperate lists for proper handling.

common/git_parser.go

@@ -1,296 +0,0 @@
-package common
-
-import (
-	"errors"
-	"fmt"
-	"io"
-)
-
-const (
-	GitStatus_Untracked = 0
-	GitStatus_Modified  = 1
-	GitStatus_Ignored   = 2
-	GitStatus_Unmerged  = 3 // States[0..3] -- Stage1, Stage2, Stage3 of merge objects
-	GitStatus_Renamed   = 4 // orig name in States[0]
-)
-
-type GitStatusData struct {
-	Path   string
-	Status int
-	States [3]string
-
-	/*
-	 <sub> A 4 character field describing the submodule state.
-	            "N..." when the entry is not a submodule.
-	            "S<c><m><u>" when the entry is a submodule.
-	            <c> is "C" if the commit changed; otherwise ".".
-	            <m> is "M" if it has tracked changes; otherwise ".".
-	            <u> is "U" if there are untracked changes; otherwise ".".
-	*/
-	SubmoduleChanges string
-}
-
-func parseGit_HexString(data io.ByteReader) (string, error) {
-	str := make([]byte, 0, 32)
-	for {
-		c, err := data.ReadByte()
-		if err != nil {
-			return "", err
-		}
-		switch {
-		case c == 0 || c == ' ':
-			return string(str), nil
-		case c >= 'a' && c <= 'f':
-		case c >= 'A' && c <= 'F':
-		case c >= '0' && c <= '9':
-		default:
-			return "", errors.New("Invalid character in hex string:" + string(c))
-		}
-		str = append(str, c)
-	}
-}
-func parseGit_String(data io.ByteReader) (string, error) {
-	str := make([]byte, 0, 100)
-	for {
-		c, err := data.ReadByte()
-		if err != nil {
-			return "", errors.New("Unexpected EOF. Expected NUL string term")
-		}
-		if c == 0 || c == ' ' {
-			return string(str), nil
-		}
-		str = append(str, c)
-	}
-}
-
-func parseGit_StringWithSpace(data io.ByteReader) (string, error) {
-	str := make([]byte, 0, 100)
-	for {
-		c, err := data.ReadByte()
-		if err != nil {
-			return "", errors.New("Unexpected EOF. Expected NUL string term")
-		}
-		if c == 0 {
-			return string(str), nil
-		}
-		str = append(str, c)
-	}
-}
-
-func skipGitStatusEntry(data io.ByteReader, skipSpaceLen int) error {
-	for skipSpaceLen > 0 {
-		c, err := data.ReadByte()
-		if err != nil {
-			return err
-		}
-		if c == ' ' {
-			skipSpaceLen--
-		}
-	}
-
-	return nil
-}
-
-func parseSingleStatusEntry(data io.ByteReader) (*GitStatusData, error) {
-	ret := GitStatusData{}
-	statusType, err := data.ReadByte()
-	if err != nil {
-		return nil, nil
-	}
-	switch statusType {
-	case '1':
-		var err error
-		if err = skipGitStatusEntry(data, 8); err != nil {
-			return nil, err
-		}
-		ret.Status = GitStatus_Modified
-		ret.Path, err = parseGit_StringWithSpace(data)
-		if err != nil {
-			return nil, err
-		}
-	case '2':
-		var err error
-		if err = skipGitStatusEntry(data, 9); err != nil {
-			return nil, err
-		}
-		ret.Status = GitStatus_Renamed
-		ret.Path, err = parseGit_StringWithSpace(data)
-		if err != nil {
-			return nil, err
-		}
-		ret.States[0], err = parseGit_StringWithSpace(data)
-		if err != nil {
-			return nil, err
-		}
-	case '?':
-		var err error
-		if err = skipGitStatusEntry(data, 1); err != nil {
-			return nil, err
-		}
-		ret.Status = GitStatus_Untracked
-		ret.Path, err = parseGit_StringWithSpace(data)
-		if err != nil {
-			return nil, err
-		}
-	case '!':
-		var err error
-		if err = skipGitStatusEntry(data, 1); err != nil {
-			return nil, err
-		}
-		ret.Status = GitStatus_Ignored
-		ret.Path, err = parseGit_StringWithSpace(data)
-		if err != nil {
-			return nil, err
-		}
-	case 'u':
-		var err error
-		if err = skipGitStatusEntry(data, 2); err != nil {
-			return nil, err
-		}
-		if ret.SubmoduleChanges, err = parseGit_String(data); err != nil {
-			return nil, err
-		}
-		if err = skipGitStatusEntry(data, 4); err != nil {
-			return nil, err
-		}
-
-		if ret.States[0], err = parseGit_HexString(data); err != nil {
-			return nil, err
-		}
-		if ret.States[1], err = parseGit_HexString(data); err != nil {
-			return nil, err
-		}
-		if ret.States[2], err = parseGit_HexString(data); err != nil {
-			return nil, err
-		}
-		ret.Status = GitStatus_Unmerged
-		ret.Path, err = parseGit_StringWithSpace(data)
-		if err != nil {
-			return nil, err
-		}
-	default:
-		return nil, errors.New("Invalid status type" + string(statusType))
-	}
-	return &ret, nil
-}
-
-func parseGitStatusData(data io.ByteReader) (Data, error) {
-	ret := make([]GitStatusData, 0, 10)
-	for {
-		data, err := parseSingleStatusEntry(data)
-		if err != nil {
-			return nil, err
-		} else if data == nil {
-			break
-		}
-
-		ret = append(ret, *data)
-	}
-	return ret, nil
-}
-
-type Data interface{}
-
-type CommitStatus int
-
-const (
-	Add CommitStatus = iota
-	Rm
-	Copy
-	Modify
-	Rename
-	TypeChange
-	Unmerged
-	Unknown
-)
-
-type GitDiffRawData struct {
-	SrcMode, DstMode     string
-	SrcCommit, DstCommit string
-	Status               CommitStatus
-	Src, Dst             string
-}
-
-func parseGit_DiffIndexStatus(data io.ByteReader, d *GitDiffRawData) error {
-	b, err := data.ReadByte()
-	if err != nil {
-		return err
-	}
-
-	switch b {
-	case 'A':
-		d.Status = Add
-	case 'C':
-		d.Status = Copy
-	case 'D':
-		d.Status = Rm
-	case 'M':
-		d.Status = Modify
-	case 'R':
-		d.Status = Rename
-	case 'T':
-		d.Status = TypeChange
-	case 'U':
-		d.Status = Unmerged
-	case 'X':
-		return fmt.Errorf("Unexpected unknown change type. This is a git bug")
-	}
-	_, err = parseGit_StringWithSpace(data)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func parseSingleGitDiffIndexRawData(data io.ByteReader) (*GitDiffRawData, error) {
-	var ret GitDiffRawData
-
-	b, err := data.ReadByte()
-	if err != nil {
-		return nil, err
-	}
-	if b != ':' {
-		return nil, fmt.Errorf("Expected ':' but got '%s'", string(b))
-	}
-
-	if ret.SrcMode, err = parseGit_String(data); err != nil {
-		return nil, err
-	}
-	if ret.DstMode, err = parseGit_String(data); err != nil {
-		return nil, err
-	}
-	if ret.Src, err = parseGit_String(data); err != nil {
-		return nil, err
-	}
-	if ret.Dst, err = parseGit_String(data); err != nil {
-		return nil, err
-	}
-	if err = parseGit_DiffIndexStatus(data, &ret); err != nil {
-		return nil, err
-	}
-	ret.Dst = ret.Src
-	switch ret.Status {
-	case Copy, Rename:
-		if ret.Src, err = parseGit_StringWithSpace(data); err != nil {
-			return nil, err
-		}
-	}
-
-	return &ret, nil
-}
-
-func parseGitDiffIndexRawData(data io.ByteReader) (Data, error) {
-	ret := make([]GitDiffRawData, 0, 10)
-	for {
-		data, err := parseSingleGitDiffIndexRawData(data)
-		if err != nil {
-			return nil, err
-		} else if data == nil {
-			break
-		}
-
-		ret = append(ret, *data)
-	}
-	return ret, nil
-}

common/git_utils.go

@@ -19,7 +19,9 @@ package common
  */
 
 import (
+	"bufio"
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -38,10 +40,6 @@ type GitSubmoduleLister interface {
 	GitSubmoduleCommitId(cwd, packageName, commitId string) (subCommitId string, valid bool)
 }
 
-type GitSubmoduleFileConflictResolver interface {
-	GitResolveSubmoduleFileConflict(cwd string) error
-}
-
 type GitStatusLister interface {
 	GitStatus(cwd string) ([]GitStatusData, error)
 }
@@ -72,7 +70,6 @@ type Git interface {
 	GitExecQuietOrPanic(cwd string, params ...string)
 
 	GitDiffLister
-	GitSubmoduleFileConflictResolver
 }
 
 type GitHandlerImpl struct {
@@ -250,26 +247,26 @@ func (e *GitHandlerImpl) GitClone(repo, branch, remoteUrl string) (string, error
 
 		e.GitExecOrPanic(repo, "fetch", "--prune", remoteName, remoteBranch)
 	}
-	/*
-		refsBytes, err := os.ReadFile(path.Join(e.GitPath, repo, ".git/refs/remotes", remoteName, "HEAD"))
-		if err != nil {
-			LogError("Cannot read HEAD of remote", remoteName)
-			return remoteName, fmt.Errorf("Cannot read HEAD of remote %s", remoteName)
-		}
+/*
+	refsBytes, err := os.ReadFile(path.Join(e.GitPath, repo, ".git/refs/remotes", remoteName, "HEAD"))
+	if err != nil {
+		LogError("Cannot read HEAD of remote", remoteName)
+		return remoteName, fmt.Errorf("Cannot read HEAD of remote %s", remoteName)
+	}
 
-		refs := string(refsBytes)
-		if refs[0:5] != "ref: " {
-			LogError("Unexpected format of remote HEAD ref:", refs)
-			return remoteName, fmt.Errorf("Unexpected format of remote HEAD ref: %s", refs)
-		}
+	refs := string(refsBytes)
+	if refs[0:5] != "ref: " {
+		LogError("Unexpected format of remote HEAD ref:", refs)
+		return remoteName, fmt.Errorf("Unexpected format of remote HEAD ref: %s", refs)
+	}
 
-		if len(branch) == 0 || branch == "HEAD" {
-			remoteRef = strings.TrimSpace(refs[5:])
-			branch = remoteRef[strings.LastIndex(remoteRef, "/")+1:]
-			LogDebug("remoteRef", remoteRef)
-			LogDebug("branch", branch)
-		}
-	*/
+	if len(branch) == 0 || branch == "HEAD" {
+		remoteRef = strings.TrimSpace(refs[5:])
+		branch = remoteRef[strings.LastIndex(remoteRef, "/")+1:]
+		LogDebug("remoteRef", remoteRef)
+		LogDebug("branch", branch)
+	}
+*/
 	args := []string{"fetch", "--prune", remoteName, branch}
 	if strings.TrimSpace(e.GitExecWithOutputOrPanic(repo, "rev-parse", "--is-shallow-repository")) == "true" {
 		args = slices.Insert(args, 1, "--unshallow")
@@ -824,6 +821,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
 				for _, te := range tree.items {
 					if te.isTree() {
 						trees[p+te.name+"/"] = te.hash
+						submoduleList[p+te.name] = te.hash
 					} else if te.isSubmodule() {
 						submoduleList[p+te.name] = te.hash
 					}
@@ -926,10 +924,193 @@ func (e *GitHandlerImpl) GitSubmoduleCommitId(cwd, packageName, commitId string)
 	return subCommitId, len(subCommitId) > 0
 }
 
-func (e *GitHandlerImpl) GitExecWithDataParse(cwd string, dataprocessor func(io.ByteReader) (Data, error), gitcmd string, args ...string) (Data, error) {
-	LogDebug("getting", gitcmd)
-	args = append([]string{gitcmd}, args...)
-	cmd := exec.Command("/usr/bin/git", args...)
+const (
+	GitStatus_Untracked = 0
+	GitStatus_Modified  = 1
+	GitStatus_Ignored   = 2
+	GitStatus_Unmerged  = 3 // States[0..3] -- Stage1, Stage2, Stage3 of merge objects
+	GitStatus_Renamed   = 4 // orig name in States[0]
+)
+
+type GitStatusData struct {
+	Path   string
+	Status int
+	States [3]string
+
+	/*
+	 <sub> A 4 character field describing the submodule state.
+	            "N..." when the entry is not a submodule.
+	            "S<c><m><u>" when the entry is a submodule.
+	            <c> is "C" if the commit changed; otherwise ".".
+	            <m> is "M" if it has tracked changes; otherwise ".".
+	            <u> is "U" if there are untracked changes; otherwise ".".
+	*/
+	SubmoduleChanges string
+}
+
+func parseGitStatusHexString(data io.ByteReader) (string, error) {
+	str := make([]byte, 0, 32)
+	for {
+		c, err := data.ReadByte()
+		if err != nil {
+			return "", err
+		}
+		switch {
+		case c == 0 || c == ' ':
+			return string(str), nil
+		case c >= 'a' && c <= 'f':
+		case c >= 'A' && c <= 'F':
+		case c >= '0' && c <= '9':
+		default:
+			return "", errors.New("Invalid character in hex string:" + string(c))
+		}
+		str = append(str, c)
+	}
+}
+func parseGitStatusString(data io.ByteReader) (string, error) {
+	str := make([]byte, 0, 100)
+	for {
+		c, err := data.ReadByte()
+		if err != nil {
+			return "", errors.New("Unexpected EOF. Expected NUL string term")
+		}
+		if c == 0 || c == ' ' {
+			return string(str), nil
+		}
+		str = append(str, c)
+	}
+}
+
+func parseGitStatusStringWithSpace(data io.ByteReader) (string, error) {
+	str := make([]byte, 0, 100)
+	for {
+		c, err := data.ReadByte()
+		if err != nil {
+			return "", errors.New("Unexpected EOF. Expected NUL string term")
+		}
+		if c == 0 {
+			return string(str), nil
+		}
+		str = append(str, c)
+	}
+}
+
+func skipGitStatusEntry(data io.ByteReader, skipSpaceLen int) error {
+	for skipSpaceLen > 0 {
+		c, err := data.ReadByte()
+		if err != nil {
+			return err
+		}
+		if c == ' ' {
+			skipSpaceLen--
+		}
+	}
+
+	return nil
+}
+
+func parseSingleStatusEntry(data io.ByteReader) (*GitStatusData, error) {
+	ret := GitStatusData{}
+	statusType, err := data.ReadByte()
+	if err != nil {
+		return nil, nil
+	}
+	switch statusType {
+	case '1':
+		var err error
+		if err = skipGitStatusEntry(data, 8); err != nil {
+			return nil, err
+		}
+		ret.Status = GitStatus_Modified
+		ret.Path, err = parseGitStatusStringWithSpace(data)
+		if err != nil {
+			return nil, err
+		}
+	case '2':
+		var err error
+		if err = skipGitStatusEntry(data, 9); err != nil {
+			return nil, err
+		}
+		ret.Status = GitStatus_Renamed
+		ret.Path, err = parseGitStatusStringWithSpace(data)
+		if err != nil {
+			return nil, err
+		}
+		ret.States[0], err = parseGitStatusStringWithSpace(data)
+		if err != nil {
+			return nil, err
+		}
+	case '?':
+		var err error
+		if err = skipGitStatusEntry(data, 1); err != nil {
+			return nil, err
+		}
+		ret.Status = GitStatus_Untracked
+		ret.Path, err = parseGitStatusStringWithSpace(data)
+		if err != nil {
+			return nil, err
+		}
+	case '!':
+		var err error
+		if err = skipGitStatusEntry(data, 1); err != nil {
+			return nil, err
+		}
+		ret.Status = GitStatus_Ignored
+		ret.Path, err = parseGitStatusStringWithSpace(data)
+		if err != nil {
+			return nil, err
+		}
+	case 'u':
+		var err error
+		if err = skipGitStatusEntry(data, 2); err != nil {
+			return nil, err
+		}
+		if ret.SubmoduleChanges, err = parseGitStatusString(data); err != nil {
+			return nil, err
+		}
+		if err = skipGitStatusEntry(data, 4); err != nil {
+			return nil, err
+		}
+
+		if ret.States[0], err = parseGitStatusHexString(data); err != nil {
+			return nil, err
+		}
+		if ret.States[1], err = parseGitStatusHexString(data); err != nil {
+			return nil, err
+		}
+		if ret.States[2], err = parseGitStatusHexString(data); err != nil {
+			return nil, err
+		}
+		ret.Status = GitStatus_Unmerged
+		ret.Path, err = parseGitStatusStringWithSpace(data)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, errors.New("Invalid status type" + string(statusType))
+	}
+	return &ret, nil
+}
+
+func parseGitStatusData(data io.ByteReader) ([]GitStatusData, error) {
+	ret := make([]GitStatusData, 0, 10)
+	for {
+		data, err := parseSingleStatusEntry(data)
+		if err != nil {
+			return nil, err
+		} else if data == nil {
+			break
+		}
+
+		ret = append(ret, *data)
+	}
+	return ret, nil
+}
+
+func (e *GitHandlerImpl) GitStatus(cwd string) (ret []GitStatusData, err error) {
+	LogDebug("getting git-status()")
+
+	cmd := exec.Command("/usr/bin/git", "status", "--porcelain=2", "-z")
 	cmd.Env = []string{
 		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
 		"GIT_LFS_SKIP_SMUDGE=1",
@@ -946,12 +1127,7 @@ func (e *GitHandlerImpl) GitExecWithDataParse(cwd string, dataprocessor func(io.
 		LogError("Error running command", cmd.Args, err)
 	}
 
-	return dataprocessor(bytes.NewReader(out))
-}
-
-func (e *GitHandlerImpl) GitStatus(cwd string) (ret []GitStatusData, err error) {
-	data, err := e.GitExecWithDataParse(cwd, parseGitStatusData, "status", "--porcelain=2", "-z")
-	return data.([]GitStatusData), err
+	return parseGitStatusData(bufio.NewReader(bytes.NewReader(out)))
 }
 
 func (e *GitHandlerImpl) GitDiff(cwd, base, head string) (string, error) {
@@ -976,94 +1152,3 @@ func (e *GitHandlerImpl) GitDiff(cwd, base, head string) (string, error) {
 
 	return string(out), nil
 }
-
-func (e *GitHandlerImpl) GitDiffIndex(cwd, commit string) ([]GitDiffRawData, error) {
-	data, err := e.GitExecWithDataParse("diff-index", parseGitDiffIndexRawData, cwd, "diff-index", "-z", "--raw", "--full-index", "--submodule=short", "HEAD")
-	return data.([]GitDiffRawData), err
-}
-
-func (git *GitHandlerImpl) GitResolveSubmoduleFileConflict(cwd string) error {
-	status, err := git.GitStatus(cwd)
-	if err != nil {
-		return fmt.Errorf("Status failed: %w", err)
-	}
-
-	// we can only resolve conflicts with .gitmodules
-	for _, s := range status {
-		if s.Status == GitStatus_Unmerged {
-			if s.Path != ".gitmodules" {
-				return err
-			}
-
-			submodules, err := git.GitSubmoduleList(cwd, "HEAD")
-			if err != nil {
-				return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
-			}
-
-			// We need to adjust the `submodules` list by the pending changes to the index
-
-			s1, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[0])
-			if err != nil {
-				return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
-			}
-			s2, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[1])
-			if err != nil {
-				return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
-			}
-			s3, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[2])
-			if err != nil {
-				return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
-			}
-
-			subs1, err := ParseSubmodulesFile(strings.NewReader(s1))
-			if err != nil {
-				return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
-			}
-			subs2, err := ParseSubmodulesFile(strings.NewReader(s2))
-			if err != nil {
-				return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
-			}
-			subs3, err := ParseSubmodulesFile(strings.NewReader(s3))
-			if err != nil {
-				return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
-			}
-
-			for r := range submodules {
-				LogError(r)
-			}
-
-			// merge from subs3 (target), subs1 (orig), subs2 (2-nd base that is missing from target base)
-			// this will update submodules
-			mergedSubs := slices.Concat(subs1, subs2, subs3)
-
-			var filteredSubs []Submodule = make([]Submodule, 0, max(len(subs1), len(subs2), len(subs3)))
-		nextSub:
-			for subName := range submodules {
-
-				for i := range mergedSubs {
-					if path.Base(mergedSubs[i].Path) == subName {
-						filteredSubs = append(filteredSubs, mergedSubs[i])
-						continue nextSub
-					}
-				}
-				return fmt.Errorf("Cannot find submodule for path: %s", subName)
-			}
-
-			out, err := os.Create(path.Join(git.GetPath(), cwd, ".gitmodules"))
-			if err != nil {
-				return fmt.Errorf("Can't open .gitmodules for writing: %w", err)
-			}
-			if err = WriteSubmodules(filteredSubs, out); err != nil {
-				return fmt.Errorf("Can't write .gitmodules: %w", err)
-			}
-			if out.Close(); err != nil {
-				return fmt.Errorf("Can't close .gitmodules: %w", err)
-			}
-
-			git.GitExecOrPanic(cwd, "add", ".gitmodules")
-			git.GitExecOrPanic(cwd, "-c", "core.editor=true", "merge", "--continue")
-		}
-	}
-
-	return nil
-}

common/git_utils_test.go

@@ -24,7 +24,6 @@ import (
 	"os"
 	"os/exec"
 	"path"
-	"runtime/debug"
 	"slices"
 	"strings"
 	"testing"
@@ -94,93 +93,6 @@ func TestGitClone(t *testing.T) {
 	}
 }
 
-func TestSubmoduleConflictResolution(t *testing.T) {
-	tests := []struct {
-		name            string
-		checkout, merge string
-		result          string
-	}{
-		{
-			name:     "adding two submodules",
-			checkout: "base_add_b1",
-			merge:    "base_add_b2",
-			result: `[submodule "pkgA"]
-    path = pkgA
-    url = ../pkgA
-[submodule "pkgB"]
-    path = pkgB
-    url = ../pkgB
-[submodule "pkgC"]
-    path = pkgC
-    url = ../pkgC
-[submodule "pkgB1"]
-    path = pkgB1
-    url = ../pkgB1
-[submodule "pkgB2"]
-    path = pkgB2
-    url = ../pkgB2
-`,
-		},
-	}
-
-	d, err := os.MkdirTemp(os.TempDir(), "submoduletests")
-	if err != nil {
-		t.Fatal(err)
-	}
-	cwd, err := os.Getwd()
-	if err != nil {
-		t.Fatal(err)
-	}
-	cmd := exec.Command(cwd + "/test_repo_setup.sh")
-	cmd.Dir = d
-	_, err = cmd.Output()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	gh, err := AllocateGitWorkTree(d, "test", "foo@example.com")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	success := true
-	noErrorOrFail := func(t *testing.T, err error) {
-		if err != nil {
-			t.Fatal(string(debug.Stack()), err)
-		}
-	}
-	for _, test := range tests {
-		success = t.Run(test.name, func(t *testing.T) {
-			git, err := gh.ReadExistingPath("prjgit")
-			if err != nil {
-				t.Fatal(err)
-			}
-			noErrorOrFail(t, git.GitExec("", "checkout", "-B", "test", "main"))
-			noErrorOrFail(t, git.GitExec("", "merge", "base_add_b1"))
-			err = git.GitExec("", "merge", "base_add_b2")
-			if err == nil {
-				t.Fatal("expected a conflict")
-			}
-			err = git.GitResolveSubmoduleFileConflict("")
-			if err != nil {
-				t.Fatal(err)
-			}
-			data, err := os.ReadFile(git.GetPath() + "/.gitmodules")
-			if err != nil {
-				t.Fatal("Cannot read .gitmodules.", err)
-			}
-
-			if string(data) != test.result {
-				t.Error("Expected", test.result, "but have", string(data))
-			}
-		}) && success
-	}
-
-	if success {
-		os.RemoveAll(d)
-	}
-}
-
 func TestGitMsgParsing(t *testing.T) {
 	t.Run("tree message with size 56", func(t *testing.T) {
 		const hdr = "f40888ea4515fe2e8eea617a16f5f50a45f652d894de3ad181d58de3aafb8f98 tree 56\x00"

common/gitea_utils.go

@@ -182,7 +182,6 @@ type Gitea interface {
 	GiteaCommitStatusGetter
 	GiteaCommitStatusSetter
 	GiteaSetRepoOptions
-	GiteaTimelineFetcher
 
 	GetNotifications(Type string, since *time.Time) ([]*models.NotificationThread, error)
 	GetDoneNotifications(Type string, page int64) ([]*models.NotificationThread, error)
@@ -310,6 +309,9 @@ func (gitea *GiteaTransport) GetPullRequests(org, repo string) ([]*models.PullRe
 			return nil, fmt.Errorf("cannot fetch PR list for %s / %s : %w", org, repo, err)
 		}
 
+		if len(req.Payload) == 0 {
+			break
+		}
 		prs = slices.Concat(prs, req.Payload)
 		if len(req.Payload) < int(limit) {
 			break
@@ -332,11 +334,11 @@ func (gitea *GiteaTransport) GetCommitStatus(org, repo, hash string) ([]*models.
 		if err != nil {
 			return res, err
 		}
-
-		res = append(res, r.Payload...)
-		if len(r.Payload) < int(limit) {
+		if len(r.Payload) == 0 {
 			break
 		}
+		res = append(res, r.Payload...)
+		page++
 	}
 
 	return res, nil
@@ -397,10 +399,10 @@ func (gitea *GiteaTransport) GetPullRequestReviews(org, project string, PRnum in
 			return nil, err
 		}
 
-		allReviews = slices.Concat(allReviews, reviews.Payload)
-		if len(reviews.Payload) < int(limit) {
+		if len(reviews.Payload) == 0 {
 			break
 		}
+		allReviews = slices.Concat(allReviews, reviews.Payload)
 		page++
 	}
 
@@ -490,6 +492,9 @@ func (gitea *GiteaTransport) GetNotifications(Type string, since *time.Time) ([]
 			return nil, err
 		}
 
+		if len(list.Payload) == 0 {
+			break
+		}
 		ret = slices.Concat(ret, list.Payload)
 		if len(list.Payload) < int(bigLimit) {
 			break
@@ -780,6 +785,9 @@ func (gitea *GiteaTransport) GetTimeline(org, repo string, idx int64) ([]*models
 
 		resCount = len(res.Payload)
 		LogDebug("page:", page, "len:", resCount)
+		if resCount == 0 {
+			break
+		}
 		page++
 
 		for _, d := range res.Payload {

common/pr.go

@@ -4,6 +4,8 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
+	"os"
+	"path"
 	"slices"
 	"strings"
 
@@ -405,8 +407,80 @@ func (rs *PRSet) Merge(gitea GiteaReviewUnrequester, git Git) error {
 
 	err = git.GitExec(DefaultGitPrj, "merge", "--no-ff", "-m", msg, prjgit.Head.Sha)
 	if err != nil {
-		if resolveError := git.GitResolveSubmoduleFileConflict(DefaultGitPrj); resolveError != nil {
-			return fmt.Errorf("Merge failed. (%w): %w", err, resolveError)
+		status, statusErr := git.GitStatus(DefaultGitPrj)
+		if statusErr != nil {
+			return fmt.Errorf("Failed to merge: %w . Status also failed: %w", err, statusErr)
+		}
+
+		// we can only resolve conflicts with .gitmodules
+		for _, s := range status {
+			if s.Status == GitStatus_Unmerged {
+				panic("Can't handle conflicts yet")
+				if s.Path != ".gitmodules" {
+					return err
+				}
+
+				submodules, err := git.GitSubmoduleList(DefaultGitPrj, "MERGE_HEAD")
+				if err != nil {
+					return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
+				}
+				s1, err := git.GitExecWithOutput(DefaultGitPrj, "cat-file", "blob", s.States[0])
+				if err != nil {
+					return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
+				}
+				s2, err := git.GitExecWithOutput(DefaultGitPrj, "cat-file", "blob", s.States[1])
+				if err != nil {
+					return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
+				}
+				s3, err := git.GitExecWithOutput(DefaultGitPrj, "cat-file", "blob", s.States[2])
+				if err != nil {
+					return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
+				}
+
+				subs1, err := ParseSubmodulesFile(strings.NewReader(s1))
+				if err != nil {
+					return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
+				}
+				subs2, err := ParseSubmodulesFile(strings.NewReader(s2))
+				if err != nil {
+					return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
+				}
+				subs3, err := ParseSubmodulesFile(strings.NewReader(s3))
+				if err != nil {
+					return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
+				}
+
+				// merge from subs3 (target), subs1 (orig), subs2 (2-nd base that is missing from target base)
+				// this will update submodules
+				mergedSubs := slices.Concat(subs1, subs2, subs3)
+
+				var filteredSubs []Submodule = make([]Submodule, 0, max(len(subs1), len(subs2), len(subs3)))
+			nextSub:
+				for subName := range submodules {
+
+					for i := range mergedSubs {
+						if path.Base(mergedSubs[i].Path) == subName {
+							filteredSubs = append(filteredSubs, mergedSubs[i])
+							continue nextSub
+						}
+					}
+					return fmt.Errorf("Cannot find submodule for path: %s", subName)
+				}
+
+				out, err := os.Create(path.Join(git.GetPath(), DefaultGitPrj, ".gitmodules"))
+				if err != nil {
+					return fmt.Errorf("Can't open .gitmodules for writing: %w", err)
+				}
+				if err = WriteSubmodules(filteredSubs, out); err != nil {
+					return fmt.Errorf("Can't write .gitmodules: %w", err)
+				}
+				if out.Close(); err != nil {
+					return fmt.Errorf("Can't close .gitmodules: %w", err)
+				}
+
+				git.GitExecOrPanic(DefaultGitPrj, "add", ".gitmodules")
+				git.GitExecOrPanic(DefaultGitPrj, "-c", "core.editor=true", "merge", "--continue")
+			}
 		}
 	}
 

common/pr_test.go

@@ -15,23 +15,22 @@ import (
 	"src.opensuse.org/autogits/common/gitea-generated/models"
 	mock_common "src.opensuse.org/autogits/common/mock"
 )
-
 /*
-	func TestCockpit(t *testing.T) {
-		common.SetLoggingLevel(common.LogLevelDebug)
-		gitea := common.AllocateGiteaTransport("https://src.opensuse.org")
-		tl, err := gitea.GetTimeline("cockpit", "cockpit", 29)
-		if err != nil {
-			t.Fatal("Fail to timeline", err)
-		}
-		t.Log(tl)
-		r, err := common.FetchGiteaReviews(gitea, []string{}, "cockpit", "cockpit", 29)
-		if err != nil {
-			t.Fatal("Error:", err)
-		}
-
-		t.Error(r)
+func TestCockpit(t *testing.T) {
+	common.SetLoggingLevel(common.LogLevelDebug)
+	gitea := common.AllocateGiteaTransport("https://src.opensuse.org")
+	tl, err := gitea.GetTimeline("cockpit", "cockpit", 29)
+	if err != nil {
+		t.Fatal("Fail to timeline", err)
 	}
+	t.Log(tl)
+	r, err := common.FetchGiteaReviews(gitea, []string{}, "cockpit", "cockpit", 29)
+	if err != nil {
+		t.Fatal("Error:", err)
+	}
+
+	t.Error(r)
+}
 */
 func reviewsToTimeline(reviews []*models.PullReview) []*models.TimelineComment {
 	timeline := make([]*models.TimelineComment, len(reviews))
@@ -940,7 +939,6 @@ func TestPRMerge(t *testing.T) {
 			pr: &models.PullRequest{
 				Base: &models.PRBranchInfo{
 					Sha: "e8b0de43d757c96a9d2c7101f4bff404e322f53a1fa4041fb85d646110c38ad4", // "base_add_b1"
-					Name: "master",
 					Repo: &models.Repository{
 						Name: "prj",
 						Owner: &models.User{
@@ -961,7 +959,6 @@ func TestPRMerge(t *testing.T) {
 			pr: &models.PullRequest{
 				Base: &models.PRBranchInfo{
 					Sha: "4fbd1026b2d7462ebe9229a49100c11f1ad6555520a21ba515122d8bc41328a8",
-					Name: "master",
 					Repo: &models.Repository{
 						Name: "prj",
 						Owner: &models.User{
@@ -981,7 +978,6 @@ func TestPRMerge(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			ctl := gomock.NewController(t)
 			mock := mock_common.NewMockGiteaPRTimelineFetcher(ctl)
-
 			reviewUnrequestMock := mock_common.NewMockGiteaReviewUnrequester(ctl)
 
 			reviewUnrequestMock.EXPECT().UnrequestReview(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)

obs-staging-bot/main.go

@@ -222,7 +222,6 @@ func ProcessRepoBuildStatus(results, ref []*common.PackageBuildStatus) (status B
 	slices.SortFunc(results, PackageBuildStatusSorter)
 	slices.SortFunc(ref, PackageBuildStatusSorter)
 
-	j := 0
 	SomeSuccess = false
 	for i := 0; i < len(results); i++ {
 		res, ok := common.ObsBuildStatusDetails[results[i].Code]
@@ -237,10 +236,11 @@ func ProcessRepoBuildStatus(results, ref []*common.PackageBuildStatus) (status B
 
 		if !res.Success {
 			// not failed if reference project also failed for same package here
-			for ; j < len(results) && strings.Compare(results[i].Package, ref[j].Package) < 0; j++ {
+			j := 0
+			for ; j < len(ref) && strings.Compare(results[i].Package, ref[j].Package) != 0; j++ {
 			}
 
-			if j < len(results) && results[i].Package == ref[j].Package {
+			if j < len(ref) {
 				refRes, ok := common.ObsBuildStatusDetails[ref[j].Code]
 				if !ok {
 					common.LogInfo("unknown ref package result code:", ref[j].Code, "package:", ref[j].Package)
@@ -322,10 +322,13 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
 		urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(pkg))
 	}
 	meta.ScmSync = pr.Head.Repo.CloneURL + "?" + strings.Join(urlPkg, "&") + "#" + pr.Head.Sha
+	if len(meta.ScmSync) >= 65535 {
+		return nil, errors.New("Reached max amount of package changes per request")
+	}
 	meta.Title = fmt.Sprintf("PR#%d to %s", pr.Index, pr.Base.Name)
-//      QE wants it published ... also we should not hardcode it here, since
-//      it is configurable via the :PullRequest project
-//	meta.PublicFlags = common.Flags{Contents: "<disable/>"}
+	//      QE wants it published ... also we should not hardcode it here, since
+	//      it is configurable via the :PullRequest project
+	//	meta.PublicFlags = common.Flags{Contents: "<disable/>"}
 
 	meta.Groups = nil
 	meta.Persons = nil
@@ -633,6 +636,14 @@ func CleanupPullNotification(gitea common.Gitea, thread *models.NotificationThre
 	return false // cleaned up now, but the cleanup was not aleady done
 }
 
+func SetStatus(gitea common.Gitea, org, repo, hash string, status *models.CommitStatus) error {
+	_, err := gitea.SetCommitStatus(org, repo, hash, status)
+	if err != nil {
+		common.LogError(err)
+	}
+	return err
+}
+
 func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, error) {
 	dir, err := os.MkdirTemp(os.TempDir(), BotName)
 	common.PanicOnError(err)
@@ -768,17 +779,22 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 		common.LogDebug(" # head submodules:", len(headSubmodules))
 		common.LogDebug(" # base submodules:", len(baseSubmodules))
 
-		modifiedOrNew := make([]string, 0, 16)
+		modifiedPackages := make([]string, 0, 16)
+		newPackages := make([]string, 0, 16)
 		if !stagingConfig.RebuildAll {
 			for pkg, headOid := range headSubmodules {
 				if baseOid, exists := baseSubmodules[pkg]; !exists || baseOid != headOid {
-					modifiedOrNew = append(modifiedOrNew, pkg)
+					if len(baseOid) > 0 {
+						modifiedPackages = append(modifiedPackages, pkg)
+					} else {
+						newPackages = append(newPackages, pkg)
+					}
 					common.LogDebug(pkg, ":", baseOid, "->", headOid)
 				}
 			}
 		}
 
-		if len(modifiedOrNew) == 0 {
+		if len(modifiedPackages) == 0 && len(newPackages) == 0 {
 			rebuild_all := false || stagingConfig.RebuildAll
 
 			reviews, err := gitea.GetPullRequestReviews(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Index)
@@ -837,6 +853,22 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 			TargetURL:   ObsWebHost + "/project/show/" + stagingProject,
 		}
 
+		if err != nil {
+			msg := "Unable to setup stage project " + stagingConfig.ObsProject
+			status.Status = common.CommitStatus_Fail
+			common.LogError(msg)
+			if !IsDryRun {
+				SetStatus(gitea, org, repo, pr.Head.Sha, status)
+				_, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, msg)
+				if err != nil {
+					common.LogError(err)
+				} else {
+					return true, nil
+				}
+			}
+			return false, nil
+		}
+
 		msg := "Changed source updated for build"
 		if change == RequestModificationProjectCreated {
 			msg = "Build is started in " + ObsWebHost + "/project/show/" +
@@ -845,8 +877,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 			if len(stagingConfig.QA) > 0 {
 				msg = msg + "\nAdditional QA builds: \n"
 			}
-			gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
-
+			SetStatus(gitea, org, repo, pr.Head.Sha, status)
 			for _, setup := range stagingConfig.QA {
 				CreateQASubProject(stagingConfig, git, gitea, pr,
 					stagingProject,
@@ -860,42 +891,44 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
 			gitea.AddComment(pr, msg)
 		}
 
-		baseResult, err := ObsClient.LastBuildResults(stagingConfig.ObsProject, modifiedOrNew...)
+		baseResult, err := ObsClient.LastBuildResults(stagingConfig.ObsProject, modifiedPackages...)
 		if err != nil {
 			common.LogError("failed fetching ref project status for", stagingConfig.ObsProject, ":", err)
 		}
 		stagingResult, err := ObsClient.BuildStatus(stagingProject)
 		if err != nil {
-			common.LogError("failed fetching ref project status for", stagingProject, ":", err)
+			common.LogError("failed fetching stage project status for", stagingProject, ":", err)
 		}
 		buildStatus := ProcessBuildStatus(stagingResult, baseResult)
 
+		done := false
 		switch buildStatus {
 		case BuildStatusSummarySuccess:
 			status.Status = common.CommitStatus_Success
+			done = true
 			if !IsDryRun {
 				_, err := gitea.AddReviewComment(pr, common.ReviewStateApproved, "Build successful")
 				if err != nil {
 					common.LogError(err)
-				} else {
-					return true, nil
 				}
 			}
 		case BuildStatusSummaryFailed:
 			status.Status = common.CommitStatus_Fail
+			done = true
 			if !IsDryRun {
 				_, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Build failed")
 				if err != nil {
 					common.LogError(err)
-				} else {
-					return true, nil
 				}
 			}
 		}
 		common.LogInfo("Build status:", buildStatus)
-		gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
-
-		// waiting for build results -- nothing to do
+		if !IsDryRun {
+			if err = SetStatus(gitea, org, repo, pr.Head.Sha, status); err != nil {
+				return false, err
+			}
+		}
+		return done, nil
 
 	} else if err == NonActionableReviewError || err == NoReviewsFoundError {
 		return true, nil

obs-status-service/main.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
@@ -268,6 +269,33 @@ func main() {
 
 		res.Write(BuildStatusSvg(nil, &common.PackageBuildStatus{Package: pkg, Code: "unknown"}))
 	})
+	http.HandleFunc("GET /search", func(res http.ResponseWriter, req *http.Request) {
+		common.LogInfo("GET /serach?" + req.URL.RawQuery)
+		queries := req.URL.Query()
+		if !queries.Has("q") {
+			res.WriteHeader(400)
+			return
+		}
+
+		names := queries["q"]
+		if len(names) != 1 {
+			res.WriteHeader(400)
+			return
+		}
+
+		packages := FindPackages(names[0])
+		data, err := json.MarshalIndent(packages, "", "   ")
+		if err != nil {
+			res.WriteHeader(500)
+			common.LogError("Error in marshalling data.", err)
+			return
+		}
+
+		res.Write(data)
+		res.Header().Add("content-type", "application/json")
+		res.WriteHeader(200)
+	})
+
 	http.HandleFunc("GET /buildlog/{Project}/{Package}/{Repository}/{Arch}", func(res http.ResponseWriter, req *http.Request) {
 		prj := req.PathValue("Project")
 		pkg := req.PathValue("Package")

obs-status-service/redis.go

@@ -29,13 +29,15 @@ func UpdateResults(r *common.BuildResult) {
 	RepoStatusLock.Lock()
 	defer RepoStatusLock.Unlock()
 
+	updateResultsWithoutLocking(r)
+}
+
+func updateResultsWithoutLocking(r *common.BuildResult) {
 	key := "result." + r.Project + "/" + r.Repository + "/" + r.Arch
-	common.LogDebug(" + Updating", key)
 	data, err := redisClient.HGetAll(context.Background(), key).Result()
 	if err != nil {
 		common.LogError("Failed fetching build results for", key, err)
 	}
-	common.LogDebug(" + Update size", len(data))
 
 	reset_time := time.Date(1000, 1, 1, 1, 1, 1, 1, time.Local)
 	for _, pkg := range r.Status {
@@ -110,6 +112,27 @@ func FindRepoResults(project, repo string) []*common.BuildResult {
 	return ret
 }
 
+func FindPackages(pkg string) []string {
+	RepoStatusLock.RLock()
+	defer RepoStatusLock.RUnlock()
+
+	data := make([]string, 0, 100)
+	for _, repo := range RepoStatus {
+		for _, status := range repo.Status {
+			if pkg == status.Package {
+				entry := repo.Project + "/" + pkg
+				if idx, found := slices.BinarySearch(data, entry); !found {
+					data = slices.Insert(data, idx, entry)
+					if len(data) >= 100 {
+						return data
+					}
+				}
+			}
+		}
+	}
+	return data
+}
+
 func FindAndUpdateProjectResults(project string) []*common.BuildResult {
 	res := FindProjectResults(project)
 	wg := &sync.WaitGroup{}
@@ -161,6 +184,8 @@ func RescanRepositories() error {
 	RepoStatusLock.Unlock()
 	var count int
 
+	projectsLooked := make([]string, 0, 10000)
+
 	for {
 		var data []string
 		data, cursor, err = redisClient.ScanType(ctx, cursor, "", 1000, "hash").Result()
@@ -169,6 +194,7 @@ func RescanRepositories() error {
 			return err
 		}
 
+		wg := &sync.WaitGroup{}
 		RepoStatusLock.Lock()
 		for _, repo := range data {
 			r := strings.Split(repo, "/")
@@ -180,14 +206,28 @@ func RescanRepositories() error {
 				Repository: r[1],
 				Arch:       r[2],
 			}
-			if pos, found := slices.BinarySearchFunc(RepoStatus, d, common.BuildResultComp); found {
+
+			var pos int
+			var found bool
+			if pos, found = slices.BinarySearchFunc(RepoStatus, d, common.BuildResultComp); found {
 				RepoStatus[pos].Dirty = true
 			} else {
 				d.Dirty = true
 				RepoStatus = slices.Insert(RepoStatus, pos, d)
 				count++
 			}
+
+			// fetch all keys, one per non-maintenance/non-home: projects, for package search
+			if idx, found := slices.BinarySearch(projectsLooked, d.Project); !found && !strings.Contains(d.Project, ":Maintenance:") && (len(d.Project) < 5 || d.Project[0:5] != "home:") {
+				projectsLooked = slices.Insert(projectsLooked, idx, d.Project)
+				wg.Add(1)
+				go func(r *common.BuildResult) {
+					updateResultsWithoutLocking(r)
+					wg.Done()
+				}(RepoStatus[pos])
+			}
 		}
+		wg.Wait()
 		RepoStatusLock.Unlock()
 
 		if cursor == 0 {

workflow-pr/pr_processor.go

@@ -317,9 +317,14 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
 	}
 
 	PrjGitTitle, PrjGitBody := PrjGitDescription(prset)
-	if PrjGitPR.PR.Title != PrjGitTitle || PrjGitPR.PR.Body != PrjGitBody {
-		common.LogDebug("New title:", PrjGitTitle)
-		common.LogDebug(PrjGitBody)
+	if PrjGitPR.PR.User.UserName == CurrentUser.UserName {
+		if PrjGitPR.PR.Title != PrjGitTitle || PrjGitPR.PR.Body != PrjGitBody {
+			common.LogDebug("New title:", PrjGitTitle)
+			common.LogDebug(PrjGitBody)
+		}
+	} else {
+		// TODO: find our first comment in timeline
+
 	}
 
 	if !common.IsDryRun {
@@ -436,10 +441,10 @@ func (pr *PRProcessor) Process(req *models.PullRequest) error {
 					continue
 				}
 			}
-		}
 
-		if err = pr.UpdatePrjGitPR(prset); err != nil {
-			return err
+			if err = pr.UpdatePrjGitPR(prset); err != nil {
+				return err
+			}
 		}
 	}