Enhance IssueProcessor and PRProcessor with additional logging and error handling

common/git_utils.go

@@ -396,12 +396,17 @@ func (e *GitHandlerImpl) GitExecQuietOrPanic(cwd string, params ...string) {
 }
 
 type ChanIO struct {
-	ch chan byte
+	ch   chan byte
+	done chan struct{}
 }
 
 func (c *ChanIO) Write(p []byte) (int, error) {
 	for _, b := range p {
-		c.ch <- b
+		select {
+		case c.ch <- b:
+		case <-c.done:
+			return 0, io.EOF
+		}
 	}
 	return len(p), nil
 }
@@ -410,21 +415,32 @@ func (c *ChanIO) Write(p []byte) (int, error) {
 func (c *ChanIO) Read(data []byte) (idx int, err error) {
 	var ok bool
 
-	data[idx], ok = <-c.ch
-	if !ok {
-		err = io.EOF
-		return
-	}
-	idx++
-
-	for len(c.ch) > 0 && idx < len(data) {
-		data[idx], ok = <-c.ch
+	select {
+	case data[idx], ok = <-c.ch:
 		if !ok {
 			err = io.EOF
 			return
 		}
-
 		idx++
+	case <-c.done:
+		err = io.EOF
+		return
+	}
+
+	for len(c.ch) > 0 && idx < len(data) {
+		select {
+		case data[idx], ok = <-c.ch:
+			if !ok {
+				err = io.EOF
+				return
+			}
+			idx++
+		case <-c.done:
+			err = io.EOF
+			return
+		default:
+			return
+		}
 	}
 
 	return
@@ -471,7 +487,14 @@ func parseGitMsg(data <-chan byte) (GitMsg, error) {
 	var size int
 
 	pos := 0
-	for c := <-data; c != ' '; c = <-data {
+	for {
+		c, ok := <-data
+		if !ok {
+			return GitMsg{}, io.EOF
+		}
+		if c == ' ' {
+			break
+		}
 		if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') {
 			id[pos] = c
 			pos++
@@ -483,7 +506,15 @@ func parseGitMsg(data <-chan byte) (GitMsg, error) {
 
 	pos = 0
 	var c byte
-	for c = <-data; c != ' ' && c != '\x00'; c = <-data {
+	for {
+		var ok bool
+		c, ok = <-data
+		if !ok {
+			return GitMsg{}, io.EOF
+		}
+		if c == ' ' || c == '\x00' {
+			break
+		}
 		if c >= 'a' && c <= 'z' {
 			msgType[pos] = c
 			pos++
@@ -509,7 +540,14 @@ func parseGitMsg(data <-chan byte) (GitMsg, error) {
 		return GitMsg{}, fmt.Errorf("Invalid object type: '%s'", string(msgType))
 	}
 
-	for c = <-data; c != '\000'; c = <-data {
+	for {
+		c, ok := <-data
+		if !ok {
+			return GitMsg{}, io.EOF
+		}
+		if c == '\x00' {
+			break
+		}
 		if c >= '0' && c <= '9' {
 			size = size*10 + (int(c) - '0')
 		} else {
@@ -528,18 +566,37 @@ func parseGitCommitHdr(oldHdr [2]string, data <-chan byte) ([2]string, int, erro
 	hdr := make([]byte, 0, 60)
 	val := make([]byte, 0, 1000)
 
-	c := <-data
+	c, ok := <-data
+	if !ok {
+		return [2]string{}, 0, io.EOF
+	}
 	size := 1
 	if c != '\n' { // end of header marker
-		for ; c != ' '; c = <-data {
+		for {
+			if c == ' ' {
+				break
+			}
 			hdr = append(hdr, c)
 			size++
+			var ok bool
+			c, ok = <-data
+			if !ok {
+				return [2]string{}, size, io.EOF
+			}
 		}
 		if size == 1 { // continuation header here
 			hdr = []byte(oldHdr[0])
 			val = append([]byte(oldHdr[1]), '\n')
 		}
-		for c := <-data; c != '\n'; c = <-data {
+		for {
+			var ok bool
+			c, ok = <-data
+			if !ok {
+				return [2]string{}, size, io.EOF
+			}
+			if c == '\n' {
+				break
+			}
 			val = append(val, c)
 			size++
 		}
@@ -552,7 +609,14 @@ func parseGitCommitHdr(oldHdr [2]string, data <-chan byte) ([2]string, int, erro
 func parseGitCommitMsg(data <-chan byte, l int) (string, error) {
 	msg := make([]byte, 0, l)
 
-	for c := <-data; c != '\x00'; c = <-data {
+	for {
+		c, ok := <-data
+		if !ok {
+			return string(msg), io.EOF
+		}
+		if c == '\x00' {
+			break
+		}
 		msg = append(msg, c)
 		l--
 	}
@@ -578,7 +642,7 @@ func parseGitCommit(data <-chan byte) (GitCommit, error) {
 		var hdr [2]string
 		hdr, size, err := parseGitCommitHdr(hdr, data)
 		if err != nil {
-			return GitCommit{}, nil
+			return GitCommit{}, err
 		}
 		l -= size
 
@@ -599,14 +663,28 @@ func parseGitCommit(data <-chan byte) (GitCommit, error) {
 func parseTreeEntry(data <-chan byte, hashLen int) (GitTreeEntry, error) {
 	var e GitTreeEntry
 
-	for c := <-data; c != ' '; c = <-data {
+	for {
+		c, ok := <-data
+		if !ok {
+			return e, io.EOF
+		}
+		if c == ' ' {
+			break
+		}
 		e.mode = e.mode*8 + int(c-'0')
 		e.size++
 	}
 	e.size++
 
 	name := make([]byte, 0, 128)
-	for c := <-data; c != '\x00'; c = <-data {
+	for {
+		c, ok := <-data
+		if !ok {
+			return e, io.EOF
+		}
+		if c == '\x00' {
+			break
+		}
 		name = append(name, c)
 		e.size++
 	}
@@ -617,7 +695,10 @@ func parseTreeEntry(data <-chan byte, hashLen int) (GitTreeEntry, error) {
 
 	hash := make([]byte, 0, hashLen*2)
 	for range hashLen {
-		c := <-data
+		c, ok := <-data
+		if !ok {
+			return e, io.EOF
+		}
 		hash = append(hash, hexBinToAscii[((c&0xF0)>>4)], hexBinToAscii[c&0xF])
 	}
 	e.hash = string(hash)
@@ -638,13 +719,16 @@ func parseGitTree(data <-chan byte) (GitTree, error) {
 	for parsedLen < hdr.size {
 		entry, err := parseTreeEntry(data, len(hdr.hash)/2)
 		if err != nil {
-			return GitTree{}, nil
+			return GitTree{}, err
 		}
 
 		t.items = append(t.items, entry)
 		parsedLen += entry.size
 	}
-	c := <-data // \0 read
+	c, ok := <-data // \0 read
+	if !ok {
+		return t, io.EOF
+	}
 
 	if c != '\x00' {
 		return t, fmt.Errorf("Unexpected character during git tree data read")
@@ -665,9 +749,16 @@ func parseGitBlob(data <-chan byte) ([]byte, error) {
 
 	d := make([]byte, hdr.size)
 	for l := 0; l < hdr.size; l++ {
-		d[l] = <-data
+		var ok bool
+		d[l], ok = <-data
+		if !ok {
+			return d, io.EOF
+		}
+	}
+	eob, ok := <-data
+	if !ok {
+		return d, io.EOF
 	}
-	eob := <-data
 	if eob != '\x00' {
 		return d, fmt.Errorf("invalid byte read in parseGitBlob")
 	}
@@ -679,16 +770,25 @@ func (e *GitHandlerImpl) GitParseCommits(cwd string, commitIDs []string) (parsed
 	var done sync.Mutex
 
 	done.Lock()
-	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}
+	done_signal := make(chan struct{})
+	var once sync.Once
+	close_done := func() {
+		once.Do(func() {
+			close(done_signal)
+		})
+	}
+
+	data_in, data_out := ChanIO{make(chan byte), done_signal}, ChanIO{make(chan byte), done_signal}
 	parsedCommits = make([]GitCommit, 0, len(commitIDs))
 
 	go func() {
 		defer done.Unlock()
+		defer close_done()
 		defer close(data_out.ch)
 
 		for _, id := range commitIDs {
 			data_out.Write([]byte(id))
-			data_out.ch <- '\x00'
+			data_out.Write([]byte{0})
 			c, e := parseGitCommit(data_in.ch)
 			if e != nil {
 				err = fmt.Errorf("Error parsing git commit: %w", e)
@@ -715,12 +815,14 @@ func (e *GitHandlerImpl) GitParseCommits(cwd string, commitIDs []string) (parsed
 	LogDebug("command run:", cmd.Args)
 	if e := cmd.Run(); e != nil {
 		LogError(e)
+		close_done()
 		close(data_in.ch)
-		close(data_out.ch)
 		return nil, e
 	}
 
 	done.Lock()
+	close_done()
+	close(data_in.ch)
 	return
 }
 
@@ -729,15 +831,21 @@ func (e *GitHandlerImpl) GitCatFile(cwd, commitId, filename string) (data []byte
 	var done sync.Mutex
 
 	done.Lock()
-	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}
+	done_signal := make(chan struct{})
+	var once sync.Once
+	close_done := func() {
+		once.Do(func() {
+			close(done_signal)
+		})
+	}
+	data_in, data_out := ChanIO{make(chan byte), done_signal}, ChanIO{make(chan byte), done_signal}
 
 	go func() {
 		defer done.Unlock()
+		defer close_done()
 		defer close(data_out.ch)
-
 		data_out.Write([]byte(commitId))
-		data_out.ch <- '\x00'
-
+		data_out.Write([]byte{0})
 		var c GitCommit
 		c, err = parseGitCommit(data_in.ch)
 		if err != nil {
@@ -745,11 +853,9 @@ func (e *GitHandlerImpl) GitCatFile(cwd, commitId, filename string) (data []byte
 			return
 		}
 		data_out.Write([]byte(c.Tree))
-		data_out.ch <- '\x00'
-
+		data_out.Write([]byte{0})
 		var tree GitTree
 		tree, err = parseGitTree(data_in.ch)
-
 		if err != nil {
 			LogError("Error parsing git tree:", err)
 			return
@@ -759,7 +865,7 @@ func (e *GitHandlerImpl) GitCatFile(cwd, commitId, filename string) (data []byte
 			if te.isBlob() && te.name == filename {
 				LogInfo("blob", te.hash)
 				data_out.Write([]byte(te.hash))
-				data_out.ch <- '\x00'
+				data_out.Write([]byte{0})
 				data, err = parseGitBlob(data_in.ch)
 				return
 			}
@@ -784,11 +890,13 @@ func (e *GitHandlerImpl) GitCatFile(cwd, commitId, filename string) (data []byte
 	LogDebug("command run:", cmd.Args)
 	if e := cmd.Run(); e != nil {
 		LogError(e)
+		close_done()
 		close(data_in.ch)
-		close(data_out.ch)
 		return nil, e
 	}
 	done.Lock()
+	close_done()
+	close(data_in.ch)
 	return
 }
 
@@ -798,16 +906,24 @@ func (e *GitHandlerImpl) GitDirectoryList(gitPath, commitId string) (directoryLi
 	directoryList = make(map[string]string)
 
 	done.Lock()
-	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}
+	done_signal := make(chan struct{})
+	var once sync.Once
+	close_done := func() {
+		once.Do(func() {
+			close(done_signal)
+		})
+	}
+	data_in, data_out := ChanIO{make(chan byte), done_signal}, ChanIO{make(chan byte), done_signal}
 
 	LogDebug("Getting directory for:", commitId)
 
 	go func() {
 		defer done.Unlock()
+		defer close_done()
 		defer close(data_out.ch)
 
 		data_out.Write([]byte(commitId))
-		data_out.ch <- '\x00'
+		data_out.Write([]byte{0})
 		var c GitCommit
 		c, err = parseGitCommit(data_in.ch)
 		if err != nil {
@@ -823,7 +939,7 @@ func (e *GitHandlerImpl) GitDirectoryList(gitPath, commitId string) (directoryLi
 				delete(trees, p)
 
 				data_out.Write([]byte(tree))
-				data_out.ch <- '\x00'
+				data_out.Write([]byte{0})
 				var tree GitTree
 				tree, err = parseGitTree(data_in.ch)
 
@@ -857,12 +973,14 @@ func (e *GitHandlerImpl) GitDirectoryList(gitPath, commitId string) (directoryLi
 	LogDebug("command run:", cmd.Args)
 	if e := cmd.Run(); e != nil {
 		LogError(e)
+		close_done()
 		close(data_in.ch)
-		close(data_out.ch)
 		return directoryList, e
 	}
 
 	done.Lock()
+	close_done()
+	close(data_in.ch)
 	return directoryList, err
 }
 
@@ -872,7 +990,14 @@ func (e *GitHandlerImpl) GitDirectoryContentList(gitPath, commitId string) (dire
 	directoryList = make(map[string]string)
 
 	done.Lock()
-	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}
+	done_signal := make(chan struct{})
+	var once sync.Once
+	close_done := func() {
+		once.Do(func() {
+			close(done_signal)
+		})
+	}
+	data_in, data_out := ChanIO{make(chan byte), done_signal}, ChanIO{make(chan byte), done_signal}
 
 	LogDebug("Getting directory content for:", commitId)
 
@@ -881,7 +1006,7 @@ func (e *GitHandlerImpl) GitDirectoryContentList(gitPath, commitId string) (dire
 		defer close(data_out.ch)
 
 		data_out.Write([]byte(commitId))
-		data_out.ch <- '\x00'
+		data_out.Write([]byte{0})
 		var c GitCommit
 		c, err = parseGitCommit(data_in.ch)
 		if err != nil {
@@ -897,7 +1022,7 @@ func (e *GitHandlerImpl) GitDirectoryContentList(gitPath, commitId string) (dire
 				delete(trees, p)
 
 				data_out.Write([]byte(tree))
-				data_out.ch <- '\x00'
+				data_out.Write([]byte{0})
 				var tree GitTree
 				tree, err = parseGitTree(data_in.ch)
 
@@ -933,12 +1058,14 @@ func (e *GitHandlerImpl) GitDirectoryContentList(gitPath, commitId string) (dire
 	LogDebug("command run:", cmd.Args)
 	if e := cmd.Run(); e != nil {
 		LogError(e)
+		close_done()
 		close(data_in.ch)
-		close(data_out.ch)
 		return directoryList, e
 	}
 
 	done.Lock()
+	close_done()
+	close(data_in.ch)
 	return directoryList, err
 }
 
@@ -948,16 +1075,24 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
 	submoduleList = make(map[string]string)
 
 	done.Lock()
-	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}
+	done_signal := make(chan struct{})
+	var once sync.Once
+	close_done := func() {
+		once.Do(func() {
+			close(done_signal)
+		})
+	}
+	data_in, data_out := ChanIO{make(chan byte), done_signal}, ChanIO{make(chan byte), done_signal}
 
 	LogDebug("Getting submodules for:", commitId)
 
 	go func() {
 		defer done.Unlock()
+		defer close_done()
 		defer close(data_out.ch)
 
 		data_out.Write([]byte(commitId))
-		data_out.ch <- '\x00'
+		data_out.Write([]byte{0})
 		var c GitCommit
 		c, err = parseGitCommit(data_in.ch)
 		if err != nil {
@@ -973,7 +1108,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
 				delete(trees, p)
 
 				data_out.Write([]byte(tree))
-				data_out.ch <- '\x00'
+				data_out.Write([]byte{0})
 				var tree GitTree
 				tree, err = parseGitTree(data_in.ch)
 
@@ -1010,17 +1145,26 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
 	LogDebug("command run:", cmd.Args)
 	if e := cmd.Run(); e != nil {
 		LogError(e)
+		close_done()
 		close(data_in.ch)
-		close(data_out.ch)
 		return submoduleList, e
 	}
 
 	done.Lock()
+	close_done()
+	close(data_in.ch)
 	return submoduleList, err
 }
 
 func (e *GitHandlerImpl) GitSubmoduleCommitId(cwd, packageName, commitId string) (subCommitId string, valid bool) {
-	data_in, data_out := ChanIO{make(chan byte)}, ChanIO{make(chan byte)}
+	done_signal := make(chan struct{})
+	var once sync.Once
+	close_done := func() {
+		once.Do(func() {
+			close(done_signal)
+		})
+	}
+	data_in, data_out := ChanIO{make(chan byte), done_signal}, ChanIO{make(chan byte), done_signal}
 	var wg sync.WaitGroup
 
 	wg.Add(1)
@@ -1036,17 +1180,18 @@ func (e *GitHandlerImpl) GitSubmoduleCommitId(cwd, packageName, commitId string)
 		}()
 
 		defer wg.Done()
+		defer close_done()
 		defer close(data_out.ch)
 
 		data_out.Write([]byte(commitId))
-		data_out.ch <- '\x00'
+		data_out.Write([]byte{0})
 		c, err := parseGitCommit(data_in.ch)
 		if err != nil {
 			LogError("Error parsing git commit:", err)
 			panic(err)
 		}
 		data_out.Write([]byte(c.Tree))
-		data_out.ch <- '\x00'
+		data_out.Write([]byte{0})
 		tree, err := parseGitTree(data_in.ch)
 
 		if err != nil {
@@ -1078,12 +1223,14 @@ func (e *GitHandlerImpl) GitSubmoduleCommitId(cwd, packageName, commitId string)
 	LogDebug("command run:", cmd.Args)
 	if e := cmd.Run(); e != nil {
 		LogError(e)
+		close_done()
 		close(data_in.ch)
-		close(data_out.ch)
 		return subCommitId, false
 	}
 
 	wg.Wait()
+	close_done()
+	close(data_in.ch)
 	return subCommitId, len(subCommitId) > 0
 }
 

common/git_utils_test.go

@@ -28,6 +28,7 @@ import (
 	"slices"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestGitClone(t *testing.T) {
@@ -717,3 +718,44 @@ func TestGitDirectoryListRepro(t *testing.T) {
 		t.Errorf("Expected 'subdir' in directory list, got %v", dirs)
 	}
 }
+
+func TestGitDeadlockFix(t *testing.T) {
+	gitDir := t.TempDir()
+	testDir, _ := os.Getwd()
+
+	cmd := exec.Command("/usr/bin/bash", path.Join(testDir, "tsetup.sh"))
+	cmd.Dir = gitDir
+	_, err := cmd.CombinedOutput()
+
+	gh, err := AllocateGitWorkTree(gitDir, "Test", "test@example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	h, err := gh.ReadExistingPath(".")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer h.Close()
+
+	// Use a blob ID to trigger error in GitParseCommits
+	// This ensures that the function returns error immediately and doesn't deadlock
+	blobId := "81aba862107f1e2f5312e165453955485f424612f313d6c2fb1b31fef9f82a14"
+
+	done := make(chan error)
+	go func() {
+		_, err := h.GitParseCommits("", []string{blobId})
+		done <- err
+	}()
+
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Error("Expected error from GitParseCommits with blob ID, got nil")
+		} else {
+			// This is expected
+			t.Logf("Got expected error: %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("GitParseCommits deadlocked! Fix is NOT working.")
+	}
+}

common/utils.go

@@ -44,6 +44,42 @@ type NewRepos struct {
 const maintainership_line = "MAINTAINER"
 
 var true_lines []string = []string{"1", "TRUE", "YES", "OK", "T"}
+var InvalidUrlError error = errors.New("PrjGit or PackageGit URLs cannot be empty.")
+var AbsoluteUrlError error = errors.New("PrjGit or PackageGit URLs cannot be relative.")
+var HostsNotEqualError error = errors.New("PrjGit or PackageGit are not the same hosts.")
+var AbsoluteUrlWithQuery error = errors.New("PrjGit or PackageGit with query parameter. Unsupported.")
+var InvalidPath error = errors.New("PrjGit or PackageGit path has unsupported format.")
+
+
+func RelativeRepositoryPath(prjgit_org, packagegit string) (string, error) {
+	if len(packagegit) == 0 {
+		return "", InvalidUrlError
+	}
+
+	pkggiturl, err := url.Parse(packagegit)
+	if err != nil {
+		return "", err
+	}
+	if !pkggiturl.IsAbs() {
+		return "", AbsoluteUrlError
+	}
+	if len(pkggiturl.RawQuery) != 0 {
+		return "", AbsoluteUrlWithQuery
+	}
+
+	pkggitpath := SplitStringNoEmpty(pkggiturl.Path, "/")
+
+	if len(pkggitpath) != 2 {
+		return "", InvalidPath
+	}
+
+	pkggitpath[1] = strings.TrimSuffix(pkggitpath[1], ".git")
+	if prjgit_org == pkggitpath[0] {
+		return "../" + pkggitpath[1], nil
+	}
+
+	return "../../" + pkggitpath[0] + "/" + pkggitpath[1], nil
+}
 
 func HasSpace(s string) bool {
 	return strings.IndexFunc(s, unicode.IsSpace) >= 0

common/utils_test.go

@@ -8,6 +8,82 @@ import (
 	"src.opensuse.org/autogits/common"
 )
 
+func TestRelativeRepositoryPath(t *testing.T) {
+	tests := []struct {
+		name         string
+		prjorg, repo string
+
+		hasError bool
+		relative string
+	}{
+		{
+			name:     "Empty packagegit",
+			prjorg:   "org1",
+			repo:     "",
+			hasError: true,
+		},
+		{
+			name:     "Invalid URL",
+			prjorg:   "org1",
+			repo:     ":",
+			hasError: true,
+		},
+		{
+			name:     "Relative packagegit",
+			prjorg:   "org1",
+			repo:     "/path/to/repo",
+			hasError: true,
+		},
+		{
+			name:     "Packagegit with query",
+			prjorg:   "org1",
+			repo:     "https://host/org1/repo?query=1",
+			hasError: true,
+		},
+		{
+			name:     "Invalid path (too short)",
+			prjorg:   "org1",
+			repo:     "https://host/repo",
+			hasError: true,
+		},
+		{
+			name:     "Invalid path (too long)",
+			prjorg:   "org1",
+			repo:     "https://host/org/repo/extra",
+			hasError: true,
+		},
+		{
+			name:     "Same org",
+			prjorg:   "org1",
+			repo:     "https://host/org1/repo.git",
+			relative: "../repo",
+		},
+		{
+			name:     "Different org",
+			prjorg:   "org1",
+			repo:     "https://host/org2/repo.git",
+			relative: "../../org2/repo",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			r, err := common.RelativeRepositoryPath(test.prjorg, test.repo)
+
+			if err != nil && !test.hasError {
+				t.Error("Expected no error but have one", err)
+			}
+			if err == nil && test.hasError {
+				t.Error("Expected an error but had none. Returned:", r)
+			}
+
+			if err == nil && test.relative != r {
+				t.Error("Expected", test.relative, "but have", r)
+			}
+		})
+	}
+}
+
 func TestGitUrlParse(t *testing.T) {
 	tests := []struct {
 		name     string

obs-staging-bot/main.go

@@ -1171,7 +1171,6 @@ var IsDryRun bool
 var ProcessPROnly string
 var ObsClient common.ObsClientInterface
 var BotUser string
-var PollInterval = 5 * time.Minute
 
 func ObsWebHostFromApiHost(apihost string) string {
 	u, err := url.Parse(apihost)
@@ -1194,18 +1193,9 @@ func main() {
 	flag.StringVar(&ObsApiHost, "obs", "", "API for OBS instance")
 	flag.StringVar(&ObsWebHost, "obs-web", "", "Web OBS instance, if not derived from the obs config")
 	flag.BoolVar(&IsDryRun, "dry", false, "Dry-run, don't actually create any build projects or review changes")
-	pollIntervalStr := flag.String("poll-interval", common.GetEnvOverrideString(os.Getenv("AUTOGITS_STAGING_BOT_POLL_INTERVAL"), ""), "Polling interval for notifications (e.g. 5m, 10s)")
 	debug := flag.Bool("debug", false, "Turns on debug logging")
 	flag.Parse()
 
-	if len(*pollIntervalStr) > 0 {
-		if d, err := time.ParseDuration(*pollIntervalStr); err == nil {
-			PollInterval = d
-		} else {
-			common.LogError("Invalid poll interval:", err)
-		}
-	}
-
 	if *debug {
 		common.SetLoggingLevel(common.LogLevelDebug)
 	} else {
@@ -1274,6 +1264,6 @@ func main() {
 	for {
 		PollWorkNotifications(ObsClient, gitea)
 		common.LogInfo("Poll cycle finished")
-		time.Sleep(PollInterval)
+		time.Sleep(5 * time.Minute)
 	}
 }

workflow-direct/main.go

@@ -19,6 +19,7 @@ package main
  */
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
 	"io/fs"
@@ -123,7 +124,9 @@ func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, co
 			common.LogError("   - ", action.Repository.Name, "repo is not sha256. Ignoring.")
 			return
 		}
-		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
+		relpath, err := common.RelativeRepositoryPath(gitOrg, action.Repository.Clone_Url)
+		common.PanicOnError(err)
+		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", relpath, action.Repository.Name))
 		defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all", "-f")
 
 		branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current"))
@@ -268,6 +271,12 @@ func verifyProjectState(git common.Git, org string, config *common.AutogitConfig
 	sub, err := git.GitSubmoduleList(gitPrj, "HEAD")
 	common.PanicOnError(err)
 
+	submodulesData, err := git.GitCatFile(gitPrj, "HEAD", ".gitmodules")
+	var submoduleEntries []common.Submodule
+	if err == nil {
+		submoduleEntries, _ = common.ParseSubmodulesFile(bytes.NewReader(submodulesData))
+	}
+
 	common.LogDebug("   * Getting package links")
 	var pkgLinks []*PackageRebaseLink
 	if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), gitPrj)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 {
@@ -395,8 +404,27 @@ next_repo:
 		}
 		//		}
 
+		relpath, err := common.RelativeRepositoryPath(gitOrg, r.CloneURL)
+		common.PanicOnError(err)
+
 		for repo := range sub {
 			if repo == r.Name {
+				// verify we are using relative repository paths, and if not, adjust them
+				sidx := slices.IndexFunc(submoduleEntries, func(s common.Submodule) bool {
+					if path.Base(s.Path) == r.Name {
+						return true
+					}
+					return false
+				})
+				if sidx >= 0 && submoduleEntries[sidx].Url != relpath {
+					submoduleEntries[sidx].Url = relpath
+					f, err := os.OpenFile(path.Join(git.GetPath(), ".gitmodules"), os.O_CREATE|os.O_TRUNC, 0o6400)
+					common.PanicOnError(err)
+					defer f.Close()
+					common.PanicOnError(common.WriteSubmodules(submoduleEntries, f))
+					isGitUpdated = true
+				}
+
 				// not missing
 				continue next_repo
 			}
@@ -420,7 +448,7 @@ next_repo:
 		}
 
 		// add repository to git project
-		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", r.CloneURL, r.Name))
+		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--force", "--depth", "1", relpath, r.Name))
 
 		curBranch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current"))
 		if branch != curBranch {

workflow-pr/issue_processor.go

@@ -131,6 +131,7 @@ func (i *IssueProcessor) ProcessAddIssue(config *common.AutogitConfig) error {
 		common.LogDebug(" - Processing new repository src:", nr.Organization+"/"+nr.PackageName+"#"+nr.Branch)
 
 		targetRepo, err := Gitea.GetRepository(config.Organization, nr.PackageName)
+		common.LogDebug(" - Target repository:", config.Organization+"/"+nr.PackageName, "exists?", targetRepo != nil, "error?", err)
 		if err != nil {
 			return err
 		}
@@ -148,6 +149,7 @@ func (i *IssueProcessor) ProcessAddIssue(config *common.AutogitConfig) error {
 		// TODO, we need to filter by project config permissions of target project, not just assume bot here.
 		users := []string{CurrentUser.UserName}
 		prs := i.IssueTimeline.FindIssuePullRequestRererences(config.Organization, nr.PackageName, 0, users)
+		common.LogDebug(" - Existing PR references in timeline:", len(prs))
 		for _, t := range prs {
 			pr, err := Gitea.GetPullRequest(config.Organization, nr.PackageName, t.RefIssue.Index)
 			if err != nil {
@@ -171,7 +173,9 @@ func (i *IssueProcessor) ProcessAddIssue(config *common.AutogitConfig) error {
 		}
 
 		srcRepo, err := FindSourceRepository(nr.Organization, nr.Repository)
+		common.LogDebug(" - FindSourceRepository:", nr.Organization+"/"+nr.Repository, "err?", err)
 		if err != nil {
+			common.LogError(" - Skipping: cannot find source repository:", nr.Organization+"/"+nr.Repository, err)
 			continue
 		}
 
@@ -183,12 +187,15 @@ func (i *IssueProcessor) ProcessAddIssue(config *common.AutogitConfig) error {
 		if err != nil {
 			return err
 		}
-		remoteName, err := git.GitClone(nr.PackageName, nr.Branch, targetRepo.SSHURL)
+		// Clone the target using its default branch — the target branch may not exist yet
+		// and will be created by the push below.
+		remoteName, err := git.GitClone(nr.PackageName, targetRepo.DefaultBranch, targetRepo.SSHURL)
 		if err != nil {
 			return err
 		}
 
 		// Check that fork/parent repository relationship exists
+		common.LogDebug(" - Fork check: src.Parent=", srcRepo.Parent.Owner.UserName+"/"+srcRepo.Parent.Name, "target=", targetRepo.Owner.UserName+"/"+targetRepo.Name)
 		if srcRepo.Parent.Name != targetRepo.Name || srcRepo.Parent.Owner.UserName != targetRepo.Owner.UserName {
 			common.LogError("Source repository is not fork of the Target repository. Fork of:", srcRepo.Parent.Owner.UserName+"/"+srcRepo.Parent.Name)
 			continue
@@ -229,6 +236,7 @@ func (i *IssueProcessor) ProcessAddIssue(config *common.AutogitConfig) error {
 		if err == nil && strings.Contains(out, "refs/heads/"+srcBranch) {
 			isBranch = true
 		}
+		common.LogDebug(" - head:", head, "isBranch:", isBranch)
 
 		if !isBranch {
 			tempBranch := fmt.Sprintf("new_package_%d_%s", issue.Index, nr.PackageName)
@@ -254,6 +262,7 @@ func (i *IssueProcessor) ProcessAddIssue(config *common.AutogitConfig) error {
 		if len(br) == 0 {
 			br = targetRepo.DefaultBranch
 		}
+		common.LogDebug(" - Creating PR: head=", head, "base=", br, "title=", title)
 		pr, err, isNew := Gitea.CreatePullRequestIfNotExist(targetRepo, head, br, title, body)
 		if err != nil {
 			common.LogError(targetRepo.Name, head, i.TargetBranch, title, body)
@@ -285,6 +294,7 @@ func (i *IssueProcessor) ProcessIssue(configs common.AutogitConfigs) error {
 
 	//	out, _ := json.MarshalIndent(issue, "", "   ")
 	//	common.LogDebug(string(out))
+	common.LogInfo("Processing issue:", common.IssueToString(issue))
 
 	var err error
 	i.IssueTimeline, err = Gitea.GetTimeline(org, repo, idx)

workflow-pr/main.go

@@ -185,6 +185,7 @@ func main() {
 			common.RequestType_PRReviewAccepted: req,
 			common.RequestType_PRReviewRejected: req,
 			common.RequestType_PRComment:        req,
+			common.RequestType_Issue:            req,
 		},
 	}
 	listenDefs.Connection().RabbitURL, _ = url.Parse(*rabbitUrl)

workflow-pr/pr_processor.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"path"
 	"runtime/debug"
 	"slices"
@@ -144,6 +145,7 @@ func (pr *PRProcessor) SetSubmodulesToMatchPRSet(prset *common.PRSet) error {
 		return err
 	}
 
+	PrjGitOrg, _, _ := prset.Config.GetPrjGit()
 	for _, pr := range prset.PRs {
 		if prset.IsPrjGitPR(pr.PR) {
 			continue
@@ -198,7 +200,26 @@ func (pr *PRProcessor) SetSubmodulesToMatchPRSet(prset *common.PRSet) error {
 			ref := fmt.Sprintf(common.PrPattern, org, repo, idx)
 			commitMsg := fmt.Sprintln("Add package", repo, "\n\nThis commit was autocreated by", GitAuthor, "\n\nreferencing PRs:\n", ref)
 
-			git.GitExecOrPanic(common.DefaultGitPrj, "submodule", "add", "-b", pr.PR.Base.Name, pr.PR.Base.Repo.SSHURL, repo)
+			relpath, err := common.RelativeRepositoryPath(PrjGitOrg, pr.PR.Base.Repo.CloneURL)
+			if err != nil {
+				common.LogError("Cannot calculate relative path for repository", pr.PR.Base.Repo.CloneURL, err)
+				return err
+			}
+			// git submodule add refuses to proceed if the repo already exists
+			// locally (left by a prior GitClone). Two locations must be cleared:
+			//   1. the working tree dir
+			//   2. .git/modules/<repo> cached by git from a previous submodule init
+			staleWorkTree := path.Join(git.GetPath(), common.DefaultGitPrj, repo)
+			staleGitModules := path.Join(git.GetPath(), common.DefaultGitPrj, ".git", "modules", repo)
+			for _, staleDir := range []string{staleWorkTree, staleGitModules} {
+				if _, err := os.Stat(staleDir); err == nil {
+					common.LogDebug("Removing stale dir before submodule add:", staleDir)
+					if err := os.RemoveAll(staleDir); err != nil {
+						return err
+					}
+				}
+			}
+			git.GitExecOrPanic(common.DefaultGitPrj, "submodule", "add", "-b", pr.PR.Base.Name, relpath, repo)
 
 			updateSubmoduleInPR(repo, prHead, git)
 			common.PanicOnError(git.GitExec(common.DefaultGitPrj, "commit", "-a", "-m", commitMsg))
@@ -482,11 +503,11 @@ func (pr *PRProcessor) Process(req *models.PullRequest) error {
 						if _, ok := err.(*repository.RepoMergePullRequestConflict); !ok {
 							common.PanicOnError(err)
 						}
-//					} else {
-//						Gitea.AddComment(pr.PR, "Closing here because the associated Project PR has been closed.")
-//						Gitea.UpdatePullRequest(org, repo, idx, &models.EditPullRequestOption{
-//							State: "closed",
-//						})
+						//					} else {
+						//						Gitea.AddComment(pr.PR, "Closing here because the associated Project PR has been closed.")
+						//						Gitea.UpdatePullRequest(org, repo, idx, &models.EditPullRequestOption{
+						//							State: "closed",
+						//						})
 					}
 				}
 			}

workflow-pr/pr_processor_test.go

@@ -28,8 +28,10 @@ func TestPrjGitDescription(t *testing.T) {
 					Base: &models.PRBranchInfo{
 						Ref: "main",
 						Repo: &models.Repository{
-							Name:  "pkg-a",
-							Owner: &models.User{UserName: "test-org"},
+							Name:     "pkg-a",
+							Owner:    &models.User{UserName: "test-org"},
+							CloneURL: "http://example.com/test-org/pkg-a.git",
+							SSHURL:   "git@example.com:test-org/pkg-a.git",
 						},
 					},
 				},
@@ -202,8 +204,10 @@ func TestSetSubmodulesToMatchPRSet(t *testing.T) {
 					Base: &models.PRBranchInfo{
 						Ref: "main",
 						Repo: &models.Repository{
-							Name:  "pkg-a",
-							Owner: &models.User{UserName: "test-org"},
+							Name:     "pkg-a",
+							Owner:    &models.User{UserName: "test-org"},
+							CloneURL: "http://example.com/test-org/pkg-a.git",
+							SSHURL:   "git@example.com:test-org/pkg-a.git",
 						},
 					},
 					Head: &models.PRBranchInfo{
@@ -630,7 +634,12 @@ func TestCreatePRjGitPR_Integration(t *testing.T) {
 				PR: &models.PullRequest{
 					State: "open",
 					Base: &models.PRBranchInfo{
-						Repo: &models.Repository{Name: "pkg-a", Owner: &models.User{UserName: "test-org"}},
+						Repo: &models.Repository{
+							Name:     "pkg-a",
+							Owner:    &models.User{UserName: "test-org"},
+							CloneURL: "http://example.com/test-org/pkg-a.git",
+							SSHURL:   "git@example.com:test-org/pkg-a.git",
+						},
 					},
 					Head: &models.PRBranchInfo{Sha: "pkg-sha"},
 				},
@@ -653,14 +662,23 @@ func TestCreatePRjGitPR_Integration(t *testing.T) {
 			Base: &models.PRBranchInfo{
 				Name:   "main",
 				RepoID: 1,
-				Repo:   &models.Repository{Name: "test-prj", Owner: &models.User{UserName: "test-org"}},
+				Repo: &models.Repository{
+					Name:     "test-prj",
+					Owner:    &models.User{UserName: "test-org"},
+					CloneURL: "http://example.com/test-org/test-prj.git",
+					SSHURL:   "git@example.com:test-org/test-prj.git",
+				},
 			},
 			Head: &models.PRBranchInfo{
 				Sha: "prj-head-sha",
 			},
 		}
 
-		gitea.EXPECT().GetRepository(gomock.Any(), gomock.Any()).Return(&models.Repository{Owner: &models.User{UserName: "test-org"}}, nil).AnyTimes()
+		gitea.EXPECT().GetRepository(gomock.Any(), gomock.Any()).Return(&models.Repository{
+			Owner:    &models.User{UserName: "test-org"},
+			CloneURL: "http://example.com/test-org/test-prj.git",
+			SSHURL:   "git@example.com:test-org/test-prj.git",
+		}, nil).AnyTimes()
 		// CreatePullRequestIfNotExist returns isNew=true
 		gitea.EXPECT().CreatePullRequestIfNotExist(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(prjPR, nil, true).AnyTimes()
 		// Expect SetLabels to be called for new PR