git: one generator per app, multiple instances

this allows locking for access for each org

common/config.go

@@ -177,3 +177,7 @@ func (config *AutogitConfig) GetPrjGit() (string, string, string) {
 
 	return org, repo, branch
 }
+
+func (config *AutogitConfig) GetRemoteBranch() string {
+	return "origin_" + config.Branch
+}

common/git_utils.go

@@ -25,12 +25,10 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"net/url"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"sync"
 )
@@ -48,7 +46,7 @@ type GitStatusLister interface {
 
 type Git interface {
 	// error if git, but wrong remote
-	GitClone(remoteUrl string) (string, error) // clone, or chekc if path is already checked out remote and pulls, error otherwise. Return relative path
+	GitClone(repo, branch, remoteName, remoteUrl string) error // clone, or chekc if path is already checked out remote and force pulls, error otherwise. Return relative path
 
 	GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error)
 	GitCatFile(cwd, commitId, filename string) (data []byte, err error)
@@ -81,45 +79,41 @@ func (s *GitHandlerImpl) GetPath() string {
 }
 
 type GitHandlerGenerator interface {
-	CreateGitHandler(repo string) (Git, error)
-	ReadExistingPath(repo string) (Git, error)
+	CreateGitHandler(org, branch string) (Git, error)
+	ReadExistingPath(org, branch string) (Git, error)
 
 	ReleaseLock(path string)
 }
 
-type GitHandlerGeneratorImpl struct {
+type gitHandlerGeneratorImpl struct {
 	// TODO: add mutex to lock paths so only one workflow per path and others wait
-	path        string
-	org, branch string
-	git_author  string
-	email       string
+	path       string
+	git_author string
+	email      string
 
 	lock_lock sync.Mutex
 	lock      map[string]sync.Mutex
 }
 
-func AllocateGitWorkTree(basePath, org, branch, gitAuthor, email string) (GitHandlerGenerator, error) {
+func AllocateGitWorkTree(basePath, gitAuthor, email string) (GitHandlerGenerator, error) {
 	if fi, err := os.Stat(basePath); err != nil || !fi.IsDir() {
 		return nil, fmt.Errorf("Git basepath not a valid directory: %s %w", basePath, err)
 	}
 
-	path := path.Join(basePath, org, branch)
-	if fi, err := os.Stat(path); err != nil {
+	if fi, err := os.Stat(basePath); err != nil {
 		if os.IsNotExist(err) {
-			if err = os.MkdirAll(path, 0o700); err != nil {
-				return nil, fmt.Errorf("Cannot create git directory structure: %s: %w", path, err)
+			if err = os.MkdirAll(basePath, 0o700); err != nil {
+				return nil, fmt.Errorf("Cannot create git directory structure: %s: %w", basePath, err)
 			}
 		} else {
-			return nil, fmt.Errorf("Error checking git directory strcture: %s: %w", path, err)
+			return nil, fmt.Errorf("Error checking git directory strcture: %s: %w", basePath, err)
 		}
 	} else if !fi.IsDir() {
-		return nil, fmt.Errorf("Invalid git directory structure: %s != directory", path)
+		return nil, fmt.Errorf("Invalid git directory structure: %s != directory", basePath)
 	}
 
-	return &GitHandlerGeneratorImpl{
+	return &gitHandlerGeneratorImpl{
 		path:       basePath,
-		org:        org,
-		branch:     branch,
 		git_author: gitAuthor,
 		email:      email,
 
@@ -127,27 +121,30 @@ func AllocateGitWorkTree(basePath, org, branch, gitAuthor, email string) (GitHan
 	}, nil
 }
 
-func (s *GitHandlerGeneratorImpl) CreateGitHandler(repoName string) (Git, error) {
+func (s *gitHandlerGeneratorImpl) CreateGitHandler(org, branch string) (Git, error) {
 
-	path := path.Join(s.path, repoName)
-	if err := os.Mkdir(path, 0o777); err != nil {
+	path := path.Join(s.path, org, branch)
+	if fs, err := os.Stat(s.path); err != nil || !fs.IsDir() {
+		return nil, err
+	}
+	if err := os.MkdirAll(path, 0o777); err != nil && !os.IsExist(err) {
 		return nil, err
 	}
 
-	return s.ReadExistingPath(repoName)
+	return s.ReadExistingPath(org, branch)
 }
 
-func (s *GitHandlerGeneratorImpl) ReadExistingPath(repo string) (Git, error) {
+func (s *gitHandlerGeneratorImpl) ReadExistingPath(org, branch string) (Git, error) {
 	git := &GitHandlerImpl{
 		GitCommiter: s.git_author,
 		GitEmail:    s.email,
-		GitPath:     path.Join(s.path, s.org, s.branch, repo),
+		GitPath:     path.Join(s.path, org, branch),
 	}
 
 	return git, nil
 }
 
-func (s *GitHandlerGeneratorImpl) ReleaseLock(repo string) {
+func (s *gitHandlerGeneratorImpl) ReleaseLock(repo string) {
 
 }
 
@@ -189,45 +186,26 @@ func (refs *GitReferences) addReference(id, branch string) {
 	refs.refs = append(refs.refs, GitReference{Branch: branch, Id: id})
 }
 
-var sshStyleUrl = regexp.MustCompile("^([\\w\\.]+@)?[\\w\\.]+:.*$")
-func GitClonePathFromUrl(remoteUrl string) (string, error) {
-	if sshStyleUrl.MatchString(remoteUrl) {
-		remoteUrl = "ssh://" + strings.Replace(remoteUrl, ":", "/", 1)
-	}
-	u, err := url.Parse(remoteUrl)
-	if err != nil {
-		return "", err
-	}
-
-	idx := strings.LastIndex(u.Path, "/")
-	p := strings.TrimSuffix(u.Path[idx+1:], ".git")
-	return p, nil
-}
-
-func (e *GitHandlerImpl) GitClone(remoteUrl string) (string, error) {
-	p, err := GitClonePathFromUrl(remoteUrl)
-	if err != nil {
-		return "", err
-	}
-
-	if fi, err := os.Stat(path.Join(e.GitPath, p)); os.IsNotExist(err) {
-		return p, e.GitExec("", "clone", remoteUrl, p)
+func (e *GitHandlerImpl) GitClone(repo, branch, remoteName, remoteUrl string) error {
+	if fi, err := os.Stat(path.Join(e.GitPath, repo)); os.IsNotExist(err) {
+		return e.GitExec("", "clone", remoteUrl, repo)
 	} else if err != nil || !fi.IsDir() {
-		return p, fmt.Errorf("Clone location not a directory or Stat error: %w", err)
+		return fmt.Errorf("Clone location not a directory or Stat error: %w", err)
 	}
 
-	clonedRemote:= strings.TrimSpace(e.GitExecWithOutputOrPanic(p, "remote", "get-url", "origin"))
+	clonedRemote := strings.TrimSpace(e.GitExecWithOutputOrPanic(repo, "remote", "get-url", remoteName))
 	if clonedRemote != remoteUrl {
-		return p, fmt.Errorf("Path at '%s' remote %s != %s", p, clonedRemote, remoteUrl)
+		e.GitExecOrPanic(repo, "remote", "set-url", remoteName, remoteUrl)
 	}
 
-	return p, err
+	e.GitExecOrPanic(repo, "fetch", remoteName)
+	return e.GitExec(repo, "checkout", "-B", branch, remoteName+"/"+branch)
 }
 
 func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error) {
-	id, err := e.GitExecWithOutput(gitDir, "rev-list", "-1", branchName)
+	id, err := e.GitExecWithOutput(gitDir, "show-ref", "-s", branchName)
 	if err != nil {
-		return "", fmt.Errorf("Can't find default remote branch: %s", branchName)
+		return "", fmt.Errorf("Can't find default branch: %s", branchName)
 	}
 
 	return strings.TrimSpace(id), nil

common/git_utils_test.go

@@ -29,53 +29,63 @@ import (
 	"testing"
 )
 
-func TestGitClonePath(t *testing.T) {
+func TestGitClone(t *testing.T) {
 	tests := []struct {
-		name  string
-		url   string
-		path  string
-		error string
+		name string
+
+		repo       string
+		branch     string
+		remoteName string
+		remoteUrl  string
 	}{
 		{
-			name:  "git style clone",
-			url:   "gitea@src.opensuse.org:foo/bar",
-			path:  "bar",
-			error: "",
+			name:       "Basic clone",
+			repo:       "pkgAclone",
+			branch:     "main",
+			remoteName: "pkgA_main",
+			remoteUrl:  "/pkgA",
 		},
 		{
-			name:  "https:// style clone",
-			url:   "https://gitea@foo.bar:900/org/repo.git",
-			path:  "repo",
-			error: "",
-		},
-		{
-			name:  "ssh:// style clone",
-			url:   "ssh://foo.bar/org/repo",
-			path:  "repo",
-			error: "",
-		},
-		{
-			name:  "broken git style clone",
-			url:   "gi-tea@src.opensuse.org:foo/bar",
-			path:  "",
-			error: "first path segment in URL cannot contain colon",
+			name:       "Remote branch is non-existent",
+			repo:       "pkgAclone",
+			branch:     "main_not_here",
+			remoteName: "pkgA_main",
+			remoteUrl:  "/pkgA",
 		},
 	}
 
+	execPath, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	d := t.TempDir()
+	os.Chdir(d)
+	cmd := exec.Command("/usr/bin/bash", path.Join(execPath, "test_clone_setup.sh"))
+	if _, err := cmd.Output(); err != nil {
+		t.Fatal(err)
+	}
+
+	gh, err := AllocateGitWorkTree(d, "Test", "test@example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			p, e := GitClonePathFromUrl(test.url)
-			if e != nil {
-				if test.error == "" || !strings.Contains(e.Error(), test.error) {
-					t.Errorf("Expected .Contains('%s') but got '%s'", test.error, e.Error())
-				}
+			g, err := gh.CreateGitHandler("org", test.branch)
+			if err != nil {
+				t.Fatal(err)
 			}
-			if test.error != "" && e == nil {
-				t.Fatal("expected error, but not none")
+
+			if err := g.GitClone(test.repo, test.branch, test.remoteName, "file://"+d+test.remoteUrl); err != nil {
+				t.Fatal(err)
 			}
-			if e == nil && p != test.path {
-				t.Error("Expected target path:", test.path, ", but got:", p)
+
+			id, err := g.GitBranchHead(test.repo, test.branch)
+			if err != nil {
+				t.Fatal(err)
 			}
+			t.Fatal(id)
 		})
 	}
 }

common/pr.go

@@ -204,14 +204,13 @@ func (rs *PRSet) IsApproved(gitea GiteaPRChecker, maintainers MaintainershipData
 	return is_reviewed
 }
 
-func (rs *PRSet) Merge(author, email string) error {
+func (rs *PRSet) Merge(gh GitHandlerGenerator) error {
 	prjgit, err := rs.GetPrjGitPR()
 	if err != nil {
 		return err
 	}
 
-	gh := GitHandlerGeneratorImpl{}
-	git, err := gh.CreateGitHandler(prjgit.Base.Name)
+	git, err := gh.CreateGitHandler(rs.Config.Organization, rs.Config.Branch)
 	if err != nil {
 		return err
 	}

common/pr_test.go

@@ -546,7 +546,8 @@ func TestPRMerge(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if err = set.Merge("test", "test@example.com"); err != nil && (test.mergeError == "" || (len(test.mergeError) > 0 && !strings.Contains(err.Error(), test.mergeError))) {
+			gh, _ := common.AllocateGitWorkTree("", "", "")
+			if err = set.Merge(gh); err != nil && (test.mergeError == "" || (len(test.mergeError) > 0 && !strings.Contains(err.Error(), test.mergeError))) {
 				t.Fatal(err)
 			}
 		})

common/utils.go

@@ -24,6 +24,10 @@ import (
 	"strings"
 )
 
+func SplitLines(str string) []string {
+	return SplitStringNoEmpty(str, "\n")
+}
+
 func SplitStringNoEmpty(str, sep string) []string {
 	ret := slices.DeleteFunc(strings.Split(str, sep), func(s string) bool {
 		return len(strings.TrimSpace(s)) == 0

workflow-pr/main.go

@@ -57,13 +57,6 @@ var PRID int64
 var CurrentUser *models.User
 
 func main() {
-	if err := common.RequireGiteaSecretToken(); err != nil {
-		log.Fatal(err)
-	}
-	if err := common.RequireRabbitSecrets(); err != nil {
-		log.Fatal(err)
-	}
-
 	workflowConfig := flag.String("config", "", "Repository and workflow definition file")
 	giteaUrl := flag.String("gitea-url", "https://src.opensuse.org", "Gitea instance")
 	rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
@@ -73,8 +66,16 @@ func main() {
 	flag.BoolVar(&ListPROnly, "list-prs-only", false, "Only lists PRs without acting on them")
 	flag.Int64Var(&PRID, "id", -1, "Process only the specific ID and ignore the rest. Use for debugging")
 
+	gitPath := flag.String("git-path", "", "Path to work location. Using temp path by default")
 	flag.Parse()
 
+	if err := common.RequireGiteaSecretToken(); err != nil {
+		log.Fatal(err)
+	}
+	if err := common.RequireRabbitSecrets(); err != nil {
+		log.Fatal(err)
+	}
+
 	if len(*workflowConfig) == 0 {
 		log.Fatalln("No configuratio file specified. Aborting")
 	}
@@ -93,7 +94,11 @@ func main() {
 	req := new(RequestProcessor)
 
 	req.configuredRepos = make(map[string][]*common.AutogitConfig)
-	req.git = &common.GitHandlerGeneratorImpl{}
+	req.git, err = common.AllocateGitWorkTree(*gitPath, GitAuthor, GitEmail)
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	orgs := make([]string, 0, 1)
 	for _, c := range configs {
 		if slices.Contains(c.Workflows, "pr") {

workflow-pr/pr_processor.go

@@ -44,7 +44,7 @@ func (w *RequestProcessor) ProcessFunc(request *common.Request) error {
 		return fmt.Errorf("Cannot find config for branch '%s'", req.Pull_Request.Base.Ref)
 	}
 
-	git, err := w.git.CreateGitHandler(GitAuthor, GitEmail, AppName)
+	git, err := w.git.CreateGitHandler(config.Organization, config.Branch)
 	if err != nil {
 		return fmt.Errorf("Error allocating GitHandler. Err: %w", err)
 	}

workflow-pr/pr_processor_opened.go

@@ -23,8 +23,9 @@ func (o *PullRequestOpened) CreateOrUpdatePrjGitPR(req *common.PullRequestWebhoo
 	}
 	log.Println("Done Creating repo", org, prj)
 
-	common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGit.SSHURL, common.DefaultGitPrj))
-	err = git.GitExec(common.DefaultGitPrj, "fetch", "origin", branchName+":"+branchName)
+	remoteName := config.GetRemoteBranch()
+	git.GitClone(common.DefaultGitPrj, config.Branch, remoteName, prjGit.SSHURL)
+	err = git.GitExec(common.DefaultGitPrj, "fetch", remoteName, branchName+":"+branchName)
 	if err != nil {
 		common.PanicOnError(git.GitExec(common.DefaultGitPrj, "checkout", "-B", branchName, prjGit.DefaultBranch))
 	} else {
@@ -50,7 +51,7 @@ referencing
 	if id := subList[req.Repository.Name]; id != req.Pull_Request.Head.Sha {
 		updateSubmoduleInPR(req, git)
 		common.PanicOnError(git.GitExec(common.DefaultGitPrj, "commit", "-a", "-m", commitMsg))
-		common.PanicOnError(git.GitExec(common.DefaultGitPrj, "push", "origin", "+HEAD:"+branchName))
+		common.PanicOnError(git.GitExec(common.DefaultGitPrj, "push", remoteName, "+HEAD:"+branchName))
 	}
 
 	_, err = o.gitea.CreatePullRequestIfNotExist(prjGit, branchName, prjGit.DefaultBranch,

workflow-pr/pr_processor_reviewed.go

@@ -18,7 +18,7 @@ func (o *PullRequestReviewed) Process(req *common.PullRequestWebhookEvent, git c
 	}
 
 	if prset.IsApproved(o.gitea, maintainers) {
-		prset.Merge(GitAuthor, GitEmail)
+		prset.Merge(git)
 	}
 
 	return nil

workflow-pr/repo_check.go

@@ -55,7 +55,7 @@ func (s *DefaultStateChecker) ProcessPR(pr *models.PullRequest, config *common.A
 	event.Sender = *common.UserFromModel(pr.User)
 	event.Requested_reviewer = nil
 
-	git, err := s.git.CreateGitHandler(GitAuthor, GitEmail, AppName)
+	git, err := s.git.CreateGitHandler(config.Organization, config.Branch)
 	if err != nil {
 		return fmt.Errorf("Error allocating GitHandler. Err: %w", err)
 	}
@@ -89,13 +89,14 @@ func (s *DefaultStateChecker) VerifyProjectState(config *common.AutogitConfig) e
 		}
 	}()
 
-	git, err := s.git.CreateGitHandler(GitAuthor, GitEmail, AppName)
+	prjGitOrg, prjGitRepo, prjGitBranch := config.GetPrjGit()
+	log.Printf("  checking %s/%s#%s", prjGitOrg, prjGitRepo, prjGitBranch)
+
+	git, err := s.git.CreateGitHandler(prjGitOrg, prjGitBranch)
 	if err != nil {
 		return fmt.Errorf("Cannot create git handler: %w", err)
 	}
 
-	prjGitOrg, prjGitRepo, prjGitBranch := config.GetPrjGit()
-	log.Printf("  checking %s/%s#%s", prjGitOrg, prjGitRepo, prjGitBranch)
 	repo, err := s.gitea.CreateRepositoryIfNotExist(git, prjGitOrg, prjGitRepo)
 	if err != nil {
 		return fmt.Errorf("Error fetching or creating '%s/%s#%s' -- aborting verifyProjectState(). Err: %w", prjGitBranch, prjGitRepo, prjGitBranch, err)