diff --git a/common/config.go b/common/config.go index 99d460d..dfa71e6 100644 --- a/common/config.go +++ b/common/config.go @@ -177,3 +177,7 @@ func (config *AutogitConfig) GetPrjGit() (string, string, string) { return org, repo, branch } + +func (config *AutogitConfig) GetRemoteBranch() string { + return "origin_" + config.Branch +} diff --git a/common/git_utils.go b/common/git_utils.go index cc26f98..2f577b2 100644 --- a/common/git_utils.go +++ b/common/git_utils.go @@ -25,12 +25,10 @@ import ( "fmt" "io" "log" - "net/url" "os" "os/exec" "path" "path/filepath" - "regexp" "strings" "sync" ) @@ -48,7 +46,7 @@ type GitStatusLister interface { type Git interface { // error if git, but wrong remote - GitClone(remoteUrl string) (string, error) // clone, or chekc if path is already checked out remote and pulls, error otherwise. Return relative path + GitClone(repo, branch, remoteName, remoteUrl string) error // clone, or chekc if path is already checked out remote and force pulls, error otherwise. Return relative path GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error) GitCatFile(cwd, commitId, filename string) (data []byte, err error) @@ -81,45 +79,41 @@ func (s *GitHandlerImpl) GetPath() string { } type GitHandlerGenerator interface { - CreateGitHandler(repo string) (Git, error) - ReadExistingPath(repo string) (Git, error) + CreateGitHandler(org, branch string) (Git, error) + ReadExistingPath(org, branch string) (Git, error) ReleaseLock(path string) } -type GitHandlerGeneratorImpl struct { +type gitHandlerGeneratorImpl struct { // TODO: add mutex to lock paths so only one workflow per path and others wait - path string - org, branch string - git_author string - email string + path string + git_author string + email string lock_lock sync.Mutex lock map[string]sync.Mutex } -func AllocateGitWorkTree(basePath, org, branch, gitAuthor, email string) (GitHandlerGenerator, error) { +func AllocateGitWorkTree(basePath, gitAuthor, email string) (GitHandlerGenerator, error) { if fi, err := os.Stat(basePath); err != nil || !fi.IsDir() { return nil, fmt.Errorf("Git basepath not a valid directory: %s %w", basePath, err) } - path := path.Join(basePath, org, branch) - if fi, err := os.Stat(path); err != nil { + if fi, err := os.Stat(basePath); err != nil { if os.IsNotExist(err) { - if err = os.MkdirAll(path, 0o700); err != nil { - return nil, fmt.Errorf("Cannot create git directory structure: %s: %w", path, err) + if err = os.MkdirAll(basePath, 0o700); err != nil { + return nil, fmt.Errorf("Cannot create git directory structure: %s: %w", basePath, err) } } else { - return nil, fmt.Errorf("Error checking git directory strcture: %s: %w", path, err) + return nil, fmt.Errorf("Error checking git directory strcture: %s: %w", basePath, err) } } else if !fi.IsDir() { - return nil, fmt.Errorf("Invalid git directory structure: %s != directory", path) + return nil, fmt.Errorf("Invalid git directory structure: %s != directory", basePath) } - return &GitHandlerGeneratorImpl{ + return &gitHandlerGeneratorImpl{ path: basePath, - org: org, - branch: branch, git_author: gitAuthor, email: email, @@ -127,27 +121,30 @@ func AllocateGitWorkTree(basePath, org, branch, gitAuthor, email string) (GitHan }, nil } -func (s *GitHandlerGeneratorImpl) CreateGitHandler(repoName string) (Git, error) { +func (s *gitHandlerGeneratorImpl) CreateGitHandler(org, branch string) (Git, error) { - path := path.Join(s.path, repoName) - if err := os.Mkdir(path, 0o777); err != nil { + path := path.Join(s.path, org, branch) + if fs, err := os.Stat(s.path); err != nil || !fs.IsDir() { + return nil, err + } + if err := os.MkdirAll(path, 0o777); err != nil && !os.IsExist(err) { return nil, err } - return s.ReadExistingPath(repoName) + return s.ReadExistingPath(org, branch) } -func (s *GitHandlerGeneratorImpl) ReadExistingPath(repo string) (Git, error) { +func (s *gitHandlerGeneratorImpl) ReadExistingPath(org, branch string) (Git, error) { git := &GitHandlerImpl{ GitCommiter: s.git_author, GitEmail: s.email, - GitPath: path.Join(s.path, s.org, s.branch, repo), + GitPath: path.Join(s.path, org, branch), } return git, nil } -func (s *GitHandlerGeneratorImpl) ReleaseLock(repo string) { +func (s *gitHandlerGeneratorImpl) ReleaseLock(repo string) { } @@ -189,45 +186,26 @@ func (refs *GitReferences) addReference(id, branch string) { refs.refs = append(refs.refs, GitReference{Branch: branch, Id: id}) } -var sshStyleUrl = regexp.MustCompile("^([\\w\\.]+@)?[\\w\\.]+:.*$") -func GitClonePathFromUrl(remoteUrl string) (string, error) { - if sshStyleUrl.MatchString(remoteUrl) { - remoteUrl = "ssh://" + strings.Replace(remoteUrl, ":", "/", 1) - } - u, err := url.Parse(remoteUrl) - if err != nil { - return "", err - } - - idx := strings.LastIndex(u.Path, "/") - p := strings.TrimSuffix(u.Path[idx+1:], ".git") - return p, nil -} - -func (e *GitHandlerImpl) GitClone(remoteUrl string) (string, error) { - p, err := GitClonePathFromUrl(remoteUrl) - if err != nil { - return "", err - } - - if fi, err := os.Stat(path.Join(e.GitPath, p)); os.IsNotExist(err) { - return p, e.GitExec("", "clone", remoteUrl, p) +func (e *GitHandlerImpl) GitClone(repo, branch, remoteName, remoteUrl string) error { + if fi, err := os.Stat(path.Join(e.GitPath, repo)); os.IsNotExist(err) { + return e.GitExec("", "clone", remoteUrl, repo) } else if err != nil || !fi.IsDir() { - return p, fmt.Errorf("Clone location not a directory or Stat error: %w", err) + return fmt.Errorf("Clone location not a directory or Stat error: %w", err) } - clonedRemote:= strings.TrimSpace(e.GitExecWithOutputOrPanic(p, "remote", "get-url", "origin")) + clonedRemote := strings.TrimSpace(e.GitExecWithOutputOrPanic(repo, "remote", "get-url", remoteName)) if clonedRemote != remoteUrl { - return p, fmt.Errorf("Path at '%s' remote %s != %s", p, clonedRemote, remoteUrl) + e.GitExecOrPanic(repo, "remote", "set-url", remoteName, remoteUrl) } - return p, err + e.GitExecOrPanic(repo, "fetch", remoteName) + return e.GitExec(repo, "checkout", "-B", branch, remoteName+"/"+branch) } func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error) { - id, err := e.GitExecWithOutput(gitDir, "rev-list", "-1", branchName) + id, err := e.GitExecWithOutput(gitDir, "show-ref", "-s", branchName) if err != nil { - return "", fmt.Errorf("Can't find default remote branch: %s", branchName) + return "", fmt.Errorf("Can't find default branch: %s", branchName) } return strings.TrimSpace(id), nil diff --git a/common/git_utils_test.go b/common/git_utils_test.go index 014a74f..30a602f 100644 --- a/common/git_utils_test.go +++ b/common/git_utils_test.go @@ -29,53 +29,63 @@ import ( "testing" ) -func TestGitClonePath(t *testing.T) { +func TestGitClone(t *testing.T) { tests := []struct { - name string - url string - path string - error string + name string + + repo string + branch string + remoteName string + remoteUrl string }{ { - name: "git style clone", - url: "gitea@src.opensuse.org:foo/bar", - path: "bar", - error: "", + name: "Basic clone", + repo: "pkgAclone", + branch: "main", + remoteName: "pkgA_main", + remoteUrl: "/pkgA", }, { - name: "https:// style clone", - url: "https://gitea@foo.bar:900/org/repo.git", - path: "repo", - error: "", - }, - { - name: "ssh:// style clone", - url: "ssh://foo.bar/org/repo", - path: "repo", - error: "", - }, - { - name: "broken git style clone", - url: "gi-tea@src.opensuse.org:foo/bar", - path: "", - error: "first path segment in URL cannot contain colon", + name: "Remote branch is non-existent", + repo: "pkgAclone", + branch: "main_not_here", + remoteName: "pkgA_main", + remoteUrl: "/pkgA", }, } + execPath, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + d := t.TempDir() + os.Chdir(d) + cmd := exec.Command("/usr/bin/bash", path.Join(execPath, "test_clone_setup.sh")) + if _, err := cmd.Output(); err != nil { + t.Fatal(err) + } + + gh, err := AllocateGitWorkTree(d, "Test", "test@example.com") + if err != nil { + t.Fatal(err) + } + for _, test := range tests { t.Run(test.name, func(t *testing.T) { - p, e := GitClonePathFromUrl(test.url) - if e != nil { - if test.error == "" || !strings.Contains(e.Error(), test.error) { - t.Errorf("Expected .Contains('%s') but got '%s'", test.error, e.Error()) - } + g, err := gh.CreateGitHandler("org", test.branch) + if err != nil { + t.Fatal(err) } - if test.error != "" && e == nil { - t.Fatal("expected error, but not none") + + if err := g.GitClone(test.repo, test.branch, test.remoteName, "file://"+d+test.remoteUrl); err != nil { + t.Fatal(err) } - if e == nil && p != test.path { - t.Error("Expected target path:", test.path, ", but got:", p) + + id, err := g.GitBranchHead(test.repo, test.branch) + if err != nil { + t.Fatal(err) } + t.Fatal(id) }) } } diff --git a/common/pr.go b/common/pr.go index db97bed..adb0f71 100644 --- a/common/pr.go +++ b/common/pr.go @@ -204,14 +204,13 @@ func (rs *PRSet) IsApproved(gitea GiteaPRChecker, maintainers MaintainershipData return is_reviewed } -func (rs *PRSet) Merge(author, email string) error { +func (rs *PRSet) Merge(gh GitHandlerGenerator) error { prjgit, err := rs.GetPrjGitPR() if err != nil { return err } - gh := GitHandlerGeneratorImpl{} - git, err := gh.CreateGitHandler(prjgit.Base.Name) + git, err := gh.CreateGitHandler(rs.Config.Organization, rs.Config.Branch) if err != nil { return err } diff --git a/common/pr_test.go b/common/pr_test.go index 67ce695..edce4e5 100644 --- a/common/pr_test.go +++ b/common/pr_test.go @@ -546,7 +546,8 @@ func TestPRMerge(t *testing.T) { t.Fatal(err) } - if err = set.Merge("test", "test@example.com"); err != nil && (test.mergeError == "" || (len(test.mergeError) > 0 && !strings.Contains(err.Error(), test.mergeError))) { + gh, _ := common.AllocateGitWorkTree("", "", "") + if err = set.Merge(gh); err != nil && (test.mergeError == "" || (len(test.mergeError) > 0 && !strings.Contains(err.Error(), test.mergeError))) { t.Fatal(err) } }) diff --git a/common/utils.go b/common/utils.go index 147a49e..a937a18 100644 --- a/common/utils.go +++ b/common/utils.go @@ -24,6 +24,10 @@ import ( "strings" ) +func SplitLines(str string) []string { + return SplitStringNoEmpty(str, "\n") +} + func SplitStringNoEmpty(str, sep string) []string { ret := slices.DeleteFunc(strings.Split(str, sep), func(s string) bool { return len(strings.TrimSpace(s)) == 0 diff --git a/workflow-pr/main.go b/workflow-pr/main.go index 4071ed1..332ad6b 100644 --- a/workflow-pr/main.go +++ b/workflow-pr/main.go @@ -57,13 +57,6 @@ var PRID int64 var CurrentUser *models.User func main() { - if err := common.RequireGiteaSecretToken(); err != nil { - log.Fatal(err) - } - if err := common.RequireRabbitSecrets(); err != nil { - log.Fatal(err) - } - workflowConfig := flag.String("config", "", "Repository and workflow definition file") giteaUrl := flag.String("gitea-url", "https://src.opensuse.org", "Gitea instance") rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance") @@ -73,8 +66,16 @@ func main() { flag.BoolVar(&ListPROnly, "list-prs-only", false, "Only lists PRs without acting on them") flag.Int64Var(&PRID, "id", -1, "Process only the specific ID and ignore the rest. Use for debugging") + gitPath := flag.String("git-path", "", "Path to work location. Using temp path by default") flag.Parse() + if err := common.RequireGiteaSecretToken(); err != nil { + log.Fatal(err) + } + if err := common.RequireRabbitSecrets(); err != nil { + log.Fatal(err) + } + if len(*workflowConfig) == 0 { log.Fatalln("No configuratio file specified. Aborting") } @@ -85,7 +86,7 @@ func main() { log.Fatal(err) } - configs, err := common.ResolveWorkflowConfigs(gitea, config) + configs, err := common.ResolveWorkflowConfigs(gitea, config) if err != nil { log.Fatal(err) } @@ -93,7 +94,11 @@ func main() { req := new(RequestProcessor) req.configuredRepos = make(map[string][]*common.AutogitConfig) - req.git = &common.GitHandlerGeneratorImpl{} + req.git, err = common.AllocateGitWorkTree(*gitPath, GitAuthor, GitEmail) + if err != nil { + log.Fatal(err) + } + orgs := make([]string, 0, 1) for _, c := range configs { if slices.Contains(c.Workflows, "pr") { diff --git a/workflow-pr/pr_processor.go b/workflow-pr/pr_processor.go index dccb926..fa8d507 100644 --- a/workflow-pr/pr_processor.go +++ b/workflow-pr/pr_processor.go @@ -44,7 +44,7 @@ func (w *RequestProcessor) ProcessFunc(request *common.Request) error { return fmt.Errorf("Cannot find config for branch '%s'", req.Pull_Request.Base.Ref) } - git, err := w.git.CreateGitHandler(GitAuthor, GitEmail, AppName) + git, err := w.git.CreateGitHandler(config.Organization, config.Branch) if err != nil { return fmt.Errorf("Error allocating GitHandler. Err: %w", err) } diff --git a/workflow-pr/pr_processor_opened.go b/workflow-pr/pr_processor_opened.go index 2b7f74b..14d4cb9 100644 --- a/workflow-pr/pr_processor_opened.go +++ b/workflow-pr/pr_processor_opened.go @@ -23,8 +23,9 @@ func (o *PullRequestOpened) CreateOrUpdatePrjGitPR(req *common.PullRequestWebhoo } log.Println("Done Creating repo", org, prj) - common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGit.SSHURL, common.DefaultGitPrj)) - err = git.GitExec(common.DefaultGitPrj, "fetch", "origin", branchName+":"+branchName) + remoteName := config.GetRemoteBranch() + git.GitClone(common.DefaultGitPrj, config.Branch, remoteName, prjGit.SSHURL) + err = git.GitExec(common.DefaultGitPrj, "fetch", remoteName, branchName+":"+branchName) if err != nil { common.PanicOnError(git.GitExec(common.DefaultGitPrj, "checkout", "-B", branchName, prjGit.DefaultBranch)) } else { @@ -50,7 +51,7 @@ referencing if id := subList[req.Repository.Name]; id != req.Pull_Request.Head.Sha { updateSubmoduleInPR(req, git) common.PanicOnError(git.GitExec(common.DefaultGitPrj, "commit", "-a", "-m", commitMsg)) - common.PanicOnError(git.GitExec(common.DefaultGitPrj, "push", "origin", "+HEAD:"+branchName)) + common.PanicOnError(git.GitExec(common.DefaultGitPrj, "push", remoteName, "+HEAD:"+branchName)) } _, err = o.gitea.CreatePullRequestIfNotExist(prjGit, branchName, prjGit.DefaultBranch, diff --git a/workflow-pr/pr_processor_reviewed.go b/workflow-pr/pr_processor_reviewed.go index ec4442b..268bd42 100644 --- a/workflow-pr/pr_processor_reviewed.go +++ b/workflow-pr/pr_processor_reviewed.go @@ -18,7 +18,7 @@ func (o *PullRequestReviewed) Process(req *common.PullRequestWebhookEvent, git c } if prset.IsApproved(o.gitea, maintainers) { - prset.Merge(GitAuthor, GitEmail) + prset.Merge(git) } return nil diff --git a/workflow-pr/repo_check.go b/workflow-pr/repo_check.go index 7b0442a..d417676 100644 --- a/workflow-pr/repo_check.go +++ b/workflow-pr/repo_check.go @@ -55,7 +55,7 @@ func (s *DefaultStateChecker) ProcessPR(pr *models.PullRequest, config *common.A event.Sender = *common.UserFromModel(pr.User) event.Requested_reviewer = nil - git, err := s.git.CreateGitHandler(GitAuthor, GitEmail, AppName) + git, err := s.git.CreateGitHandler(config.Organization, config.Branch) if err != nil { return fmt.Errorf("Error allocating GitHandler. Err: %w", err) } @@ -89,13 +89,14 @@ func (s *DefaultStateChecker) VerifyProjectState(config *common.AutogitConfig) e } }() - git, err := s.git.CreateGitHandler(GitAuthor, GitEmail, AppName) + prjGitOrg, prjGitRepo, prjGitBranch := config.GetPrjGit() + log.Printf(" checking %s/%s#%s", prjGitOrg, prjGitRepo, prjGitBranch) + + git, err := s.git.CreateGitHandler(prjGitOrg, prjGitBranch) if err != nil { return fmt.Errorf("Cannot create git handler: %w", err) } - prjGitOrg, prjGitRepo, prjGitBranch := config.GetPrjGit() - log.Printf(" checking %s/%s#%s", prjGitOrg, prjGitRepo, prjGitBranch) repo, err := s.gitea.CreateRepositoryIfNotExist(git, prjGitOrg, prjGitRepo) if err != nil { return fmt.Errorf("Error fetching or creating '%s/%s#%s' -- aborting verifyProjectState(). Err: %w", prjGitBranch, prjGitRepo, prjGitBranch, err)