From 50c642250a1d65d864dd3eeee639bfdafbc83a1d9f047a256b91e34f9a931191 Mon Sep 17 00:00:00 2001 From: Adam Majer Date: Mon, 2 Sep 2024 17:27:23 +0200 Subject: [PATCH] . --- bots-common/git_utils.go | 25 ++++++++--- bots-common/gitea_utils.go | 33 +++++++++++++- bots-common/request_handler.go | 13 ++---- prjgit-updater/main.go | 78 ++++++++++++++++++++++++++-------- 4 files changed, 115 insertions(+), 34 deletions(-) diff --git a/bots-common/git_utils.go b/bots-common/git_utils.go index fc834e8..33a7e4e 100644 --- a/bots-common/git_utils.go +++ b/bots-common/git_utils.go @@ -19,6 +19,23 @@ type GitHandler struct { GitCommiter string } +func CreateGitHandler(git_author, name string) (*GitHandler, error) { + var err error + + git := new(GitHandler) + git.GitCommiter = git_author + git.GitPath, err = os.MkdirTemp("", name) + if err != nil { + return nil, fmt.Errorf("Cannot create temp dir: %w", err) + } + + if err = os.Chmod(git.GitPath, 0700); err != nil { + return nil, fmt.Errorf("Cannot fix permissions of temp dir: %w", err) + } + + return git, nil +} + //func (h *GitHandler) ProcessBranchList() []string { // if h.HasError() { // return make([]string, 0) @@ -147,12 +164,6 @@ func (e *GitHandler) GitBranchHead(gitDir, branchName string) (string, error) { return "", fmt.Errorf("Can't find default remote branch: %s", branchName) } -type ExecStream interface { - Close() - HasError() bool - GitExec(cwd string, param ...string) ExecStream -} - func (e *GitHandler) Close() error { if err := os.RemoveAll(e.GitPath); err != nil { return err @@ -546,6 +557,8 @@ func (e *GitHandler) GitCatFile(cwd, commitId, filename string) (data []byte, er return } +// return (filename) -> (hash) map for all submodules +// TODO: recursive? map different orgs, not just assume '.' for path func (e *GitHandler) GitSubmoduleList(cwd, commitId string) (submoduleList map[string]string, err error) { var done sync.Mutex submoduleList = make(map[string]string) diff --git a/bots-common/gitea_utils.go b/bots-common/gitea_utils.go index 8e89b77..402a950 100644 --- a/bots-common/gitea_utils.go +++ b/bots-common/gitea_utils.go @@ -121,7 +121,18 @@ func (gitea *GiteaTransport) SetNotificationRead(notificationId int64) error { return nil } -func (gitea *GiteaTransport) CreateRepositoryIfNotExist(git GitHandler, org Organization, repoName string) (*models.Repository, error) { +func (gitea *GiteaTransport) GetOrganization(orgName string) (*models.Organization, error) { + org, err := gitea.client.Organization.OrgGet( + organization.NewOrgGetParams().WithOrg(orgName), + gitea.transport.DefaultAuthentication, + ) + if err != nil { + return nil, fmt.Errorf("Error fetching org: '%s' data. Err: %w", orgName, err) + } + return org.Payload, nil +} + +func (gitea *GiteaTransport) CreateRepositoryIfNotExist(git *GitHandler, org Organization, repoName string) (*models.Repository, error) { repo, err := gitea.client.Repository.RepoGet( repository.NewRepoGetParams().WithDefaults().WithOwner(org.Username).WithRepo(repoName), gitea.transport.DefaultAuthentication) @@ -349,7 +360,7 @@ func (gitea *GiteaTransport) GetAssociatedPrjGitPR(pr *PullRequestWebhookEvent) } prLine := fmt.Sprintf(PrPattern, pr.Repository.Owner.Username, pr.Repository.Name, pr.Number) -// h.StdLogger.Printf("attemping to match line: '%s'\n", prLine) + // h.StdLogger.Printf("attemping to match line: '%s'\n", prLine) // payload_processing: for _, pr := range prs.Payload { @@ -396,3 +407,21 @@ func (gitea *GiteaTransport) GetRepositoryFileContent(repo *models.Repository, h func (gitea *GiteaTransport) GetPullRequestFileContent(pr *models.PullRequest, path string) ([]byte, error) { return gitea.GetRepositoryFileContent(pr.Head.Repo, pr.Head.Sha, path) } + +func (gitea *GiteaTransport) GetRecentCommits(org, repo, branch string, commitNo int64) ([]*models.Commit, error) { + not := false + commits, err := gitea.client.Repository.RepoGetAllCommits( + repository.NewRepoGetAllCommitsParams(). + WithStat(¬). + WithFiles(¬). + WithVerification(¬). + WithLimit(&commitNo), + gitea.transport.DefaultAuthentication, + ) + + if err != nil { + return nil, err + } + + return commits.Payload, nil +} diff --git a/bots-common/request_handler.go b/bots-common/request_handler.go index 16c152d..c64573b 100644 --- a/bots-common/request_handler.go +++ b/bots-common/request_handler.go @@ -74,27 +74,22 @@ type RequestHandler struct { StdLogger, ErrLogger *log.Logger Request *Request - Git GitHandler + Git *GitHandler } func (r *RequestHandler) WriteError() { r.ErrLogger.Println("internal error sent") } - func CreateRequestHandler(git_author, name string) (*RequestHandler, error) { +func CreateRequestHandler(git_author, name string) (*RequestHandler, error) { var h *RequestHandler = new(RequestHandler) h.StdLogger, h.ErrLogger = CreateStdoutLogger(os.Stdout, os.Stderr) var err error - h.Git.GitCommiter = git_author - h.Git.GitPath, err = os.MkdirTemp("", name) + h.Git, err = CreateGitHandler(git_author, name) if err != nil { - return nil, fmt.Errorf("Cannot create temp dir: %w", err) - } - - if err = os.Chmod(h.Git.GitPath, 0700); err != nil { - return nil, fmt.Errorf("Cannot fix permissions of temp dir: %w", err) + return nil, err } return h, nil } diff --git a/prjgit-updater/main.go b/prjgit-updater/main.go index 5bed583..05b4d82 100644 --- a/prjgit-updater/main.go +++ b/prjgit-updater/main.go @@ -14,12 +14,7 @@ import ( "src.opensuse.org/autogits/common" ) -type ConfigRepos struct { - prjgit string - branch string // "" == default branch -} - -var configuredRepos map[string]ConfigRepos +var configuredRepos map[string]*common.AutogitConfig var gitea *common.GiteaTransport func isConfiguredOrg(org *common.Organization) bool { @@ -38,7 +33,7 @@ func processRepositoryAction(h *common.RequestHandler) error { return nil } - prjgit := config.prjgit + prjgit := config.GitProjectName if action.Repository.Name == prjgit { if h.Git.DebugLogger { h.StdLogger.Printf("repository event %s for PrjGit '%s'. Ignoring\n", common.DefaultGitPrj, action.Action) @@ -87,7 +82,7 @@ func processPushAction(h *common.RequestHandler) error { return nil } - prjgit := config.prjgit + prjgit := config.GitProjectName if action.Repository.Name == prjgit { if h.Git.DebugLogger { h.StdLogger.Printf("push to %s -- ignoring\n", prjgit) @@ -122,14 +117,66 @@ func processPushAction(h *common.RequestHandler) error { return nil } -func verifyProjectState(org string, config ConfigRepos) error { +func verifyProjectState(git *common.GitHandler, orgName string, config *common.AutogitConfig) error { + org := common.Organization{ + Username: orgName, + } + repo, err := gitea.CreateRepositoryIfNotExist(git, org, config.GitProjectName) + if err != nil { + return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", orgName, config.GitProjectName, err) + } + + if err := git.GitExec("", "clone", "--depth", "1", repo.SSHURL, config.GitProjectName); err != nil { + return fmt.Errorf("Error closing projectgit for %s, Err: %w", config.GitProjectName, err) + } + + sub, err := git.GitSubmoduleList(config.GitProjectName, "HEAD") + if err != nil { + return fmt.Errorf("Failed to fetch submodule list... Err: %w", err) + } + + isGitUpdated := false + for filename, commitId := range sub { + commits, err := gitea.GetRecentCommits(orgName, filename, config.Branch, 10) + if err != nil { + return fmt.Errorf("Failed to fetch recent commits for package: '%s'. Err: %w", filename, err) + } + + idx := 1000 + for i, c := range commits { + if c.SHA == commitId { + idx = i + break + } + } + + if idx == 0 { + // up-to-date + continue + } else if idx < len(commits) { // update + git.GitExec(config.GitProjectName, "submodule", "update", "--init", "--depth", "1", "--checkout", filename) + git.GitExec(filepath.Join(config.GitProjectName, filename), "fetch", "--depth", "1", "origin", commits[0].SHA) + git.GitExec(filepath.Join(config.GitProjectName, filename), "checkout", commits[0].SHA) + isGitUpdated = true + } else { + // probably need `merge-base` or `rev-list` here instead, or the project updated already + return fmt.Errorf("Cannot find SHA of last matching update for package: '%s'. idx: %d", filename, idx) + } + } + + if isGitUpdated { + git.GitExec(common.DefaultGitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC") + git.GitExec(common.DefaultGitPrj, "push") + // commit changes and push + } + return nil } var checkOnStart bool var checkInterval time.Duration -func consistencyCheckProcess(repos map[string]ConfigRepos) { +func consistencyCheckProcess(git *common.GitHandler, repos map[string]*common.AutogitConfig) { if checkOnStart { var wg sync.WaitGroup for org, conf := range repos { @@ -138,7 +185,7 @@ func consistencyCheckProcess(repos map[string]ConfigRepos) { go func() { defer wg.Done() - verifyProjectState(org, conf) + verifyProjectState(git, org, conf) }() } wg.Wait() @@ -149,7 +196,7 @@ func consistencyCheckProcess(repos map[string]ConfigRepos) { time.Sleep(checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))) log.Printf(" ++ starting verification, org: `%s`\n", org) - if err := verifyProjectState(org, conf); err != nil { + if err := verifyProjectState(git, org, conf); err != nil { log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err) } log.Printf(" ++ verification complete, org: `%s`\n", org) @@ -178,17 +225,14 @@ func main() { log.Fatalf("Error reading config file. err: %v", err) } - configuredRepos = make(map[string]ConfigRepos) + configuredRepos = make(map[string]*common.AutogitConfig) orgs := make([]string, 0, 10) for _, c := range configs { if slices.Contains(c.Workflows, "push") { if debugMode { log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName) } - configuredRepos[c.Organization] = ConfigRepos{ - prjgit: c.GitProjectName, - branch: c.Branch, - } + configuredRepos[c.Organization] = c orgs = append(orgs, c.Organization) } }