package main /* * This file is part of Autogits. * * Copyright © 2024 SUSE LLC * * Autogits is free software: you can redistribute it and/or modify it under * the terms of the GNU General Public License as published by the Free Software * Foundation, either version 2 of the License, or (at your option) any later * version. * * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A * PARTICULAR PURPOSE. See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along with * Foobar. If not, see . */ import ( "flag" "fmt" "io/fs" "log" "math/rand" "net/url" "os" "os/signal" "path" "path/filepath" "slices" "strings" "syscall" "time" "src.opensuse.org/autogits/common" ) const ( AppName = "direct_workflow" GitAuthor = "AutoGits prjgit-updater" GitEmail = "adam+autogits-direct@zombino.com" ) var configuredRepos map[string][]*common.AutogitConfig var gitea common.Gitea var orgLinks map[string]*PackageRebaseLink func isConfiguredOrg(org *common.Organization) bool { _, found := configuredRepos[org.Username] return found } func concatenateErrors(err1, err2 error) error { if err1 == nil { return err2 } if err2 == nil { return err1 } return fmt.Errorf("%w\n%w", err1, err2) } type RepositoryActionProcessor struct{} func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error { action := request.Data.(*common.RepositoryWebhookEvent) configs, configFound := configuredRepos[action.Organization.Username] if !configFound { log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username) return nil } for _, config := range configs { if org, repo, _ := config.GetPrjGit(); org == action.Repository.Owner.Username && repo == action.Repository.Name { log.Println("+ ignoring repo event for PrjGit repository", config.GitProjectName) return nil } } var err error for _, config := range configs { err = concatenateErrors(err, processConfiguredRepositoryAction(action, config)) } return err } func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error { gitOrg, gitPrj, gitBranch := config.GetPrjGit() git, err := gh.CreateGitHandler(config.Organization) common.PanicOnError(err) defer git.Close() if len(config.Branch) == 0 { config.Branch = action.Repository.Default_Branch } prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj) if err != nil { return fmt.Errorf("Error accessing/creating prjgit: %s/%s#%s err: %w", gitOrg, gitPrj, gitBranch, err) } remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL) common.PanicOnError(err) switch action.Action { case "created": if action.Repository.Object_Format_Name != "sha256" { return fmt.Errorf(" - '%s' repo is not sha256. Ignoring.", action.Repository.Name) } common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name)) defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all") branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current")) if branch != config.Branch { if err := git.GitExec(path.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil { return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here } common.PanicOnError(git.GitExec(path.Join(gitPrj, action.Repository.Name), "checkout", config.Branch)) } common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package inclusion via Direct Workflow")) if !noop { common.PanicOnError(git.GitExec(gitPrj, "push")) } case "deleted": if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() { if DebugMode { log.Println("delete event for", action.Repository.Name, "-- not in project. Ignoring") } return nil } common.PanicOnError(git.GitExec(gitPrj, "rm", action.Repository.Name)) common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package removal via Direct Workflow")) if !noop { git.GitExecOrPanic(gitPrj, "push", remoteName) } default: return fmt.Errorf("%s: %s", "Unknown action type", action.Action) } return nil } type PushActionProcessor struct{} func (*PushActionProcessor) ProcessFunc(request *common.Request) error { action := request.Data.(*common.PushWebhookEvent) configs, configFound := configuredRepos[action.Repository.Owner.Username] if !configFound { log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Repository.Owner.Username) return nil } for _, config := range configs { if gitOrg, gitPrj, _ := config.GetPrjGit(); gitOrg == action.Repository.Owner.Username && gitPrj == action.Repository.Name { log.Println("+ ignoring push to PrjGit repository", config.GitProjectName) return nil } } var err error for _, config := range configs { err = concatenateErrors(err, processConfiguredPushAction(action, config)) } return err } func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) error { gitOrg, gitPrj, gitBranch := config.GetPrjGit() git, err := gh.CreateGitHandler(config.Organization) common.PanicOnError(err) defer git.Close() log.Printf("push to: %s/%s for %s/%s#%s", action.Repository.Owner.Username, action.Repository.Name, gitOrg, gitPrj, gitBranch) if len(config.Branch) == 0 { config.Branch = action.Repository.Default_Branch log.Println(" + default branch", action.Repository.Default_Branch) } prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj) if err != nil { return fmt.Errorf("Error accessing/creating prjgit: %s/%s err: %w", gitOrg, gitPrj, err) } remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL) common.PanicOnError(err) headCommitId, err := git.GitRemoteHead(gitPrj, remoteName, gitBranch) common.PanicOnError(err) commit, ok := git.GitSubmoduleCommitId(gitPrj, action.Repository.Name, headCommitId) for ok && action.Head_Commit.Id == commit { log.Println(" -- nothing to do, commit already in ProjectGit") return nil } if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() { if DebugMode { log.Println("Pushed to package that is not part of the project. Ignoring:", err) } return nil } git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name) defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all") if err := git.GitExec(filepath.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "--force", remoteName, config.Branch+":"+config.Branch); err != nil { return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here } id, err := git.GitRemoteHead(filepath.Join(gitPrj, action.Repository.Name), remoteName, config.Branch) common.PanicOnError(err) if action.Head_Commit.Id == id { git.GitExecOrPanic(filepath.Join(gitPrj, action.Repository.Name), "checkout", id) git.GitExecOrPanic(gitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow") if !noop { git.GitExecOrPanic(gitPrj, "push", remoteName) } return nil } log.Println("push of refs not on the configured branch", config.Branch, ". ignoring.") return nil } func verifyProjectState(git common.Git, org string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) { defer func() { e := recover() if e != nil { errCast, ok := e.(error) if ok { err = errCast } } }() gitOrg, gitPrj, gitBranch := config.GetPrjGit() repo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj) if err != nil { return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", gitOrg, gitPrj, err) } remoteName, err := git.GitClone(gitPrj, gitBranch, repo.SSHURL) common.PanicOnError(err) defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all") log.Println(" * Getting submodule list") sub, err := git.GitSubmoduleList(gitPrj, "HEAD") common.PanicOnError(err) log.Println(" * Getting package links") var pkgLinks []*PackageRebaseLink if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), gitPrj)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 { if data, err := os.ReadFile(path.Join(git.GetPath(), gitPrj, common.PrjLinksFile)); err == nil { pkgLinks, err = parseProjectLinks(data) if err != nil { log.Println("Cannot parse project links file:", err.Error()) pkgLinks = nil } else { ResolveLinks(org, pkgLinks, gitea) } } } else { log.Println(" - No package links defined") } /* Check existing submodule that they are updated */ isGitUpdated := false next_package: for filename, commitId := range sub { // ignore project gits //for _, c := range configs { if gitPrj == filename { log.Println(" prjgit as package? ignoring project git:", filename) continue next_package } //} log.Printf(" verifying package: %s -> %s(%s)", commitId, filename, config.Branch) commits, err := gitea.GetRecentCommits(org, filename, config.Branch, 10) if len(commits) == 0 { if repo, err := gitea.GetRepository(org, filename); repo == nil && err == nil { git.GitExecOrPanic(gitPrj, "rm", filename) isGitUpdated = true } } if err != nil { log.Println(" -> failed to fetch recent commits for package:", filename, " Err:", err) continue } idx := 1000 for i, c := range commits { if c.SHA == commitId { idx = i break } } var link *PackageRebaseLink for _, l := range pkgLinks { if l.Pkg == filename { link = l log.Println(" -> linked package") // so, we need to rebase here. Can't really optimize, so clone entire package tree and remote pkgPath := path.Join(gitPrj, filename) git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--checkout", filename) git.GitExecOrPanic(pkgPath, "fetch", "origin", commits[0].SHA) git.GitExecOrPanic(pkgPath, "tag", "NOW") git.GitExecOrPanic(pkgPath, "fetch", "origin") git.GitExecOrPanic(pkgPath, "remote", "add", "parent", link.parentRepo.SSHURL) git.GitExecOrPanic(pkgPath, "fetch", "parent") git.GitExecOrPanic(pkgPath, "rebase", "--onto", "parent", link.SourceBranch) nCommits := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgPath, "rev-list", "^NOW", "HEAD"), "\n")) if nCommits > 0 { if !noop { git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+config.Branch) } isGitUpdated = true } break } } if link == nil { if idx == 0 { // up-to-date continue } else if idx < len(commits) { // update common.PanicOnError(git.GitExec(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", filename)) common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "fetch", "--depth", "1", "origin", commits[0].SHA)) common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "checkout", commits[0].SHA)) log.Println(" -> updated to", commits[0].SHA) isGitUpdated = true } else { // probably need `merge-base` or `rev-list` here instead, or the project updated already log.Println(" *** Cannot find SHA of last matching update for package:", filename, " Ignoring") } } } // find all missing repositories, and add them if DebugMode { log.Println("checking for missing repositories...") } repos, err := gitea.GetOrganizationRepositories(org) if err != nil { return err } if DebugMode { log.Println(" nRepos:", len(repos)) } /* Check repositories in org to make sure they are included in project git */ next_repo: for _, r := range repos { if DebugMode { log.Println(" -- checking", r.Name) } if r.ObjectFormatName != "sha256" { if DebugMode { log.Println(" + ", r.ObjectFormatName, ". Needs to be sha256. Ignoring") } continue next_repo } // for _, c := range configs { if gitPrj == r.Name { // ignore project gits continue next_repo } // } for repo := range sub { if repo == r.Name { // not missing continue next_repo } } if DebugMode { log.Println(" -- checking repository:", r.Name) } if _, err := gitea.GetRecentCommits(org, r.Name, config.Branch, 1); err != nil { // assumption that package does not exist, so not part of project // https://github.com/go-gitea/gitea/issues/31976 continue } // add repository to git project common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name)) if len(config.Branch) > 0 { branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current")) if branch != config.Branch { if err := git.GitExec(path.Join(gitPrj, r.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil { return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", config.Branch, repo.Owner.UserName, r.Name) } common.PanicOnError(git.GitExec(path.Join(gitPrj, r.Name), "checkout", config.Branch)) } } isGitUpdated = true } if isGitUpdated { common.PanicOnError(git.GitExec(gitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC")) if !noop { git.GitExecOrPanic(gitPrj, "push", remoteName) } } if DebugMode { log.Println("Verification finished for ", org, ", prjgit:", config.GitProjectName) } return nil } var checkOnStart bool var noop bool var checkInterval time.Duration func checkOrg(org string, configs []*common.AutogitConfig) { git, err := gh.CreateGitHandler(org) if err != nil { log.Println("Faield to allocate GitHandler:", err) return } defer git.Close() for _, config := range configs { log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName) if err := verifyProjectState(git, org, config, configs); err != nil { log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err) } else { log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName) } } } func checkRepos() { for org, configs := range configuredRepos { if checkInterval > 0 { sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval))) log.Println(" - sleep interval", sleepInterval, "until next check") time.Sleep(sleepInterval) } checkOrg(org, configs) } } func consistencyCheckProcess() { if checkOnStart { savedCheckInterval := checkInterval checkInterval = 0 log.Println("== Startup consistency check begin...") checkRepos() log.Println("== Startup consistency check done...") checkInterval = savedCheckInterval } for { checkRepos() } } var DebugMode bool var gh common.GitHandlerGenerator func updateConfiguration(configFilename string, orgs *[]string) { configFile, err := common.ReadConfigFile(configFilename) if err != nil { log.Fatal(err) } configs, _ := common.ResolveWorkflowConfigs(gitea, configFile) configuredRepos = make(map[string][]*common.AutogitConfig) *orgs = make([]string, 0, 1) for _, c := range configs { if slices.Contains(c.Workflows, "direct") { if DebugMode { log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName) } configs := configuredRepos[c.Organization] if configs == nil { configs = make([]*common.AutogitConfig, 0, 1) } configs = append(configs, c) configuredRepos[c.Organization] = configs *orgs = append(*orgs, c.Organization) } } } func main() { configFilename := flag.String("config", "", "List of PrjGit") giteaUrl := flag.String("gitea-url", "https://src.opensuse.org", "Gitea instance") rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance") flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information") flag.BoolVar(&noop, "dry", false, "Dry mode. Do not push changes to remote repo.") flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays") checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours") basePath := flag.String("repo-path", "", "Repository path. Default is temporary directory") flag.Parse() if err := common.RequireGiteaSecretToken(); err != nil { log.Fatal(err) } if err := common.RequireRabbitSecrets(); err != nil { log.Fatal(err) } var defs common.ListenDefinitions var err error if len(*basePath) == 0 { *basePath, err = os.MkdirTemp(os.TempDir(), AppName) if err != nil { log.Fatal(err) } } gh, err = common.AllocateGitWorkTree(*basePath, GitAuthor, GitEmail) if err != nil { log.Fatal(err) } // handle reconfiguration signalChannel := make(chan os.Signal, 1) defer close(signalChannel) go func() { for { sig, ok := <-signalChannel if !ok { return } if sig != syscall.SIGHUP { log.Println("Unexpected signal received:", sig) continue } log.Println("*** Reconfiguring ***") updateConfiguration(*configFilename, &defs.Orgs) defs.UpdateTopics() } }() signal.Notify(signalChannel, syscall.SIGHUP) checkInterval = time.Duration(*checkIntervalHours) * time.Hour gitea = common.AllocateGiteaTransport(*giteaUrl) CurrentUser, err := gitea.GetCurrentUser() if err != nil { log.Fatalln("Cannot fetch current user:", err) } log.Println("Current User:", CurrentUser.UserName) updateConfiguration(*configFilename, &defs.Orgs) defs.GitAuthor = GitAuthor defs.RabbitURL, err = url.Parse(*rabbitUrl) if err != nil { log.Panicf("cannot parse server URL. Err: %#v\n", err) } go consistencyCheckProcess() log.Println("defs:", defs) defs.Handlers = make(map[string]common.RequestProcessor) defs.Handlers[common.RequestType_Push] = &PushActionProcessor{} defs.Handlers[common.RequestType_Repository] = &RepositoryActionProcessor{} log.Fatal(defs.ProcessRabbitMQEvents()) }