package main /* * This file is part of Autogits. * * Copyright © 2024 SUSE LLC * * Autogits is free software: you can redistribute it and/or modify it under * the terms of the GNU General Public License as published by the Free Software * Foundation, either version 2 of the License, or (at your option) any later * version. * * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A * PARTICULAR PURPOSE. See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along with * Foobar. If not, see . */ import ( "flag" "fmt" "io/fs" "math/rand" "net/url" "os" "os/signal" "path" "path/filepath" "slices" "strings" "syscall" "time" "src.opensuse.org/autogits/common" ) const ( AppName = "direct_workflow" GitAuthor = "AutoGits prjgit-updater" GitEmail = "autogits-direct@noreply@src.opensuse.org" ) var configuredRepos map[string][]*common.AutogitConfig var gitea common.Gitea var orgLinks map[string]*PackageRebaseLink func isConfiguredOrg(org *common.Organization) bool { _, found := configuredRepos[org.Username] return found } type RepositoryActionProcessor struct{} func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error { action := request.Data.(*common.RepositoryWebhookEvent) configs, configFound := configuredRepos[action.Organization.Username] if !configFound { common.LogInfo("Repository event for", action.Organization.Username, ". Not configured. Ignoring.", action.Organization.Username) return nil } for _, config := range configs { if org, repo, _ := config.GetPrjGit(); org == action.Repository.Owner.Username && repo == action.Repository.Name { common.LogError("+ ignoring repo event for PrjGit repository", config.GitProjectName) return nil } } for _, config := range configs { processConfiguredRepositoryAction(action, config) } return nil } func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) { gitOrg, gitPrj, gitBranch := config.GetPrjGit() git, err := gh.CreateGitHandler(config.Organization) common.PanicOnError(err) defer git.Close() configBranch := config.Branch if len(configBranch) == 0 { configBranch = action.Repository.Default_Branch if common.IsRemovedBranch(configBranch) { common.LogDebug(" - default branch has deleted suffix. Skipping") return } } prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj) if err != nil { common.LogError("Error accessing/creating prjgit:", gitOrg, gitPrj, gitBranch, err) return } remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL) common.PanicOnError(err) switch action.Action { case "created": if action.Repository.Object_Format_Name != "sha256" { common.LogError(" - '%s' repo is not sha256. Ignoring.", action.Repository.Name) return } common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name)) defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all") branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current")) if branch != configBranch { if err := git.GitExec(path.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", configBranch+":"+configBranch); err != nil { common.LogError("error fetching branch", configBranch, ". ignoring as non-existent.", err) // no branch? so ignore repo here return } common.PanicOnError(git.GitExec(path.Join(gitPrj, action.Repository.Name), "checkout", configBranch)) } common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Auto-inclusion "+action.Repository.Name)) if !noop { common.PanicOnError(git.GitExec(gitPrj, "push")) } case "deleted": if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() { common.LogDebug("delete event for", action.Repository.Name, "-- not in project. Ignoring") return } common.PanicOnError(git.GitExec(gitPrj, "rm", action.Repository.Name)) common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package removal via Direct Workflow")) if !noop { git.GitExecOrPanic(gitPrj, "push", remoteName) } default: common.LogError("Unknown action type:", action.Action) return } } type PushActionProcessor struct{} func (*PushActionProcessor) ProcessFunc(request *common.Request) error { action := request.Data.(*common.PushWebhookEvent) configs, configFound := configuredRepos[action.Repository.Owner.Username] if !configFound { common.LogDebug("Repository event for", action.Repository.Owner.Username, ". Not configured. Ignoring.") return nil } for _, config := range configs { if gitOrg, gitPrj, _ := config.GetPrjGit(); gitOrg == action.Repository.Owner.Username && gitPrj == action.Repository.Name { common.LogInfo("+ ignoring push to PrjGit repository", config.GitProjectName) return nil } } for _, config := range configs { processConfiguredPushAction(action, config) } return nil } func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) { gitOrg, gitPrj, gitBranch := config.GetPrjGit() git, err := gh.CreateGitHandler(config.Organization) common.PanicOnError(err) defer git.Close() common.LogDebug("push to:", action.Repository.Owner.Username, action.Repository.Name, "for:", gitOrg, gitPrj, gitBranch) branch := config.Branch if len(branch) == 0 { if common.IsRemovedBranch(branch) { common.LogDebug(" + default branch has removed suffix:", branch, "Skipping.") return } branch = action.Repository.Default_Branch common.LogDebug(" + using default branch", branch) } prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj) if err != nil { common.LogError("Error accessing/creating prjgit:", gitOrg, gitPrj, err) return } remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL) common.PanicOnError(err) headCommitId, err := git.GitRemoteHead(gitPrj, remoteName, gitBranch) common.PanicOnError(err) commit, ok := git.GitSubmoduleCommitId(gitPrj, action.Repository.Name, headCommitId) for ok && action.Head_Commit.Id == commit { common.LogDebug(" -- nothing to do, commit already in ProjectGit") return } if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil { git.GitExecOrPanic(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name) common.LogDebug("Pushed to package that is not part of the project. Re-adding...", err) } else if !stat.IsDir() { common.LogError("Pushed to a package that is not a submodule but exists in the project. Ignoring.") return } git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name) defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all") if err := git.GitExec(filepath.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "--force", remoteName, branch+":"+branch); err != nil { common.LogError("Error fetching branch:", branch, "Ignoring as non-existent.", err) return } id, err := git.GitRemoteHead(filepath.Join(gitPrj, action.Repository.Name), remoteName, branch) common.PanicOnError(err) if action.Head_Commit.Id == id { git.GitExecOrPanic(filepath.Join(gitPrj, action.Repository.Name), "checkout", id) git.GitExecOrPanic(gitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow") if !noop { git.GitExecOrPanic(gitPrj, "push", remoteName) } return } common.LogDebug("push of refs not on the configured branch", branch, ". ignoring.") } func verifyProjectState(git common.Git, org string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) { defer func() { e := recover() if e != nil { errCast, ok := e.(error) if ok { err = errCast } } }() gitOrg, gitPrj, gitBranch := config.GetPrjGit() repo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj) if err != nil { return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", gitOrg, gitPrj, err) } remoteName, err := git.GitClone(gitPrj, gitBranch, repo.SSHURL) common.PanicOnError(err) defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all") common.LogDebug(" * Getting submodule list") sub, err := git.GitSubmoduleList(gitPrj, "HEAD") common.PanicOnError(err) common.LogDebug(" * Getting package links") var pkgLinks []*PackageRebaseLink if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), gitPrj)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 { if data, err := os.ReadFile(path.Join(git.GetPath(), gitPrj, common.PrjLinksFile)); err == nil { pkgLinks, err = parseProjectLinks(data) if err != nil { common.LogError("Cannot parse project links file:", err.Error()) pkgLinks = nil } else { ResolveLinks(org, pkgLinks, gitea) } } } else { common.LogInfo(" - No package links defined") } /* Check existing submodule that they are updated */ isGitUpdated := false next_package: for filename, commitId := range sub { // ignore project gits //for _, c := range configs { if gitPrj == filename { common.LogDebug(" prjgit as package? ignoring project git:", filename) continue next_package } //} branch := config.Branch common.LogDebug(" verifying package: %s -> %s(%s)", commitId, filename, branch) if repo, err := gitea.GetRepository(org, filename); repo == nil && err == nil { common.LogDebug(" repository removed...") git.GitExecOrPanic(gitPrj, "rm", filename) isGitUpdated = true continue } if len(branch) == 0 { branch = repo.DefaultBranch if common.IsRemovedBranch(branch) { common.LogDebug(" Default branch for", filename, "is excluded") git.GitExecOrPanic(gitPrj, "rm", filename) isGitUpdated = true continue } } commits, err := gitea.GetRecentCommits(org, filename, branch, 10) if err != nil { common.LogDebug(" -> failed to fetch recent commits for package:", filename, " Err:", err) continue } idx := 1000 for i, c := range commits { if c.SHA == commitId { idx = i break } } var link *PackageRebaseLink for _, l := range pkgLinks { if l.Pkg == filename { link = l common.LogDebug(" -> linked package") // so, we need to rebase here. Can't really optimize, so clone entire package tree and remote pkgPath := path.Join(gitPrj, filename) git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--checkout", filename) git.GitExecOrPanic(pkgPath, "fetch", "origin", commits[0].SHA) git.GitExecOrPanic(pkgPath, "tag", "NOW") git.GitExecOrPanic(pkgPath, "fetch", "origin") git.GitExecOrPanic(pkgPath, "remote", "add", "parent", link.parentRepo.SSHURL) git.GitExecOrPanic(pkgPath, "fetch", "parent") git.GitExecOrPanic(pkgPath, "rebase", "--onto", "parent", link.SourceBranch) nCommits := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgPath, "rev-list", "^NOW", "HEAD"), "\n")) if nCommits > 0 { if !noop { git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+branch) } isGitUpdated = true } break } } if link == nil { if idx == 0 { // up-to-date continue } else if idx < len(commits) { // update common.PanicOnError(git.GitExec(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", filename)) common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "fetch", "--depth", "1", "origin", commits[0].SHA)) common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "checkout", commits[0].SHA)) common.LogDebug(" -> updated to", commits[0].SHA) isGitUpdated = true } else { // probably need `merge-base` or `rev-list` here instead, or the project updated already common.LogInfo(" *** Cannot find SHA of last matching update for package:", filename, " Ignoring") } } } // find all missing repositories, and add them common.LogDebug("checking for missing repositories...") repos, err := gitea.GetOrganizationRepositories(org) if err != nil { return err } common.LogDebug(" nRepos:", len(repos)) /* Check repositories in org to make sure they are included in project git */ next_repo: for _, r := range repos { // for _, c := range configs { if gitPrj == r.Name { // ignore project gits continue next_repo } // } for repo := range sub { if repo == r.Name { // not missing continue next_repo } } common.LogDebug(" -- checking repository:", r.Name) branch := config.Branch if len(branch) == 0 { branch = r.DefaultBranch if common.IsRemovedBranch(branch) { continue } } if commits, err := gitea.GetRecentCommits(org, r.Name, branch, 1); err != nil || len(commits) == 0 { // assumption that package does not exist, so not part of project // https://github.com/go-gitea/gitea/issues/31976 // or, we do not have commits here continue } // add repository to git project common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name)) curBranch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current")) if branch != curBranch { if err := git.GitExec(path.Join(gitPrj, r.Name), "fetch", "--depth", "1", "origin", branch+":"+branch); err != nil { return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", branch, repo.Owner.UserName, r.Name) } common.PanicOnError(git.GitExec(path.Join(gitPrj, r.Name), "checkout", branch)) } isGitUpdated = true } if isGitUpdated { common.PanicOnError(git.GitExec(gitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC")) if !noop { git.GitExecOrPanic(gitPrj, "push", remoteName) } } common.LogInfo("Verification finished for ", org, ", prjgit:", config.GitProjectName) return nil } var checkOnStart bool var noop bool var checkInterval time.Duration func checkOrg(org string, configs []*common.AutogitConfig) { git, err := gh.CreateGitHandler(org) if err != nil { common.LogError("Failed to allocate GitHandler:", err) return } defer git.Close() for _, config := range configs { common.LogInfo(" ++ starting verification, org:", org, "config:", config.GitProjectName) if err := verifyProjectState(git, org, config, configs); err != nil { common.LogError(" *** verification failed, org:", org, err) } else { common.LogError(" ++ verification complete, org:", org, config.GitProjectName) } } } func checkRepos() { for org, configs := range configuredRepos { if checkInterval > 0 { sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval))) common.LogInfo(" - sleep interval", sleepInterval, "until next check") time.Sleep(sleepInterval) } checkOrg(org, configs) } } func consistencyCheckProcess() { if checkOnStart { savedCheckInterval := checkInterval checkInterval = 0 common.LogInfo("== Startup consistency check begin...") checkRepos() common.LogInfo("== Startup consistency check done...") checkInterval = savedCheckInterval } for { checkRepos() } } var DebugMode bool var gh common.GitHandlerGenerator func updateConfiguration(configFilename string, orgs *[]string) { configFile, err := common.ReadConfigFile(configFilename) if err != nil { common.LogError(err) os.Exit(4) } configs, _ := common.ResolveWorkflowConfigs(gitea, configFile) configuredRepos = make(map[string][]*common.AutogitConfig) *orgs = make([]string, 0, 1) for _, c := range configs { if slices.Contains(c.Workflows, "direct") { common.LogDebug(" + adding org:", c.Organization, ", branch:", c.Branch, ", prjgit:", c.GitProjectName) configs := configuredRepos[c.Organization] if configs == nil { configs = make([]*common.AutogitConfig, 0, 1) } configs = append(configs, c) configuredRepos[c.Organization] = configs *orgs = append(*orgs, c.Organization) } } } func main() { configFilename := flag.String("config", "", "List of PrjGit") giteaUrl := flag.String("gitea-url", "https://src.opensuse.org", "Gitea instance") rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance") flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information") flag.BoolVar(&noop, "dry", false, "Dry mode. Do not push changes to remote repo.") flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays") checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours") basePath := flag.String("repo-path", "", "Repository path. Default is temporary directory") flag.Parse() if err := common.RequireGiteaSecretToken(); err != nil { common.LogError(err) os.Exit(1) } if err := common.RequireRabbitSecrets(); err != nil { common.LogError(err) os.Exit(1) } defs := &common.RabbitMQGiteaEventsProcessor{} var err error if len(*basePath) == 0 { *basePath, err = os.MkdirTemp(os.TempDir(), AppName) if err != nil { common.LogError(err) os.Exit(1) } } gh, err = common.AllocateGitWorkTree(*basePath, GitAuthor, GitEmail) if err != nil { common.LogError(err) os.Exit(1) } // handle reconfiguration signalChannel := make(chan os.Signal, 1) defer close(signalChannel) go func() { for { sig, ok := <-signalChannel if !ok { return } if sig != syscall.SIGHUP { common.LogError("Unexpected signal received:", sig) continue } common.LogError("*** Reconfiguring ***") updateConfiguration(*configFilename, &defs.Orgs) defs.Connection().UpdateTopics(defs) } }() signal.Notify(signalChannel, syscall.SIGHUP) checkInterval = time.Duration(*checkIntervalHours) * time.Hour gitea = common.AllocateGiteaTransport(*giteaUrl) CurrentUser, err := gitea.GetCurrentUser() if err != nil { common.LogError("Cannot fetch current user:", err) os.Exit(2) } common.LogInfo("Current User:", CurrentUser.UserName) updateConfiguration(*configFilename, &defs.Orgs) defs.Connection().RabbitURL, err = url.Parse(*rabbitUrl) if err != nil { common.LogError("cannot parse server URL. Err:", err) os.Exit(3) } go consistencyCheckProcess() common.LogInfo("defs:", *defs) defs.Handlers = make(map[string]common.RequestProcessor) defs.Handlers[common.RequestType_Push] = &PushActionProcessor{} defs.Handlers[common.RequestType_Repository] = &RepositoryActionProcessor{} common.LogError(common.ProcessRabbitMQEvents(defs)) }