package main /* * This file is part of Autogits. * * Copyright © 2024 SUSE LLC * * Autogits is free software: you can redistribute it and/or modify it under * the terms of the GNU General Public License as published by the Free Software * Foundation, either version 2 of the License, or (at your option) any later * version. * * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A * PARTICULAR PURPOSE. See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along with * Foobar. If not, see . */ import ( "flag" "fmt" "log" "math/rand" "os" "path" "path/filepath" "slices" "time" "src.opensuse.org/autogits/common" ) const ( AppName = "direct_workflow" GitAuthor = "AutoGits prjgit-updater" GitEmail = "adam+autogits-direct@zombino.com" ) var configuredRepos map[string][]*common.AutogitConfig var gitea common.Gitea func isConfiguredOrg(org *common.Organization) bool { _, found := configuredRepos[org.Username] return found } func concatenateErrors(err1, err2 error) error { if err1 == nil { return err2 } if err2 == nil { return err1 } return fmt.Errorf("%w\n%w", err1, err2) } func processRepositoryAction(request *common.Request) error { action := request.Data.(*common.RepositoryWebhookEvent) configs, configFound := configuredRepos[action.Organization.Username] if !configFound { log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username) return nil } for _, config := range configs { if config.GitProjectName == action.Repository.Name { log.Println("+ ignoring repo event for PrjGit repository", config.GitProjectName) return nil } } var err error for _, config := range configs { err = concatenateErrors(err, processConfiguredRepositoryAction(action, config)) } return err } func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error { prjgit := config.GitProjectName git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName) common.PanicOnError(err) if !DebugMode { defer git.Close() } if len(config.Branch) == 0 { config.Branch = action.Repository.Default_Branch } prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, *action.Organization, prjgit) if err != nil { return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err) } common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, prjgit)) switch action.Action { case "created": common.PanicOnError(git.GitExec(prjgit, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name)) if _, err := git.GitBranchHead(path.Join(prjgit, action.Repository.Name), config.Branch); err != nil { if err := git.GitExec(path.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil { return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here } } common.PanicOnError(git.GitExec(path.Join(prjgit, action.Repository.Name), "checkout", config.Branch)) common.PanicOnError(git.GitExec(prjgit, "commit", "-m", "Automatic package inclusion via Direct Workflow")) common.PanicOnError(git.GitExec(prjgit, "push")) case "deleted": if stat, err := os.Stat(filepath.Join(git.GitPath, prjgit, action.Repository.Name)); err != nil || !stat.IsDir() { if git.DebugLogger { log.Println("delete event for", action.Repository.Name, "-- not in project. Ignoring") } return nil } common.PanicOnError(git.GitExec(prjgit, "rm", action.Repository.Name)) common.PanicOnError(git.GitExec(prjgit, "commit", "-m", "Automatic package removal via Direct Workflow")) common.PanicOnError(git.GitExec(prjgit, "push")) default: return fmt.Errorf("%s: %s", "Unknown action type", action.Action) } return nil } func processPushAction(request *common.Request) error { action := request.Data.(*common.PushWebhookEvent) configs, configFound := configuredRepos[action.Repository.Owner.Username] if !configFound { log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Repository.Owner.Username) return nil } for _, config := range configs { if config.GitProjectName == action.Repository.Name { log.Println("+ ignoring push to PrjGit repository", config.GitProjectName) return nil } } var err error for _, config := range configs { err = concatenateErrors(err, processConfiguredPushAction(action, config)) } return err } func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) error { prjgit := config.GitProjectName git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName) common.PanicOnError(err) if !DebugMode { defer git.Close() } if len(config.Branch) == 0 { config.Branch = action.Repository.Default_Branch log.Println(" + default branch", action.Repository.Default_Branch) } prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, *action.Repository.Owner, prjgit) if err != nil { return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err) } common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, prjgit)) if stat, err := os.Stat(filepath.Join(git.GitPath, prjgit, action.Repository.Name)); err != nil || !stat.IsDir() { if git.DebugLogger { log.Println("Pushed to package that is not part of the project. Ignoring:", err) } return nil } common.PanicOnError(git.GitExec(prjgit, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name)) if err := git.GitExec(filepath.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil { return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here } id, err := git.GitBranchHead(filepath.Join(prjgit, action.Repository.Name), config.Branch) common.PanicOnError(err) for _, commitId := range action.Commits { if commitId.Id == id { common.PanicOnError(git.GitExec(filepath.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", id)) common.PanicOnError(git.GitExec(filepath.Join(prjgit, action.Repository.Name), "checkout", id)) common.PanicOnError(git.GitExec(prjgit, "commit", "-a", "-m", "Automatic update via push via Direct Workflow")) common.PanicOnError(git.GitExec(prjgit, "push")) return nil } } log.Println("push of refs not on the configured branch", config.Branch, ". ignoring.") return nil } func verifyProjectState(git *common.GitHandler, orgName string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) { defer func() { e := recover() if e != nil { errCast, ok := e.(error) if ok { err = errCast } } }() org := common.Organization{ Username: orgName, } repo, err := gitea.CreateRepositoryIfNotExist(git, org, config.GitProjectName) if err != nil { return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", orgName, config.GitProjectName, err) } common.PanicOnError(git.GitExec("", "clone", "--depth", "1", repo.SSHURL, config.GitProjectName)) log.Println("getting submodule list") sub, err := git.GitSubmoduleList(config.GitProjectName, "HEAD") common.PanicOnError(err) isGitUpdated := false next_package: for filename, commitId := range sub { // ignore project gits for _, c := range configs { if c.GitProjectName == filename { log.Println(" prjgit as package? ignoring project git:", filename) continue next_package } } log.Println(" verifying package:", filename, commitId, config.Branch) commits, err := gitea.GetRecentCommits(orgName, filename, config.Branch, 10) if err != nil { // assumption that package does not exist, remove from project // https://github.com/go-gitea/gitea/issues/31976 if err := git.GitExec(config.GitProjectName, "rm", filename); err != nil { return fmt.Errorf("Failed to remove deleted submodule. Err: %w", err) } isGitUpdated = true continue } // if err != nil { // return fmt.Errorf("Failed to fetch recent commits for package: '%s'. Err: %w", filename, err) // } idx := 1000 for i, c := range commits { if c.SHA == commitId { idx = i break } } if idx == 0 { // up-to-date continue } else if idx < len(commits) { // update common.PanicOnError(git.GitExec(config.GitProjectName, "submodule", "update", "--init", "--depth", "1", "--checkout", filename)) common.PanicOnError(git.GitExec(filepath.Join(config.GitProjectName, filename), "fetch", "--depth", "1", "origin", commits[0].SHA)) common.PanicOnError(git.GitExec(filepath.Join(config.GitProjectName, filename), "checkout", commits[0].SHA)) isGitUpdated = true } else { // probably need `merge-base` or `rev-list` here instead, or the project updated already return fmt.Errorf("Cannot find SHA of last matching update for package: '%s'. idx: %d", filename, idx) } } // find all missing repositories, and add them if DebugMode { log.Println("checking for missing repositories...") } repos, err := gitea.GetOrganizationRepositories(orgName) if err != nil { return err } if DebugMode { log.Println(" nRepos:", len(repos)) } next_repo: for _, r := range repos { if DebugMode { log.Println(" -- checking", r.Name) } for _, c := range configs { if c.Organization == orgName && c.GitProjectName == r.Name { // ignore project gits continue next_repo } } for repo := range sub { if repo == r.Name { // not missing continue next_repo } } if DebugMode { log.Println(" -- checking repository:", r.Name) } if _, err := gitea.GetRecentCommits(orgName, r.Name, config.Branch, 1); err != nil { // assumption that package does not exist, so not part of project // https://github.com/go-gitea/gitea/issues/31976 continue } // add repository to git project common.PanicOnError(git.GitExec(config.GitProjectName, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name)) if len(config.Branch) > 0 { if err := git.GitExec(path.Join(config.GitProjectName, r.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil { return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", config.Branch, repo.Owner.UserName, r.Name) } common.PanicOnError(git.GitExec(path.Join(config.GitProjectName, r.Name), "checkout", config.Branch)) } isGitUpdated = true } if isGitUpdated { common.PanicOnError(git.GitExec(config.GitProjectName, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC")) common.PanicOnError(git.GitExec(config.GitProjectName, "push")) } if DebugMode { log.Println("Verification finished for ", orgName, ", config", config.GitProjectName) } return nil } var checkOnStart bool var checkInterval time.Duration func checkRepos() { for org, configs := range configuredRepos { for _, config := range configs { if checkInterval > 0 { sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval))) log.Println(" - sleep interval", sleepInterval, "until next check") time.Sleep(sleepInterval) } log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName) git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName) if err != nil { log.Println("Faield to allocate GitHandler:", err) return } if err := verifyProjectState(git, org, config, configs); err != nil { log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err) } log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName) } } } func consistencyCheckProcess() { if checkOnStart { savedCheckInterval := checkInterval checkInterval = 0 log.Println("== Startup consistency check begin...") checkRepos() log.Println("== Startup consistency check done...") checkInterval = savedCheckInterval } for { checkRepos() } } var DebugMode bool func main() { if err := common.RequireGiteaSecretToken(); err != nil { log.Fatal(err) } if err := common.RequireRabbitSecrets(); err != nil { log.Fatal(err) } workflowConfig := flag.String("config", "", "Repository and workflow definition file") giteaHost := flag.String("gitea", "src.opensuse.org", "Gitea instance") rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance") flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information") flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays") checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours") flag.Parse() checkInterval = time.Duration(*checkIntervalHours) * time.Hour if len(*workflowConfig) == 0 { log.Fatalln("No configuratio file specified. Aborting") } configs, err := common.ReadWorkflowConfigsFile(*workflowConfig) if err != nil { log.Fatal(err) } configuredRepos = make(map[string][]*common.AutogitConfig) orgs := make([]string, 0, 1) for _, c := range configs { if slices.Contains(c.Workflows, "direct") { if DebugMode { log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName) } configs := configuredRepos[c.Organization] if configs == nil { configs = make([]*common.AutogitConfig, 0, 1) } configs = append(configs, c) configuredRepos[c.Organization] = configs orgs = append(orgs, c.Organization) } } gitea = common.AllocateGiteaTransport(*giteaHost) go consistencyCheckProcess() var defs common.ListenDefinitions defs.GitAuthor = GitAuthor defs.RabbitURL = *rabbitUrl defs.Handlers = make(map[string]common.RequestProcessor) defs.Handlers[common.RequestType_Push] = processPushAction defs.Handlers[common.RequestType_Repository] = processRepositoryAction log.Fatal(common.ProcessRabbitMQEvents(defs, orgs)) }