Adam Majer
b7ec9a9ffb
Handle default branch name in push and branch create handlers Don't panic in this case in case the project has multiple configs
440 lines
14 KiB
Go
440 lines
14 KiB
Go
package main
|
|
|
|
/*
|
|
* This file is part of Autogits.
|
|
*
|
|
* Copyright © 2024 SUSE LLC
|
|
*
|
|
* Autogits is free software: you can redistribute it and/or modify it under
|
|
* the terms of the GNU General Public License as published by the Free Software
|
|
* Foundation, either version 2 of the License, or (at your option) any later
|
|
* version.
|
|
*
|
|
* Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
|
|
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along with
|
|
* Foobar. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"math/rand"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"slices"
|
|
"time"
|
|
|
|
"src.opensuse.org/autogits/common"
|
|
)
|
|
|
|
const AppName = "direct_workflow"
|
|
const GitAuthor = "AutoGits prjgit-updater"
|
|
const GitEmail = "adam+autogits-direct@zombino.com"
|
|
|
|
var configuredRepos map[string][]*common.AutogitConfig
|
|
var gitea *common.GiteaTransport
|
|
|
|
func isConfiguredOrg(org *common.Organization) bool {
|
|
_, found := configuredRepos[org.Username]
|
|
return found
|
|
}
|
|
|
|
func concatenateErrors(err1, err2 error) error {
|
|
if err1 == nil {
|
|
return err2
|
|
}
|
|
|
|
if err2 == nil {
|
|
return err1
|
|
}
|
|
|
|
return fmt.Errorf("%w\n%w", err1, err2)
|
|
}
|
|
|
|
func processRepositoryAction(h *common.RequestHandler) error {
|
|
action := h.Request.Data.(*common.RepositoryWebhookEvent)
|
|
configs, configFound := configuredRepos[action.Organization.Username]
|
|
|
|
if !configFound {
|
|
h.StdLogger.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username)
|
|
return nil
|
|
}
|
|
|
|
for _, config := range configs {
|
|
if config.GitProjectName == action.Repository.Name {
|
|
h.StdLogger.Println("+ ignoring repo event for PrjGit repository", config.GitProjectName)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
var err error
|
|
for _, config := range configs {
|
|
err = concatenateErrors(err, processConfiguredRepositoryAction(h, action, config))
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func processConfiguredRepositoryAction(h *common.RequestHandler, action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error {
|
|
prjgit := config.GitProjectName
|
|
git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
|
|
common.PanicOnError(err)
|
|
if !debugMode {
|
|
defer git.Close()
|
|
}
|
|
|
|
if len(config.Branch) == 0 {
|
|
config.Branch = action.Repository.Default_Branch
|
|
}
|
|
|
|
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, *action.Organization, prjgit)
|
|
if err != nil {
|
|
return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
|
|
}
|
|
|
|
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, prjgit))
|
|
|
|
switch action.Action {
|
|
case "created":
|
|
common.PanicOnError(git.GitExec(prjgit, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
|
|
if _, err := git.GitBranchHead(path.Join(prjgit, action.Repository.Name), config.Branch); err != nil {
|
|
if err := git.GitExec(path.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
|
|
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
|
|
}
|
|
}
|
|
common.PanicOnError(git.GitExec(path.Join(prjgit, action.Repository.Name), "checkout", config.Branch))
|
|
common.PanicOnError(git.GitExec(prjgit, "commit", "-m", "Automatic package inclusion via Direct Workflow"))
|
|
common.PanicOnError(git.GitExec(prjgit, "push"))
|
|
|
|
case "deleted":
|
|
if stat, err := os.Stat(filepath.Join(git.GitPath, prjgit, action.Repository.Name)); err != nil || !stat.IsDir() {
|
|
if git.DebugLogger {
|
|
h.StdLogger.Printf("delete event for %s -- not in project. Ignoring\n", action.Repository.Name)
|
|
}
|
|
return nil
|
|
}
|
|
common.PanicOnError(git.GitExec(prjgit, "rm", action.Repository.Name))
|
|
common.PanicOnError(git.GitExec(prjgit, "commit", "-m", "Automatic package removal via Direct Workflow"))
|
|
common.PanicOnError(git.GitExec(prjgit, "push"))
|
|
|
|
default:
|
|
return fmt.Errorf("%s: %s", "Unknown action type", action.Action)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func processPushAction(h *common.RequestHandler) error {
|
|
action := h.Request.Data.(*common.PushWebhookEvent)
|
|
configs, configFound := configuredRepos[action.Repository.Owner.Username]
|
|
|
|
if !configFound {
|
|
h.StdLogger.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Repository.Owner.Username)
|
|
return nil
|
|
}
|
|
|
|
for _, config := range configs {
|
|
if config.GitProjectName == action.Repository.Name {
|
|
h.StdLogger.Println("+ ignoring push to PrjGit repository", config.GitProjectName)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
var err error
|
|
for _, config := range configs {
|
|
err = concatenateErrors(err, processConfiguredPushAction(h, action, config))
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func processConfiguredPushAction(h *common.RequestHandler, action *common.PushWebhookEvent, config *common.AutogitConfig) error {
|
|
prjgit := config.GitProjectName
|
|
git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
|
|
common.PanicOnError(err)
|
|
if !debugMode {
|
|
defer git.Close()
|
|
}
|
|
|
|
if len(config.Branch) == 0 {
|
|
config.Branch = action.Repository.Default_Branch
|
|
}
|
|
|
|
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, *action.Repository.Owner, prjgit)
|
|
if err != nil {
|
|
return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
|
|
}
|
|
|
|
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, prjgit))
|
|
if stat, err := os.Stat(filepath.Join(git.GitPath, prjgit, action.Repository.Name)); err != nil || !stat.IsDir() {
|
|
if git.DebugLogger {
|
|
h.StdLogger.Println("Pushed to package that is not part of the project. Ignoring:", err)
|
|
}
|
|
return nil
|
|
}
|
|
common.PanicOnError(git.GitExec(prjgit, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name))
|
|
if err := git.GitExec(filepath.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
|
|
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
|
|
}
|
|
id, err := git.GitBranchHead(filepath.Join(prjgit, action.Repository.Name), config.Branch)
|
|
common.PanicOnError(err)
|
|
for _, commitId := range action.Commits {
|
|
if commitId.Id == id {
|
|
common.PanicOnError(git.GitExec(filepath.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", id))
|
|
common.PanicOnError(git.GitExec(filepath.Join(prjgit, action.Repository.Name), "checkout", id))
|
|
common.PanicOnError(git.GitExec(prjgit, "commit", "-a", "-m", "Automatic update via push via Direct Workflow"))
|
|
common.PanicOnError(git.GitExec(prjgit, "push"))
|
|
return nil
|
|
}
|
|
}
|
|
|
|
h.StdLogger.Println("push of refs the configured branch", config.Branch, ". ignoring.")
|
|
return nil
|
|
}
|
|
|
|
func verifyProjectState(git *common.GitHandler, orgName string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) {
|
|
defer func() {
|
|
e := recover()
|
|
if e != nil {
|
|
errCast, ok := e.(error)
|
|
if ok {
|
|
err = errCast
|
|
}
|
|
}
|
|
}()
|
|
|
|
org := common.Organization{
|
|
Username: orgName,
|
|
}
|
|
repo, err := gitea.CreateRepositoryIfNotExist(git, org, config.GitProjectName)
|
|
if err != nil {
|
|
return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", orgName, config.GitProjectName, err)
|
|
}
|
|
|
|
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", repo.SSHURL, config.GitProjectName))
|
|
log.Println("getting submodule list")
|
|
sub, err := git.GitSubmoduleList(config.GitProjectName, "HEAD")
|
|
common.PanicOnError(err)
|
|
|
|
isGitUpdated := false
|
|
next_package:
|
|
for filename, commitId := range sub {
|
|
// ignore project gits
|
|
for _, c := range configs {
|
|
if c.GitProjectName == filename {
|
|
log.Println(" prjgit as package? ignoring project git:", filename)
|
|
continue next_package
|
|
}
|
|
}
|
|
|
|
log.Println(" verifying package:", filename, commitId, config.Branch)
|
|
commits, err := gitea.GetRecentCommits(orgName, filename, config.Branch, 10)
|
|
if err != nil {
|
|
// assumption that package does not exist, remove from project
|
|
// https://github.com/go-gitea/gitea/issues/31976
|
|
if err := git.GitExec(config.GitProjectName, "rm", filename); err != nil {
|
|
return fmt.Errorf("Failed to remove deleted submodule. Err: %w", err)
|
|
}
|
|
isGitUpdated = true
|
|
continue
|
|
}
|
|
// if err != nil {
|
|
// return fmt.Errorf("Failed to fetch recent commits for package: '%s'. Err: %w", filename, err)
|
|
// }
|
|
|
|
idx := 1000
|
|
for i, c := range commits {
|
|
if c.SHA == commitId {
|
|
idx = i
|
|
break
|
|
}
|
|
}
|
|
|
|
if idx == 0 {
|
|
// up-to-date
|
|
continue
|
|
} else if idx < len(commits) { // update
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "submodule", "update", "--init", "--depth", "1", "--checkout", filename))
|
|
common.PanicOnError(git.GitExec(filepath.Join(config.GitProjectName, filename), "fetch", "--depth", "1", "origin", commits[0].SHA))
|
|
common.PanicOnError(git.GitExec(filepath.Join(config.GitProjectName, filename), "checkout", commits[0].SHA))
|
|
isGitUpdated = true
|
|
} else {
|
|
// probably need `merge-base` or `rev-list` here instead, or the project updated already
|
|
return fmt.Errorf("Cannot find SHA of last matching update for package: '%s'. idx: %d", filename, idx)
|
|
}
|
|
}
|
|
|
|
// find all missing repositories, and add them
|
|
if debugMode {
|
|
log.Println("checking for missing repositories...")
|
|
}
|
|
repos, err := gitea.GetOrganizationRepositories(orgName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if debugMode {
|
|
log.Println(" nRepos:", len(repos))
|
|
}
|
|
|
|
next_repo:
|
|
for _, r := range repos {
|
|
if debugMode {
|
|
log.Println(" -- checking", r.Name)
|
|
}
|
|
|
|
for _, c := range configs {
|
|
if c.Organization == orgName && c.GitProjectName == r.Name {
|
|
// ignore project gits
|
|
continue next_repo
|
|
}
|
|
}
|
|
|
|
for repo := range sub {
|
|
if repo == r.Name {
|
|
// not missing
|
|
continue next_repo
|
|
}
|
|
}
|
|
|
|
if debugMode {
|
|
log.Println(" -- checking repository:", r.Name)
|
|
}
|
|
|
|
if _, err := gitea.GetRecentCommits(orgName, r.Name, config.Branch, 1); err != nil {
|
|
// assumption that package does not exist, so not part of project
|
|
// https://github.com/go-gitea/gitea/issues/31976
|
|
continue
|
|
}
|
|
|
|
// add repository to git project
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "submodule", "--quiet", "add", "--depth", "1", r.SSHURL, r.Name))
|
|
|
|
if len(config.Branch) > 0 {
|
|
common.PanicOnError(git.GitExec(path.Join(config.GitProjectName, r.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch))
|
|
common.PanicOnError(git.GitExec(path.Join(config.GitProjectName, r.Name), "checkout", config.Branch))
|
|
}
|
|
|
|
isGitUpdated = true
|
|
}
|
|
|
|
if isGitUpdated {
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC"))
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "push"))
|
|
}
|
|
|
|
if debugMode {
|
|
log.Println("Verification finished for ", orgName, ", config", config.GitProjectName)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
var checkOnStart bool
|
|
var checkInterval time.Duration
|
|
|
|
func checkRepos() {
|
|
for org, configs := range configuredRepos {
|
|
for _, config := range configs {
|
|
if checkInterval > 0 {
|
|
sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))
|
|
log.Println(" - sleep interval", sleepInterval, "until next check")
|
|
time.Sleep(sleepInterval)
|
|
}
|
|
|
|
log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName)
|
|
git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
|
|
if err != nil {
|
|
log.Println("Faield to allocate GitHandler:", err)
|
|
return
|
|
}
|
|
if err := verifyProjectState(git, org, config, configs); err != nil {
|
|
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
|
|
}
|
|
log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName)
|
|
}
|
|
}
|
|
}
|
|
|
|
func consistencyCheckProcess() {
|
|
if checkOnStart {
|
|
savedCheckInterval := checkInterval
|
|
checkInterval = 0
|
|
log.Println("== Startup consistency check begin...")
|
|
checkRepos()
|
|
log.Println("== Startup consistency check done...")
|
|
checkInterval = savedCheckInterval
|
|
}
|
|
|
|
for {
|
|
checkRepos()
|
|
}
|
|
}
|
|
|
|
var debugMode bool
|
|
|
|
func main() {
|
|
if err := common.RequireGiteaSecretToken(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
if err := common.RequireRabbitSecrets(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
workflowConfig := flag.String("config", "", "Repository and workflow definition file")
|
|
giteaHost := flag.String("gitea", "src.opensuse.org", "Gitea instance")
|
|
rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
|
|
flag.BoolVar(&debugMode, "debug", false, "Extra debugging information")
|
|
flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays")
|
|
checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
|
|
flag.Parse()
|
|
|
|
checkInterval = time.Duration(*checkIntervalHours) * time.Hour
|
|
|
|
if len(*workflowConfig) == 0 {
|
|
log.Fatalln("No configuratio file specified. Aborting")
|
|
}
|
|
|
|
configs, err := common.ReadWorkflowConfigsFile(*workflowConfig)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
configuredRepos = make(map[string][]*common.AutogitConfig)
|
|
orgs := make([]string, 0, 1)
|
|
for _, c := range configs {
|
|
if slices.Contains(c.Workflows, "direct") {
|
|
if debugMode {
|
|
log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName)
|
|
}
|
|
configs := configuredRepos[c.Organization]
|
|
if configs == nil {
|
|
configs = make([]*common.AutogitConfig, 0, 1)
|
|
}
|
|
configs = append(configs, c)
|
|
configuredRepos[c.Organization] = configs
|
|
|
|
orgs = append(orgs, c.Organization)
|
|
}
|
|
}
|
|
|
|
gitea = common.AllocateGiteaTransport(*giteaHost)
|
|
go consistencyCheckProcess()
|
|
|
|
var defs common.ListenDefinitions
|
|
|
|
defs.GitAuthor = GitAuthor
|
|
defs.RabbitURL = *rabbitUrl
|
|
|
|
defs.Handlers = make(map[string]common.RequestProcessor)
|
|
defs.Handlers[common.RequestType_Push] = processPushAction
|
|
defs.Handlers[common.RequestType_Repository] = processRepositoryAction
|
|
|
|
log.Fatal(common.ProcessRabbitMQEvents(defs, orgs))
|
|
}
|