Files
autogits/workflow-direct/main.go
Adam Majer 8db558891a direct: remove config.Branch clobbering
use our own copy of branch instead of writing it in the config.
This should fix handling of default branches where the default
branch differs between repositories.
2025-11-04 18:00:21 +01:00

588 lines
19 KiB
Go

package main
/*
* This file is part of Autogits.
*
* Copyright © 2024 SUSE LLC
*
* Autogits is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 2 of the License, or (at your option) any later
* version.
*
* Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Foobar. If not, see <https://www.gnu.org/licenses/>.
*/
import (
"flag"
"fmt"
"io/fs"
"math/rand"
"net/url"
"os"
"os/signal"
"path"
"path/filepath"
"slices"
"strings"
"syscall"
"time"
"src.opensuse.org/autogits/common"
)
const (
AppName = "direct_workflow"
GitAuthor = "AutoGits prjgit-updater"
GitEmail = "autogits-direct@noreply@src.opensuse.org"
)
var configuredRepos map[string][]*common.AutogitConfig
var gitea common.Gitea
var orgLinks map[string]*PackageRebaseLink
func isConfiguredOrg(org *common.Organization) bool {
_, found := configuredRepos[org.Username]
return found
}
type RepositoryActionProcessor struct{}
func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error {
action := request.Data.(*common.RepositoryWebhookEvent)
configs, configFound := configuredRepos[action.Organization.Username]
if !configFound {
common.LogInfo("Repository event for", action.Organization.Username, ". Not configured. Ignoring.", action.Organization.Username)
return nil
}
for _, config := range configs {
if org, repo, _ := config.GetPrjGit(); org == action.Repository.Owner.Username && repo == action.Repository.Name {
common.LogError("+ ignoring repo event for PrjGit repository", config.GitProjectName)
return nil
}
}
for _, config := range configs {
processConfiguredRepositoryAction(action, config)
}
return nil
}
func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) {
gitOrg, gitPrj, gitBranch := config.GetPrjGit()
git, err := gh.CreateGitHandler(config.Organization)
common.PanicOnError(err)
defer git.Close()
configBranch := config.Branch
if len(configBranch) == 0 {
configBranch = action.Repository.Default_Branch
if common.IsRemovedBranch(configBranch) {
common.LogDebug(" - default branch has deleted suffix. Skipping")
return
}
}
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
if err != nil {
common.LogError("Error accessing/creating prjgit:", gitOrg, gitPrj, gitBranch, err)
return
}
remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL)
common.PanicOnError(err)
switch action.Action {
case "created":
if action.Repository.Object_Format_Name != "sha256" {
common.LogError(" - '%s' repo is not sha256. Ignoring.", action.Repository.Name)
return
}
common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all")
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current"))
if branch != configBranch {
if err := git.GitExec(path.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", configBranch+":"+configBranch); err != nil {
common.LogError("error fetching branch", configBranch, ". ignoring as non-existent.", err) // no branch? so ignore repo here
return
}
common.PanicOnError(git.GitExec(path.Join(gitPrj, action.Repository.Name), "checkout", configBranch))
}
common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Auto-inclusion "+action.Repository.Name))
if !noop {
common.PanicOnError(git.GitExec(gitPrj, "push"))
}
case "deleted":
if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
common.LogDebug("delete event for", action.Repository.Name, "-- not in project. Ignoring")
return
}
common.PanicOnError(git.GitExec(gitPrj, "rm", action.Repository.Name))
common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package removal via Direct Workflow"))
if !noop {
git.GitExecOrPanic(gitPrj, "push", remoteName)
}
default:
common.LogError("Unknown action type:", action.Action)
return
}
}
type PushActionProcessor struct{}
func (*PushActionProcessor) ProcessFunc(request *common.Request) error {
action := request.Data.(*common.PushWebhookEvent)
configs, configFound := configuredRepos[action.Repository.Owner.Username]
if !configFound {
common.LogDebug("Repository event for", action.Repository.Owner.Username, ". Not configured. Ignoring.")
return nil
}
for _, config := range configs {
if gitOrg, gitPrj, _ := config.GetPrjGit(); gitOrg == action.Repository.Owner.Username && gitPrj == action.Repository.Name {
common.LogInfo("+ ignoring push to PrjGit repository", config.GitProjectName)
return nil
}
}
for _, config := range configs {
processConfiguredPushAction(action, config)
}
return nil
}
func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) {
gitOrg, gitPrj, gitBranch := config.GetPrjGit()
git, err := gh.CreateGitHandler(config.Organization)
common.PanicOnError(err)
defer git.Close()
common.LogDebug("push to:", action.Repository.Owner.Username, action.Repository.Name, "for:", gitOrg, gitPrj, gitBranch)
branch := config.Branch
if len(branch) == 0 {
if common.IsRemovedBranch(branch) {
common.LogDebug(" + default branch has removed suffix:", branch, "Skipping.")
return
}
branch = action.Repository.Default_Branch
common.LogDebug(" + using default branch", branch)
}
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
if err != nil {
common.LogError("Error accessing/creating prjgit:", gitOrg, gitPrj, err)
return
}
remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL)
common.PanicOnError(err)
headCommitId, err := git.GitRemoteHead(gitPrj, remoteName, gitBranch)
common.PanicOnError(err)
commit, ok := git.GitSubmoduleCommitId(gitPrj, action.Repository.Name, headCommitId)
for ok && action.Head_Commit.Id == commit {
common.LogDebug(" -- nothing to do, commit already in ProjectGit")
return
}
if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil {
git.GitExecOrPanic(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name)
common.LogDebug("Pushed to package that is not part of the project. Re-adding...", err)
} else if !stat.IsDir() {
common.LogError("Pushed to a package that is not a submodule but exists in the project. Ignoring.")
return
}
git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name)
defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all")
if err := git.GitExec(filepath.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "--force", remoteName, branch+":"+branch); err != nil {
common.LogError("Error fetching branch:", branch, "Ignoring as non-existent.", err)
return
}
id, err := git.GitRemoteHead(filepath.Join(gitPrj, action.Repository.Name), remoteName, branch)
common.PanicOnError(err)
if action.Head_Commit.Id == id {
git.GitExecOrPanic(filepath.Join(gitPrj, action.Repository.Name), "checkout", id)
git.GitExecOrPanic(gitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow")
if !noop {
git.GitExecOrPanic(gitPrj, "push", remoteName)
}
return
}
common.LogDebug("push of refs not on the configured branch", branch, ". ignoring.")
}
func verifyProjectState(git common.Git, org string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) {
defer func() {
e := recover()
if e != nil {
errCast, ok := e.(error)
if ok {
err = errCast
}
}
}()
gitOrg, gitPrj, gitBranch := config.GetPrjGit()
repo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
if err != nil {
return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", gitOrg, gitPrj, err)
}
remoteName, err := git.GitClone(gitPrj, gitBranch, repo.SSHURL)
common.PanicOnError(err)
defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all")
common.LogDebug(" * Getting submodule list")
sub, err := git.GitSubmoduleList(gitPrj, "HEAD")
common.PanicOnError(err)
common.LogDebug(" * Getting package links")
var pkgLinks []*PackageRebaseLink
if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), gitPrj)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 {
if data, err := os.ReadFile(path.Join(git.GetPath(), gitPrj, common.PrjLinksFile)); err == nil {
pkgLinks, err = parseProjectLinks(data)
if err != nil {
common.LogError("Cannot parse project links file:", err.Error())
pkgLinks = nil
} else {
ResolveLinks(org, pkgLinks, gitea)
}
}
} else {
common.LogInfo(" - No package links defined")
}
/* Check existing submodule that they are updated */
isGitUpdated := false
next_package:
for filename, commitId := range sub {
// ignore project gits
//for _, c := range configs {
if gitPrj == filename {
common.LogDebug(" prjgit as package? ignoring project git:", filename)
continue next_package
}
//}
branch := config.Branch
common.LogDebug(" verifying package: %s -> %s(%s)", commitId, filename, branch)
if repo, err := gitea.GetRepository(org, filename); repo == nil && err == nil {
common.LogDebug(" repository removed...")
git.GitExecOrPanic(gitPrj, "rm", filename)
isGitUpdated = true
continue
}
if len(branch) == 0 {
branch = repo.DefaultBranch
if common.IsRemovedBranch(branch) {
common.LogDebug(" Default branch for", filename, "is excluded")
git.GitExecOrPanic(gitPrj, "rm", filename)
isGitUpdated = true
continue
}
}
commits, err := gitea.GetRecentCommits(org, filename, branch, 10)
if err != nil {
common.LogDebug(" -> failed to fetch recent commits for package:", filename, " Err:", err)
continue
}
idx := 1000
for i, c := range commits {
if c.SHA == commitId {
idx = i
break
}
}
var link *PackageRebaseLink
for _, l := range pkgLinks {
if l.Pkg == filename {
link = l
common.LogDebug(" -> linked package")
// so, we need to rebase here. Can't really optimize, so clone entire package tree and remote
pkgPath := path.Join(gitPrj, filename)
git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--checkout", filename)
git.GitExecOrPanic(pkgPath, "fetch", "origin", commits[0].SHA)
git.GitExecOrPanic(pkgPath, "tag", "NOW")
git.GitExecOrPanic(pkgPath, "fetch", "origin")
git.GitExecOrPanic(pkgPath, "remote", "add", "parent", link.parentRepo.SSHURL)
git.GitExecOrPanic(pkgPath, "fetch", "parent")
git.GitExecOrPanic(pkgPath, "rebase", "--onto", "parent", link.SourceBranch)
nCommits := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgPath, "rev-list", "^NOW", "HEAD"), "\n"))
if nCommits > 0 {
if !noop {
git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+branch)
}
isGitUpdated = true
}
break
}
}
if link == nil {
if idx == 0 {
// up-to-date
continue
} else if idx < len(commits) { // update
common.PanicOnError(git.GitExec(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", filename))
common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "fetch", "--depth", "1", "origin", commits[0].SHA))
common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "checkout", commits[0].SHA))
common.LogDebug(" -> updated to", commits[0].SHA)
isGitUpdated = true
} else {
// probably need `merge-base` or `rev-list` here instead, or the project updated already
common.LogInfo(" *** Cannot find SHA of last matching update for package:", filename, " Ignoring")
}
}
}
// find all missing repositories, and add them
common.LogDebug("checking for missing repositories...")
repos, err := gitea.GetOrganizationRepositories(org)
if err != nil {
return err
}
common.LogDebug(" nRepos:", len(repos))
/* Check repositories in org to make sure they are included in project git */
next_repo:
for _, r := range repos {
// for _, c := range configs {
if gitPrj == r.Name {
// ignore project gits
continue next_repo
}
// }
for repo := range sub {
if repo == r.Name {
// not missing
continue next_repo
}
}
common.LogDebug(" -- checking repository:", r.Name)
branch := config.Branch
if len(branch) == 0 {
branch = r.DefaultBranch
if common.IsRemovedBranch(branch) {
continue
}
}
if commits, err := gitea.GetRecentCommits(org, r.Name, branch, 1); err != nil || len(commits) == 0 {
// assumption that package does not exist, so not part of project
// https://github.com/go-gitea/gitea/issues/31976
// or, we do not have commits here
continue
}
// add repository to git project
common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name))
curBranch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current"))
if branch != curBranch {
if err := git.GitExec(path.Join(gitPrj, r.Name), "fetch", "--depth", "1", "origin", branch+":"+branch); err != nil {
return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", branch, repo.Owner.UserName, r.Name)
}
common.PanicOnError(git.GitExec(path.Join(gitPrj, r.Name), "checkout", branch))
}
isGitUpdated = true
}
if isGitUpdated {
common.PanicOnError(git.GitExec(gitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC"))
if !noop {
git.GitExecOrPanic(gitPrj, "push", remoteName)
}
}
common.LogInfo("Verification finished for ", org, ", prjgit:", config.GitProjectName)
return nil
}
var checkOnStart bool
var noop bool
var checkInterval time.Duration
func checkOrg(org string, configs []*common.AutogitConfig) {
git, err := gh.CreateGitHandler(org)
if err != nil {
common.LogError("Failed to allocate GitHandler:", err)
return
}
defer git.Close()
for _, config := range configs {
common.LogInfo(" ++ starting verification, org:", org, "config:", config.GitProjectName)
if err := verifyProjectState(git, org, config, configs); err != nil {
common.LogError(" *** verification failed, org:", org, err)
} else {
common.LogError(" ++ verification complete, org:", org, config.GitProjectName)
}
}
}
func checkRepos() {
for org, configs := range configuredRepos {
if checkInterval > 0 {
sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))
common.LogInfo(" - sleep interval", sleepInterval, "until next check")
time.Sleep(sleepInterval)
}
checkOrg(org, configs)
}
}
func consistencyCheckProcess() {
if checkOnStart {
savedCheckInterval := checkInterval
checkInterval = 0
common.LogInfo("== Startup consistency check begin...")
checkRepos()
common.LogInfo("== Startup consistency check done...")
checkInterval = savedCheckInterval
}
for {
checkRepos()
}
}
var DebugMode bool
var gh common.GitHandlerGenerator
func updateConfiguration(configFilename string, orgs *[]string) {
configFile, err := common.ReadConfigFile(configFilename)
if err != nil {
common.LogError(err)
os.Exit(4)
}
configs, _ := common.ResolveWorkflowConfigs(gitea, configFile)
configuredRepos = make(map[string][]*common.AutogitConfig)
*orgs = make([]string, 0, 1)
for _, c := range configs {
if slices.Contains(c.Workflows, "direct") {
common.LogDebug(" + adding org:", c.Organization, ", branch:", c.Branch, ", prjgit:", c.GitProjectName)
configs := configuredRepos[c.Organization]
if configs == nil {
configs = make([]*common.AutogitConfig, 0, 1)
}
configs = append(configs, c)
configuredRepos[c.Organization] = configs
*orgs = append(*orgs, c.Organization)
}
}
}
func main() {
configFilename := flag.String("config", "", "List of PrjGit")
giteaUrl := flag.String("gitea-url", "https://src.opensuse.org", "Gitea instance")
rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information")
flag.BoolVar(&noop, "dry", false, "Dry mode. Do not push changes to remote repo.")
flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays")
checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
basePath := flag.String("repo-path", "", "Repository path. Default is temporary directory")
flag.Parse()
if err := common.RequireGiteaSecretToken(); err != nil {
common.LogError(err)
os.Exit(1)
}
if err := common.RequireRabbitSecrets(); err != nil {
common.LogError(err)
os.Exit(1)
}
defs := &common.RabbitMQGiteaEventsProcessor{}
var err error
if len(*basePath) == 0 {
*basePath, err = os.MkdirTemp(os.TempDir(), AppName)
if err != nil {
common.LogError(err)
os.Exit(1)
}
}
gh, err = common.AllocateGitWorkTree(*basePath, GitAuthor, GitEmail)
if err != nil {
common.LogError(err)
os.Exit(1)
}
// handle reconfiguration
signalChannel := make(chan os.Signal, 1)
defer close(signalChannel)
go func() {
for {
sig, ok := <-signalChannel
if !ok {
return
}
if sig != syscall.SIGHUP {
common.LogError("Unexpected signal received:", sig)
continue
}
common.LogError("*** Reconfiguring ***")
updateConfiguration(*configFilename, &defs.Orgs)
defs.Connection().UpdateTopics(defs)
}
}()
signal.Notify(signalChannel, syscall.SIGHUP)
checkInterval = time.Duration(*checkIntervalHours) * time.Hour
gitea = common.AllocateGiteaTransport(*giteaUrl)
CurrentUser, err := gitea.GetCurrentUser()
if err != nil {
common.LogError("Cannot fetch current user:", err)
os.Exit(2)
}
common.LogInfo("Current User:", CurrentUser.UserName)
updateConfiguration(*configFilename, &defs.Orgs)
defs.Connection().RabbitURL, err = url.Parse(*rabbitUrl)
if err != nil {
common.LogError("cannot parse server URL. Err:", err)
os.Exit(3)
}
go consistencyCheckProcess()
common.LogInfo("defs:", *defs)
defs.Handlers = make(map[string]common.RequestProcessor)
defs.Handlers[common.RequestType_Push] = &PushActionProcessor{}
defs.Handlers[common.RequestType_Repository] = &RepositoryActionProcessor{}
common.LogError(common.ProcessRabbitMQEvents(defs))
}