Adam Majer
940e5be2c1
config now only has reference to org or prjgits and the rest is defined in the "workflow.config" in the prjgit itself. This allows the config to be updated in the project.
533 lines
17 KiB
Go
533 lines
17 KiB
Go
package main
|
|
|
|
/*
|
|
* This file is part of Autogits.
|
|
*
|
|
* Copyright © 2024 SUSE LLC
|
|
*
|
|
* Autogits is free software: you can redistribute it and/or modify it under
|
|
* the terms of the GNU General Public License as published by the Free Software
|
|
* Foundation, either version 2 of the License, or (at your option) any later
|
|
* version.
|
|
*
|
|
* Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
|
|
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along with
|
|
* Foobar. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
import (
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io/fs"
|
|
"log"
|
|
"math/rand"
|
|
"net/url"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"src.opensuse.org/autogits/common"
|
|
)
|
|
|
|
const (
|
|
AppName = "direct_workflow"
|
|
GitAuthor = "AutoGits prjgit-updater"
|
|
GitEmail = "adam+autogits-direct@zombino.com"
|
|
)
|
|
|
|
var configuredRepos map[string][]*common.AutogitConfig
|
|
var gitea common.Gitea
|
|
|
|
var orgLinks map[string]*PackageRebaseLink
|
|
|
|
func isConfiguredOrg(org *common.Organization) bool {
|
|
_, found := configuredRepos[org.Username]
|
|
return found
|
|
}
|
|
|
|
func concatenateErrors(err1, err2 error) error {
|
|
if err1 == nil {
|
|
return err2
|
|
}
|
|
|
|
if err2 == nil {
|
|
return err1
|
|
}
|
|
|
|
return fmt.Errorf("%w\n%w", err1, err2)
|
|
}
|
|
|
|
type RepositoryActionProcessor struct{}
|
|
|
|
func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error {
|
|
action := request.Data.(*common.RepositoryWebhookEvent)
|
|
configs, configFound := configuredRepos[action.Organization.Username]
|
|
|
|
if !configFound {
|
|
log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username)
|
|
return nil
|
|
}
|
|
|
|
for _, config := range configs {
|
|
if config.GitProjectName == action.Repository.Name {
|
|
log.Println("+ ignoring repo event for PrjGit repository", config.GitProjectName)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
var err error
|
|
for _, config := range configs {
|
|
err = concatenateErrors(err, processConfiguredRepositoryAction(action, config))
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error {
|
|
prjgit := config.GitProjectName
|
|
ghi := common.GitHandlerGeneratorImpl{}
|
|
git, err := ghi.CreateGitHandler(GitAuthor, GitEmail, AppName)
|
|
common.PanicOnError(err)
|
|
if !DebugMode {
|
|
defer git.Close()
|
|
}
|
|
|
|
if len(config.Branch) == 0 {
|
|
config.Branch = action.Repository.Default_Branch
|
|
}
|
|
|
|
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, action.Organization.Username, prjgit)
|
|
if err != nil {
|
|
return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
|
|
}
|
|
|
|
if _, err := fs.Stat(os.DirFS(git.GetPath()), config.GitProjectName); errors.Is(err, os.ErrNotExist) {
|
|
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, prjgit))
|
|
}
|
|
|
|
switch action.Action {
|
|
case "created":
|
|
if action.Repository.Object_Format_Name != "sha256" {
|
|
return fmt.Errorf(" - '%s' repo is not sha256. Ignoring.", action.Repository.Name)
|
|
}
|
|
common.PanicOnError(git.GitExec(prjgit, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
|
|
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(prjgit, action.Repository.Name), "branch", "--show-current"))
|
|
if branch != config.Branch {
|
|
if err := git.GitExec(path.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
|
|
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
|
|
}
|
|
common.PanicOnError(git.GitExec(path.Join(prjgit, action.Repository.Name), "checkout", config.Branch))
|
|
}
|
|
common.PanicOnError(git.GitExec(prjgit, "commit", "-m", "Automatic package inclusion via Direct Workflow"))
|
|
common.PanicOnError(git.GitExec(prjgit, "push"))
|
|
|
|
case "deleted":
|
|
if stat, err := os.Stat(filepath.Join(git.GetPath(), prjgit, action.Repository.Name)); err != nil || !stat.IsDir() {
|
|
if DebugMode {
|
|
log.Println("delete event for", action.Repository.Name, "-- not in project. Ignoring")
|
|
}
|
|
return nil
|
|
}
|
|
common.PanicOnError(git.GitExec(prjgit, "rm", action.Repository.Name))
|
|
common.PanicOnError(git.GitExec(prjgit, "commit", "-m", "Automatic package removal via Direct Workflow"))
|
|
common.PanicOnError(git.GitExec(prjgit, "push"))
|
|
|
|
default:
|
|
return fmt.Errorf("%s: %s", "Unknown action type", action.Action)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type PushActionProcessor struct{}
|
|
|
|
func (*PushActionProcessor) ProcessFunc(request *common.Request) error {
|
|
action := request.Data.(*common.PushWebhookEvent)
|
|
configs, configFound := configuredRepos[action.Repository.Owner.Username]
|
|
|
|
if !configFound {
|
|
log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Repository.Owner.Username)
|
|
return nil
|
|
}
|
|
|
|
for _, config := range configs {
|
|
if config.GitProjectName == action.Repository.Name {
|
|
log.Println("+ ignoring push to PrjGit repository", config.GitProjectName)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
var err error
|
|
for _, config := range configs {
|
|
err = concatenateErrors(err, processConfiguredPushAction(action, config))
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) error {
|
|
prjgit := config.GitProjectName
|
|
ghi := common.GitHandlerGeneratorImpl{}
|
|
git, err := ghi.CreateGitHandler(GitAuthor, GitEmail, AppName)
|
|
common.PanicOnError(err)
|
|
if !DebugMode {
|
|
defer git.Close()
|
|
}
|
|
|
|
if len(config.Branch) == 0 {
|
|
config.Branch = action.Repository.Default_Branch
|
|
log.Println(" + default branch", action.Repository.Default_Branch)
|
|
}
|
|
|
|
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, action.Repository.Owner.Username, prjgit)
|
|
if err != nil {
|
|
return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
|
|
}
|
|
|
|
if _, err := fs.Stat(os.DirFS(git.GetPath()), config.GitProjectName); errors.Is(err, os.ErrNotExist) {
|
|
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, prjgit))
|
|
}
|
|
if stat, err := os.Stat(filepath.Join(git.GetPath(), prjgit, action.Repository.Name)); err != nil || !stat.IsDir() {
|
|
if DebugMode {
|
|
log.Println("Pushed to package that is not part of the project. Ignoring:", err)
|
|
}
|
|
return nil
|
|
}
|
|
common.PanicOnError(git.GitExec(prjgit, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name))
|
|
if err := git.GitExec(filepath.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
|
|
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
|
|
}
|
|
id, err := git.GitBranchHead(filepath.Join(prjgit, action.Repository.Name), config.Branch)
|
|
common.PanicOnError(err)
|
|
for _, commitId := range action.Commits {
|
|
if commitId.Id == id {
|
|
common.PanicOnError(git.GitExec(filepath.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", id))
|
|
common.PanicOnError(git.GitExec(filepath.Join(prjgit, action.Repository.Name), "checkout", id))
|
|
common.PanicOnError(git.GitExec(prjgit, "commit", "-a", "-m", "Automatic update via push via Direct Workflow"))
|
|
common.PanicOnError(git.GitExec(prjgit, "push"))
|
|
return nil
|
|
}
|
|
}
|
|
|
|
log.Println("push of refs not on the configured branch", config.Branch, ". ignoring.")
|
|
return nil
|
|
}
|
|
|
|
func verifyProjectState(git common.Git, org string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) {
|
|
defer func() {
|
|
e := recover()
|
|
if e != nil {
|
|
errCast, ok := e.(error)
|
|
if ok {
|
|
err = errCast
|
|
}
|
|
}
|
|
}()
|
|
|
|
repo, err := gitea.CreateRepositoryIfNotExist(git, org, config.GitProjectName)
|
|
if err != nil {
|
|
return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", org, config.GitProjectName, err)
|
|
}
|
|
|
|
if _, err := fs.Stat(os.DirFS(git.GetPath()), config.GitProjectName); errors.Is(err, os.ErrNotExist) {
|
|
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", repo.SSHURL, config.GitProjectName))
|
|
}
|
|
log.Println(" * Getting submodule list")
|
|
sub, err := git.GitSubmoduleList(config.GitProjectName, "HEAD")
|
|
common.PanicOnError(err)
|
|
|
|
log.Println(" * Getting package links")
|
|
var pkgLinks []*PackageRebaseLink
|
|
if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), config.GitProjectName)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 {
|
|
if data, err := os.ReadFile(path.Join(git.GetPath(), config.GitProjectName, common.PrjLinksFile)); err == nil {
|
|
pkgLinks, err = parseProjectLinks(data)
|
|
if err != nil {
|
|
log.Println("Cannot parse project links file:", err.Error())
|
|
pkgLinks = nil
|
|
} else {
|
|
ResolveLinks(org, pkgLinks, gitea)
|
|
}
|
|
}
|
|
} else {
|
|
log.Println(" - No package links defined")
|
|
}
|
|
|
|
/* Check existing submodule that they are updated */
|
|
|
|
isGitUpdated := false
|
|
next_package:
|
|
for filename, commitId := range sub {
|
|
// ignore project gits
|
|
for _, c := range configs {
|
|
if c.GitProjectName == filename {
|
|
log.Println(" prjgit as package? ignoring project git:", filename)
|
|
continue next_package
|
|
}
|
|
}
|
|
|
|
log.Println(" verifying package:", filename, commitId, config.Branch)
|
|
commits, err := gitea.GetRecentCommits(org, filename, config.Branch, 10)
|
|
if err != nil {
|
|
// assumption that package does not exist, remove from project
|
|
// https://github.com/go-gitea/gitea/issues/31976
|
|
if err := git.GitExec(config.GitProjectName, "rm", filename); err != nil {
|
|
return fmt.Errorf("Failed to remove deleted submodule. Err: %w", err)
|
|
}
|
|
isGitUpdated = true
|
|
continue
|
|
}
|
|
// if err != nil {
|
|
// return fmt.Errorf("Failed to fetch recent commits for package: '%s'. Err: %w", filename, err)
|
|
// }
|
|
|
|
idx := 1000
|
|
for i, c := range commits {
|
|
if c.SHA == commitId {
|
|
idx = i
|
|
break
|
|
}
|
|
}
|
|
|
|
var link *PackageRebaseLink
|
|
for _, l := range pkgLinks {
|
|
if l.Pkg == filename {
|
|
link = l
|
|
|
|
log.Println(" -> linked package")
|
|
// so, we need to rebase here. Can't really optimize, so clone entire package tree and remote
|
|
pkgPath := path.Join(config.GitProjectName, filename)
|
|
git.GitExecOrPanic(config.GitProjectName, "submodule", "update", "--init", "--checkout", filename)
|
|
git.GitExecOrPanic(pkgPath, "fetch", "origin", commits[0].SHA)
|
|
git.GitExecOrPanic(pkgPath, "tag", "NOW")
|
|
git.GitExecOrPanic(pkgPath, "fetch", "origin")
|
|
git.GitExecOrPanic(pkgPath, "remote", "add", "parent", link.parentRepo.SSHURL)
|
|
git.GitExecOrPanic(pkgPath, "fetch", "parent")
|
|
git.GitExecOrPanic(pkgPath, "rebase", "--onto", "parent", link.SourceBranch)
|
|
|
|
nCommits := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgPath, "rev-list", "^NOW", "HEAD"), "\n"))
|
|
if nCommits > 0 {
|
|
git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+config.Branch)
|
|
isGitUpdated = true
|
|
}
|
|
|
|
break
|
|
}
|
|
}
|
|
|
|
if link == nil {
|
|
if idx == 0 {
|
|
// up-to-date
|
|
continue
|
|
} else if idx < len(commits) { // update
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "submodule", "update", "--init", "--depth", "1", "--checkout", filename))
|
|
common.PanicOnError(git.GitExec(filepath.Join(config.GitProjectName, filename), "fetch", "--depth", "1", "origin", commits[0].SHA))
|
|
common.PanicOnError(git.GitExec(filepath.Join(config.GitProjectName, filename), "checkout", commits[0].SHA))
|
|
isGitUpdated = true
|
|
} else {
|
|
// probably need `merge-base` or `rev-list` here instead, or the project updated already
|
|
return fmt.Errorf("Cannot find SHA of last matching update for package: '%s'. idx: %d", filename, idx)
|
|
}
|
|
}
|
|
}
|
|
|
|
// find all missing repositories, and add them
|
|
if DebugMode {
|
|
log.Println("checking for missing repositories...")
|
|
}
|
|
repos, err := gitea.GetOrganizationRepositories(org)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if DebugMode {
|
|
log.Println(" nRepos:", len(repos))
|
|
}
|
|
|
|
/* Check repositories in org to make sure they are included in project git */
|
|
next_repo:
|
|
for _, r := range repos {
|
|
if DebugMode {
|
|
log.Println(" -- checking", r.Name)
|
|
}
|
|
|
|
if r.ObjectFormatName != "sha256" {
|
|
if DebugMode {
|
|
log.Println(" + ", r.ObjectFormatName, ". Needs to be sha256. Ignoring")
|
|
}
|
|
continue next_repo
|
|
}
|
|
|
|
for _, c := range configs {
|
|
if c.Organization == org && c.GitProjectName == r.Name {
|
|
// ignore project gits
|
|
continue next_repo
|
|
}
|
|
}
|
|
|
|
for repo := range sub {
|
|
if repo == r.Name {
|
|
// not missing
|
|
continue next_repo
|
|
}
|
|
}
|
|
|
|
if DebugMode {
|
|
log.Println(" -- checking repository:", r.Name)
|
|
}
|
|
|
|
if _, err := gitea.GetRecentCommits(org, r.Name, config.Branch, 1); err != nil {
|
|
// assumption that package does not exist, so not part of project
|
|
// https://github.com/go-gitea/gitea/issues/31976
|
|
continue
|
|
}
|
|
|
|
// add repository to git project
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name))
|
|
|
|
if len(config.Branch) > 0 {
|
|
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(config.GitProjectName, r.Name), "branch", "--show-current"))
|
|
if branch != config.Branch {
|
|
if err := git.GitExec(path.Join(config.GitProjectName, r.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
|
|
return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", config.Branch, repo.Owner.UserName, r.Name)
|
|
}
|
|
common.PanicOnError(git.GitExec(path.Join(config.GitProjectName, r.Name), "checkout", config.Branch))
|
|
}
|
|
}
|
|
|
|
isGitUpdated = true
|
|
}
|
|
|
|
if isGitUpdated {
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC"))
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "push"))
|
|
}
|
|
|
|
if DebugMode {
|
|
log.Println("Verification finished for ", org, ", config", config.GitProjectName)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
var checkOnStart bool
|
|
var checkInterval time.Duration
|
|
|
|
func checkRepos() {
|
|
for org, configs := range configuredRepos {
|
|
for _, config := range configs {
|
|
if checkInterval > 0 {
|
|
sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))
|
|
log.Println(" - sleep interval", sleepInterval, "until next check")
|
|
time.Sleep(sleepInterval)
|
|
}
|
|
|
|
log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName)
|
|
ghi := common.GitHandlerGeneratorImpl{}
|
|
git, err := ghi.CreateGitHandler(GitAuthor, GitEmail, AppName)
|
|
if err != nil {
|
|
log.Println("Faield to allocate GitHandler:", err)
|
|
return
|
|
}
|
|
if err := verifyProjectState(git, org, config, configs); err != nil {
|
|
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
|
|
}
|
|
log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName)
|
|
}
|
|
}
|
|
}
|
|
|
|
func consistencyCheckProcess() {
|
|
if checkOnStart {
|
|
savedCheckInterval := checkInterval
|
|
checkInterval = 0
|
|
log.Println("== Startup consistency check begin...")
|
|
checkRepos()
|
|
log.Println("== Startup consistency check done...")
|
|
checkInterval = savedCheckInterval
|
|
}
|
|
|
|
for {
|
|
checkRepos()
|
|
}
|
|
}
|
|
|
|
var DebugMode bool
|
|
|
|
func updateConfiguration(configFilename string, orgs *[]string) {
|
|
configFile, err := common.ReadConfigFile(configFilename)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
configs, _ := common.ResolveWorkflowConfigs(gitea, configFile)
|
|
configuredRepos = make(map[string][]*common.AutogitConfig)
|
|
*orgs = make([]string, 0, 1)
|
|
for _, c := range configs {
|
|
if slices.Contains(c.Workflows, "direct") {
|
|
if DebugMode {
|
|
log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName)
|
|
}
|
|
configs := configuredRepos[c.Organization]
|
|
if configs == nil {
|
|
configs = make([]*common.AutogitConfig, 0, 1)
|
|
}
|
|
configs = append(configs, c)
|
|
configuredRepos[c.Organization] = configs
|
|
|
|
*orgs = append(*orgs, c.Organization)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func main() {
|
|
configFilename := flag.String("config", "", "List of PrjGit")
|
|
giteaHost := flag.String("gitea", "src.opensuse.org", "Gitea instance")
|
|
rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
|
|
flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information")
|
|
flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays")
|
|
checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
|
|
flag.Parse()
|
|
|
|
if err := common.RequireGiteaSecretToken(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
if err := common.RequireRabbitSecrets(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
checkInterval = time.Duration(*checkIntervalHours) * time.Hour
|
|
|
|
gitea = common.AllocateGiteaTransport(*giteaHost)
|
|
CurrentUser, err := gitea.GetCurrentUser()
|
|
if err != nil {
|
|
log.Fatalln("Cannot fetch current user:", err)
|
|
}
|
|
log.Println("Current User:", CurrentUser.UserName)
|
|
|
|
var defs common.ListenDefinitions
|
|
updateConfiguration(*configFilename, &defs.Orgs)
|
|
|
|
defs.GitAuthor = GitAuthor
|
|
defs.RabbitURL, err = url.Parse(*rabbitUrl)
|
|
if err != nil {
|
|
log.Panicf("cannot parse server URL. Err: %#v\n", err)
|
|
}
|
|
|
|
go consistencyCheckProcess()
|
|
log.Println("defs:", defs)
|
|
|
|
defs.Handlers = make(map[string]common.RequestProcessor)
|
|
defs.Handlers[common.RequestType_Push] = &PushActionProcessor{}
|
|
defs.Handlers[common.RequestType_Repository] = &RepositoryActionProcessor{}
|
|
|
|
log.Fatal(defs.ProcessRabbitMQEvents())
|
|
}
|