445 lines
14 KiB
Go
445 lines
14 KiB
Go
package main
|
|
|
|
/*
|
|
* This file is part of Autogits.
|
|
*
|
|
* Copyright © 2024 SUSE LLC
|
|
*
|
|
* Autogits is free software: you can redistribute it and/or modify it under
|
|
* the terms of the GNU General Public License as published by the Free Software
|
|
* Foundation, either version 2 of the License, or (at your option) any later
|
|
* version.
|
|
*
|
|
* Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
|
|
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along with
|
|
* Foobar. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"math/rand"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"slices"
|
|
"time"
|
|
|
|
"src.opensuse.org/autogits/common"
|
|
)
|
|
|
|
const (
|
|
AppName = "direct_workflow"
|
|
GitAuthor = "AutoGits prjgit-updater"
|
|
GitEmail = "adam+autogits-direct@zombino.com"
|
|
)
|
|
|
|
var configuredRepos map[string][]*common.AutogitConfig
|
|
var gitea common.Gitea
|
|
|
|
func isConfiguredOrg(org *common.Organization) bool {
|
|
_, found := configuredRepos[org.Username]
|
|
return found
|
|
}
|
|
|
|
func concatenateErrors(err1, err2 error) error {
|
|
if err1 == nil {
|
|
return err2
|
|
}
|
|
|
|
if err2 == nil {
|
|
return err1
|
|
}
|
|
|
|
return fmt.Errorf("%w\n%w", err1, err2)
|
|
}
|
|
|
|
func processRepositoryAction(request *common.Request) error {
|
|
action := request.Data.(*common.RepositoryWebhookEvent)
|
|
configs, configFound := configuredRepos[action.Organization.Username]
|
|
|
|
if !configFound {
|
|
log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username)
|
|
return nil
|
|
}
|
|
|
|
for _, config := range configs {
|
|
if config.GitProjectName == action.Repository.Name {
|
|
log.Println("+ ignoring repo event for PrjGit repository", config.GitProjectName)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
var err error
|
|
for _, config := range configs {
|
|
err = concatenateErrors(err, processConfiguredRepositoryAction(action, config))
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error {
|
|
prjgit := config.GitProjectName
|
|
git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
|
|
common.PanicOnError(err)
|
|
if !DebugMode {
|
|
defer git.Close()
|
|
}
|
|
|
|
if len(config.Branch) == 0 {
|
|
config.Branch = action.Repository.Default_Branch
|
|
}
|
|
|
|
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, *action.Organization, prjgit)
|
|
if err != nil {
|
|
return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
|
|
}
|
|
|
|
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, prjgit))
|
|
|
|
switch action.Action {
|
|
case "created":
|
|
common.PanicOnError(git.GitExec(prjgit, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
|
|
if _, err := git.GitBranchHead(path.Join(prjgit, action.Repository.Name), config.Branch); err != nil {
|
|
if err := git.GitExec(path.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
|
|
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
|
|
}
|
|
}
|
|
common.PanicOnError(git.GitExec(path.Join(prjgit, action.Repository.Name), "checkout", config.Branch))
|
|
common.PanicOnError(git.GitExec(prjgit, "commit", "-m", "Automatic package inclusion via Direct Workflow"))
|
|
common.PanicOnError(git.GitExec(prjgit, "push"))
|
|
|
|
case "deleted":
|
|
if stat, err := os.Stat(filepath.Join(git.GitPath, prjgit, action.Repository.Name)); err != nil || !stat.IsDir() {
|
|
if git.DebugLogger {
|
|
log.Println("delete event for", action.Repository.Name, "-- not in project. Ignoring")
|
|
}
|
|
return nil
|
|
}
|
|
common.PanicOnError(git.GitExec(prjgit, "rm", action.Repository.Name))
|
|
common.PanicOnError(git.GitExec(prjgit, "commit", "-m", "Automatic package removal via Direct Workflow"))
|
|
common.PanicOnError(git.GitExec(prjgit, "push"))
|
|
|
|
default:
|
|
return fmt.Errorf("%s: %s", "Unknown action type", action.Action)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func processPushAction(request *common.Request) error {
|
|
action := request.Data.(*common.PushWebhookEvent)
|
|
configs, configFound := configuredRepos[action.Repository.Owner.Username]
|
|
|
|
if !configFound {
|
|
log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Repository.Owner.Username)
|
|
return nil
|
|
}
|
|
|
|
for _, config := range configs {
|
|
if config.GitProjectName == action.Repository.Name {
|
|
log.Println("+ ignoring push to PrjGit repository", config.GitProjectName)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
var err error
|
|
for _, config := range configs {
|
|
err = concatenateErrors(err, processConfiguredPushAction(action, config))
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) error {
|
|
prjgit := config.GitProjectName
|
|
git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
|
|
common.PanicOnError(err)
|
|
if !DebugMode {
|
|
defer git.Close()
|
|
}
|
|
|
|
if len(config.Branch) == 0 {
|
|
config.Branch = action.Repository.Default_Branch
|
|
log.Println(" + default branch", action.Repository.Default_Branch)
|
|
}
|
|
|
|
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, *action.Repository.Owner, prjgit)
|
|
if err != nil {
|
|
return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
|
|
}
|
|
|
|
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, prjgit))
|
|
if stat, err := os.Stat(filepath.Join(git.GitPath, prjgit, action.Repository.Name)); err != nil || !stat.IsDir() {
|
|
if git.DebugLogger {
|
|
log.Println("Pushed to package that is not part of the project. Ignoring:", err)
|
|
}
|
|
return nil
|
|
}
|
|
common.PanicOnError(git.GitExec(prjgit, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name))
|
|
if err := git.GitExec(filepath.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
|
|
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
|
|
}
|
|
id, err := git.GitBranchHead(filepath.Join(prjgit, action.Repository.Name), config.Branch)
|
|
common.PanicOnError(err)
|
|
for _, commitId := range action.Commits {
|
|
if commitId.Id == id {
|
|
common.PanicOnError(git.GitExec(filepath.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", id))
|
|
common.PanicOnError(git.GitExec(filepath.Join(prjgit, action.Repository.Name), "checkout", id))
|
|
common.PanicOnError(git.GitExec(prjgit, "commit", "-a", "-m", "Automatic update via push via Direct Workflow"))
|
|
common.PanicOnError(git.GitExec(prjgit, "push"))
|
|
return nil
|
|
}
|
|
}
|
|
|
|
log.Println("push of refs not on the configured branch", config.Branch, ". ignoring.")
|
|
return nil
|
|
}
|
|
|
|
func verifyProjectState(git *common.GitHandler, orgName string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) {
|
|
defer func() {
|
|
e := recover()
|
|
if e != nil {
|
|
errCast, ok := e.(error)
|
|
if ok {
|
|
err = errCast
|
|
}
|
|
}
|
|
}()
|
|
|
|
org := common.Organization{
|
|
Username: orgName,
|
|
}
|
|
repo, err := gitea.CreateRepositoryIfNotExist(git, org, config.GitProjectName)
|
|
if err != nil {
|
|
return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", orgName, config.GitProjectName, err)
|
|
}
|
|
|
|
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", repo.SSHURL, config.GitProjectName))
|
|
log.Println("getting submodule list")
|
|
sub, err := git.GitSubmoduleList(config.GitProjectName, "HEAD")
|
|
common.PanicOnError(err)
|
|
|
|
isGitUpdated := false
|
|
next_package:
|
|
for filename, commitId := range sub {
|
|
// ignore project gits
|
|
for _, c := range configs {
|
|
if c.GitProjectName == filename {
|
|
log.Println(" prjgit as package? ignoring project git:", filename)
|
|
continue next_package
|
|
}
|
|
}
|
|
|
|
log.Println(" verifying package:", filename, commitId, config.Branch)
|
|
commits, err := gitea.GetRecentCommits(orgName, filename, config.Branch, 10)
|
|
if err != nil {
|
|
// assumption that package does not exist, remove from project
|
|
// https://github.com/go-gitea/gitea/issues/31976
|
|
if err := git.GitExec(config.GitProjectName, "rm", filename); err != nil {
|
|
return fmt.Errorf("Failed to remove deleted submodule. Err: %w", err)
|
|
}
|
|
isGitUpdated = true
|
|
continue
|
|
}
|
|
// if err != nil {
|
|
// return fmt.Errorf("Failed to fetch recent commits for package: '%s'. Err: %w", filename, err)
|
|
// }
|
|
|
|
idx := 1000
|
|
for i, c := range commits {
|
|
if c.SHA == commitId {
|
|
idx = i
|
|
break
|
|
}
|
|
}
|
|
|
|
if idx == 0 {
|
|
// up-to-date
|
|
continue
|
|
} else if idx < len(commits) { // update
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "submodule", "update", "--init", "--depth", "1", "--checkout", filename))
|
|
common.PanicOnError(git.GitExec(filepath.Join(config.GitProjectName, filename), "fetch", "--depth", "1", "origin", commits[0].SHA))
|
|
common.PanicOnError(git.GitExec(filepath.Join(config.GitProjectName, filename), "checkout", commits[0].SHA))
|
|
isGitUpdated = true
|
|
} else {
|
|
// probably need `merge-base` or `rev-list` here instead, or the project updated already
|
|
return fmt.Errorf("Cannot find SHA of last matching update for package: '%s'. idx: %d", filename, idx)
|
|
}
|
|
}
|
|
|
|
// find all missing repositories, and add them
|
|
if DebugMode {
|
|
log.Println("checking for missing repositories...")
|
|
}
|
|
repos, err := gitea.GetOrganizationRepositories(orgName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if DebugMode {
|
|
log.Println(" nRepos:", len(repos))
|
|
}
|
|
|
|
next_repo:
|
|
for _, r := range repos {
|
|
if DebugMode {
|
|
log.Println(" -- checking", r.Name)
|
|
}
|
|
|
|
for _, c := range configs {
|
|
if c.Organization == orgName && c.GitProjectName == r.Name {
|
|
// ignore project gits
|
|
continue next_repo
|
|
}
|
|
}
|
|
|
|
for repo := range sub {
|
|
if repo == r.Name {
|
|
// not missing
|
|
continue next_repo
|
|
}
|
|
}
|
|
|
|
if DebugMode {
|
|
log.Println(" -- checking repository:", r.Name)
|
|
}
|
|
|
|
if _, err := gitea.GetRecentCommits(orgName, r.Name, config.Branch, 1); err != nil {
|
|
// assumption that package does not exist, so not part of project
|
|
// https://github.com/go-gitea/gitea/issues/31976
|
|
continue
|
|
}
|
|
|
|
// add repository to git project
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name))
|
|
|
|
if len(config.Branch) > 0 {
|
|
if err := git.GitExec(path.Join(config.GitProjectName, r.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
|
|
return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", config.Branch, repo.Owner.UserName, r.Name)
|
|
}
|
|
common.PanicOnError(git.GitExec(path.Join(config.GitProjectName, r.Name), "checkout", config.Branch))
|
|
}
|
|
|
|
isGitUpdated = true
|
|
}
|
|
|
|
if isGitUpdated {
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC"))
|
|
common.PanicOnError(git.GitExec(config.GitProjectName, "push"))
|
|
}
|
|
|
|
if DebugMode {
|
|
log.Println("Verification finished for ", orgName, ", config", config.GitProjectName)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
var checkOnStart bool
|
|
var checkInterval time.Duration
|
|
|
|
func checkRepos() {
|
|
for org, configs := range configuredRepos {
|
|
for _, config := range configs {
|
|
if checkInterval > 0 {
|
|
sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))
|
|
log.Println(" - sleep interval", sleepInterval, "until next check")
|
|
time.Sleep(sleepInterval)
|
|
}
|
|
|
|
log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName)
|
|
git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
|
|
if err != nil {
|
|
log.Println("Faield to allocate GitHandler:", err)
|
|
return
|
|
}
|
|
if err := verifyProjectState(git, org, config, configs); err != nil {
|
|
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
|
|
}
|
|
log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName)
|
|
}
|
|
}
|
|
}
|
|
|
|
func consistencyCheckProcess() {
|
|
if checkOnStart {
|
|
savedCheckInterval := checkInterval
|
|
checkInterval = 0
|
|
log.Println("== Startup consistency check begin...")
|
|
checkRepos()
|
|
log.Println("== Startup consistency check done...")
|
|
checkInterval = savedCheckInterval
|
|
}
|
|
|
|
for {
|
|
checkRepos()
|
|
}
|
|
}
|
|
|
|
var DebugMode bool
|
|
|
|
func main() {
|
|
if err := common.RequireGiteaSecretToken(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
if err := common.RequireRabbitSecrets(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
workflowConfig := flag.String("config", "", "Repository and workflow definition file")
|
|
giteaHost := flag.String("gitea", "src.opensuse.org", "Gitea instance")
|
|
rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
|
|
flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information")
|
|
flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays")
|
|
checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
|
|
flag.Parse()
|
|
|
|
checkInterval = time.Duration(*checkIntervalHours) * time.Hour
|
|
|
|
if len(*workflowConfig) == 0 {
|
|
log.Fatalln("No configuratio file specified. Aborting")
|
|
}
|
|
|
|
configs, err := common.ReadWorkflowConfigsFile(*workflowConfig)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
configuredRepos = make(map[string][]*common.AutogitConfig)
|
|
orgs := make([]string, 0, 1)
|
|
for _, c := range configs {
|
|
if slices.Contains(c.Workflows, "direct") {
|
|
if DebugMode {
|
|
log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName)
|
|
}
|
|
configs := configuredRepos[c.Organization]
|
|
if configs == nil {
|
|
configs = make([]*common.AutogitConfig, 0, 1)
|
|
}
|
|
configs = append(configs, c)
|
|
configuredRepos[c.Organization] = configs
|
|
|
|
orgs = append(orgs, c.Organization)
|
|
}
|
|
}
|
|
|
|
gitea = common.AllocateGiteaTransport(*giteaHost)
|
|
go consistencyCheckProcess()
|
|
|
|
var defs common.ListenDefinitions
|
|
|
|
defs.GitAuthor = GitAuthor
|
|
defs.RabbitURL = *rabbitUrl
|
|
|
|
defs.Handlers = make(map[string]common.RequestProcessor)
|
|
defs.Handlers[common.RequestType_Push] = processPushAction
|
|
defs.Handlers[common.RequestType_Repository] = processRepositoryAction
|
|
|
|
log.Fatal(common.ProcessRabbitMQEvents(defs, orgs))
|
|
}
|