autogits/workflow-direct/main.go

package main

/*
 * This file is part of Autogits.
 *
 * Copyright © 2024 SUSE LLC
 *
 * Autogits is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License as published by the Free Software
 * Foundation, either version 2 of the License, or (at your option) any later
 * version.
 *
 * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 * PARTICULAR PURPOSE. See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * Foobar. If not, see <https://www.gnu.org/licenses/>.
 */

import (
	"flag"
	"fmt"
	"io/fs"
	"log"
	"math/rand"
	"net/url"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"slices"
	"strings"
	"syscall"
	"time"

	"src.opensuse.org/autogits/common"
)

const (
	AppName   = "direct_workflow"
	GitAuthor = "AutoGits prjgit-updater"
	GitEmail  = "adam+autogits-direct@zombino.com"
)

var configuredRepos map[string][]*common.AutogitConfig
var gitea common.Gitea

var orgLinks map[string]*PackageRebaseLink

func isConfiguredOrg(org *common.Organization) bool {
	_, found := configuredRepos[org.Username]
	return found
}

func concatenateErrors(err1, err2 error) error {
	if err1 == nil {
		return err2
	}

	if err2 == nil {
		return err1
	}

	return fmt.Errorf("%w\n%w", err1, err2)
}

type RepositoryActionProcessor struct{}

func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error {
	action := request.Data.(*common.RepositoryWebhookEvent)
	configs, configFound := configuredRepos[action.Organization.Username]

	if !configFound {
		log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username)
		return nil
	}

	for _, config := range configs {
		if org, repo, _ := config.GetPrjGit(); org == action.Repository.Owner.Username && repo == action.Repository.Name {
			log.Println("+ ignoring repo event for PrjGit repository", config.GitProjectName)
			return nil
		}
	}

	var err error
	for _, config := range configs {
		err = concatenateErrors(err, processConfiguredRepositoryAction(action, config))
	}

	return err
}

func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error {
	gitOrg, gitPrj, gitBranch := config.GetPrjGit()
	git, err := gh.CreateGitHandler(config.Organization)
	common.PanicOnError(err)
	defer git.Close()

	if len(config.Branch) == 0 {
		config.Branch = action.Repository.Default_Branch
	}

	prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
	if err != nil {
		return fmt.Errorf("Error accessing/creating prjgit: %s/%s#%s err: %w", gitOrg, gitPrj, gitBranch, err)
	}

	remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL)
	common.PanicOnError(err)

	switch action.Action {
	case "created":
		if action.Repository.Object_Format_Name != "sha256" {
			return fmt.Errorf("   - '%s' repo is not sha256. Ignoring.", action.Repository.Name)
		}
		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
		defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all")

		branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current"))
		if branch != config.Branch {
			if err := git.GitExec(path.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
				return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
			}
			common.PanicOnError(git.GitExec(path.Join(gitPrj, action.Repository.Name), "checkout", config.Branch))
		}
		common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package inclusion via Direct Workflow"))
		if !noop {
			common.PanicOnError(git.GitExec(gitPrj, "push"))
		}

	case "deleted":
		if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
			if DebugMode {
				log.Println("delete event for", action.Repository.Name, "-- not in project. Ignoring")
			}
			return nil
		}
		common.PanicOnError(git.GitExec(gitPrj, "rm", action.Repository.Name))
		common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package removal via Direct Workflow"))
		if !noop {
			git.GitExecOrPanic(gitPrj, "push", remoteName)
		}

	default:
		return fmt.Errorf("%s: %s", "Unknown action type", action.Action)
	}

	return nil
}

type PushActionProcessor struct{}

func (*PushActionProcessor) ProcessFunc(request *common.Request) error {
	action := request.Data.(*common.PushWebhookEvent)
	configs, configFound := configuredRepos[action.Repository.Owner.Username]

	if !configFound {
		log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Repository.Owner.Username)
		return nil
	}

	for _, config := range configs {
		if gitOrg, gitPrj, _ := config.GetPrjGit(); gitOrg == action.Repository.Owner.Username && gitPrj == action.Repository.Name {
			log.Println("+ ignoring push to PrjGit repository", config.GitProjectName)
			return nil
		}
	}

	var err error
	for _, config := range configs {
		err = concatenateErrors(err, processConfiguredPushAction(action, config))
	}

	return err
}

func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) error {
	gitOrg, gitPrj, gitBranch := config.GetPrjGit()
	git, err := gh.CreateGitHandler(config.Organization)
	common.PanicOnError(err)
	defer git.Close()

	log.Printf("push to: %s/%s for %s/%s#%s", action.Repository.Owner.Username, action.Repository.Name, gitOrg, gitPrj, gitBranch)
	if len(config.Branch) == 0 {
		config.Branch = action.Repository.Default_Branch
		log.Println("  + default branch", action.Repository.Default_Branch)
	}

	prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
	if err != nil {
		return fmt.Errorf("Error accessing/creating prjgit: %s/%s err: %w", gitOrg, gitPrj, err)
	}

	remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL)
	common.PanicOnError(err)
	headCommitId, err := git.GitRemoteHead(gitPrj, remoteName, gitBranch)
	common.PanicOnError(err)
	commit, ok := git.GitSubmoduleCommitId(gitPrj, action.Repository.Name, headCommitId)
	for ok && action.Head_Commit.Id == commit {
		log.Println("  -- nothing to do, commit already in ProjectGit")
		return nil
	}

	if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
		if DebugMode {
			log.Println("Pushed to package that is not part of the project. Ignoring:", err)
		}
		return nil
	}
	git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name)
	defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all")

	if err := git.GitExec(filepath.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "--force", remoteName, config.Branch+":"+config.Branch); err != nil {
		return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
	}
	id, err := git.GitRemoteHead(filepath.Join(gitPrj, action.Repository.Name), remoteName, config.Branch)
	common.PanicOnError(err)
	if action.Head_Commit.Id == id {
		git.GitExecOrPanic(filepath.Join(gitPrj, action.Repository.Name), "checkout", id)
		git.GitExecOrPanic(gitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow")
		if !noop {
			git.GitExecOrPanic(gitPrj, "push", remoteName)
		}
		return nil
	}

	log.Println("push of refs not on the configured branch", config.Branch, ". ignoring.")
	return nil
}

func verifyProjectState(git common.Git, org string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) {
	defer func() {
		e := recover()
		if e != nil {
			errCast, ok := e.(error)
			if ok {
				err = errCast
			}
		}
	}()

	gitOrg, gitPrj, gitBranch := config.GetPrjGit()
	repo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
	if err != nil {
		return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", gitOrg, gitPrj, err)
	}

	remoteName, err := git.GitClone(gitPrj, gitBranch, repo.SSHURL)
	common.PanicOnError(err)
	defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all")

	log.Println("   * Getting submodule list")
	sub, err := git.GitSubmoduleList(gitPrj, "HEAD")
	common.PanicOnError(err)

	log.Println("   * Getting package links")
	var pkgLinks []*PackageRebaseLink
	if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), gitPrj)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 {
		if data, err := os.ReadFile(path.Join(git.GetPath(), gitPrj, common.PrjLinksFile)); err == nil {
			pkgLinks, err = parseProjectLinks(data)
			if err != nil {
				log.Println("Cannot parse project links file:", err.Error())
				pkgLinks = nil
			} else {
				ResolveLinks(org, pkgLinks, gitea)
			}
		}
	} else {
		log.Println("   - No package links defined")
	}

	/* Check existing submodule that they are updated */

	isGitUpdated := false
next_package:
	for filename, commitId := range sub {
		// ignore project gits
		//for _, c := range configs {
		if gitPrj == filename {
			log.Println("     prjgit as package? ignoring project git:", filename)
			continue next_package
		}
		//}

		log.Printf("     verifying package: %s -> %s(%s)", commitId, filename, config.Branch)
		commits, err := gitea.GetRecentCommits(org, filename, config.Branch, 10)
		if len(commits) == 0 {
			if repo, err := gitea.GetRepository(org, filename); repo == nil && err == nil {
				git.GitExecOrPanic(gitPrj, "rm", filename)
				isGitUpdated = true
			}
		}
		if err != nil {
			log.Println("   -> failed to fetch recent commits for package:", filename, " Err:", err)
			continue
		}

		idx := 1000
		for i, c := range commits {
			if c.SHA == commitId {
				idx = i
				break
			}
		}

		var link *PackageRebaseLink
		for _, l := range pkgLinks {
			if l.Pkg == filename {
				link = l

				log.Println("      -> linked package")
				// so, we need to rebase here. Can't really optimize, so clone entire package tree and remote
				pkgPath := path.Join(gitPrj, filename)
				git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--checkout", filename)
				git.GitExecOrPanic(pkgPath, "fetch", "origin", commits[0].SHA)
				git.GitExecOrPanic(pkgPath, "tag", "NOW")
				git.GitExecOrPanic(pkgPath, "fetch", "origin")
				git.GitExecOrPanic(pkgPath, "remote", "add", "parent", link.parentRepo.SSHURL)
				git.GitExecOrPanic(pkgPath, "fetch", "parent")
				git.GitExecOrPanic(pkgPath, "rebase", "--onto", "parent", link.SourceBranch)

				nCommits := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgPath, "rev-list", "^NOW", "HEAD"), "\n"))
				if nCommits > 0 {
					if !noop {
						git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+config.Branch)
					}
					isGitUpdated = true
				}

				break
			}
		}

		if link == nil {
			if idx == 0 {
				// up-to-date
				continue
			} else if idx < len(commits) { // update
				common.PanicOnError(git.GitExec(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", filename))
				common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "fetch", "--depth", "1", "origin", commits[0].SHA))
				common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "checkout", commits[0].SHA))
				log.Println("    -> updated to", commits[0].SHA)
				isGitUpdated = true
			} else {
				// probably need `merge-base` or `rev-list` here instead, or the project updated already
				log.Println("     *** Cannot find SHA of last matching update for package:", filename, " Ignoring")
			}
		}
	}

	// find all missing repositories, and add them
	if DebugMode {
		log.Println("checking for missing repositories...")
	}
	repos, err := gitea.GetOrganizationRepositories(org)
	if err != nil {
		return err
	}

	if DebugMode {
		log.Println("  nRepos:", len(repos))
	}

	/* Check repositories in org to make sure they are included in project git */
next_repo:
	for _, r := range repos {
		if DebugMode {
			log.Println(" -- checking", r.Name)
		}

		if r.ObjectFormatName != "sha256" {
			if DebugMode {
				log.Println("    + ", r.ObjectFormatName, ". Needs to be sha256. Ignoring")
			}
			continue next_repo
		}

		//		for _, c := range configs {
		if gitPrj == r.Name {
			// ignore project gits
			continue next_repo
		}
		//		}

		for repo := range sub {
			if repo == r.Name {
				// not missing
				continue next_repo
			}
		}

		if DebugMode {
			log.Println(" -- checking repository:", r.Name)
		}

		if _, err := gitea.GetRecentCommits(org, r.Name, config.Branch, 1); err != nil {
			// assumption that package does not exist, so not part of project
			// https://github.com/go-gitea/gitea/issues/31976
			continue
		}

		// add repository to git project
		common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name))

		if len(config.Branch) > 0 {
			branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current"))
			if branch != config.Branch {
				if err := git.GitExec(path.Join(gitPrj, r.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
					return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", config.Branch, repo.Owner.UserName, r.Name)
				}
				common.PanicOnError(git.GitExec(path.Join(gitPrj, r.Name), "checkout", config.Branch))
			}
		}

		isGitUpdated = true
	}

	if isGitUpdated {
		common.PanicOnError(git.GitExec(gitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC"))
		if !noop {
			git.GitExecOrPanic(gitPrj, "push", remoteName)
		}
	}

	if DebugMode {
		log.Println("Verification finished for ", org, ", prjgit:", config.GitProjectName)
	}

	return nil
}

var checkOnStart bool
var noop bool
var checkInterval time.Duration

func checkOrg(org string, configs []*common.AutogitConfig) {
	git, err := gh.CreateGitHandler(org)
	if err != nil {
		log.Println("Faield to allocate GitHandler:", err)
		return
	}
	defer git.Close()

	for _, config := range configs {
		log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName)
		if err := verifyProjectState(git, org, config, configs); err != nil {
			log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
		} else {
			log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName)
		}
	}
}

func checkRepos() {
	for org, configs := range configuredRepos {
		if checkInterval > 0 {
			sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))
			log.Println(" - sleep interval", sleepInterval, "until next check")
			time.Sleep(sleepInterval)
		}

		checkOrg(org, configs)
	}
}

func consistencyCheckProcess() {
	if checkOnStart {
		savedCheckInterval := checkInterval
		checkInterval = 0
		log.Println("== Startup consistency check begin...")
		checkRepos()
		log.Println("== Startup consistency check done...")
		checkInterval = savedCheckInterval
	}

	for {
		checkRepos()
	}
}

var DebugMode bool
var gh common.GitHandlerGenerator

func updateConfiguration(configFilename string, orgs *[]string) {
	configFile, err := common.ReadConfigFile(configFilename)
	if err != nil {
		log.Fatal(err)
	}

	configs, _ := common.ResolveWorkflowConfigs(gitea, configFile)
	configuredRepos = make(map[string][]*common.AutogitConfig)
	*orgs = make([]string, 0, 1)
	for _, c := range configs {
		if slices.Contains(c.Workflows, "direct") {
			if DebugMode {
				log.Printf("  + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName)
			}
			configs := configuredRepos[c.Organization]
			if configs == nil {
				configs = make([]*common.AutogitConfig, 0, 1)
			}
			configs = append(configs, c)
			configuredRepos[c.Organization] = configs

			*orgs = append(*orgs, c.Organization)
		}
	}
}

func main() {
	configFilename := flag.String("config", "", "List of PrjGit")
	giteaUrl := flag.String("gitea-url", "https://src.opensuse.org", "Gitea instance")
	rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
	flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information")
	flag.BoolVar(&noop, "dry", false, "Dry mode. Do not push changes to remote repo.")
	flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays")
	checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
	basePath := flag.String("repo-path", "", "Repository path. Default is temporary directory")
	flag.Parse()

	if err := common.RequireGiteaSecretToken(); err != nil {
		log.Fatal(err)
	}
	if err := common.RequireRabbitSecrets(); err != nil {
		log.Fatal(err)
	}

	var defs common.ListenDefinitions
	var err error

	if len(*basePath) == 0 {
		*basePath, err = os.MkdirTemp(os.TempDir(), AppName)
		if err != nil {
			log.Fatal(err)
		}
	}
	gh, err = common.AllocateGitWorkTree(*basePath, GitAuthor, GitEmail)
	if err != nil {
		log.Fatal(err)
	}

	// handle reconfiguration
	signalChannel := make(chan os.Signal, 1)
	defer close(signalChannel)
	go func() {
		for {
			sig, ok := <-signalChannel

			if !ok {
				return
			}

			if sig != syscall.SIGHUP {
				log.Println("Unexpected signal received:", sig)
				continue
			}
			log.Println("*** Reconfiguring ***")
			updateConfiguration(*configFilename, &defs.Orgs)
			defs.UpdateTopics()
		}
	}()
	signal.Notify(signalChannel, syscall.SIGHUP)

	checkInterval = time.Duration(*checkIntervalHours) * time.Hour

	gitea = common.AllocateGiteaTransport(*giteaUrl)
	CurrentUser, err := gitea.GetCurrentUser()
	if err != nil {
		log.Fatalln("Cannot fetch current user:", err)
	}
	log.Println("Current User:", CurrentUser.UserName)

	updateConfiguration(*configFilename, &defs.Orgs)

	defs.GitAuthor = GitAuthor
	defs.RabbitURL, err = url.Parse(*rabbitUrl)
	if err != nil {
		log.Panicf("cannot parse server URL. Err: %#v\n", err)
	}

	go consistencyCheckProcess()
	log.Println("defs:", defs)

	defs.Handlers = make(map[string]common.RequestProcessor)
	defs.Handlers[common.RequestType_Push] = &PushActionProcessor{}
	defs.Handlers[common.RequestType_Repository] = &RepositoryActionProcessor{}

	log.Fatal(defs.ProcessRabbitMQEvents())
}