autogits/prjgit-updater/main.go

package main

/*
 * This file is part of Autogits.
 *
 * Copyright © 2024 SUSE LLC
 *
 * Autogits is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License as published by the Free Software
 * Foundation, either version 2 of the License, or (at your option) any later
 * version.
 *
 * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 * PARTICULAR PURPOSE. See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * Foobar. If not, see <https://www.gnu.org/licenses/>.
 */

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"path"
	"path/filepath"
	"slices"
	"time"

	"src.opensuse.org/autogits/common"
)

const AppName = "direct_workflow"
const GitAuthor = "AutoGits prjgit-updater"
const GitEmail = "adam+autogits-direct@zombino.com"

var configuredRepos map[string][]*common.AutogitConfig
var gitea *common.GiteaTransport

func isConfiguredOrg(org *common.Organization) bool {
	_, found := configuredRepos[org.Username]
	return found
}

func concatenateErrors(err1, err2 error) error {
	if err1 == nil {
		return err2
	}

	if err2 == nil {
		return err1
	}

	return fmt.Errorf("%w\n%w", err1, err2)
}

func processRepositoryAction(h *common.RequestHandler) error {
	action := h.Request.Data.(*common.RepositoryWebhookEvent)
	configs, configFound := configuredRepos[action.Organization.Username]

	if !configFound {
		h.StdLogger.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username)
		return nil
	}

	for _, config := range configs {
		if config.GitProjectName == action.Repository.Name {
			h.StdLogger.Println("+ ignoring repo event for PrjGit repository", config.GitProjectName)
		}
	}

	var err error
	for _, config := range configs {
		err = concatenateErrors(err, processConfiguredRepositoryAction(h, action, config))
	}

	return err
}
func processConfiguredRepositoryAction(h *common.RequestHandler, action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error {
	prjgit := config.GitProjectName
	git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
	if err != nil {
		return err
	}
	// defer git.Close()

	prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, *action.Organization, prjgit)
	if err != nil {
		return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
	}

	if err := git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, common.DefaultGitPrj); err != nil {
		return err
	}

	switch action.Action {
	case "created":
		if err := git.GitExec(common.DefaultGitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url); err != nil {
			return err
		}
		if err := git.GitExec(common.DefaultGitPrj, "commit", "-m", "Automatic package inclusion via Direct Workflow"); err != nil {
			return err
		}
		if err := git.GitExec(common.DefaultGitPrj, "push"); err != nil {
			return err
		}

	case "deleted":
		if stat, err := os.Stat(filepath.Join(git.GitPath, common.DefaultGitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
			if git.DebugLogger {
				h.StdLogger.Printf("delete event for %s -- not in project. Ignoring\n", action.Repository.Name)
			}
			return nil
		}
		if err := git.GitExec(common.DefaultGitPrj, "rm", action.Repository.Name); err != nil {
			return err
		}
		if err := git.GitExec(common.DefaultGitPrj, "commit", "-m", "Automatic package removal via Direct Workflow"); err != nil {
			return err
		}
		if err := git.GitExec(common.DefaultGitPrj, "push"); err != nil {
			return err
		}

	default:
		return fmt.Errorf("%s: %s", "Unknown action type", action.Action)
	}

	return nil
}

func processPushAction(h *common.RequestHandler) error {
	action := h.Request.Data.(*common.PushWebhookEvent)
	configs, configFound := configuredRepos[action.Repository.Owner.Username]

	if !configFound {
		h.StdLogger.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Repository.Owner.Username)
		return nil
	}

	for _, config := range configs {
		if config.GitProjectName == action.Repository.Name {
			h.StdLogger.Println("+ ignoring push to PrjGit repository", config.GitProjectName)
		}
	}

	var err error
	for _, config := range configs {
		err = concatenateErrors(err, processConfiguredPushAction(h, action, config))
	}

	return err
}

func processConfiguredPushAction(h *common.RequestHandler, action *common.PushWebhookEvent, config *common.AutogitConfig) error {
	prjgit := config.GitProjectName
	git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
	if err != nil {
		return err
	}
	defer git.Close()

	prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, *action.Repository.Owner, prjgit)
	if err != nil {
		return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
	}

	if err := git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, common.DefaultGitPrj); err != nil {
		return err
	}
	if stat, err := os.Stat(filepath.Join(git.GitPath, common.DefaultGitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
		if git.DebugLogger {
			h.StdLogger.Printf("Pushed to package that is not part of the project. Ignoring: %v\n", err)
		}
		return nil
	}
	if err := git.GitExec(common.DefaultGitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name); err != nil {
		return err
	}
	id, err := git.GitBranchHead(filepath.Join(common.DefaultGitPrj, action.Repository.Name), action.Repository.Default_Branch)
	if err != nil {
		return err
	}
	for _, commitId := range action.Commits {
		if commitId.Id == id {
			if err := git.GitExec(filepath.Join(common.DefaultGitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", id); err != nil {
				return err
			}
			if err := git.GitExec(filepath.Join(common.DefaultGitPrj, action.Repository.Name), "checkout", id); err != nil {
				return err
			}
			if err := git.GitExec(common.DefaultGitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow"); err != nil {
				return err
			}
			if err := git.GitExec(common.DefaultGitPrj, "push"); err != nil {
				return err
			}
			return nil
		}
	}

	h.StdLogger.Println("push of refs not on the main branch. ignoring.")
	return nil
}

func verifyProjectState(git *common.GitHandler, orgName string, config *common.AutogitConfig, configs []*common.AutogitConfig) error {
	org := common.Organization{
		Username: orgName,
	}
	repo, err := gitea.CreateRepositoryIfNotExist(git, org, config.GitProjectName)
	if err != nil {
		return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", orgName, config.GitProjectName, err)
	}

	if err := git.GitExec("", "clone", "--depth", "1", repo.SSHURL, config.GitProjectName); err != nil {
		return fmt.Errorf("Error closing projectgit for %s, Err: %w", config.GitProjectName, err)
	}

	log.Println("getting submodule list")
	sub, err := git.GitSubmoduleList(config.GitProjectName, "HEAD")
	if err != nil {
		return fmt.Errorf("Failed to fetch submodule list... Err: %w", err)
	}

	isGitUpdated := false
next_package:
	for filename, commitId := range sub {
		// ignore project gits
		for _, c := range configs {
			if c.GitProjectName == filename {
				log.Println("     prjgit as package? ignoring project git:", filename)
				continue next_package
			}
		}

		log.Println("     verifying package:", filename, commitId, config.Branch)
		commits, err := gitea.GetRecentCommits(orgName, filename, config.Branch, 10)
		if err != nil {
			// assumption that package does not exist, remove from project
			// https://github.com/go-gitea/gitea/issues/31976
			if err := git.GitExec(config.GitProjectName, "rm", filename); err != nil {
				return fmt.Errorf("Failed to remove deleted submodule. Err: %w", err)
			}
			isGitUpdated = true
			continue
		}
		//		if err != nil {
		//			return fmt.Errorf("Failed to fetch recent commits for package: '%s'. Err: %w", filename, err)
		//		}

		idx := 1000
		for i, c := range commits {
			if c.SHA == commitId {
				idx = i
				break
			}
		}

		if idx == 0 {
			// up-to-date
			continue
		} else if idx < len(commits) { // update
			if err := git.GitExec(config.GitProjectName, "submodule", "update", "--init", "--depth", "1", "--checkout", filename); err != nil {
				return err
			}
			if err := git.GitExec(filepath.Join(config.GitProjectName, filename), "fetch", "--depth", "1", "origin", commits[0].SHA); err != nil {
				return err
			}
			if err := git.GitExec(filepath.Join(config.GitProjectName, filename), "checkout", commits[0].SHA); err != nil {
				return err
			}
			isGitUpdated = true
		} else {
			// probably need `merge-base` or `rev-list` here instead, or the project updated already
			return fmt.Errorf("Cannot find SHA of last matching update for package: '%s'.  idx: %d", filename, idx)
		}
	}

	// find all missing repositories, and add them
	if debugMode {
		log.Println("checking for missing repositories...")
	}
	repos, err := gitea.GetOrganizationRepositories(orgName)
	if err != nil {
		return err
	}

	if debugMode {
		log.Println("  nRepos:", len(repos))
	}

next_repo:
	for _, r := range repos {
		if debugMode {
			log.Println(" -- checking", r.Name)
		}

		for _, c := range configs {
			if c.Organization == orgName && c.GitProjectName == r.Name {
				// ignore project gits
				break
			}
		}

		for repo := range sub {
			if repo == r.Name {
				continue next_repo
			}
		}

		if debugMode {
			log.Println(" -- checking repository:", r.Name)
		}

		if _, err := gitea.GetRecentCommits(orgName, r.Name, config.Branch, 10); err != nil {
			// assumption that package does not exist, so not part of project
			// https://github.com/go-gitea/gitea/issues/31976
			break
		}

		// add repository to git project
		if err := git.GitExec(config.GitProjectName, "submodule", "--quiet", "add", "--depth", "1", r.SSHURL); err != nil {
			return fmt.Errorf("Cannot add submodule '%s' to project '%s'. Err: %w", r.Name, config.GitProjectName, err)
		}

		if len(config.Branch) > 0 {
			if err := git.GitExec(path.Join(config.GitProjectName, r.Name), "fetch", "--depth", "1", "origin", config.Branch); err != nil {
				return fmt.Errorf("Failed to fetch branch '%s' from '%s'/'%s'. Err: %w", config.Branch, orgName, r.Name, err)
			}
			if err := git.GitExec(path.Join(config.GitProjectName, r.Name), "checkout", config.Branch); err != nil {
				return fmt.Errorf("Failed to checkout fetched branch '%s' from '%s'/'%s'. Err: %w", config.Branch, orgName, r.Name, err)
			}
		}

		isGitUpdated = true
	}

	if isGitUpdated {
		if err := git.GitExec(config.GitProjectName, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC"); err != nil {
			return err
		}
		if err := git.GitExec(config.GitProjectName, "push"); err != nil {
			return err
		}
	}

	if debugMode {
		log.Println("Verification finished for ", orgName, ", config", config.GitProjectName)
	}

	return nil
}

var checkOnStart bool
var checkInterval time.Duration

func consistencyCheckProcess() {
	if checkOnStart {
		log.Println("== Startup consistency check begin...")
		for org, configs := range configuredRepos {
			for _, config := range configs {
				log.Println("  - org: ", org, " - config: ", config.GitProjectName)
				git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
				if err != nil {
					log.Println("Failed to allocate GitHandler:", err)
					return
				}
				if err := verifyProjectState(git, org, config, configs); err != nil {
					log.Println("Failed to verify state of org:", org, err)
					return
				}
			}
		}
		log.Println("== Startup consistency check done...")
	}

	for org, configs := range configuredRepos {
		for _, config := range configs {
			sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))
			log.Println(" - sleep interval", sleepInterval)
			time.Sleep(sleepInterval)

			log.Printf(" ++ starting verification, org: `%s`\n", org)
			git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
			if err != nil {
				log.Println("Faield to allocate GitHandler:", err)
				return
			}
			if err := verifyProjectState(git, org, config, configs); err != nil {
				log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
			}
			log.Printf(" ++ verification complete, org: `%s`\n", org)
		}
	}
}

var debugMode bool

func main() {
	if err := common.RequireGiteaSecretToken(); err != nil {
		log.Fatal(err)
	}
	if err := common.RequireRabbitSecrets(); err != nil {
		log.Fatal(err)
	}

	workflowConfig := flag.String("config", "", "Repository and workflow definition file")
	giteaHost := flag.String("gitea", "src.opensuse.org", "Gitea instance")
	rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
	flag.BoolVar(&debugMode, "debug", false, "Extra debugging information")
	flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays")
	checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
	flag.Parse()

	checkInterval = time.Duration(*checkIntervalHours) * time.Hour

	if len(*workflowConfig) == 0 {
		log.Fatalln("No configuratio file specified. Aborting")
	}

	configs, err := common.ReadWorkflowConfigsFile(*workflowConfig)
	if err != nil {
		log.Fatal(err)
	}

	configuredRepos = make(map[string][]*common.AutogitConfig)
	orgs := make([]string, 0, 1)
	for _, c := range configs {
		if slices.Contains(c.Workflows, "direct") {
			if debugMode {
				log.Printf("  + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName)
			}
			configs := configuredRepos[c.Organization]
			if configs == nil {
				configs = make([]*common.AutogitConfig, 0, 1)
			}
			configs = append(configs, c)
			configuredRepos[c.Organization] = configs

			orgs = append(orgs, c.Organization)
		}
	}

	gitea = common.AllocateGiteaTransport(*giteaHost)
	go consistencyCheckProcess()

	var defs common.ListenDefinitions

	defs.GitAuthor = GitAuthor
	defs.RabbitURL = *rabbitUrl

	defs.Handlers = make(map[string]common.RequestProcessor)
	defs.Handlers[common.RequestType_Push] = processPushAction
	defs.Handlers[common.RequestType_Repository] = processRepositoryAction

	log.Fatal(common.ProcessRabbitMQEvents(defs, orgs))
}