This commit is contained in:
Adam Majer 2024-09-04 14:04:13 +02:00
parent 09e0b48927
commit df5e073893
7 changed files with 271 additions and 102 deletions

View File

@ -17,9 +17,10 @@ type GitHandler struct {
GitPath string GitPath string
GitCommiter string GitCommiter string
GitEmail string
} }
func CreateGitHandler(git_author, name string) (*GitHandler, error) { func CreateGitHandler(git_author, email, name string) (*GitHandler, error) {
var err error var err error
git := new(GitHandler) git := new(GitHandler)
@ -194,6 +195,7 @@ func (e *GitHandler) GitExec(cwd string, params ...string) error {
"GIT_CEILING_DIRECTORIES=" + e.GitPath, "GIT_CEILING_DIRECTORIES=" + e.GitPath,
"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_GLOBAL=/dev/null",
"GIT_AUTHOR_NAME=" + e.GitCommiter, "GIT_AUTHOR_NAME=" + e.GitCommiter,
"GIT_COMMITTER_NAME=" + e.GitCommiter,
"EMAIL=not@exist@src.opensuse.org", "EMAIL=not@exist@src.opensuse.org",
"GIT_LFS_SKIP_SMUDGE=1", "GIT_LFS_SKIP_SMUDGE=1",
"GIT_SSH_COMMAND=/usr/bin/ssh -o StrictHostKeyChecking=yes", "GIT_SSH_COMMAND=/usr/bin/ssh -o StrictHostKeyChecking=yes",
@ -212,7 +214,7 @@ func (e *GitHandler) GitExec(cwd string, params ...string) error {
if e.DebugLogger { if e.DebugLogger {
log.Printf(" *** error: %v\n", err) log.Printf(" *** error: %v\n", err)
} }
return err return fmt.Errorf("error executing: git %#v \n%s\n err: %w", cmd.Args, out, err)
} }
return nil return nil
@ -549,7 +551,7 @@ func (e *GitHandler) GitCatFile(cwd, commitId, filename string) (data []byte, er
return len(data), nil return len(data), nil
}) })
if e.DebugLogger { if e.DebugLogger {
log.Printf("command run: %v\n", cmd.Args) log.Printf("command run: %v\n", cmd.Args)
} }
err = cmd.Run() err = cmd.Run()
@ -601,6 +603,8 @@ func (e *GitHandler) GitSubmoduleList(cwd, commitId string) (submoduleList map[s
"GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_GLOBAL=/dev/null",
} }
cmd.Dir = filepath.Join(e.GitPath, cwd) cmd.Dir = filepath.Join(e.GitPath, cwd)
cmd.Stdout = &data_in
cmd.Stdin = &data_out
cmd.Stderr = writeFunc(func(data []byte) (int, error) { cmd.Stderr = writeFunc(func(data []byte) (int, error) {
if e.DebugLogger { if e.DebugLogger {
log.Println(string(data)) log.Println(string(data))
@ -608,7 +612,7 @@ func (e *GitHandler) GitSubmoduleList(cwd, commitId string) (submoduleList map[s
return len(data), nil return len(data), nil
}) })
if e.DebugLogger { if e.DebugLogger {
log.Printf("command run: %v\n", cmd.Args) log.Printf("command run: %v\n", cmd.Args)
} }
err = cmd.Run() err = cmd.Run()

View File

@ -253,7 +253,7 @@ func TestCommitTreeParsingOfHead(t *testing.T) {
} }
}) })
t.Run("reads README.md", func (t *testing.T) { t.Run("reads README.md", func(t *testing.T) {
h := GitHandler{ h := GitHandler{
GitPath: gitDir, GitPath: gitDir,
} }
@ -266,7 +266,21 @@ func TestCommitTreeParsingOfHead(t *testing.T) {
} }
}) })
t.Run("read HEAD", func(t *testing.T) {
h := GitHandler{
GitPath: gitDir,
}
data, err := h.GitSubmoduleList("", "HEAD")
if err != nil {
t.Error("failed to get submodule list", err)
}
if len(data) != 5 {
t.Error("Invalid len of submodules", len(data))
}
})
t.Run("try to parse unknown item", func(t *testing.T) { t.Run("try to parse unknown item", func(t *testing.T) {
}) })
} }

View File

@ -164,8 +164,12 @@ func (gitea *GiteaTransport) CreateRepositoryIfNotExist(git *GitHandler, org Org
if err = os.Mkdir(filepath.Join(git.GitPath, DefaultGitPrj), 0700); err != nil { if err = os.Mkdir(filepath.Join(git.GitPath, DefaultGitPrj), 0700); err != nil {
return nil, err return nil, err
} }
git.GitExec(DefaultGitPrj, "init", "--object-format="+repo.Payload.ObjectFormatName) if err = git.GitExec(DefaultGitPrj, "init", "--object-format="+repo.Payload.ObjectFormatName); err != nil {
git.GitExec(DefaultGitPrj, "checkout", "-b", repo.Payload.DefaultBranch) return nil, err
}
if err = git.GitExec(DefaultGitPrj, "checkout", "-b", repo.Payload.DefaultBranch); err != nil {
return nil, err
}
readmeFilename := filepath.Join(git.GitPath, DefaultGitPrj, "README.md") readmeFilename := filepath.Join(git.GitPath, DefaultGitPrj, "README.md")
{ {
file, _ := os.Create(readmeFilename) file, _ := os.Create(readmeFilename)
@ -173,9 +177,15 @@ func (gitea *GiteaTransport) CreateRepositoryIfNotExist(git *GitHandler, org Org
io.WriteString(file, ReadmeBoilerplate) io.WriteString(file, ReadmeBoilerplate)
} }
git.GitExec(DefaultGitPrj, "add", "README.md") if err = git.GitExec(DefaultGitPrj, "add", "README.md"); err != nil {
git.GitExec(DefaultGitPrj, "commit", "-m", "Automatic devel project creation") return nil, err
git.GitExec(DefaultGitPrj, "remote", "add", "origin", repo.Payload.SSHURL) }
if err = git.GitExec(DefaultGitPrj, "commit", "-m", "Automatic devel project creation"); err != nil {
return nil, err
}
if err = git.GitExec(DefaultGitPrj, "remote", "add", "origin", repo.Payload.SSHURL); err != nil {
return nil, err
}
return repo.Payload, nil return repo.Payload, nil
default: default:
@ -410,8 +420,14 @@ func (gitea *GiteaTransport) GetPullRequestFileContent(pr *models.PullRequest, p
func (gitea *GiteaTransport) GetRecentCommits(org, repo, branch string, commitNo int64) ([]*models.Commit, error) { func (gitea *GiteaTransport) GetRecentCommits(org, repo, branch string, commitNo int64) ([]*models.Commit, error) {
not := false not := false
var page int64
page = 1
commits, err := gitea.client.Repository.RepoGetAllCommits( commits, err := gitea.client.Repository.RepoGetAllCommits(
repository.NewRepoGetAllCommitsParams(). repository.NewRepoGetAllCommitsParams().
WithOwner(org).
WithRepo(repo).
WithSha(&branch).
WithPage(&page).
WithStat(&not). WithStat(&not).
WithFiles(&not). WithFiles(&not).
WithVerification(&not). WithVerification(&not).

View File

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"log" "log"
"net/url" "net/url"
"slices"
"strings" "strings"
"time" "time"
@ -48,7 +49,7 @@ func processRabbitMQ(msgCh chan<- RabbitMessage, server url.URL, topics []string
queueName := server.Path queueName := server.Path
server.Path = "" server.Path = ""
if queueName[0] == '/' { if len(queueName) > 0 && queueName[0] == '/' {
queueName = queueName[1:] queueName = queueName[1:]
} }
@ -100,8 +101,10 @@ func processRabbitMQ(msgCh chan<- RabbitMessage, server url.URL, topics []string
} }
// log.Printf("queue: %s:%d", q.Name, q.Consumers) // log.Printf("queue: %s:%d", q.Name, q.Consumers)
log.Println(" -- listening to topics:")
for _, topic := range topics { for _, topic := range topics {
err = ch.QueueBind(q.Name, topic, "pubsub", false, nil) err = ch.QueueBind(q.Name, topic, "pubsub", false, nil)
log.Println(" +", topic)
if err != nil { if err != nil {
return fmt.Errorf("Cannot find queue to exchange with topic %s. Err: %w", topic, err) return fmt.Errorf("Cannot find queue to exchange with topic %s. Err: %w", topic, err)
} }
@ -126,6 +129,7 @@ func processRabbitMQ(msgCh chan<- RabbitMessage, server url.URL, topics []string
func connectAndProcessRabbitMQ(log *log.Logger, ch chan<- RabbitMessage, server url.URL, topics []string) { func connectAndProcessRabbitMQ(log *log.Logger, ch chan<- RabbitMessage, server url.URL, topics []string) {
defer func() { defer func() {
if r := recover(); r != nil { if r := recover(); r != nil {
log.Println(r)
log.Println("'crash' RabbitMQ worker. Recovering... reconnecting...") log.Println("'crash' RabbitMQ worker. Recovering... reconnecting...")
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
go connectAndProcessRabbitMQ(log, ch, server, topics) go connectAndProcessRabbitMQ(log, ch, server, topics)
@ -150,24 +154,38 @@ func connectToRabbitMQ(log *log.Logger, server url.URL, topics []string) chan Ra
} }
func ProcessEvent(f RequestProcessor, h *RequestHandler) { func ProcessEvent(f RequestProcessor, h *RequestHandler) {
defer func(){ defer func() {
recover() if r := recover(); r != nil {
log.Println(r)
}
}() }()
f(h) if err := f(h); err != nil {
log.Println(err)
}
} }
func ProcessRabbitMQEvents(listenDefs ListenDefinitions, orgs []string) error { func ProcessRabbitMQEvents(listenDefs ListenDefinitions, orgs []string) error {
server, err := url.Parse(listenDefs.RabbitURL) server, err := url.Parse(listenDefs.RabbitURL)
if err != nil { if err != nil {
log.Panicf("cannot parse server URL. Err: %#v", err) log.Panicf("cannot parse server URL. Err: %#v\n", err)
} }
log.Println("RabbitMQ connection:", *server)
topics := make([]string, 0, len(listenDefs.Handlers)*len(orgs)) topics := make([]string, 0, len(listenDefs.Handlers)*len(orgs))
log.Println(len(listenDefs.Handlers), len(orgs))
server.User = url.UserPassword(rabbitUser, rabbitPassword)
domain := "suse"
if server.Hostname() == "rabbit.opensuse.org" {
domain = "opensuse"
}
for _, org := range orgs { for _, org := range orgs {
for k := range listenDefs.Handlers { for k := range listenDefs.Handlers {
topics = append(topics, fmt.Sprintf("*suse.gitea.%s.%s#", org, k)) topics = append(topics, fmt.Sprintf("%s.gitea.%s.%s.#", domain, org, k))
} }
} }
@ -179,13 +197,22 @@ func ProcessRabbitMQEvents(listenDefs ListenDefinitions, orgs []string) error {
return nil return nil
} }
log.Println("event:", msg.RoutingKey)
route := strings.Split(msg.RoutingKey, ".") route := strings.Split(msg.RoutingKey, ".")
if len(route) > 3 { if len(route) > 3 {
reqType := route[3] reqType := route[3]
org := route[2] org := route[2]
if handler, found := listenDefs.Handlers[org]; found { if !slices.Contains(orgs, org) {
h, err := CreateRequestHandler(listenDefs.GitAuthor, listenDefs.GitAuthor) log.Println("Got even for unhandeled org:", org)
continue
}
log.Println("org:", org, "type:", reqType)
if handler, found := listenDefs.Handlers[reqType]; found {
log.Println("handler found", handler)
h, err := CreateRequestHandler()
if err != nil { if err != nil {
log.Printf("Cannot create request handler: %v\n", err) log.Printf("Cannot create request handler: %v\n", err)
continue continue
@ -195,11 +222,12 @@ func ProcessRabbitMQEvents(listenDefs ListenDefinitions, orgs []string) error {
log.Printf("Error parsing request JSON: %v\n", err) log.Printf("Error parsing request JSON: %v\n", err)
continue continue
} else { } else {
log.Println("processing req", req.Type)
h.Request = req h.Request = req
ProcessEvent(handler, h) ProcessEvent(handler, h)
} }
} }
} }
} }
} }

View File

@ -74,22 +74,23 @@ type RequestHandler struct {
StdLogger, ErrLogger *log.Logger StdLogger, ErrLogger *log.Logger
Request *Request Request *Request
Git *GitHandler // Git *GitHandler
} }
func (r *RequestHandler) WriteError() { func (r *RequestHandler) WriteError() {
r.ErrLogger.Println("internal error sent") r.ErrLogger.Println("internal error sent")
} }
func CreateRequestHandler(git_author, name string) (*RequestHandler, error) { func CreateRequestHandler() (*RequestHandler, error) {
var h *RequestHandler = new(RequestHandler) var h *RequestHandler = new(RequestHandler)
h.StdLogger, h.ErrLogger = CreateStdoutLogger(os.Stdout, os.Stderr) h.StdLogger, h.ErrLogger = CreateStdoutLogger(os.Stdout, os.Stderr)
var err error /* var err error
h.Git, err = CreateGitHandler(git_author, name) h.Git, err = CreateGitHandler(git_author, name)
if err != nil { if err != nil {
return nil, err return nil, err
} }
*/
return h, nil return h, nil
} }

View File

@ -1,14 +1,14 @@
[ [
{ {
"workflows": ["direct"], "Workflows": ["direct"],
"organization": "autogits", "Organization": "autogits",
"git_project_name": "MyPrj" "GitProjectName": "MyPrj"
}, },
{ {
"workflows": ["direct"], "Workflows": ["direct"],
"organization": "autogits", "Organization": "autogits",
"git_project_name": "HiddenPrj", "GitProjectName": "HiddenPrj",
"branch": "hidden" "Branch": "hidden"
} }
] ]

View File

@ -8,13 +8,16 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"slices" "slices"
"sync"
"time" "time"
"src.opensuse.org/autogits/common" "src.opensuse.org/autogits/common"
) )
var configuredRepos map[string]*common.AutogitConfig const AppName = "direct_workflow"
const GitAuthor = "AutoGits prjgit-updater"
const GitEmail = "adam+autogits-direct@zombino.com"
var configuredRepos map[string][]*common.AutogitConfig
var gitea *common.GiteaTransport var gitea *common.GiteaTransport
func isConfiguredOrg(org *common.Organization) bool { func isConfiguredOrg(org *common.Organization) bool {
@ -22,47 +25,85 @@ func isConfiguredOrg(org *common.Organization) bool {
return found return found
} }
func concatenateErrors(err1, err2 error) error {
if err1 == nil {
return err2
}
if err2 == nil {
return err1
}
return fmt.Errorf("%w\n%w", err1, err2)
}
func processRepositoryAction(h *common.RequestHandler) error { func processRepositoryAction(h *common.RequestHandler) error {
action := h.Request.Data.(*common.RepositoryWebhookEvent) action := h.Request.Data.(*common.RepositoryWebhookEvent)
config, configFound := configuredRepos[action.Organization.Username] configs, configFound := configuredRepos[action.Organization.Username]
if !configFound { if !configFound {
if h.Git.DebugLogger { h.StdLogger.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username)
h.StdLogger.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username)
}
return nil return nil
} }
log.Println("num configs found:", len(configs))
var err error
for _, config := range configs {
err = concatenateErrors(err, processConfiguredRepositoryAction(h, action, config))
}
return err
}
func processConfiguredRepositoryAction(h *common.RequestHandler, action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error {
prjgit := config.GitProjectName prjgit := config.GitProjectName
if action.Repository.Name == prjgit { if action.Repository.Name == prjgit {
if h.Git.DebugLogger { h.StdLogger.Printf("repository event %s for PrjGit '%s'. Ignoring\n", common.DefaultGitPrj, action.Action)
h.StdLogger.Printf("repository event %s for PrjGit '%s'. Ignoring\n", common.DefaultGitPrj, action.Action)
}
return nil return nil
} }
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(h.Git, *action.Organization, prjgit) git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
if err != nil {
return err
}
// defer git.Close()
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, *action.Organization, prjgit)
if err != nil { if err != nil {
return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err) return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
} }
h.Git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, common.DefaultGitPrj)
if err := git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, common.DefaultGitPrj); err != nil {
return err
}
switch action.Action { switch action.Action {
case "created": case "created":
h.Git.GitExec(common.DefaultGitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url) if err := git.GitExec(common.DefaultGitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url); err != nil {
h.Git.GitExec(common.DefaultGitPrj, "commit", "-m", "Automatic package inclusion via Direct Workflow") return err
h.Git.GitExec(common.DefaultGitPrj, "push") }
if err := git.GitExec(common.DefaultGitPrj, "commit", "-m", "Automatic package inclusion via Direct Workflow"); err != nil {
return err
}
if err := git.GitExec(common.DefaultGitPrj, "push"); err != nil {
return err
}
case "deleted": case "deleted":
if stat, err := os.Stat(filepath.Join(h.Git.GitPath, common.DefaultGitPrj, action.Repository.Name)); err != nil || !stat.IsDir() { if stat, err := os.Stat(filepath.Join(git.GitPath, common.DefaultGitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
if h.Git.DebugLogger { if git.DebugLogger {
h.StdLogger.Printf("delete event for %s -- not in project. Ignoring\n", action.Repository.Name) h.StdLogger.Printf("delete event for %s -- not in project. Ignoring\n", action.Repository.Name)
} }
return nil return nil
} }
h.Git.GitExec(common.DefaultGitPrj, "rm", action.Repository.Name) if err := git.GitExec(common.DefaultGitPrj, "rm", action.Repository.Name); err != nil {
h.Git.GitExec(common.DefaultGitPrj, "commit", "-m", "Automatic package removal via Direct Workflow") return err
h.Git.GitExec(common.DefaultGitPrj, "push") }
if err := git.GitExec(common.DefaultGitPrj, "commit", "-m", "Automatic package removal via Direct Workflow"); err != nil {
return err
}
if err := git.GitExec(common.DefaultGitPrj, "push"); err != nil {
return err
}
default: default:
return fmt.Errorf("%s: %s", "Unknown action type", action.Action) return fmt.Errorf("%s: %s", "Unknown action type", action.Action)
@ -73,42 +114,69 @@ func processRepositoryAction(h *common.RequestHandler) error {
func processPushAction(h *common.RequestHandler) error { func processPushAction(h *common.RequestHandler) error {
action := h.Request.Data.(*common.PushWebhookEvent) action := h.Request.Data.(*common.PushWebhookEvent)
config, configFound := configuredRepos[action.Repository.Owner.Username] configs, configFound := configuredRepos[action.Repository.Owner.Username]
if !configFound { if !configFound {
if h.Git.DebugLogger { h.StdLogger.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Repository.Owner.Username)
h.StdLogger.Printf("Push event to Organizationr '%s'. Not configured. Ignoring.\n", action.Repository.Owner.Username)
}
return nil return nil
} }
var err error
for _, config := range configs {
err = concatenateErrors(err, processConfiguredPushAction(h, action, config))
}
return err
}
func processConfiguredPushAction(h *common.RequestHandler, action *common.PushWebhookEvent, config *common.AutogitConfig) error {
prjgit := config.GitProjectName prjgit := config.GitProjectName
if action.Repository.Name == prjgit { if action.Repository.Name == prjgit {
if h.Git.DebugLogger { h.StdLogger.Printf("push to %s -- ignoring\n", prjgit)
h.StdLogger.Printf("push to %s -- ignoring\n", prjgit)
}
return nil return nil
} }
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(h.Git, *action.Repository.Owner, prjgit) git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
if err != nil {
return err
}
defer git.Close()
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, *action.Repository.Owner, prjgit)
if err != nil { if err != nil {
return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err) return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
} }
h.Git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, common.DefaultGitPrj)
if stat, err := os.Stat(filepath.Join(h.Git.GitPath, common.DefaultGitPrj, action.Repository.Name)); err != nil || !stat.IsDir() { if err := git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, common.DefaultGitPrj); err != nil {
if h.Git.DebugLogger { return err
}
if stat, err := os.Stat(filepath.Join(git.GitPath, common.DefaultGitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
if git.DebugLogger {
h.StdLogger.Printf("Pushed to package that is not part of the project. Ignoring: %v\n", err) h.StdLogger.Printf("Pushed to package that is not part of the project. Ignoring: %v\n", err)
} }
return nil return nil
} }
h.Git.GitExec(common.DefaultGitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name) if err := git.GitExec(common.DefaultGitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name); err != nil {
id, _ := h.Git.GitBranchHead(filepath.Join(common.DefaultGitPrj, action.Repository.Name), action.Repository.Default_Branch) return err
}
id, err := git.GitBranchHead(filepath.Join(common.DefaultGitPrj, action.Repository.Name), action.Repository.Default_Branch)
if err != nil {
return err
}
for _, commitId := range action.Commits { for _, commitId := range action.Commits {
if commitId.Id == id { if commitId.Id == id {
h.Git.GitExec(filepath.Join(common.DefaultGitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", id) if err := git.GitExec(filepath.Join(common.DefaultGitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", id); err != nil {
h.Git.GitExec(filepath.Join(common.DefaultGitPrj, action.Repository.Name), "checkout", id) return err
h.Git.GitExec(common.DefaultGitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow") }
h.Git.GitExec(common.DefaultGitPrj, "push") if err := git.GitExec(filepath.Join(common.DefaultGitPrj, action.Repository.Name), "checkout", id); err != nil {
return err
}
if err := git.GitExec(common.DefaultGitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow"); err != nil {
return err
}
if err := git.GitExec(common.DefaultGitPrj, "push"); err != nil {
return err
}
return nil return nil
} }
} }
@ -130,6 +198,7 @@ func verifyProjectState(git *common.GitHandler, orgName string, config *common.A
return fmt.Errorf("Error closing projectgit for %s, Err: %w", config.GitProjectName, err) return fmt.Errorf("Error closing projectgit for %s, Err: %w", config.GitProjectName, err)
} }
log.Println("getting submodule list")
sub, err := git.GitSubmoduleList(config.GitProjectName, "HEAD") sub, err := git.GitSubmoduleList(config.GitProjectName, "HEAD")
if err != nil { if err != nil {
return fmt.Errorf("Failed to fetch submodule list... Err: %w", err) return fmt.Errorf("Failed to fetch submodule list... Err: %w", err)
@ -137,10 +206,19 @@ func verifyProjectState(git *common.GitHandler, orgName string, config *common.A
isGitUpdated := false isGitUpdated := false
for filename, commitId := range sub { for filename, commitId := range sub {
log.Println(" verifying package:", filename, commitId, config.Branch)
commits, err := gitea.GetRecentCommits(orgName, filename, config.Branch, 10) commits, err := gitea.GetRecentCommits(orgName, filename, config.Branch, 10)
if err != nil { if err != nil {
return fmt.Errorf("Failed to fetch recent commits for package: '%s'. Err: %w", filename, err) // assumption that package does not exist, remove from project
if err := git.GitExec(config.GitProjectName, "rm", filename); err != nil {
return fmt.Errorf("Failed to remove deleted submodule. Err: %w", err)
}
isGitUpdated = true
continue
} }
// if err != nil {
// return fmt.Errorf("Failed to fetch recent commits for package: '%s'. Err: %w", filename, err)
// }
idx := 1000 idx := 1000
for i, c := range commits { for i, c := range commits {
@ -154,9 +232,15 @@ func verifyProjectState(git *common.GitHandler, orgName string, config *common.A
// up-to-date // up-to-date
continue continue
} else if idx < len(commits) { // update } else if idx < len(commits) { // update
git.GitExec(config.GitProjectName, "submodule", "update", "--init", "--depth", "1", "--checkout", filename) if err := git.GitExec(config.GitProjectName, "submodule", "update", "--init", "--depth", "1", "--checkout", filename); err != nil {
git.GitExec(filepath.Join(config.GitProjectName, filename), "fetch", "--depth", "1", "origin", commits[0].SHA) return err
git.GitExec(filepath.Join(config.GitProjectName, filename), "checkout", commits[0].SHA) }
if err := git.GitExec(filepath.Join(config.GitProjectName, filename), "fetch", "--depth", "1", "origin", commits[0].SHA); err != nil {
return err
}
if err := git.GitExec(filepath.Join(config.GitProjectName, filename), "checkout", commits[0].SHA); err != nil {
return err
}
isGitUpdated = true isGitUpdated = true
} else { } else {
// probably need `merge-base` or `rev-list` here instead, or the project updated already // probably need `merge-base` or `rev-list` here instead, or the project updated already
@ -165,9 +249,12 @@ func verifyProjectState(git *common.GitHandler, orgName string, config *common.A
} }
if isGitUpdated { if isGitUpdated {
git.GitExec(common.DefaultGitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC") if err := git.GitExec(config.GitProjectName, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC"); err != nil {
git.GitExec(common.DefaultGitPrj, "push") return err
// commit changes and push }
if err := git.GitExec(config.GitProjectName, "push"); err != nil {
return err
}
} }
return nil return nil
@ -176,36 +263,54 @@ func verifyProjectState(git *common.GitHandler, orgName string, config *common.A
var checkOnStart bool var checkOnStart bool
var checkInterval time.Duration var checkInterval time.Duration
func consistencyCheckProcess(git *common.GitHandler, repos map[string]*common.AutogitConfig) { func consistencyCheckProcess() {
if checkOnStart { if checkOnStart {
var wg sync.WaitGroup log.Println("== Startup consistency check begin...")
for org, conf := range repos { for org, configs := range configuredRepos {
for _, config := range configs {
wg.Add(1) log.Println(" - org: ", org, " - config: ", config.GitProjectName)
go func() { git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
defer wg.Done() if err != nil {
log.Println("Failed to allocate GitHandler:", err)
verifyProjectState(git, org, conf) return
}() }
if err := verifyProjectState(git, org, config); err != nil {
log.Println("Failed to verify state of org:", org, err)
return
}
}
} }
wg.Wait() log.Println("== Startup consistency check done...")
log.Printf("== Startup consistency check done...")
} }
for org, conf := range repos { for org, configs := range configuredRepos {
time.Sleep(checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))) for _, config := range configs {
time.Sleep(checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval))))
log.Printf(" ++ starting verification, org: `%s`\n", org) log.Printf(" ++ starting verification, org: `%s`\n", org)
if err := verifyProjectState(git, org, conf); err != nil { git, err := common.CreateGitHandler(GitAuthor, GitEmail, AppName)
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err) if err != nil {
log.Println("Faield to allocate GitHandler:", err)
return
}
if err := verifyProjectState(git, org, config); err != nil {
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
}
log.Printf(" ++ verification complete, org: `%s`\n", org)
} }
log.Printf(" ++ verification complete, org: `%s`\n", org)
} }
} }
var debugMode bool var debugMode bool
func main() { func main() {
if err := common.RequireGiteaSecretToken(); err != nil {
log.Fatal(err)
}
if err := common.RequireRabbitSecrets(); err != nil {
log.Fatal(err)
}
workflowConfig := flag.String("config", "", "Repository and workflow definition file") workflowConfig := flag.String("config", "", "Repository and workflow definition file")
giteaHost := flag.String("gitea", "src.opensuse.org", "Gitea instance") giteaHost := flag.String("gitea", "src.opensuse.org", "Gitea instance")
rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance") rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
@ -225,34 +330,35 @@ func main() {
log.Fatal(err) log.Fatal(err)
} }
configuredRepos = make(map[string]*common.AutogitConfig) configuredRepos = make(map[string][]*common.AutogitConfig)
orgs := make([]string, 0, 10) orgs := make([]string, 0, 1)
for _, c := range configs { for _, c := range configs {
if slices.Contains(c.Workflows, "push") { if slices.Contains(c.Workflows, "direct") {
if debugMode { if debugMode {
log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName) log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName)
} }
configuredRepos[c.Organization] = c configs := configuredRepos[c.Organization]
if configs == nil {
configs = make([]*common.AutogitConfig, 0, 1)
}
configs = append(configs, c)
configuredRepos[c.Organization] = configs
orgs = append(orgs, c.Organization) orgs = append(orgs, c.Organization)
} }
} }
gitea = common.AllocateGiteaTransport(*giteaHost) gitea = common.AllocateGiteaTransport(*giteaHost)
go consistencyCheckProcess()
var defs common.ListenDefinitions var defs common.ListenDefinitions
defs.GitAuthor = "GiteaBot - AutoDevel" defs.GitAuthor = GitAuthor
defs.RabbitURL = *rabbitUrl defs.RabbitURL = *rabbitUrl
defs.Handlers = make(map[string]common.RequestProcessor) defs.Handlers = make(map[string]common.RequestProcessor)
defs.Handlers[common.RequestType_Push] = processPushAction defs.Handlers[common.RequestType_Push] = processPushAction
defs.Handlers[common.RequestType_Repository] = processRepositoryAction defs.Handlers[common.RequestType_Repository] = processRepositoryAction
if err := common.RequireGiteaSecretToken(); err != nil {
log.Fatal(err)
}
if err := common.RequireRabbitSecrets(); err != nil {
log.Fatal(err)
}
log.Fatal(common.ProcessRabbitMQEvents(defs, orgs)) log.Fatal(common.ProcessRabbitMQEvents(defs, orgs))
} }