From 8b7f552e77039172907761134d91ddad18c9f61be5c1090c5d9a7072231ee9aa Mon Sep 17 00:00:00 2001
From: Adam Majer <amajer@suse.com>
Date: Wed, 28 Aug 2024 12:39:32 +0200
Subject: [PATCH] wip

---
 bots-common/git_utils.go | 142 ++++++++++++++++-----------------------
 prjgit-updater/main.go   |  42 ++++++++++++
 2 files changed, 101 insertions(+), 83 deletions(-)

diff --git a/bots-common/git_utils.go b/bots-common/git_utils.go
index 7de381b..0b312be 100644
--- a/bots-common/git_utils.go
+++ b/bots-common/git_utils.go
@@ -3,6 +3,7 @@ package common
 import (
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"os/exec"
 	"path"
@@ -11,7 +12,14 @@ import (
 	"sync"
 )
 
-//func (h *RequestHandler) ProcessBranchList() []string {
+type GitHandler struct {
+	DebugLogger bool
+
+	GitPath     string
+	GitCommiter string
+}
+
+//func (h *GitHandler) ProcessBranchList() []string {
 //	if h.HasError() {
 //		return make([]string, 0)
 //	}
@@ -119,22 +127,15 @@ func findGitDir(p string) (string, error) {
 	return "", fmt.Errorf("Can't find git subdirectory in '%s'", p)
 }
 
-func (e *RequestHandler) GitBranchHead(gitDir, branchName string) (string, error) {
-	if e.HasError() {
-		return "", e.Error
-	}
-
+func (e *GitHandler) GitBranchHead(gitDir, branchName string) (string, error) {
 	path, err := findGitDir(path.Join(e.GitPath, gitDir))
 	if err != nil {
-		e.ErrLogger.Printf("Error identifying gitdir in `%s`: %v\n", gitDir, err)
-		e.Error = err
+		return "", fmt.Errorf("Error identifying gitdir in `%s`: %w", gitDir, err)
 	}
 
 	refs, err := processRefs(path)
 	if err != nil {
-		e.ErrLogger.Printf("Error finding branches (%s): %v\n", branchName, err)
-		e.Error = err
-		return "", e.Error
+		return "", fmt.Errorf("Error finding branches (%s): %w\n", branchName, err)
 	}
 
 	for _, ref := range refs {
@@ -143,9 +144,7 @@ func (e *RequestHandler) GitBranchHead(gitDir, branchName string) (string, error
 		}
 	}
 
-	e.Error = fmt.Errorf("Can't find default remote branch: %s", branchName)
-	e.ErrLogger.Println(e.Error.Error())
-	return "", e.Error
+	return "", fmt.Errorf("Can't find default remote branch: %s", branchName)
 }
 
 type ExecStream interface {
@@ -154,18 +153,12 @@ type ExecStream interface {
 	GitExec(cwd string, param ...string) ExecStream
 }
 
-func (e *RequestHandler) Close() {
-	if e.GitPath == "" {
-		return
+func (e *GitHandler) Close() error {
+	if err := os.RemoveAll(e.GitPath); err != nil {
+		return err
 	}
-
-	e.Error = os.RemoveAll(e.GitPath)
 	e.GitPath = ""
-	return
-}
-
-func (e *RequestHandler) HasError() bool {
-	return e.Error != nil
+	return nil
 }
 
 type writeFunc func(data []byte) (int, error)
@@ -184,12 +177,7 @@ func (h writeFunc) Close() error {
 	return err
 }
 
-
-func (e *RequestHandler) GitExec(cwd string, params ...string) ExecStream {
-	if e.Error != nil {
-		return e
-	}
-
+func (e *GitHandler) GitExec(cwd string, params ...string) ExecStream {
 	cmd := exec.Command("/usr/bin/git", params...)
 	cmd.Env = []string{
 		"GIT_CEILING_DIRECTORIES=" + e.GitPath,
@@ -200,18 +188,12 @@ func (e *RequestHandler) GitExec(cwd string, params ...string) ExecStream {
 		"GIT_SSH_COMMAND=/usr/bin/ssh -o StrictHostKeyChecking=yes",
 	}
 	cmd.Dir = filepath.Join(e.GitPath, cwd)
-	cmd.Stdout = writeFunc(func(data []byte) (int, error) {
-		e.StdLogger.Println(data)
-		return len(data), nil
-	})
-	cmd.Stderr = writeFunc(func(data []byte) (int, error) {
-		e.ErrLogger.Println(data)
-		return len(data), nil
-	})
 	cmd.Stdin = nil
 
-	e.StdLogger.Printf("git execute: %#v\n", cmd.Args)
-	e.Error = cmd.Run()
+	if e.DebugLogger {
+		log.Printf("git execute: %#v\n", cmd.Args)
+	}
+	out, err := cmd.CombinedOutput()
 
 	return e
 }
@@ -239,7 +221,7 @@ func (c *ChanIO) Read(data []byte) (idx int, err error) {
 	idx++
 
 	for len(c.ch) > 0 && idx < len(data) {
-		data[idx], ok = <- c.ch
+		data[idx], ok = <-c.ch
 		if !ok {
 			err = io.EOF
 			return
@@ -322,9 +304,9 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) {
 			return gitMsg{}, fmt.Errorf("Missing format weird")
 		}
 		return gitMsg{
-			hash: string(id[:]),
+			hash:     string(id[:]),
 			itemType: "missing",
-			size: 0,
+			size:     0,
 		}, fmt.Errorf("Object not found: '%s'", string(id))
 	default:
 		return gitMsg{}, fmt.Errorf("Invalid object type: '%s'", string(msgType))
@@ -369,7 +351,7 @@ func parseGitCommitMsg(data <-chan byte, l int) (string, error) {
 		msg = append(msg, c)
 		l--
 	}
-//	l--
+	//	l--
 
 	if l != 0 {
 		return "", fmt.Errorf("Unexpected data in the git commit msg: l=%d", l)
@@ -480,7 +462,7 @@ func parseGitBlob(data <-chan byte) ([]byte, error) {
 	}
 
 	d := make([]byte, hdr.size)
-	for l:=0; l<hdr.size; l++ {
+	for l := 0; l < hdr.size; l++ {
 		d[l] = <-data
 	}
 	eob := <-data
@@ -492,9 +474,8 @@ func parseGitBlob(data <-chan byte) ([]byte, error) {
 }
 
 // TODO: support sub-trees
-func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
+func (e *GitHandler) GitCatFile(cwd, commitId, filename string) (data []byte, err error) {
 	var done sync.Mutex
-	var data []byte
 
 	done.Lock()
 	data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
@@ -507,8 +488,7 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
 		data_out.ch <- '\x00'
 		c, err := parseGitCommit(data_in.ch)
 		if err != nil {
-			e.Error = err
-			e.ErrLogger.Printf("Error parsing git commit: %v\n", err)
+			log.Printf("Error parsing git commit: %v\n", err)
 			return
 		}
 		data_out.Write([]byte(c.Tree))
@@ -516,8 +496,9 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
 		tree, err := parseGitTree(data_in.ch)
 
 		if err != nil {
-			e.Error = err
-			e.ErrLogger.Printf("Error parsing git tree: %v\n", err)
+			if e.DebugLogger {
+				log.Printf("Error parsing git tree: %v\n", err)
+			}
 			return
 		}
 
@@ -526,16 +507,11 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
 				data_out.Write([]byte(te.hash))
 				data_out.ch <- '\x00'
 				data, err = parseGitBlob(data_in.ch)
-				if err != nil {
-					e.Error = err
-					e.ErrLogger.Printf("Error reading blob data: %v\n", err)
-				}
 				return
 			}
 		}
 
-		e.Error = fmt.Errorf("file not found: '%s'", filename)
-		e.ErrLogger.Println(e.Error.Error())
+		err = fmt.Errorf("file not found: '%s'", filename)
 	}()
 
 	cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z")
@@ -554,16 +530,12 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
 	e.Error = cmd.Run()
 
 	done.Lock()
-	return data
+	return
 }
 
-func (e *RequestHandler) GitSubmoduleList(cwd, commitId string) map[string]string {
+func (e *GitHandler) GitSubmoduleList(cwd, commitId string) (submoduleList map[string]string, err error) {
 	var done sync.Mutex
-	submoduleList := make(map[string]string)
-
-	if e.HasError() {
-		return submoduleList
-	}
+	submoduleList = make(map[string]string)
 
 	done.Lock()
 	data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
@@ -616,39 +588,39 @@ func (e *RequestHandler) GitSubmoduleList(cwd, commitId string) map[string]strin
 	return submoduleList
 }
 
-func (e *RequestHandler) GitSubmoduleCommitId(cwd, packageName, commitId string) (string, bool) {
-	if e.Error != nil {
-		return "", false
-	}
+func (e *GitHandler) GitSubmoduleCommitId(cwd, packageName, commitId string) (subCommitId string, valid bool) {
+	defer func() {
+		if recover() != nil {
+			commitId = ""
+			valid = false
+		}
+	}()
 
 	data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
-	var subCommitId string
-	var foundLock sync.Mutex
+	var wg sync.WaitGroup
 
-	foundLock.Lock()
+	wg.Add(1)
 
-	e.StdLogger.Printf("getting commit id '%s' from git at '%s' with packageName: %s\n", commitId, cwd, packageName)
+	if e.DebugLogger {
+		log.Printf("getting commit id '%s' from git at '%s' with packageName: %s\n", commitId, cwd, packageName)
+	}
 
 	go func() {
-		defer foundLock.Unlock()
+		defer wg.Done()
 		defer close(data_out.ch)
 
 		data_out.Write([]byte(commitId))
 		data_out.ch <- '\x00'
 		c, err := parseGitCommit(data_in.ch)
 		if err != nil {
-			e.Error = err
-			e.ErrLogger.Printf("Error parsing git commit: %v\n", err)
-			return
+			log.Panic("Error parsing git commit: %v\n", err)
 		}
 		data_out.Write([]byte(c.Tree))
 		data_out.ch <- '\x00'
 		tree, err := parseGitTree(data_in.ch)
 
 		if err != nil {
-			e.Error = err
-			e.ErrLogger.Printf("Error parsing git tree: %v\n", err)
-			return
+			log.Panicf("Error parsing git tree: %v\n", err)
 		}
 
 		for _, te := range tree.items {
@@ -668,12 +640,16 @@ func (e *RequestHandler) GitSubmoduleCommitId(cwd, packageName, commitId string)
 	cmd.Stdout = &data_in
 	cmd.Stdin = &data_out
 	cmd.Stderr = writeFunc(func(data []byte) (int, error) {
-		e.ErrLogger.Println(data)
+		log.Println(string(data))
 		return len(data), nil
 	})
-	e.StdLogger.Printf("command run: %v\n", cmd.Args)
-	e.Error = cmd.Run()
+	if e.DebugLogger {
+		log.Printf("command run: %v\n", cmd.Args)
+	}
+	if err := cmd.Run(); err != nil {
+		log.Printf("Error running command %v, err: %v", cmd.Args, err)
+	}
 
-	foundLock.Lock()
+	wg.Wait()
 	return subCommitId, len(subCommitId) == len(commitId)
 }
diff --git a/prjgit-updater/main.go b/prjgit-updater/main.go
index f76481a..77703cb 100644
--- a/prjgit-updater/main.go
+++ b/prjgit-updater/main.go
@@ -4,10 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"slices"
+	"sync"
+	"time"
 
+	"golang.org/x/tools/go/analysis/passes/defers"
 	"src.opensuse.org/autogits/common"
 )
 
@@ -77,14 +81,52 @@ func processPushAction(h *common.RequestHandler) error {
 	return nil
 }
 
+func verifyProjectState(org string, config ConfigRepos) error {
+
+}
+
+var checkOnStart bool
+var checkInterval time.Duration 
+
+func consistencyCheckProcess(repos map[string]ConfigRepos) {
+	if checkOnStart {
+			var wg sync.WaitGroup
+		for org, conf := range repos {
+
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				
+				verifyProjectState(org, conf)
+			}()
+		}
+		wg.Wait()
+		log.Printf("== Startup consistency check done...")
+	}
+
+	for org, conf := range repos {
+		time.Sleep(checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval))))
+
+		log.Printf(" ++ starting verification, org: `%s`\n", org)
+		if err := verifyProjectState(org, conf); err != nil {
+			log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
+		}
+		log.Printf(" ++ verification complete, org: `%s`\n", org)
+	}
+}
+
 var debugMode bool
 
 func main() {
 	workflowConfig := flag.String("config", "", "Repository and workflow definition file")
 	rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
 	flag.BoolVar(&debugMode, "debug", false, "Extra debugging information")
+	flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays")
+	checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
 	flag.Parse()
 
+	checkInterval = time.Duration(*checkIntervalHours) * time.Hour
+
 	if len(*workflowConfig) == 0 {
 		log.Fatalln("No configuratio file specified. Aborting")
 	}