adamm/autogits
SHA256
1
0
Fork 0
Code Issues 8 Projects 2 Activity

wip

Browse Source
This commit is contained in:
Adam Majer 2024-08-28 12:39:32 +02:00
parent e125031384
commit 8b7f552e77
2 changed files with 101 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
bots-common/git_utils.go
View File

@ -3,6 +3,7 @@ package common
import (
"fmt"
"io"
"log"
"os"
"os/exec"
"path"
@ -11,7 +12,14 @@ import (
"sync"
)
//func (h *RequestHandler) ProcessBranchList() []string {
type GitHandler struct {
DebugLogger bool
GitPath string
GitCommiter string
}
//func (h *GitHandler) ProcessBranchList() []string {
// if h.HasError() {
// return make([]string, 0)
// }
@ -119,22 +127,15 @@ func findGitDir(p string) (string, error) {
return "", fmt.Errorf("Can't find git subdirectory in '%s'", p)
}
func (e *RequestHandler) GitBranchHead(gitDir, branchName string) (string, error) {
if e.HasError() {
return "", e.Error
}
func (e *GitHandler) GitBranchHead(gitDir, branchName string) (string, error) {
path, err := findGitDir(path.Join(e.GitPath, gitDir))
if err != nil {
e.ErrLogger.Printf("Error identifying gitdir in `%s`: %v\n", gitDir, err)
e.Error = err
return "", fmt.Errorf("Error identifying gitdir in `%s`: %w", gitDir, err)
}
refs, err := processRefs(path)
if err != nil {
e.ErrLogger.Printf("Error finding branches (%s): %v\n", branchName, err)
e.Error = err
return "", e.Error
return "", fmt.Errorf("Error finding branches (%s): %w\n", branchName, err)
}
for _, ref := range refs {
@ -143,9 +144,7 @@ func (e *RequestHandler) GitBranchHead(gitDir, branchName string) (string, error
}
}
e.Error = fmt.Errorf("Can't find default remote branch: %s", branchName)
e.ErrLogger.Println(e.Error.Error())
return "", e.Error
return "", fmt.Errorf("Can't find default remote branch: %s", branchName)
}
type ExecStream interface {
@ -154,18 +153,12 @@ type ExecStream interface {
GitExec(cwd string, param ...string) ExecStream
}
func (e *RequestHandler) Close() {
if e.GitPath == "" {
return
func (e *GitHandler) Close() error {
if err := os.RemoveAll(e.GitPath); err != nil {
return err
}
e.Error = os.RemoveAll(e.GitPath)
e.GitPath = ""
return
}
func (e *RequestHandler) HasError() bool {
return e.Error != nil
return nil
}
type writeFunc func(data []byte) (int, error)
@ -184,12 +177,7 @@ func (h writeFunc) Close() error {
return err
}
func (e *RequestHandler) GitExec(cwd string, params ...string) ExecStream {
if e.Error != nil {
return e
}
func (e *GitHandler) GitExec(cwd string, params ...string) ExecStream {
cmd := exec.Command("/usr/bin/git", params...)
cmd.Env = []string{
"GIT_CEILING_DIRECTORIES=" + e.GitPath,
@ -200,18 +188,12 @@ func (e *RequestHandler) GitExec(cwd string, params ...string) ExecStream {
"GIT_SSH_COMMAND=/usr/bin/ssh -o StrictHostKeyChecking=yes",
}
cmd.Dir = filepath.Join(e.GitPath, cwd)
cmd.Stdout = writeFunc(func(data []byte) (int, error) {
e.StdLogger.Println(data)
return len(data), nil
})
cmd.Stderr = writeFunc(func(data []byte) (int, error) {
e.ErrLogger.Println(data)
return len(data), nil
})
cmd.Stdin = nil
e.StdLogger.Printf("git execute: %#v\n", cmd.Args)
e.Error = cmd.Run()
if e.DebugLogger {
log.Printf("git execute: %#v\n", cmd.Args)
}
out, err := cmd.CombinedOutput()
return e
}
@ -239,7 +221,7 @@ func (c *ChanIO) Read(data []byte) (idx int, err error) {
idx++
for len(c.ch) > 0 && idx < len(data) {
data[idx], ok = <- c.ch
data[idx], ok = <-c.ch
if !ok {
err = io.EOF
return
@ -322,9 +304,9 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) {
return gitMsg{}, fmt.Errorf("Missing format weird")
}
return gitMsg{
hash: string(id[:]),
hash: string(id[:]),
itemType: "missing",
size: 0,
size: 0,
}, fmt.Errorf("Object not found: '%s'", string(id))
default:
return gitMsg{}, fmt.Errorf("Invalid object type: '%s'", string(msgType))
@ -369,7 +351,7 @@ func parseGitCommitMsg(data <-chan byte, l int) (string, error) {
msg = append(msg, c)
l--
}
// l--
// l--
if l != 0 {
return "", fmt.Errorf("Unexpected data in the git commit msg: l=%d", l)
@ -480,7 +462,7 @@ func parseGitBlob(data <-chan byte) ([]byte, error) {
}
d := make([]byte, hdr.size)
for l:=0; l<hdr.size; l++ {
for l := 0; l < hdr.size; l++ {
d[l] = <-data
}
eob := <-data
@ -492,9 +474,8 @@ func parseGitBlob(data <-chan byte) ([]byte, error) {
}
// TODO: support sub-trees
func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
func (e *GitHandler) GitCatFile(cwd, commitId, filename string) (data []byte, err error) {
var done sync.Mutex
var data []byte
done.Lock()
data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
@ -507,8 +488,7 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
data_out.ch <- '\x00'
c, err := parseGitCommit(data_in.ch)
if err != nil {
e.Error = err
e.ErrLogger.Printf("Error parsing git commit: %v\n", err)
log.Printf("Error parsing git commit: %v\n", err)
return
}
data_out.Write([]byte(c.Tree))
@ -516,8 +496,9 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
tree, err := parseGitTree(data_in.ch)
if err != nil {
e.Error = err
e.ErrLogger.Printf("Error parsing git tree: %v\n", err)
if e.DebugLogger {
log.Printf("Error parsing git tree: %v\n", err)
}
return
}
@ -526,16 +507,11 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
data_out.Write([]byte(te.hash))
data_out.ch <- '\x00'
data, err = parseGitBlob(data_in.ch)
if err != nil {
e.Error = err
e.ErrLogger.Printf("Error reading blob data: %v\n", err)
}
return
}
}
e.Error = fmt.Errorf("file not found: '%s'", filename)
e.ErrLogger.Println(e.Error.Error())
err = fmt.Errorf("file not found: '%s'", filename)
}()
cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z")
@ -554,16 +530,12 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
e.Error = cmd.Run()
done.Lock()
return data
return
}
func (e *RequestHandler) GitSubmoduleList(cwd, commitId string) map[string]string {
func (e *GitHandler) GitSubmoduleList(cwd, commitId string) (submoduleList map[string]string, err error) {
var done sync.Mutex
submoduleList := make(map[string]string)
if e.HasError() {
return submoduleList
}
submoduleList = make(map[string]string)
done.Lock()
data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
@ -616,39 +588,39 @@ func (e *RequestHandler) GitSubmoduleList(cwd, commitId string) map[string]strin
return submoduleList
}
func (e *RequestHandler) GitSubmoduleCommitId(cwd, packageName, commitId string) (string, bool) {
if e.Error != nil {
return "", false
}
func (e *GitHandler) GitSubmoduleCommitId(cwd, packageName, commitId string) (subCommitId string, valid bool) {
defer func() {
if recover() != nil {
commitId = ""
valid = false
}
}()
data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
var subCommitId string
var foundLock sync.Mutex
var wg sync.WaitGroup
foundLock.Lock()
wg.Add(1)
e.StdLogger.Printf("getting commit id '%s' from git at '%s' with packageName: %s\n", commitId, cwd, packageName)
if e.DebugLogger {
log.Printf("getting commit id '%s' from git at '%s' with packageName: %s\n", commitId, cwd, packageName)
}
go func() {
defer foundLock.Unlock()
defer wg.Done()
defer close(data_out.ch)
data_out.Write([]byte(commitId))
data_out.ch <- '\x00'
c, err := parseGitCommit(data_in.ch)
if err != nil {
e.Error = err
e.ErrLogger.Printf("Error parsing git commit: %v\n", err)
return
log.Panic("Error parsing git commit: %v\n", err)
}
data_out.Write([]byte(c.Tree))
data_out.ch <- '\x00'
tree, err := parseGitTree(data_in.ch)
if err != nil {
e.Error = err
e.ErrLogger.Printf("Error parsing git tree: %v\n", err)
return
log.Panicf("Error parsing git tree: %v\n", err)
}
for _, te := range tree.items {
@ -668,12 +640,16 @@ func (e *RequestHandler) GitSubmoduleCommitId(cwd, packageName, commitId string)
cmd.Stdout = &data_in
cmd.Stdin = &data_out
cmd.Stderr = writeFunc(func(data []byte) (int, error) {
e.ErrLogger.Println(data)
log.Println(string(data))
return len(data), nil
})
e.StdLogger.Printf("command run: %v\n", cmd.Args)
e.Error = cmd.Run()
if e.DebugLogger {
log.Printf("command run: %v\n", cmd.Args)
}
if err := cmd.Run(); err != nil {
log.Printf("Error running command %v, err: %v", cmd.Args, err)
}
foundLock.Lock()
wg.Wait()
return subCommitId, len(subCommitId) == len(commitId)
}

42
prjgit-updater/main.go
View File

@ -4,10 +4,14 @@ import (
"flag"
"fmt"
"log"
"math/rand"
"os"
"path/filepath"
"slices"
"sync"
"time"
"golang.org/x/tools/go/analysis/passes/defers"
"src.opensuse.org/autogits/common"
)
@ -77,14 +81,52 @@ func processPushAction(h *common.RequestHandler) error {
return nil
}
func verifyProjectState(org string, config ConfigRepos) error {
}
var checkOnStart bool
var checkInterval time.Duration
func consistencyCheckProcess(repos map[string]ConfigRepos) {
if checkOnStart {
var wg sync.WaitGroup
for org, conf := range repos {
wg.Add(1)
go func() {
defer wg.Done()
verifyProjectState(org, conf)
}()
}
wg.Wait()
log.Printf("== Startup consistency check done...")
}
for org, conf := range repos {
time.Sleep(checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval))))
log.Printf(" ++ starting verification, org: `%s`\n", org)
if err := verifyProjectState(org, conf); err != nil {
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
}
log.Printf(" ++ verification complete, org: `%s`\n", org)
}
}
var debugMode bool
func main() {
workflowConfig := flag.String("config", "", "Repository and workflow definition file")
rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
flag.BoolVar(&debugMode, "debug", false, "Extra debugging information")
flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays")
checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
flag.Parse()
checkInterval = time.Duration(*checkIntervalHours) * time.Hour
if len(*workflowConfig) == 0 {
log.Fatalln("No configuratio file specified. Aborting")
}