adamm/autogits
SHA256
1
0
Fork 0
Code Issues 8 Projects 2 Activity

wip

Browse Source
This commit is contained in:
Adam Majer 2024-08-28 12:39:32 +02:00
parent e125031384
commit 8b7f552e77
2 changed files with 101 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
bots-common/git_utils.go
View File

@ -3,6 +3,7 @@ package common
import ( import (
"fmt" "fmt"
"io" "io"
"log"
"os" "os"
"os/exec" "os/exec"
"path" "path"
@ -11,7 +12,14 @@ import (
"sync" "sync"
) )
//func (h *RequestHandler) ProcessBranchList() []string { type GitHandler struct {
DebugLogger bool
GitPath string
GitCommiter string
}
//func (h *GitHandler) ProcessBranchList() []string {
// if h.HasError() { // if h.HasError() {
// return make([]string, 0) // return make([]string, 0)
// } // }
@ -119,22 +127,15 @@ func findGitDir(p string) (string, error) {
return "", fmt.Errorf("Can't find git subdirectory in '%s'", p) return "", fmt.Errorf("Can't find git subdirectory in '%s'", p)
} }
func (e *RequestHandler) GitBranchHead(gitDir, branchName string) (string, error) { func (e *GitHandler) GitBranchHead(gitDir, branchName string) (string, error) {
if e.HasError() {
return "", e.Error
}
path, err := findGitDir(path.Join(e.GitPath, gitDir)) path, err := findGitDir(path.Join(e.GitPath, gitDir))
if err != nil { if err != nil {
e.ErrLogger.Printf("Error identifying gitdir in `%s`: %v\n", gitDir, err) return "", fmt.Errorf("Error identifying gitdir in `%s`: %w", gitDir, err)
e.Error = err
} }
refs, err := processRefs(path) refs, err := processRefs(path)
if err != nil { if err != nil {
e.ErrLogger.Printf("Error finding branches (%s): %v\n", branchName, err) return "", fmt.Errorf("Error finding branches (%s): %w\n", branchName, err)
e.Error = err
return "", e.Error
} }
for _, ref := range refs { for _, ref := range refs {
@ -143,9 +144,7 @@ func (e *RequestHandler) GitBranchHead(gitDir, branchName string) (string, error
} }
} }
e.Error = fmt.Errorf("Can't find default remote branch: %s", branchName) return "", fmt.Errorf("Can't find default remote branch: %s", branchName)
e.ErrLogger.Println(e.Error.Error())
return "", e.Error
} }
type ExecStream interface { type ExecStream interface {
@ -154,18 +153,12 @@ type ExecStream interface {
GitExec(cwd string, param ...string) ExecStream GitExec(cwd string, param ...string) ExecStream
} }
func (e *RequestHandler) Close() { func (e *GitHandler) Close() error {
if e.GitPath == "" { if err := os.RemoveAll(e.GitPath); err != nil {
return return err
} }
e.Error = os.RemoveAll(e.GitPath)
e.GitPath = "" e.GitPath = ""
return return nil
}
func (e *RequestHandler) HasError() bool {
return e.Error != nil
} }
type writeFunc func(data []byte) (int, error) type writeFunc func(data []byte) (int, error)
@ -184,12 +177,7 @@ func (h writeFunc) Close() error {
return err return err
} }
func (e *GitHandler) GitExec(cwd string, params ...string) ExecStream {
func (e *RequestHandler) GitExec(cwd string, params ...string) ExecStream {
if e.Error != nil {
return e
}
cmd := exec.Command("/usr/bin/git", params...) cmd := exec.Command("/usr/bin/git", params...)
cmd.Env = []string{ cmd.Env = []string{
"GIT_CEILING_DIRECTORIES=" + e.GitPath, "GIT_CEILING_DIRECTORIES=" + e.GitPath,
@ -200,18 +188,12 @@ func (e *RequestHandler) GitExec(cwd string, params ...string) ExecStream {
"GIT_SSH_COMMAND=/usr/bin/ssh -o StrictHostKeyChecking=yes", "GIT_SSH_COMMAND=/usr/bin/ssh -o StrictHostKeyChecking=yes",
} }
cmd.Dir = filepath.Join(e.GitPath, cwd) cmd.Dir = filepath.Join(e.GitPath, cwd)
cmd.Stdout = writeFunc(func(data []byte) (int, error) {
e.StdLogger.Println(data)
return len(data), nil
})
cmd.Stderr = writeFunc(func(data []byte) (int, error) {
e.ErrLogger.Println(data)
return len(data), nil
})
cmd.Stdin = nil cmd.Stdin = nil
e.StdLogger.Printf("git execute: %#v\n", cmd.Args) if e.DebugLogger {
e.Error = cmd.Run() log.Printf("git execute: %#v\n", cmd.Args)
}
out, err := cmd.CombinedOutput()
return e return e
} }
@ -239,7 +221,7 @@ func (c *ChanIO) Read(data []byte) (idx int, err error) {
idx++ idx++
for len(c.ch) > 0 && idx < len(data) { for len(c.ch) > 0 && idx < len(data) {
data[idx], ok = <- c.ch data[idx], ok = <-c.ch
if !ok { if !ok {
err = io.EOF err = io.EOF
return return
@ -369,7 +351,7 @@ func parseGitCommitMsg(data <-chan byte, l int) (string, error) {
msg = append(msg, c) msg = append(msg, c)
l-- l--
} }
// l-- // l--
if l != 0 { if l != 0 {
return "", fmt.Errorf("Unexpected data in the git commit msg: l=%d", l) return "", fmt.Errorf("Unexpected data in the git commit msg: l=%d", l)
@ -480,7 +462,7 @@ func parseGitBlob(data <-chan byte) ([]byte, error) {
} }
d := make([]byte, hdr.size) d := make([]byte, hdr.size)
for l:=0; l<hdr.size; l++ { for l := 0; l < hdr.size; l++ {
d[l] = <-data d[l] = <-data
} }
eob := <-data eob := <-data
@ -492,9 +474,8 @@ func parseGitBlob(data <-chan byte) ([]byte, error) {
} }
// TODO: support sub-trees // TODO: support sub-trees
func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte { func (e *GitHandler) GitCatFile(cwd, commitId, filename string) (data []byte, err error) {
var done sync.Mutex var done sync.Mutex
var data []byte
done.Lock() done.Lock()
data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)} data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
@ -507,8 +488,7 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
data_out.ch <- '\x00' data_out.ch <- '\x00'
c, err := parseGitCommit(data_in.ch) c, err := parseGitCommit(data_in.ch)
if err != nil { if err != nil {
e.Error = err log.Printf("Error parsing git commit: %v\n", err)
e.ErrLogger.Printf("Error parsing git commit: %v\n", err)
return return
} }
data_out.Write([]byte(c.Tree)) data_out.Write([]byte(c.Tree))
@ -516,8 +496,9 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
tree, err := parseGitTree(data_in.ch) tree, err := parseGitTree(data_in.ch)
if err != nil { if err != nil {
e.Error = err if e.DebugLogger {
e.ErrLogger.Printf("Error parsing git tree: %v\n", err) log.Printf("Error parsing git tree: %v\n", err)
}
return return
} }
@ -526,16 +507,11 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
data_out.Write([]byte(te.hash)) data_out.Write([]byte(te.hash))
data_out.ch <- '\x00' data_out.ch <- '\x00'
data, err = parseGitBlob(data_in.ch) data, err = parseGitBlob(data_in.ch)
if err != nil {
e.Error = err
e.ErrLogger.Printf("Error reading blob data: %v\n", err)
}
return return
} }
} }
e.Error = fmt.Errorf("file not found: '%s'", filename) err = fmt.Errorf("file not found: '%s'", filename)
e.ErrLogger.Println(e.Error.Error())
}() }()
cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z") cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z")
@ -554,16 +530,12 @@ func (e *RequestHandler) GitCatFile(cwd, commitId, filename string) []byte {
e.Error = cmd.Run() e.Error = cmd.Run()
done.Lock() done.Lock()
return data return
} }
func (e *RequestHandler) GitSubmoduleList(cwd, commitId string) map[string]string { func (e *GitHandler) GitSubmoduleList(cwd, commitId string) (submoduleList map[string]string, err error) {
var done sync.Mutex var done sync.Mutex
submoduleList := make(map[string]string) submoduleList = make(map[string]string)
if e.HasError() {
return submoduleList
}
done.Lock() done.Lock()
data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)} data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
@ -616,39 +588,39 @@ func (e *RequestHandler) GitSubmoduleList(cwd, commitId string) map[string]strin
return submoduleList return submoduleList
} }
func (e *RequestHandler) GitSubmoduleCommitId(cwd, packageName, commitId string) (string, bool) { func (e *GitHandler) GitSubmoduleCommitId(cwd, packageName, commitId string) (subCommitId string, valid bool) {
if e.Error != nil { defer func() {
return "", false if recover() != nil {
commitId = ""
valid = false
} }
}()
data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)} data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
var subCommitId string var wg sync.WaitGroup
var foundLock sync.Mutex
foundLock.Lock() wg.Add(1)
e.StdLogger.Printf("getting commit id '%s' from git at '%s' with packageName: %s\n", commitId, cwd, packageName) if e.DebugLogger {
log.Printf("getting commit id '%s' from git at '%s' with packageName: %s\n", commitId, cwd, packageName)
}
go func() { go func() {
defer foundLock.Unlock() defer wg.Done()
defer close(data_out.ch) defer close(data_out.ch)
data_out.Write([]byte(commitId)) data_out.Write([]byte(commitId))
data_out.ch <- '\x00' data_out.ch <- '\x00'
c, err := parseGitCommit(data_in.ch) c, err := parseGitCommit(data_in.ch)
if err != nil { if err != nil {
e.Error = err log.Panic("Error parsing git commit: %v\n", err)
e.ErrLogger.Printf("Error parsing git commit: %v\n", err)
return
} }
data_out.Write([]byte(c.Tree)) data_out.Write([]byte(c.Tree))
data_out.ch <- '\x00' data_out.ch <- '\x00'
tree, err := parseGitTree(data_in.ch) tree, err := parseGitTree(data_in.ch)
if err != nil { if err != nil {
e.Error = err log.Panicf("Error parsing git tree: %v\n", err)
e.ErrLogger.Printf("Error parsing git tree: %v\n", err)
return
} }
for _, te := range tree.items { for _, te := range tree.items {
@ -668,12 +640,16 @@ func (e *RequestHandler) GitSubmoduleCommitId(cwd, packageName, commitId string)
cmd.Stdout = &data_in cmd.Stdout = &data_in
cmd.Stdin = &data_out cmd.Stdin = &data_out
cmd.Stderr = writeFunc(func(data []byte) (int, error) { cmd.Stderr = writeFunc(func(data []byte) (int, error) {
e.ErrLogger.Println(data) log.Println(string(data))
return len(data), nil return len(data), nil
}) })
e.StdLogger.Printf("command run: %v\n", cmd.Args) if e.DebugLogger {
e.Error = cmd.Run() log.Printf("command run: %v\n", cmd.Args)
}
if err := cmd.Run(); err != nil {
log.Printf("Error running command %v, err: %v", cmd.Args, err)
}
foundLock.Lock() wg.Wait()
return subCommitId, len(subCommitId) == len(commitId) return subCommitId, len(subCommitId) == len(commitId)
} }

42
prjgit-updater/main.go
View File

@ -4,10 +4,14 @@ import (
"flag" "flag"
"fmt" "fmt"
"log" "log"
"math/rand"
"os" "os"
"path/filepath" "path/filepath"
"slices" "slices"
"sync"
"time"
"golang.org/x/tools/go/analysis/passes/defers"
"src.opensuse.org/autogits/common" "src.opensuse.org/autogits/common"
) )
@ -77,14 +81,52 @@ func processPushAction(h *common.RequestHandler) error {
return nil return nil
} }
func verifyProjectState(org string, config ConfigRepos) error {
}
var checkOnStart bool
var checkInterval time.Duration
func consistencyCheckProcess(repos map[string]ConfigRepos) {
if checkOnStart {
var wg sync.WaitGroup
for org, conf := range repos {
wg.Add(1)
go func() {
defer wg.Done()
verifyProjectState(org, conf)
}()
}
wg.Wait()
log.Printf("== Startup consistency check done...")
}
for org, conf := range repos {
time.Sleep(checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval))))
log.Printf(" ++ starting verification, org: `%s`\n", org)
if err := verifyProjectState(org, conf); err != nil {
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
}
log.Printf(" ++ verification complete, org: `%s`\n", org)
}
}
var debugMode bool var debugMode bool
func main() { func main() {
workflowConfig := flag.String("config", "", "Repository and workflow definition file") workflowConfig := flag.String("config", "", "Repository and workflow definition file")
rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance") rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
flag.BoolVar(&debugMode, "debug", false, "Extra debugging information") flag.BoolVar(&debugMode, "debug", false, "Extra debugging information")
flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays")
checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
flag.Parse() flag.Parse()
checkInterval = time.Duration(*checkIntervalHours) * time.Hour
if len(*workflowConfig) == 0 { if len(*workflowConfig) == 0 {
log.Fatalln("No configuratio file specified. Aborting") log.Fatalln("No configuratio file specified. Aborting")
} }