2
1
forked from adamm/autogits

53 Commits

Author SHA256 Message Date
fa61af0db6 Implement detection for local repositories
Repositories which build against another repo in the same project need
to do so also in the forked project. This is eg for consuming rpms
from one repo in an image build from same project.
2025-05-05 11:26:07 +02:00
23e2566843 Fix git path compare of meta to pull request
.git is optional, but doesn't matter, so trimming it away
2025-05-05 10:54:31 +02:00
0d0fcef7ac staging: fixes 2025-05-04 20:45:33 +02:00
62a597718b fix parsing test 2025-05-03 14:34:33 +02:00
327cb4ceaf fixes if git cat-file has error 2025-05-02 22:46:31 +02:00
aac475ad16 wip 2025-05-02 16:57:13 +02:00
046a60a6ed move staging config to its own config file 2025-05-02 11:18:23 +02:00
dcf964bf7a wip 2025-04-30 17:26:31 +02:00
bff5f1cab7 common: handle case of missing remote
If repo present, but remote not setup, just set it up
2025-04-30 12:29:32 +02:00
6d1ef184e0 workflow-pr: logging 2025-04-29 19:08:37 +02:00
e30d366f2f workflow-pr: logging updates 2025-04-29 18:00:37 +02:00
4a2fe06f05 staging: refactor 2025-04-28 23:47:05 +02:00
210e7588f1 common: actually remove items we process 2025-04-28 22:05:50 +02:00
72b100124d staging: list notification correctly in logging 2025-04-28 19:51:32 +02:00
996d36aaa8 staging: more refactor 2025-04-28 19:47:05 +02:00
82b5b105b1 staging: refactor 2025-04-28 19:44:32 +02:00
248ec4d03c staging: get last results for reference project 2025-04-28 17:34:54 +02:00
faa21f5453 staging: logging adjustments 2025-04-28 16:57:05 +02:00
21c4a7c1e0 wip 2025-04-28 14:23:59 +02:00
f3f76e7d5b Merge commit '96e1c26600f02a81299d4c121a2239c2a28e3184ef306cb0ac2cf00f0f97202e' into refactor 2025-04-28 12:37:38 +02:00
e341b630a2 wip 2025-04-27 22:53:19 +02:00
58532b9b60 wip 2025-04-25 17:40:44 +02:00
a697ccd0ca sync 2025-04-25 16:55:24 +02:00
4bafe0b4ef Merge branch 'refactor' of c3:gitea_test/autogits into refactor 2025-04-25 16:55:09 +02:00
7af2092ae1 wip 2025-04-24 23:51:46 +02:00
32374f76c1 status 2025-04-23 17:51:59 +02:00
9403b563f6 wip 2025-04-22 23:42:41 +02:00
bd492f8d92 no branch if default 2025-04-17 18:40:20 +02:00
fbc84d551d workflow-direct: use correct remote name instead of origin 2025-04-17 18:21:23 +02:00
874a120f88 we are using master for project git .. this may change 2025-04-17 17:58:18 +02:00
199396c210 Use HuJSON
for the comments and glory
2025-04-17 15:33:18 +02:00
f0de3ad54a workflow-direct: no panic if no changes 2025-04-17 15:12:51 +02:00
bfeac63c57 update repo parsing 2025-04-17 13:34:11 +02:00
d65f37739c fixes 2025-04-17 00:38:53 +02:00
5895e3d02c workflow-direct: add no-op mode, for debugging 2025-04-16 23:49:31 +02:00
0e036b5ec6 workflow-direct: move away from prjgit repo being just repo 2025-04-16 18:07:37 +02:00
1d1602852c direct: GitClone instead of running clone directly 2025-04-15 23:38:41 +02:00
9b5013ee45 git clone lock fixes 2025-04-15 18:15:35 +02:00
ed815c3ad1 unique org_repo remote names 2025-04-15 14:55:19 +02:00
8645063e8d git utils 2025-04-15 13:51:08 +02:00
2d044d5664 git: one generator per app, multiple instances
this allows locking for access for each org
2025-04-14 18:33:18 +02:00
51ba81f257 Merge branch 'main' into refactor 2025-04-11 14:00:00 +02:00
bb7a247f66 common: commit status api 2025-04-11 13:58:20 +02:00
c1f71253a4 devel_update: dead code removal 2025-04-11 13:57:56 +02:00
e257b113b9 devel-importer: helpful scripts 2025-04-09 18:48:30 +02:00
11e0bbaed1 devel-importer: remove remote branches 2025-04-09 16:47:19 +02:00
fb430d8c76 group-review: don't use regex for matching group name 2025-04-09 12:21:55 +02:00
7ed2a7082d Fix notification parsing regex 2025-04-09 11:43:54 +02:00
ba7686189e add GitClone for persistent git clones 2025-04-09 00:03:22 +02:00
9dcd25b69a wip 2025-04-08 19:03:33 +02:00
d89c77e22d common: use hostname:port instead of just hostname for API calls 2025-04-08 16:48:25 +02:00
f91c61cd20 tests 2025-04-08 00:23:24 +02:00
06aef50047 start refactoring PR bot 2025-04-07 19:03:02 +02:00
43 changed files with 2250 additions and 1289 deletions

View File

@@ -26,12 +26,19 @@ import (
"log"
"os"
"strings"
"github.com/tailscale/hujson"
)
//go:generate mockgen -source=config.go -destination=mock/config.go -typed
const (
ProjectConfigFile = "workflow.config"
StagingConfigFile = "staging.config"
)
type ConfigFile struct {
GitProjectName []string
GitProjectNames []string
}
type ReviewGroup struct {
@@ -39,6 +46,11 @@ type ReviewGroup struct {
Reviewers []string
}
type QAConfig struct {
Name string
Origin string
}
type AutogitConfig struct {
Workflows []string // [pr, direct, test]
Organization string
@@ -57,7 +69,11 @@ func ReadConfig(reader io.Reader) (*ConfigFile, error) {
}
config := ConfigFile{}
if err := json.Unmarshal(data, &config.GitProjectName); err != nil {
data, err = hujson.Standardize(data)
if err != nil {
return nil, fmt.Errorf("Failed to parse json: %w", err)
}
if err := json.Unmarshal(data, &config.GitProjectNames); err != nil {
return nil, fmt.Errorf("Error parsing Git Project paths: %w", err)
}
@@ -79,6 +95,19 @@ type GiteaFileContentAndRepoFetcher interface {
GiteaRepoFetcher
}
func PartiallyParseWorkflowConfig(data []byte) (*AutogitConfig, error) {
var config AutogitConfig
data, err := hujson.Standardize(data)
if err != nil {
return nil, fmt.Errorf("Failed to parse json: %w", err)
}
if err := json.Unmarshal(data, &config); err != nil {
return nil, fmt.Errorf("Error parsing workflow config file: %s: %w", string(data), err)
}
return &config, nil
}
func ReadWorkflowConfig(gitea GiteaFileContentAndRepoFetcher, git_project string) (*AutogitConfig, error) {
hash := strings.Split(git_project, "#")
branch := ""
@@ -96,14 +125,14 @@ func ReadWorkflowConfig(gitea GiteaFileContentAndRepoFetcher, git_project string
return nil, fmt.Errorf("Missing org/repo in projectgit: %s", git_project)
}
data, _, err := gitea.GetRepositoryFileContent(a[0], prjGitRepo, branch, "workflow.config")
data, _, err := gitea.GetRepositoryFileContent(a[0], prjGitRepo, branch, ProjectConfigFile)
if err != nil {
return nil, fmt.Errorf("Error fetching 'workflow.config' for %s/%s#%s: %w", a[0], prjGitRepo, branch, err)
}
var config AutogitConfig
if err := json.Unmarshal(data, &config); err != nil {
return nil, fmt.Errorf("Error parsing workflow config file: %s: %w", string(data), err)
config, err := PartiallyParseWorkflowConfig(data)
if err != nil {
return nil, err
}
if len(config.Organization) < 1 {
@@ -118,12 +147,12 @@ func ReadWorkflowConfig(gitea GiteaFileContentAndRepoFetcher, git_project string
}
}
config.GitProjectName = config.GitProjectName + "#" + branch
return &config, nil
return config, nil
}
func ResolveWorkflowConfigs(gitea GiteaFileContentAndRepoFetcher, config *ConfigFile) (AutogitConfigs, error) {
configs := make([]*AutogitConfig, 0, len(config.GitProjectName))
for _, git_project := range config.GitProjectName {
configs := make([]*AutogitConfig, 0, len(config.GitProjectNames))
for _, git_project := range config.GitProjectNames {
c, err := ReadWorkflowConfig(gitea, git_project)
if err != nil {
// can't sync, so ignore for now
@@ -159,3 +188,59 @@ func (config *AutogitConfig) GetReviewGroupMembers(reviewer string) ([]string, e
return nil, errors.New("User " + reviewer + " not found as group reviewer for " + config.GitProjectName)
}
func (config *AutogitConfig) GetPrjGit() (string, string, string) {
org := config.Organization
repo := DefaultGitPrj
branch := "master"
a := strings.Split(config.GitProjectName, "/")
if len(a[0]) > 0 {
repo = strings.TrimSpace(a[0])
}
if len(a) == 2 {
if a[0] = strings.TrimSpace(a[0]); len(a[0]) > 0 {
org = a[0]
}
repo = strings.TrimSpace(a[1])
}
b := strings.Split(repo, "#")
if len(b) == 2 {
if b[0] = strings.TrimSpace(b[0]); len(b[0]) > 0 {
repo = b[0]
} else {
repo = DefaultGitPrj
}
if b[1] = strings.TrimSpace(b[1]); len(b[1]) > 0 {
branch = strings.TrimSpace(b[1])
}
}
return org, repo, branch
}
func (config *AutogitConfig) GetRemoteBranch() string {
return "origin_" + config.Branch
}
type StagingConfig struct {
ObsProject string
RebuildAll bool
// if set, then only use pull request numbers as unique identifiers
StagingProject string
QA []QAConfig
}
func ParseStagingConfig(data []byte) (*StagingConfig, error) {
var staging StagingConfig
data, err := hujson.Standardize(data)
if err != nil {
return nil, err
}
if err := json.Unmarshal(data, &staging); err != nil {
return nil, err
}
return &staging, nil
}

View File

@@ -1,12 +1,13 @@
package common_test
import (
"slices"
"testing"
"go.uber.org/mock/gomock"
mock_common "src.opensuse.org/autogits/common/mock"
"src.opensuse.org/autogits/common"
"src.opensuse.org/autogits/common/gitea-generated/models"
mock_common "src.opensuse.org/autogits/common/mock"
)
func TestConfigWorkflowParser(t *testing.T) {
@@ -47,3 +48,83 @@ func TestConfigWorkflowParser(t *testing.T) {
})
}
}
func TestProjectGitParser(t *testing.T) {
tests := []struct {
name string
prjgit string
org string
branch string
res [3]string
}{
{
name: "repo only",
prjgit: "repo.git",
org: "org",
branch: "br",
res: [3]string{"org", "repo.git", "master"},
},
{
name: "default",
org: "org",
res: [3]string{"org", common.DefaultGitPrj, "master"},
},
{
name: "repo with branch",
org: "org2",
prjgit: "repo.git#somebranch",
res: [3]string{"org2", "repo.git", "somebranch"},
},
{
name: "repo org and branch",
org: "org3",
prjgit: "oorg/foo.bar#point",
res: [3]string{"oorg", "foo.bar", "point"},
},
{
name: "whitespace shouldn't matter",
prjgit: " oorg / \nfoo.bar\t # point ",
res: [3]string{"oorg", "foo.bar", "point"},
},
{
name: "repo org and empty branch",
org: "org3",
prjgit: "oorg/foo.bar#",
res: [3]string{"oorg", "foo.bar", "master"},
},
{
name: "only branch defined",
org: "org3",
prjgit: "#mybranch",
res: [3]string{"org3", "_ObsPrj", "mybranch"},
},
{
name: "only org and branch defined",
org: "org3",
prjgit: "org1/#mybranch",
res: [3]string{"org1", "_ObsPrj", "mybranch"},
},
{
name: "empty org and repo",
org: "org3",
prjgit: "/repo#",
res: [3]string{"org3", "repo", "master"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c := &common.AutogitConfig{
Organization: test.org,
Branch: test.branch,
GitProjectName: test.prjgit,
}
i, j, k := c.GetPrjGit()
res := []string{i, j, k}
if !slices.Equal(res, test.res[:]) {
t.Error("Expected", test.res, "but received", res)
}
})
}
}

View File

@@ -24,9 +24,9 @@ import (
"errors"
"fmt"
"io"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"sync"
@@ -44,11 +44,15 @@ type GitStatusLister interface {
}
type Git interface {
// error if git, but wrong remote
GitClone(repo, branch, remoteUrl string) (string, error) // clone, or check if path is already checked out remote and force pulls, error otherwise. Return remotename, errror
GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error)
GitCatFile(cwd, commitId, filename string) (data []byte, err error)
GetPath() string
GitBranchHead(gitDir, branchName string) (string, error)
GitRemoteHead(gitDir, remoteName, branchName string) (string, error)
io.Closer
GitSubmoduleLister
@@ -61,11 +65,11 @@ type Git interface {
}
type GitHandlerImpl struct {
DebugLogger bool
GitPath string
GitCommiter string
GitEmail string
lock *sync.Mutex
}
func (s *GitHandlerImpl) GetPath() string {
@@ -73,34 +77,86 @@ func (s *GitHandlerImpl) GetPath() string {
}
type GitHandlerGenerator interface {
CreateGitHandler(git_author, email, prjName string) (Git, error)
ReadExistingPath(git_author, email, gitPath string) (Git, error)
CreateGitHandler(org string) (Git, error)
ReadExistingPath(org string) (Git, error)
ReleaseLock(path string)
}
type GitHandlerGeneratorImpl struct{}
type gitHandlerGeneratorImpl struct {
path string
git_author string
email string
func (s *GitHandlerGeneratorImpl) CreateGitHandler(git_author, email, prj_name string) (Git, error) {
gitPath, err := os.MkdirTemp("", prj_name)
if err != nil {
return nil, fmt.Errorf("Cannot create temp dir: %w", err)
}
if err = os.Chmod(gitPath, 0700); err != nil {
return nil, fmt.Errorf("Cannot fix permissions of temp dir: %w", err)
}
return s.ReadExistingPath(git_author, email, gitPath)
lock_lock sync.Mutex
lock map[string]*sync.Mutex // per org
}
func (*GitHandlerGeneratorImpl) ReadExistingPath(git_author, email, gitPath string) (Git, error) {
func AllocateGitWorkTree(basePath, gitAuthor, email string) (*gitHandlerGeneratorImpl, error) {
if fi, err := os.Stat(basePath); err != nil || !fi.IsDir() {
return nil, fmt.Errorf("Git basepath not a valid directory: %s %w", basePath, err)
}
if fi, err := os.Stat(basePath); err != nil {
if os.IsNotExist(err) {
if err = os.MkdirAll(basePath, 0o700); err != nil {
return nil, fmt.Errorf("Cannot create git directory structure: %s: %w", basePath, err)
}
} else {
return nil, fmt.Errorf("Error checking git directory strcture: %s: %w", basePath, err)
}
} else if !fi.IsDir() {
return nil, fmt.Errorf("Invalid git directory structure: %s != directory", basePath)
}
return &gitHandlerGeneratorImpl{
path: basePath,
git_author: gitAuthor,
email: email,
lock: make(map[string]*sync.Mutex),
}, nil
}
func (s *gitHandlerGeneratorImpl) CreateGitHandler(org string) (Git, error) {
path := path.Join(s.path, org)
if fs, err := os.Stat(path); (err != nil && !os.IsNotExist(err)) || (err == nil && !fs.IsDir()) {
return nil, err
} else if err != nil && os.IsNotExist(err) {
if err := os.MkdirAll(path, 0o777); err != nil && !os.IsExist(err) {
return nil, err
}
}
return s.ReadExistingPath(org)
}
func (s *gitHandlerGeneratorImpl) ReadExistingPath(org string) (Git, error) {
s.lock_lock.Lock()
defer s.lock_lock.Unlock()
if _, ok := s.lock[org]; !ok {
s.lock[org] = &sync.Mutex{}
}
s.lock[org].Lock()
git := &GitHandlerImpl{
GitCommiter: git_author,
GitPath: gitPath,
GitCommiter: s.git_author,
GitEmail: s.email,
GitPath: path.Join(s.path, org),
lock: s.lock[org],
}
return git, nil
}
func (s *gitHandlerGeneratorImpl) ReleaseLock(org string) {
m, ok := s.lock[org]
if ok {
m.Unlock()
}
}
//func (h *GitHandler) ProcessBranchList() []string {
// if h.HasError() {
// return make([]string, 0)
@@ -139,20 +195,58 @@ func (refs *GitReferences) addReference(id, branch string) {
refs.refs = append(refs.refs, GitReference{Branch: branch, Id: id})
}
func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error) {
id, err := e.GitExecWithOutput(gitDir, "rev-list", "-1", branchName)
func (e *GitHandlerImpl) GitClone(repo, branch, remoteUrl string) (string, error) {
remoteUrlComp, err := ParseGitRemoteUrl(remoteUrl)
if err != nil {
return "", fmt.Errorf("Can't find default remote branch: %s", branchName)
return "", fmt.Errorf("Cannot parse remote URL: %w", err)
}
if len(branch) == 0 {
branch = remoteUrlComp.Commit
}
if len(branch) == 0 {
branch = "HEAD"
}
remoteName := remoteUrlComp.RemoteName()
LogDebug("Clone", *remoteUrlComp, " -> ", remoteName)
if fi, err := os.Stat(path.Join(e.GitPath, repo)); os.IsNotExist(err) {
if err = e.GitExec("", "clone", "--origin", remoteName, remoteUrl, repo); err != nil {
return remoteName, err
}
} else if err != nil || !fi.IsDir() {
return remoteName, fmt.Errorf("Clone location not a directory or Stat error: %w", err)
} else {
if u, err := e.GitExecWithOutput(repo, "remote", "get-url", remoteName); err != nil {
e.GitExecOrPanic(repo, "remote", "add", remoteName, remoteUrl)
} else if clonedRemote := strings.TrimSpace(u); clonedRemote != remoteUrl {
e.GitExecOrPanic(repo, "remote", "set-url", remoteName, remoteUrl)
}
e.GitExecOrPanic(repo, "fetch", remoteName, branch)
}
return remoteName, e.GitExec(repo, "checkout", "-B", branch, "refs/remotes/"+remoteName+"/"+branch)
}
func (e *GitHandlerImpl) GitBranchHead(gitDir, branchName string) (string, error) {
id, err := e.GitExecWithOutput(gitDir, "show-ref", "--hash", "--verify", "refs/heads/"+branchName)
if err != nil {
return "", fmt.Errorf("Can't find default branch: %s", branchName)
}
return strings.TrimSpace(id), nil
}
func (e *GitHandlerImpl) GitRemoteHead(gitDir, remote, branchName string) (string, error) {
id, err := e.GitExecWithOutput(gitDir, "show-ref", "--hash", "--verify", "refs/remotes/"+remote+"/"+branchName)
if err != nil {
return "", fmt.Errorf("Can't find default branch: %s", branchName)
}
return strings.TrimSpace(id), nil
}
func (e *GitHandlerImpl) Close() error {
if err := os.RemoveAll(e.GitPath); err != nil {
return err
}
e.GitPath = ""
e.lock.Unlock()
return nil
}
@@ -175,14 +269,16 @@ func (h writeFunc) Close() error {
func (e *GitHandlerImpl) GitExecWithOutputOrPanic(cwd string, params ...string) string {
out, err := e.GitExecWithOutput(cwd, params...)
if err != nil {
log.Panicln("git command failed:", params, "@", cwd, "err:", err)
LogError("git command failed:", params, "@", cwd, "err:", err)
panic(err)
}
return out
}
func (e *GitHandlerImpl) GitExecOrPanic(cwd string, params ...string) {
if err := e.GitExec(cwd, params...); err != nil {
log.Panicln("git command failed:", params, "@", cwd, "err:", err)
LogError("git command failed:", params, "@", cwd, "err:", err)
panic(err)
}
}
@@ -210,17 +306,11 @@ func (e *GitHandlerImpl) GitExecWithOutput(cwd string, params ...string) (string
cmd.Dir = filepath.Join(e.GitPath, cwd)
cmd.Stdin = nil
if e.DebugLogger {
log.Printf("git execute: %#v\n", cmd.Args)
}
LogDebug("git execute:", cmd.Args)
out, err := cmd.CombinedOutput()
if e.DebugLogger {
log.Println(string(out))
}
LogDebug(string(out))
if err != nil {
if e.DebugLogger {
log.Printf(" *** error: %v\n", err)
}
LogError("git", cmd.Args, " error:", err)
return "", fmt.Errorf("error executing: git %#v \n%s\n err: %w", cmd.Args, out, err)
}
@@ -453,7 +543,6 @@ func parseTreeEntry(data <-chan byte, hashLen int) (GitTreeEntry, error) {
}
func parseGitTree(data <-chan byte) (GitTree, error) {
hdr, err := parseGitMsg(data)
if err != nil {
return GitTree{}, err
@@ -536,14 +625,10 @@ func (e *GitHandlerImpl) GitParseCommits(cwd string, commitIDs []string) (parsed
cmd.Stdout = &data_in
cmd.Stdin = &data_out
cmd.Stderr = writeFunc(func(data []byte) (int, error) {
if e.DebugLogger {
log.Println(string(data))
}
LogError(string(data))
return len(data), nil
})
if e.DebugLogger {
log.Printf("command run: %v\n", cmd.Args)
}
LogDebug("command run:", cmd.Args)
err = cmd.Run()
done.Lock()
@@ -563,24 +648,27 @@ func (e *GitHandlerImpl) GitCatFile(cwd, commitId, filename string) (data []byte
data_out.Write([]byte(commitId))
data_out.ch <- '\x00'
c, err := parseGitCommit(data_in.ch)
var c GitCommit
c, err = parseGitCommit(data_in.ch)
if err != nil {
log.Printf("Error parsing git commit: %v\n", err)
LogError("Error parsing git commit:", err)
return
}
data_out.Write([]byte(c.Tree))
data_out.ch <- '\x00'
tree, err := parseGitTree(data_in.ch)
var tree GitTree
tree, err = parseGitTree(data_in.ch)
if err != nil {
if e.DebugLogger {
log.Printf("Error parsing git tree: %v\n", err)
}
LogError("Error parsing git tree:", err)
return
}
for _, te := range tree.items {
if te.isBlob() && te.name == filename {
LogInfo("blob", te.hash)
data_out.Write([]byte(te.hash))
data_out.ch <- '\x00'
data, err = parseGitBlob(data_in.ch)
@@ -588,7 +676,7 @@ func (e *GitHandlerImpl) GitCatFile(cwd, commitId, filename string) (data []byte
}
}
err = fmt.Errorf("file not found: '%s'", filename)
LogError("file not found:", filename)
}()
cmd := exec.Command("/usr/bin/git", "cat-file", "--batch", "-Z")
@@ -601,22 +689,20 @@ func (e *GitHandlerImpl) GitCatFile(cwd, commitId, filename string) (data []byte
cmd.Stdout = &data_in
cmd.Stdin = &data_out
cmd.Stderr = writeFunc(func(data []byte) (int, error) {
if e.DebugLogger {
log.Println(string(data))
}
LogError(string(data))
return len(data), nil
})
if e.DebugLogger {
log.Printf("command run: %v\n", cmd.Args)
LogDebug("command run:", cmd.Args)
if e := cmd.Run(); e != nil {
close(data_in.ch)
close(data_out.ch)
return nil, e
}
err = cmd.Run()
done.Lock()
return
}
// return (filename) -> (hash) map for all submodules
// TODO: recursive? map different orgs, not just assume '.' for path
func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleList map[string]string, err error) {
var done sync.Mutex
submoduleList = make(map[string]string)
@@ -636,19 +722,32 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
err = fmt.Errorf("Error parsing git commit. Err: %w", err)
return
}
data_out.Write([]byte(c.Tree))
data_out.ch <- '\x00'
var tree GitTree
tree, err = parseGitTree(data_in.ch)
if err != nil {
err = fmt.Errorf("Error parsing git tree: %w", err)
return
}
trees := make(map[string]string)
trees[""] = c.Tree
for _, te := range tree.items {
if te.isSubmodule() {
submoduleList[te.name] = te.hash
for len(trees) > 0 {
for p, tree := range trees {
delete(trees, p)
data_out.Write([]byte(tree))
data_out.ch <- '\x00'
var tree GitTree
tree, err = parseGitTree(data_in.ch)
if err != nil {
err = fmt.Errorf("Error parsing git tree: %w", err)
return
}
for _, te := range tree.items {
if te.isTree() {
trees[p+te.name+"/"] = te.hash
} else if te.isSubmodule() {
submoduleList[p+te.name] = te.hash
}
}
}
}
}()
@@ -663,14 +762,10 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
cmd.Stdout = &data_in
cmd.Stdin = &data_out
cmd.Stderr = writeFunc(func(data []byte) (int, error) {
if e.DebugLogger {
log.Println(string(data))
}
LogError(string(data))
return len(data), nil
})
if e.DebugLogger {
log.Printf("command run: %v\n", cmd.Args)
}
LogDebug("command run:", cmd.Args)
err = cmd.Run()
done.Lock()
@@ -682,10 +777,7 @@ func (e *GitHandlerImpl) GitSubmoduleCommitId(cwd, packageName, commitId string)
var wg sync.WaitGroup
wg.Add(1)
if e.DebugLogger {
log.Printf("getting commit id '%s' from git at '%s' with packageName: %s\n", commitId, cwd, packageName)
}
LogDebug("getting commit id", commitId, "from git at", cwd, "with packageName:", packageName)
go func() {
defer func() {
@@ -703,14 +795,16 @@ func (e *GitHandlerImpl) GitSubmoduleCommitId(cwd, packageName, commitId string)
data_out.ch <- '\x00'
c, err := parseGitCommit(data_in.ch)
if err != nil {
log.Panicf("Error parsing git commit: %v\n", err)
LogError("Error parsing git commit:", err)
panic(err)
}
data_out.Write([]byte(c.Tree))
data_out.ch <- '\x00'
tree, err := parseGitTree(data_in.ch)
if err != nil {
log.Panicf("Error parsing git tree: %v\n", err)
LogError("Error parsing git tree:", err)
panic(err)
}
for _, te := range tree.items {
@@ -731,14 +825,12 @@ func (e *GitHandlerImpl) GitSubmoduleCommitId(cwd, packageName, commitId string)
cmd.Stdout = &data_in
cmd.Stdin = &data_out
cmd.Stderr = writeFunc(func(data []byte) (int, error) {
log.Println(string(data))
LogError(string(data))
return len(data), nil
})
if e.DebugLogger {
log.Printf("command run: %v\n", cmd.Args)
}
LogDebug("command run:", cmd.Args)
if err := cmd.Run(); err != nil {
log.Printf("Error running command %v, err: %v", cmd.Args, err)
LogError("Error running command:", cmd.Args, err)
}
wg.Wait()
@@ -898,9 +990,7 @@ func parseGitStatusData(data io.ByteReader) ([]GitStatusData, error) {
}
func (e *GitHandlerImpl) GitStatus(cwd string) (ret []GitStatusData, err error) {
if e.DebugLogger {
log.Println("getting git-status()")
}
LogDebug("getting git-status()")
cmd := exec.Command("/usr/bin/git", "status", "--porcelain=2", "-z")
cmd.Env = []string{
@@ -910,15 +1000,13 @@ func (e *GitHandlerImpl) GitStatus(cwd string) (ret []GitStatusData, err error)
}
cmd.Dir = filepath.Join(e.GitPath, cwd)
cmd.Stderr = writeFunc(func(data []byte) (int, error) {
log.Println(string(data))
LogError(string(data))
return len(data), nil
})
if e.DebugLogger {
log.Printf("command run: %v\n", cmd.Args)
}
LogDebug("command run:", cmd.Args)
out, err := cmd.Output()
if err != nil {
log.Printf("Error running command %v, err: %v", cmd.Args, err)
LogError("Error running command", cmd.Args, err)
}
return parseGitStatusData(bufio.NewReader(bytes.NewReader(out)))

View File

@@ -29,6 +29,68 @@ import (
"testing"
)
func TestGitClone(t *testing.T) {
tests := []struct {
name string
repo string
branch string
remoteName string
remoteUrl string
}{
{
name: "Basic clone",
repo: "pkgAclone",
branch: "main",
remoteName: "pkgA_main",
remoteUrl: "/pkgA",
},
{
name: "Remote branch is non-existent",
repo: "pkgAclone",
branch: "main_not_here",
remoteName: "pkgA_main",
remoteUrl: "/pkgA",
},
}
execPath, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
d := t.TempDir()
os.Chdir(d)
defer os.Chdir(execPath)
cmd := exec.Command(path.Join(execPath, "test_clone_setup.sh"))
if _, err := cmd.Output(); err != nil {
t.Fatal(err)
}
gh, err := AllocateGitWorkTree(d, "Test", "test@example.com")
if err != nil {
t.Fatal(err)
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
g, err := gh.CreateGitHandler("org")
if err != nil {
t.Fatal(err)
}
if _, err := g.GitClone(test.repo, test.branch, "file://"+d+test.remoteUrl); err != nil {
t.Fatal(err)
}
id, err := g.GitBranchHead(test.repo, test.branch)
if err != nil {
t.Fatal(err)
}
t.Fatal(id)
})
}
}
func TestGitMsgParsing(t *testing.T) {
t.Run("tree message with size 56", func(t *testing.T) {
const hdr = "f40888ea4515fe2e8eea617a16f5f50a45f652d894de3ad181d58de3aafb8f98 tree 56\x00"
@@ -243,9 +305,36 @@ Reviewed-By: Marco Ippolito <marcoippolito54@gmail.com>` + "\x00"
t.Error("expected submodule not found")
}
})
t.Run("parse nested trees with subtrees", func(t *testing.T) {
const data = "873a323b262ebb3bd77b2592b2e11bdd08dbc721cbf4ac9f97637e58e1fffce7 tree 1083\x00100644\x20\x2Egitattributes\x00\xD8v\xA95\x87\xC1\xA9\xFCPn\xDD\xD4\x13\x9B\x8E\xD2\xCFs\xBD\x11q\x8A\xAE\x8A\x7Cg\xE2C\x14J\x01\xB0100644\x20\x2Egitignore\x00\xC3\xCD\x8En\x887\x3AJ\xA0P\xEEL\xD4\xF5\xD2v\x9C\xA6v\xC5D\x60\x40\x95\xD1\x0B\xA4\xB8\x86\xD4rE100644\x20COPYING\x00\x12\x2A\x28\xC8\xB9\x5D\x9B\x8A\x23\x1F\xE96\x07\x3F\xA9D\x90\xFD\xCE\x2Bi\x2D\x031\x5C\xCC\xC4fx\x00\xC22100644\x20README\x2Emd\x00\x92D\xF7\xFF\x0E0\x5C\xF2\xAC\x0DA\x06\x92\x0B\xD6z\x3CGh\x00y\x7EW1\xB9a\x8Ch\x215Fa100644\x20_service\x00\xC51\xF2\x12\xF3\x24\x9C\xD9\x9F\x0A\x93Mp\x12\xC1\xF7i\x05\x95\xC5Z\x06\x95i\x3Az\xC3\xF59\x7E\xF8\x1B100644\x20autogits\x2Echanges\x00\xF7\x8D\xBF\x0A\xCB\x5D\xB7y\x8C\xA9\x9C\xEB\x92\xAFd\x2C\x98\x23\x0C\x13\x13\xED\xDE\x5D\xBALD6\x3BR\x5B\xCA100644\x20autogits\x2Espec\x00\xD2\xBC\x20v\xD3\xE5F\xCA\xEE\xEA\x18\xC84\x0D\xA7\xCA\xD8O\xF2\x0A\xAB\x40\x2A\xFAL\x3B\xB4\xE6\x11\xE7o\xD140000\x20common\x00\xE2\xC9dg\xD0\x5D\xD1\xF1\x8ARW\xF0\x96\xD6\x29\x2F\x8F\xD9\xC7\x82\x1A\xB7\xAAw\xB0\xCE\xA8\xFE\xC8\xD7D\xF2100755\x20dev_test_helper\x2Esh\x00\xECY\xDD\xB3rz\x9Fh\xD4\x2E\x85\x02\x13\xF8\xFE\xB57\x8B\x1B6\x8E\x09dC\x1E\xE0\x90\x09\x08\xED\xBD_40000\x20devel\x2Dimporter\x00v\x98\x9B\x92\xD8\x24lu\xFC\xB2d\xC9\xCENb\xEE\x0F\x21\x8B\x92\x88\xDBs\xF8\x2E\xA8\xC8W\x1C\x20\xCF\xD440000\x20doc\x00\x8Akyq\xD0\xCF\xB8\x2F\x80Y\x2F\x11\xF0\x14\xA9\xFE\x96\x14\xE0W\x2C\xCF\xB9\x86\x7E\xFDi\xD7\x1F\x08Q\xFB40000\x20gitea\x2Devents\x2Drabbitmq\x2Dpublisher\x00\x5Cb\x3Fh\xA2\x06\x06\x0Cd\x09\xA5\xD9\xF7\x23\x5C\xF85\xF5\xB8\xBE\x7F\xD4O\x25t\xEF\xCC\xAB\x18\x7C\x0C\xF3100644\x20go\x2Emod\x00j\x85\x0B\x03\xC8\x9F\x9F\x0F\xC8\xE0\x8C\xF7\x3D\xC19\xF7\x12gk\xD6\x18JN\x24\xC0\x1C\xBE\x97oY\x02\x8D100644\x20go\x2Esum\x00h\x88\x2E\x27\xED\xD39\x8D\x12\x0F\x7D\x97\xA2\x5DE\xB9\x82o\x0Cu\xF4l\xA17s\x28\x2BQT\xE6\x12\x9040000\x20group\x2Dreview\x00\x7E\x7B\xB42\x0F\x3B\xC9o\x2C\xE79\x1DR\xE2\xE4i\xAE\xF6u\x90\x09\xD8\xC9c\xE7\xF7\xC7\x92\xFB\xD7\xDD140000\x20obs\x2Dstaging\x2Dbot\x00\x12\xE8\xAF\x09\xD4\x5D\x13\x8D\xC9\x0AvPDc\xB6\x7C\xAC4\xD9\xC5\xD4_\x98i\xBE2\xA7\x25aj\xE2k40000\x20obs\x2Dstatus\x2Dservice\x00MATY\xA3\xFA\xED\x05\xBE\xEB\x2B\x07\x9CN\xA9\xF3SB\x22MlV\xA4\x5D\xDA\x0B\x0F\x23\xA1\xA8z\xD740000\x20systemd\x00\x2D\xE2\x03\x7E\xBD\xEB6\x8F\xC5\x0E\x12\xD4\xBD\x97P\xDD\xA2\x92\xCE6n\x08Q\xCA\xE4\x15\x97\x8F\x26V\x3DW100644\x20vendor\x2Etar\x2Ezst\x00\xD9\x2Es\x03I\x91\x22\x24\xC86q\x91\x95\xEF\xA3\xC9\x3C\x06D\x90w\xAD\xCB\xAE\xEEu2i\xCE\x05\x09u40000\x20workflow\x2Ddirect\x00\x94\xDB\xDFc\xB5A\xD5\x16\xB3\xC3ng\x94J\xE7\x101jYF\x15Q\xE97\xCFg\x14\x12\x28\x3A\xFC\xDB40000\x20workflow\x2Dpr\x00\xC1\xD8Z9\x18\x60\xA2\xE2\xEF\xB0\xFC\xD7\x2Ah\xF07\x0D\xEC\x8A7\x7E\x1A\xAAn\x13\x9C\xEC\x05s\xE8\xBDf\x00"
ch := make(chan byte, 2000)
for _, b := range []byte(data) {
ch <- b
}
tree, err := parseGitTree(ch)
if err != nil {
t.Error(err)
}
found := false
for _, item := range tree.items {
t.Log(item)
if item.name == "workflow-pr" && item.hash == "c1d85a391860a2e2efb0fcd72a68f0370dec8a377e1aaa6e139cec0573e8bd66" && item.isTree() {
found = true
break
}
}
if !found {
t.Error("expected submodule not found")
}
})
}
func TestCommitTreeParsingOfHead(t *testing.T) {
func TestCommitTreeParsing(t *testing.T) {
gitDir := t.TempDir()
testDir, _ := os.Getwd()
var commitId string
@@ -260,11 +349,58 @@ func TestCommitTreeParsingOfHead(t *testing.T) {
t.Fatal(err.Error())
}
gh, err := AllocateGitWorkTree(gitDir, "", "")
if err != nil {
t.Fatal(err)
}
t.Run("GitCatFile commit", func(t *testing.T) {
h, _ := gh.ReadExistingPath(".")
defer h.Close()
file, err := h.GitCatFile("", commitId, "help")
if err != nil {
t.Error("failed", err)
}
if string(file) != "help\n" {
t.Error("expected 'help\\n' but got", string(file))
}
})
t.Run("GitCatFile commit", func(t *testing.T) {
h, _ := gh.ReadExistingPath(".")
defer h.Close()
file, err := h.GitCatFile("", "HEAD", "help")
if err != nil {
t.Error("failed", err)
}
if string(file) != "help\n" {
t.Error("expected 'help\\n' but got", string(file))
}
})
t.Run("GitCatFile bad commit", func(t *testing.T) {
h, _ := gh.ReadExistingPath(".")
defer h.Close()
file, err := h.GitCatFile("", "518b468f391bf01d5d76d497d7cbecfa8b46d185714cf8745800ae18afb21afd", "help")
if err == nil {
t.Error("expected error, but not nothing")
}
if string(file) != "" {
t.Error("expected 'help\\n' but got", file)
}
})
t.Run("reads HEAD and parses the tree", func(t *testing.T) {
const nodejs21 = "c678c57007d496a98bec668ae38f2c26a695f94af78012f15d044ccf066ccb41"
h := GitHandlerImpl{
GitPath: gitDir,
}
h, _ := gh.ReadExistingPath(".")
defer h.Close()
id, ok := h.GitSubmoduleCommitId("", "nodejs21", commitId)
if !ok {
t.Error("failed parse")
@@ -275,9 +411,9 @@ func TestCommitTreeParsingOfHead(t *testing.T) {
})
t.Run("reads README.md", func(t *testing.T) {
h := GitHandlerImpl{
GitPath: gitDir,
}
h, _ := gh.ReadExistingPath(".")
defer h.Close()
data, err := h.GitCatFile("", commitId, "README.md")
if err != nil {
t.Errorf("failed parse: %v", err)
@@ -288,9 +424,8 @@ func TestCommitTreeParsingOfHead(t *testing.T) {
})
t.Run("read HEAD", func(t *testing.T) {
h := GitHandlerImpl{
GitPath: gitDir,
}
h, _ := gh.ReadExistingPath(".")
defer h.Close()
data, err := h.GitSubmoduleList("", "HEAD")
if err != nil {
@@ -380,14 +515,13 @@ func TestGitStatusParse(t *testing.T) {
name: "Renamed file",
data: []byte("1 M. N... 100644 100644 100644 d23eb05d9ca92883ab9f4d28f3ec90c05f667f3a5c8c8e291bd65e03bac9ae3c 896cd09f36d39e782d66ae32dd5614d4f4d83fc689f132aab2dfc019a9f5b6f3 .gitmodules\x002 R. S... 160000 160000 160000 3befe051a34612530acfa84c736d2454278453ec0f78ec028f25d2980f8c3559 3befe051a34612530acfa84c736d2454278453ec0f78ec028f25d2980f8c3559 R100 pkgQ\x00pkgC\x00"),
res: []GitStatusData{
{
Path: "pkgQ",
{
Path: "pkgQ",
Status: GitStatus_Renamed,
States: [3]string{"pkgC"},
},
{
Path: ".gitmodules",
{
Path: ".gitmodules",
Status: GitStatus_Modified,
},
},

View File

@@ -68,7 +68,7 @@ const (
)
type GiteaComment interface {
AddComment(pr *models.PullRequest, comment string) (error)
AddComment(pr *models.PullRequest, comment string) error
}
type GiteaMaintainershipReader interface {
@@ -85,13 +85,19 @@ type GiteaReviewFetcher interface {
GetPullRequestReviews(org, project string, PRnum int64) ([]*models.PullReview, error)
}
type GiteaCommentFetcher interface {
GetIssueComments(org, project string, issueNo int64) ([]*models.Comment, error)
}
type GiteaPRChecker interface {
GiteaReviewFetcher
GiteaCommentFetcher
GiteaMaintainershipReader
}
type GiteaReviewFetcherAndRequester interface {
GiteaReviewFetcher
GiteaCommentFetcher
GiteaReviewRequester
}
@@ -111,6 +117,27 @@ type GiteaFileContentReader interface {
GetRepositoryFileContent(org, repo, hash, path string) ([]byte, string, error)
}
const (
CommitStatus_Pending = "pending"
CommitStatus_Success = "success"
CommitStatus_Fail = "failure"
CommitStatus_Error = "error"
)
type CommitStatus struct {
Context string
Description string
CommitStatus string
}
type GiteaCommitStatusSetter interface {
SetCommitStatus(org, repo, hash string, status *CommitStatus) error
}
type GiteaCommitStatusGetter interface {
GetCommitStatus(org, repo, hash string) ([]*CommitStatus, error)
}
type Gitea interface {
GiteaComment
GiteaRepoFetcher
@@ -118,6 +145,7 @@ type Gitea interface {
GiteaReviewer
GiteaPRFetcher
GiteaReviewFetcher
GiteaCommentFetcher
GiteaMaintainershipReader
GiteaFileContentReader
@@ -129,7 +157,7 @@ type Gitea interface {
CreatePullRequestIfNotExist(repo *models.Repository, srcId, targetId, title, body string) (*models.PullRequest, error)
GetAssociatedPrjGitPR(prjGitOrg, prjGitRepo, refOrg, refRepo string, Index int64) (*models.PullRequest, error)
GetPullRequestFileContent(pr *models.PullRequest, path string) ([]byte, string, error)
GetRecentPullRequests(org, repo string) ([]*models.PullRequest, error)
GetRecentPullRequests(org, repo, branch string) ([]*models.PullRequest, error)
GetRecentCommits(org, repo, branch string, commitNo int64) ([]*models.Commit, error)
GetCurrentUser() (*models.User, error)
@@ -148,7 +176,7 @@ func AllocateGiteaTransport(giteaUrl string) Gitea {
log.Panicln("Failed to parse gitea url:", err)
}
r.transport = transport.New(url.Hostname(), apiclient.DefaultBasePath, [](string){url.Scheme})
r.transport = transport.New(url.Host, apiclient.DefaultBasePath, [](string){url.Scheme})
r.transport.DefaultAuthentication = transport.BearerToken(giteaToken)
r.client = apiclient.New(r.transport, nil)
@@ -180,6 +208,10 @@ func (gitea *GiteaTransport) GetPullRequest(org, project string, num int64) (*mo
func (gitea *GiteaTransport) GetRepository(org, pkg string) (*models.Repository, error) {
repo, err := gitea.client.Repository.RepoGet(repository.NewRepoGetParams().WithDefaults().WithOwner(org).WithRepo(pkg), gitea.transport.DefaultAuthentication)
if err != nil {
switch err.(type) {
case *repository.RepoGetNotFound:
return nil, nil
}
return nil, err
}
@@ -217,6 +249,30 @@ func (gitea *GiteaTransport) GetPullRequestReviews(org, project string, PRnum in
return allReviews, nil
}
func (gitea *GiteaTransport) GetIssueComments(org, project string, issueNo int64) ([]*models.Comment, error) {
// limit := int64(20)
// var page int64
// var allComments []*models.Comment
// for {
c, err := gitea.client.Issue.IssueGetComments(
issue.NewIssueGetCommentsParams().
WithDefaults().
WithOwner(org).
WithRepo(project).
WithIndex(issueNo),
gitea.transport.DefaultAuthentication)
if err != nil {
return nil, err
}
return c.Payload, nil
// if len(c.Payload) < int(limit)
// }
}
func (gitea *GiteaTransport) GetPullNotifications(since *time.Time) ([]*models.NotificationThread, error) {
bigLimit := int64(100000)
@@ -292,6 +348,7 @@ func (gitea *GiteaTransport) GetOrganizationRepositories(orgName string) ([]*mod
}
func (gitea *GiteaTransport) CreateRepositoryIfNotExist(git Git, org, repoName string) (*models.Repository, error) {
log.Println(org, repoName)
repo, err := gitea.client.Repository.RepoGet(
repository.NewRepoGetParams().WithDefaults().WithOwner(org).WithRepo(repoName),
gitea.transport.DefaultAuthentication)
@@ -299,6 +356,7 @@ func (gitea *GiteaTransport) CreateRepositoryIfNotExist(git Git, org, repoName s
if err != nil {
switch err.(type) {
case *repository.RepoGetNotFound:
log.Println("not found", err)
repo, err := gitea.client.Organization.CreateOrgRepo(
organization.NewCreateOrgRepoParams().WithDefaults().WithBody(
&models.CreateRepoOption{
@@ -484,7 +542,7 @@ func (gitea *GiteaTransport) AddReviewComment(pr *models.PullRequest, state mode
return c.Payload, nil
}
func (gitea *GiteaTransport) AddComment(pr *models.PullRequest, comment string) (error) {
func (gitea *GiteaTransport) AddComment(pr *models.PullRequest, comment string) error {
_, err := gitea.client.Issue.IssueCreateComment(
issue.NewIssueCreateCommentParams().
WithDefaults().
@@ -524,7 +582,7 @@ func (gitea *GiteaTransport) GetRepositoryFileContent(org, repo, hash, path stri
}
data := make([]byte, content.Payload.Size)
n, err := base64.StdEncoding.Decode(data, []byte(content.Payload.Content))
n, err := base64.StdEncoding.Decode(data, []byte(content.Payload.Content))
if err != nil {
log.Println(content.Payload.Content[239])
log.Println(len(content.Payload.Content))
@@ -545,12 +603,13 @@ func (gitea *GiteaTransport) GetPullRequestFileContent(pr *models.PullRequest, p
return gitea.GetRepositoryFileContent(pr.Head.Repo.Owner.UserName, pr.Head.Repo.Name, pr.Head.Sha, path)
}
func (gitea *GiteaTransport) GetRecentPullRequests(org, repo string) ([]*models.PullRequest, error) {
func (gitea *GiteaTransport) GetRecentPullRequests(org, repo, branch string) ([]*models.PullRequest, error) {
prs := make([]*models.PullRequest, 0, 10)
var page int64
page = 1
sort := "recentupdate"
endPrs:
for {
res, err := gitea.client.Repository.RepoListPullRequests(
repository.NewRepoListPullRequestsParams().
@@ -563,15 +622,21 @@ func (gitea *GiteaTransport) GetRecentPullRequests(org, repo string) ([]*models.
return nil, err
}
prs = append(prs, res.Payload...)
n := len(res.Payload)
if n < 10 {
if len(res.Payload) == 0 {
break
}
// if pr is closed for more than a week, assume that we are done too
if time.Since(time.Time(res.Payload[n-1].Updated)) > 7*24*time.Hour {
break
for _, pr := range res.Payload {
if pr.Base.Name != branch {
continue
}
// if pr is closed for more than a week, assume that we are done too
if time.Since(time.Time(pr.Updated)) > 7*24*time.Hour {
break endPrs
}
prs = append(prs, pr)
}
page++
@@ -583,20 +648,26 @@ func (gitea *GiteaTransport) GetRecentPullRequests(org, repo string) ([]*models.
func (gitea *GiteaTransport) GetRecentCommits(org, repo, branch string, commitNo int64) ([]*models.Commit, error) {
not := false
var page int64 = 1
commits, err := gitea.client.Repository.RepoGetAllCommits(
repository.NewRepoGetAllCommitsParams().
WithOwner(org).
WithRepo(repo).
WithSha(&branch).
WithPage(&page).
WithStat(&not).
WithFiles(&not).
WithVerification(&not).
WithLimit(&commitNo),
gitea.transport.DefaultAuthentication,
)
params := repository.NewRepoGetAllCommitsParams().
WithOwner(org).
WithRepo(repo).
WithPage(&page).
WithStat(&not).
WithFiles(&not).
WithVerification(&not).
WithLimit(&commitNo)
if len(branch) > 0 {
params = params.WithSha(&branch)
}
commits, err := gitea.client.Repository.RepoGetAllCommits(params, gitea.transport.DefaultAuthentication)
if err != nil {
switch err.(type) {
case *repository.RepoGetAllCommitsNotFound:
return nil, nil
}
return nil, err
}

View File

@@ -21,7 +21,6 @@ package common
import (
"crypto/tls"
"fmt"
"log"
"net/url"
"runtime/debug"
"slices"
@@ -78,18 +77,18 @@ func (l *ListenDefinitions) processTopicChanges(ch *rabbitmq.Channel, queueName
return
}
log.Println(" topic change:", topic)
LogDebug(" topic change:", topic)
switch topic[0] {
case '+':
if err := ch.QueueBind(queueName, topic[1:], "pubsub", false, nil); err != nil {
log.Println(err)
LogError(err)
}
case '-':
if err := ch.QueueUnbind(queueName, topic[1:], "pubsub", nil); err != nil {
log.Println(err)
LogError(err)
}
default:
log.Println("Ignoring topic change.")
LogInfo("Ignoring unknown topic change:", topic)
}
}
}
@@ -126,7 +125,7 @@ func (l *ListenDefinitions) processRabbitMQ(msgCh chan<- RabbitMessage) error {
} else {
q, err = ch.QueueDeclarePassive(queueName, true, false, true, false, nil)
if err != nil {
log.Printf("queue not found .. trying to create it: %v\n", err)
LogInfo("queue not found .. trying to create it:", err)
if ch.IsClosed() {
ch, err = connection.Channel()
if err != nil {
@@ -136,7 +135,7 @@ func (l *ListenDefinitions) processRabbitMQ(msgCh chan<- RabbitMessage) error {
q, err = ch.QueueDeclare(queueName, true, false, true, false, nil)
if err != nil {
log.Printf("can't create persistent queue ... falling back to temporaty queue: %v\n", err)
LogInfo("can't create persistent queue ... falling back to temporaty queue:", err)
if ch.IsClosed() {
ch, err = connection.Channel()
return fmt.Errorf("Channel cannot be re-opened. Err: %w", err)
@@ -150,7 +149,7 @@ func (l *ListenDefinitions) processRabbitMQ(msgCh chan<- RabbitMessage) error {
}
// log.Printf("queue: %s:%d", q.Name, q.Consumers)
log.Println(" -- listening to topics:")
LogInfo(" -- listening to topics:")
l.topicSubChanges = make(chan string)
defer close(l.topicSubChanges)
go l.processTopicChanges(ch, q.Name)
@@ -175,29 +174,29 @@ func (l *ListenDefinitions) processRabbitMQ(msgCh chan<- RabbitMessage) error {
}
}
func (l *ListenDefinitions) connectAndProcessRabbitMQ(log *log.Logger, ch chan<- RabbitMessage) {
func (l *ListenDefinitions) connectAndProcessRabbitMQ(ch chan<- RabbitMessage) {
defer func() {
if r := recover(); r != nil {
log.Println(r)
log.Println("'crash' RabbitMQ worker. Recovering... reconnecting...")
LogError(r)
LogError("'crash' RabbitMQ worker. Recovering... reconnecting...")
time.Sleep(5 * time.Second)
go l.connectAndProcessRabbitMQ(log, ch)
go l.connectAndProcessRabbitMQ(ch)
}
}()
for {
err := l.processRabbitMQ(ch)
if err != nil {
log.Printf("Error in RabbitMQ connection. %#v", err)
log.Println("Reconnecting in 2 seconds...")
LogError("Error in RabbitMQ connection. %#v", err)
LogInfo("Reconnecting in 2 seconds...")
time.Sleep(2 * time.Second)
}
}
}
func (l *ListenDefinitions) connectToRabbitMQ(log *log.Logger) chan RabbitMessage {
func (l *ListenDefinitions) connectToRabbitMQ() chan RabbitMessage {
ch := make(chan RabbitMessage, 100)
go l.connectAndProcessRabbitMQ(log, ch)
go l.connectAndProcessRabbitMQ(ch)
return ch
}
@@ -205,16 +204,16 @@ func (l *ListenDefinitions) connectToRabbitMQ(log *log.Logger) chan RabbitMessag
func ProcessEvent(f RequestProcessor, request *Request) {
defer func() {
if r := recover(); r != nil {
log.Println("panic caught")
LogError("panic caught")
if err, ok := r.(error); !ok {
log.Println(err)
LogError(err)
}
log.Println(string(debug.Stack()))
LogError(string(debug.Stack()))
}
}()
if err := f.ProcessFunc(request); err != nil {
log.Println(err)
LogError(err)
}
}
@@ -268,12 +267,13 @@ next_new_topic:
}
func (l *ListenDefinitions) ProcessRabbitMQEvents() error {
log.Println("RabbitMQ connection:", l.RabbitURL.String())
log.Println(len(l.Handlers), len(l.Orgs))
LogInfo("RabbitMQ connection:", l.RabbitURL.String())
LogDebug("# Handlers:", len(l.Handlers))
LogDebug("# Orgs:", len(l.Orgs))
l.RabbitURL.User = url.UserPassword(rabbitUser, rabbitPassword)
l.topics = l.generateTopics()
ch := l.connectToRabbitMQ(log.Default())
ch := l.connectToRabbitMQ()
for {
msg, ok := <-ch
@@ -281,7 +281,7 @@ func (l *ListenDefinitions) ProcessRabbitMQEvents() error {
return nil
}
log.Println("event:", msg.RoutingKey)
LogDebug("event:", msg.RoutingKey)
route := strings.Split(msg.RoutingKey, ".")
if len(route) > 3 {
@@ -289,11 +289,11 @@ func (l *ListenDefinitions) ProcessRabbitMQEvents() error {
org := route[2]
if !slices.Contains(l.Orgs, org) {
log.Println("Got event for unhandeled org:", org)
LogInfo("Got event for unhandeled org:", org)
continue
}
log.Println("org:", org, "type:", reqType)
LogDebug("org:", org, "type:", reqType)
if handler, found := l.Handlers[reqType]; found {
/* h, err := CreateRequestHandler()
if err != nil {
@@ -303,10 +303,10 @@ func (l *ListenDefinitions) ProcessRabbitMQEvents() error {
*/
req, err := ParseRequestJSON(reqType, msg.Body)
if err != nil {
log.Println("Error parsing request JSON:", err)
LogError("Error parsing request JSON:", err)
continue
} else {
log.Println("processing req", req.Type)
LogDebug("processing req", req.Type)
// h.Request = req
ProcessEvent(handler, req)

View File

@@ -41,7 +41,7 @@ func TestListenDefinitionsTopicUpdate(t *testing.T) {
l.UpdateTopics()
if len(l.topicSubChanges) != len(test.topicDelta) {
t.Fatal("topicSubChanges != topicDelta")
t.Fatal("topicSubChanges", len(l.topicSubChanges), " != topicDelta", len(test.topicDelta))
}
})
}

View File

@@ -39,3 +39,40 @@ func PanicOnError(err error) {
panic(err)
}
}
func PanicOnErrorWithMsg(err error, msg string) {
if err != nil {
LogError(msg)
panic(err)
}
}
type LogLevel int
const (
LogLevelNone = 0
LogLevelInfo = 5
LogLevelDebug = 10
)
var logLevel LogLevel
func SetLoggingLevel(ll LogLevel) {
logLevel = ll
}
func LogError(params ...any) {
log.Println(append([]any{"[E]"}, params...)...)
}
func LogDebug(params ...any) {
if logLevel >= LogLevelDebug {
log.Println(append([]any{"[D]"}, params...)...)
}
}
func LogInfo(params ...any) {
if logLevel >= LogLevelInfo {
log.Println(append([]any{"[I]"}, params...)...)
}
}

View File

@@ -32,10 +32,21 @@ import (
"regexp"
"slices"
"strings"
"syscall"
"time"
)
//go:generate mockgen -source=obs_utils.go -destination=mock/obs_utils.go -typed
type BuildResultOptions struct {
BinaryList bool
OldState string
LastBuild bool
}
type ObsStatusFetcherWithState interface {
BuildStatusWithState(project string, opts *BuildResultOptions, packages ...string) (*BuildResultList, error)
}
type ObsClient struct {
baseUrl *url.URL
client *http.Client
@@ -236,7 +247,6 @@ func (c *ObsClient) ObsRequest(method string, url string, body io.Reader) (*http
}
if res.StatusCode == 401 {
// log.Printf("new authentication is needed ...")
if c.sshkey == "" {
req.SetBasicAuth(c.user, c.password)
} else {
@@ -248,25 +258,19 @@ func (c *ObsClient) ObsRequest(method string, url string, body io.Reader) (*http
return nil, errors.New("No realm found")
}
realm := string(match[1])
sshKeygenPath, err := exec.LookPath("ssh-keygen")
if err != nil {
fmt.Println("ssh-keygen not found")
}
// SSH Sign
cmd := exec.Command(sshKeygenPath, "-Y", "sign", "-f", c.sshkeyfile, "-n", realm, "-q")
cmd := exec.Command("ssh-keygen", "-Y", "sign", "-f", c.sshkeyfile, "-n", realm, "-q")
now := time.Now().Unix()
sigdata := fmt.Sprintf("(created): %d", now)
cmd.Stdin = strings.NewReader(sigdata)
stdout, err := cmd.Output()
if err != nil {
log.Panicln("SSH sign error:", cmd.Stderr)
if exitError, ok := err.(*exec.ExitError); ok {
waitStatus := exitError.Sys().(syscall.WaitStatus)
exitCode := waitStatus.ExitStatus()
return nil, errors.New(fmt.Sprintf("ssh-keygen signature creation failed: %d", exitCode))
exitCode := -1337
if cmd.ProcessState != nil {
exitCode = cmd.ProcessState.ExitCode()
}
return nil, errors.New("ssh-keygen signature creation failed")
return nil, errors.New(fmt.Sprintf("ssh-keygen signature creation failed: %d", exitCode))
}
reg := regexp.MustCompile("(?s)-----BEGIN SSH SIGNATURE-----\n(.*?)\n-----END SSH SIGNATURE-----")
match = reg.FindStringSubmatch(string(stdout))
@@ -274,7 +278,7 @@ func (c *ObsClient) ObsRequest(method string, url string, body io.Reader) (*http
return nil, errors.New("could not extract ssh signature")
}
signature, err := base64.StdEncoding.DecodeString(string(match[1]))
signature, err := base64.StdEncoding.DecodeString(match[1])
if err != nil {
return nil, err
}
@@ -283,7 +287,7 @@ func (c *ObsClient) ObsRequest(method string, url string, body io.Reader) (*http
authorization := fmt.Sprintf(`keyId="%s",algorithm="ssh",headers="(created)",created=%d,signature="%s"`,
c.user, now, signatureBase64)
// log.Printf("Add Authorization Signature ", authorization)
// log.Printf("Add Authorization Signature ", authorization)
req.Header.Add("Authorization", "Signature "+authorization)
}
}
@@ -293,7 +297,7 @@ func (c *ObsClient) ObsRequest(method string, url string, body io.Reader) (*http
res, err = c.client.Do(req)
if err != nil {
log.Panicln("Authentification failed:", res.StatusCode)
log.Println("Authentification failed:", res.StatusCode)
return nil, err
}
@@ -445,6 +449,8 @@ type BuildResult struct {
Arch string `xml:"arch,attr"`
Code string `xml:"code,attr"`
Dirty bool `xml:"dirty,attr"`
ScmSync string `xml:"scmsync"`
ScmInfo string `xml:"scminfo"`
Status []PackageBuildStatus `xml:"status"`
Binaries []BinaryList `xml:"binarylist"`
}
@@ -462,20 +468,25 @@ type BinaryList struct {
type BuildResultList struct {
XMLName xml.Name `xml:"resultlist"`
State string `xml:"state,attr"`
Result []BuildResult `xml:"result"`
}
func (r *BuildResultList) GetPackageList() []string {
pkgList := make([]string, 0, 16)
for _, res := range r.Result {
// TODO: enough to iterate over one result set?
pkgList := make([]string, 0, 3*len(r.Result[0].Status)/2)
for ridx, res := range r.Result {
for _, status := range res.Status {
if !slices.Contains(pkgList, status.Package) {
if ridx == 0 {
pkgList = append(pkgList, status.Package)
} else if idx, found := slices.BinarySearch(pkgList, status.Package); !found {
pkgList = slices.Insert(pkgList, idx, status.Package)
}
}
if ridx == 0 {
slices.Sort(pkgList)
}
}
return pkgList
@@ -605,6 +616,11 @@ var ObsBuildStatusDetails map[string]ObsBuildStatusDetail = map[string]ObsBuildS
Description: "The scheduler has not yet evaluated this package. Should be a short intermediate state for new packages.",
Finished: false,
},
"error": ObsBuildStatusDetail{
Code: "Error",
Description: "Unknown status code",
},
}
var ObsRepoStatusDetails map[string]ObsBuildStatusDetail = map[string]ObsBuildStatusDetail{
// repo status
@@ -689,13 +705,28 @@ func (c *ObsClient) ProjectConfig(project string) (string, error) {
}
func (c *ObsClient) BuildStatus(project string, packages ...string) (*BuildResultList, error) {
return c.BuildStatusWithState(project, &BuildResultOptions{}, packages...)
}
func (c *ObsClient) LastBuildResults(project string, packages ...string) (*BuildResultList, error) {
return c.BuildStatusWithState(project, &BuildResultOptions{LastBuild: true}, packages...)
}
func (c *ObsClient) BuildStatusWithState(project string, opts *BuildResultOptions, packages ...string) (*BuildResultList, error) {
u := c.baseUrl.JoinPath("build", project, "_result")
query := u.Query()
query.Add("view", "status")
query.Add("view", "binarylist")
if opts.BinaryList {
query.Add("view", "binarylist")
}
query.Add("multibuild", "1")
if len(packages) > 0 {
if len(opts.OldState) > 0 {
query.Add("oldstate", opts.OldState)
}
if opts.LastBuild {
query.Add("lastbuild", "1")
}
if len(packages) > 0 {
for _, pkg := range packages {
query.Add("package", pkg)
}

View File

@@ -53,8 +53,9 @@ func FetchPRSet(gitea GiteaPRFetcher, org, repo string, num int64, config *Autog
var pr *models.PullRequest
var err error
if org != config.Organization || repo != config.GitProjectName {
if pr, err = gitea.GetAssociatedPrjGitPR(config.Organization, config.GitProjectName, org, repo, num); err != nil {
prjGitOrg, prjGitRepo, _ := config.GetPrjGit()
if prjGitOrg != org || prjGitRepo != config.GitProjectName {
if pr, err = gitea.GetAssociatedPrjGitPR(prjGitOrg, prjGitRepo, org, repo, num); err != nil {
return nil, err
}
@@ -203,14 +204,14 @@ func (rs *PRSet) IsApproved(gitea GiteaPRChecker, maintainers MaintainershipData
return is_reviewed
}
func (rs *PRSet) Merge(author, email string) error {
func (rs *PRSet) Merge(gh GitHandlerGenerator) error {
prjgit, err := rs.GetPrjGitPR()
if err != nil {
return err
}
gh := GitHandlerGeneratorImpl{}
git, err := gh.CreateGitHandler(author, email, prjgit.Base.Name)
git, err := gh.CreateGitHandler(rs.Config.Organization)
defer git.Close()
if err != nil {
return err
}

View File

@@ -546,7 +546,8 @@ func TestPRMerge(t *testing.T) {
t.Fatal(err)
}
if err = set.Merge("test", "test@example.com"); err != nil && (test.mergeError == "" || (len(test.mergeError) > 0 && !strings.Contains(err.Error(), test.mergeError))) {
gh, _ := common.AllocateGitWorkTree("", "", "")
if err = set.Merge(gh); err != nil && (test.mergeError == "" || (len(test.mergeError) > 0 && !strings.Contains(err.Error(), test.mergeError))) {
t.Fatal(err)
}
})

View File

@@ -28,9 +28,14 @@ import (
type Commit struct {
Id string
Message string
Added []string
Removed []string
Modified []string
}
type PushWebhookEvent struct {
Ref string
Total_Commits int
Head_Commit Commit
Commits []Commit

View File

@@ -20,10 +20,16 @@ package common
import (
"fmt"
"net/url"
"regexp"
"slices"
"strings"
)
func SplitLines(str string) []string {
return SplitStringNoEmpty(str, "\n")
}
func SplitStringNoEmpty(str, sep string) []string {
ret := slices.DeleteFunc(strings.Split(str, sep), func(s string) bool {
return len(strings.TrimSpace(s)) == 0
@@ -35,16 +41,81 @@ func SplitStringNoEmpty(str, sep string) []string {
}
func TranslateHttpsToSshUrl(url string) (string, error) {
const url1 = "https://src.opensuse.org/"
const url2 = "https://src.suse.de/"
const (
url1 = "https://src.opensuse.org/"
url2 = "https://src.suse.de/"
if len(url) > len(url1) && url[0:len(url1)] == url1 {
return "gitea@src.opensuse.org:" + url[len(url1):], nil
url1_len = len(url1)
url2_len = len(url2)
)
if len(url) > url1_len && url[0:url1_len] == url1 {
return "ssh://gitea@src.opensuse.org/" + url[url1_len:], nil
}
if len(url) > len(url2) && url[0:len(url2)] == url2 {
return "gitea@src.suse.de:" + url[len(url2):], nil
if len(url) > url2_len && url[0:url2_len] == url2 {
return "ssh://gitea@src.suse.de/" + url[url2_len:], nil
}
return "", fmt.Errorf("Unknown input url %s", url)
}
func TranslateSshNativeToUrl(urlString string) (string, error) {
rx := regexp.MustCompile("^([^:@]+)@?([^:]*):(.+)$")
m := rx.FindAllStringSubmatch(urlString, -1)
if m == nil {
return "", fmt.Errorf("Cannot match expected native SSH schema: %s", urlString)
}
if len(m[0][2]) > 0 {
// with user
return "ssh://" + m[0][1] + "@" + m[0][2] + "/" + m[0][3], nil
}
// without user
return "ssh://" + m[0][1] + "/" + m[0][3], nil
}
type GitUrl struct {
Org string
Repo string
Commit string
}
var valid_schemas []string = []string{"https", "ssh", "http", "file"}
func ParseGitRemoteUrl(urlString string) (*GitUrl, error) {
url, err := url.Parse(urlString)
if err != nil || !slices.Contains(valid_schemas, url.Scheme) {
u, err := TranslateSshNativeToUrl(urlString)
if err != nil {
return nil, fmt.Errorf("Unable to parse url: %w", err)
}
return ParseGitRemoteUrl(u)
}
e := SplitStringNoEmpty(url.Path, "/")
if len(e) != 2 {
return nil, fmt.Errorf("Unexpected format for Gitea URL: %s", e)
}
org := e[0]
repo := e[1]
if len(repo) > 4 && repo[len(repo)-4:] == ".git" {
repo = repo[0 : len(repo)-4]
}
u := GitUrl{
Org: org,
Repo: repo,
Commit: url.Fragment,
}
return &u, nil
}
func (giturl *GitUrl) RemoteName() string {
if giturl == nil || len(giturl.Org) == 0 || len(giturl.Repo) == 0 {
return "origin"
}
return strings.ToLower(giturl.Org) + "_" + strings.ToLower(giturl.Repo)
}

167
common/utils_test.go Normal file
View File

@@ -0,0 +1,167 @@
package common_test
import (
"testing"
"src.opensuse.org/autogits/common"
)
func TestGitUrlParse(t *testing.T) {
tests := []struct {
name string
inputUrl string
url common.GitUrl
error bool
}{
{
name: "Empty string",
error: true,
},
{
name: "OpenSUSE HTTPS Url",
url: common.GitUrl{
Org: "foo",
Repo: "b",
},
inputUrl: "https://src.opensuse.org/foo/b.git",
},
{
name: "OpenSUSE HTTPS Url",
url: common.GitUrl{
Org: "a",
Repo: "b",
},
inputUrl: "https://src.opensuse.org/a/b",
},
{
name: "OpenSUSE HTTPS Url",
url: common.GitUrl{
Org: "foo",
Repo: "bar",
Commit: "main",
},
inputUrl: "https://src.opensuse.org/foo/bar.git#main",
},
{
name: "invalid OpenSUSE HTTPS Url",
inputUrl: "https://src.opensuse.org/bar.git#main",
error: true,
},
{
name: "OpenSUSE SSH Url",
url: common.GitUrl{
Org: "foo",
Repo: "bar",
Commit: "main",
},
inputUrl: "ssh://src.opensuse.org/foo/bar.git#main",
},
{
name: "SSH native OpenSUSE Url",
inputUrl: "gitea@src.opensuse.org:foo/bar.git#main",
url: common.GitUrl{
Org: "foo",
Repo: "bar",
Commit: "main",
},
},
{
name: "SSH native OpenSUSE Url without user",
inputUrl: "src.opensuse.org:foo/bar.git#main",
url: common.GitUrl{
Org: "foo",
Repo: "bar",
Commit: "main",
},
},
{
name: "invalid SSH native OpenSUSE Url without user",
inputUrl: "src.opensuse.org:/br.it",
error: true,
},
{
name: "SSH native OpenSUSE Url without user",
inputUrl: "src.opensuse.org:foo/bar#main",
url: common.GitUrl{
Org: "foo",
Repo: "bar",
Commit: "main",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
url, err := common.ParseGitRemoteUrl(test.inputUrl)
if test.error && err != nil {
return
}
if test.error && err == nil {
t.Fatal("Expected an error but received", *url)
} else if !test.error && err != nil {
t.Error(err)
}
if url == nil {
t.Fatal("Recieved nil. Expected", test.url)
} else if *url != test.url {
t.Fatalf("Expected %v but received %v", test.url, *url)
}
})
}
}
func TestRemoteName(t *testing.T) {
tests := []struct {
name string
giturl *common.GitUrl
remotename string
}{
{
name: "empty",
remotename: "origin",
},
{
name: "regular repo",
giturl: &common.GitUrl{
Org: "org1",
Repo: "repo2",
Commit: "main",
},
remotename: "org1_repo2",
},
{
name: "regular repo with upper case",
giturl: &common.GitUrl{
Org: "Org1",
Repo: "REPO2",
},
remotename: "org1_repo2",
},
{
name: "no org",
giturl: &common.GitUrl{
Repo: "REPO2",
},
remotename: "origin",
},
{
name: "no repo",
giturl: &common.GitUrl{
Org: "ORG2",
},
remotename: "origin",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
n := test.giturl.RemoteName()
if n != test.remotename {
t.Errorf("Expected '%s' but received '%s'", test.remotename, n)
}
})
}
}

View File

@@ -5,9 +5,12 @@ if [ "x$1" = 'x' ]; then
exit 1
fi
files=$(find .. -maxdepth 2 -name \*.go -and \! -wholename \*/gitea-generated/\*)
while true; do
go test --run "$1"
inotifywait --exclude 'node_modules' -qqr -e close_write .. && clear
inotifywait -qqr -e close_write $files
clear
sleep 0.2
done

View File

@@ -1 +1,4 @@
devel-importer
Factory
git
git-migrated

View File

@@ -0,0 +1,50 @@
#!/usr/bin/bash
# Factory meta data checked-out
export DEVEL_PACKAGES=$PWD/Factory/pkgs/_meta/devel_packages
devel=$PWD/devel_update.sh
function getorg {
osc meta prj $1 | grep scmsync | sed -e's,^.*src\.opensuse\.org/,,' -e 's,/_ObsPrj.*$,,'
}
function factory {
$devel get $1
}
function message {
org=$1
pkg=$2
dprj=$3
echo "This package is developed in git at https://src.opensuse.org/${org}/${pkg} for OBS package ${dprj}/${pkg} -- see https://en.opensuse.org/openSUSE:OBS_to_Git"
}
obs=$1
if [ -z "$1" ]; then
obs=$(cat migrated_projects)
fi
for p in $obs; do
org=$(getorg $p)
if [ -z "$org" ]; then
echo $p is not in git!
continue
fi
packages=$(osc ls $p)
for pkg in $packages; do
dprj=$(factory $pkg)
if [ "$dprj" != "$p" ]; then
# not devel project
continue
fi
msg=$(message $org $pkg $dprj)
if [ -n "$msg" ]; then
echo $msg
osc meta attribute openSUSE:Factory $pkg --attribute OBS:RejectBranch --set "$msg" > /dev/null || exit 1
fi
done
done

127
devel-importer/devel_update.sh Executable file
View File

@@ -0,0 +1,127 @@
#!/usr/bin/bash
#
# Updates devel_packages in https://src.opensuse.org/openSUSE/Factory/src/branch/main/pkgs/_meta/devel_packages
#
# syntax:
# get <pkg>
# set <prj> <pkg>
# rm <prj> <pkg>
# sync
#
# DEVEL_PACKAGES env should point to the devel_packages clone from
# repo above, otherwise will look in CWD
#
set +e
function getdevel {
local pkg="$1"
awk "{ if ( \$1 == \"$pkg\" ) print \$2 }" "$DEVEL_PACKAGES"
}
function setdevel {
local prj="$1"
local pkg="$2"
if [ x"$prj" == "x" ] || [ x"$pkg" == "x" ]; then
echo "devel_update set <prj> <pkg>"
exit 10
fi
cat <(awk "{ if ( \$1 != \"$pkg\" ) print }" "$DEVEL_PACKAGES") <(echo $pkg $prj) | sort -d > "$DEVEL_PACKAGES".$$
mv "$DEVEL_PACKAGES".$$ "$DEVEL_PACKAGES"
}
function rmdevel {
local prj="$1"
local pkg="$2"
if [ x"$prj" == "x" ] || [ x"$pkg" == "x" ]; then
echo "devel_update rm <prj> <pkg>"
exit 10
fi
awk "{ if ( ! ( \$1 == \"$pkg\" && \$2 == \"$prj\" ) ) print }" "$DEVEL_PACKAGES" > "$DEVEL_PACKAGES".$$
mv "$DEVEL_PACKAGES".$$ "$DEVEL_PACKAGES"
}
if [ -z "$DEVEL_PACKAGES" ]; then
DEVEL_PACKAGES=./devel_packages
fi
if ! [ -w "$DEVEL_PACKAGES" ] || ! [ -e "$DEVEL_PACKAGES" ] ; then
echo "The DEVEL_PACKAGES ($DEVEL_PACKAGES) file is not writable or doesn't exist"
exit 0
fi
case "$1" in
get)
shift
getdevel "$@"
;;
set)
shift
setdevel "$@"
;;
rm)
shift
rmdevel "$@"
;;
sync)
warning=0
badpkgs=""
pkgs=$(osc ls openSUSE:Factory)
# add new packages
for pkg in $pkgs; do
if [ "${pkg/*:*/IGNORE}" == "IGNORE" ]; then
continue
fi
grep -q "^$pkg\( \|\$\)" "$DEVEL_PACKAGES"
if [ $? -ne 0 ]; then
echo -n "$pkg -> "
devel=$(osc develproject openSUSE:Factory $pkg 2> /dev/null)
devel=${devel/\/*/}
if [ -z "$devel" ]; then
devel=$(osc rq list -s accepted -P openSUSE:Factory -p $pkg -t submit | grep "^\s*submit:.* -> openSUSE:Factory\$" | sed -e "s,^\s*submit:\s*\([^/]\+\)/${pkg}@.*,\1," | uniq)
c=$(echo "$devel" | grep -c .)
if [ $c -ne 1 ]; then
badpkgs="$badpkgs $pkg"
warning=1
devel="***** UNKNOWN"
fi
fi
setdevel "$devel" "$pkg"
echo "$devel"
fi
done
# remove deleted packages
for pkg in $(awk '{ print $1 }' < "$DEVEL_PACKAGES"); do
if [[ " $pkgs " != *[[:space:]]"$pkg"[[:space:]]* ]]; then
echo "removing $pkg"
d=$(getdevel "$pkg")
if [ -n "$d" ]; then
rmdevel "$d" "$pkg"
fi
fi
done
# set devel change in last 10 days
osc rq list -t change_devel -D 10 -P openSUSE:Factory -s accepted |
grep 'change_devel:\s\+openSUSE:Factory/' |
sed -e 's,^\s*change_devel:\s*openSUSE:Factory/\([a-zA-Z0-9_+-]\+\)\s*developed in \([a-zA-Z0-9_+:-]\+\)/\1\s*$,\2 \1,' |
while read line; do
setdevel ${line/ */} ${line/* /};
done
if [ $warning -ne 0 ]; then
echo " **** WARNING ****" > /dev/stderr
echo "Could not fix some packages. Manual intervention required:$badpkgs" > /dev/stderr
fi
;;
*)
echo " devel_update (get,set,rm,sync) ...."
esac

View File

@@ -457,15 +457,16 @@ func importRepos(packages []string) {
// branchName := repo.DefaultBranch
remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg.Name, "remote", "show"), "\n")
if !slices.Contains(remotes, "devel") {
git.GitExecOrPanic(pkg.Name, "remote", "add", "devel", repo.SSHURL)
if !slices.Contains(remotes, "develorigin") {
git.GitExecOrPanic(pkg.Name, "remote", "add", "develorigin", repo.SSHURL)
// git.GitExecOrPanic(pkg.Name, "fetch", "devel")
}
if slices.Contains(remotes, "origin") {
git.GitExecOrPanic(pkg.Name, "lfs", "fetch", "--all")
git.GitExecOrPanic(pkg.Name, "lfs", "push", "devel", "--all")
git.GitExecOrPanic(pkg.Name, "lfs", "push", "develorigin", "--all")
}
git.GitExecOrPanic(pkg.Name, "push", "devel", "main", "-f")
git.GitExecOrPanic(pkg.Name, "push", "develorigin", "main", "-f")
git.GitExec(pkg.Name, "push", "develorigin", "--delete", "factory", "devel")
// git.GitExecOrPanic(pkg.Name, "checkout", "-B", "main", "devel/main")
_, err := client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(&models.EditRepoOption{
DefaultBranch: "main",
@@ -479,7 +480,8 @@ func importRepos(packages []string) {
AllowSquash: false,
AllowFastForwardOnly: true,
AllowRebaseUpdate: false,
AllowManualMerge: false,
AllowManualMerge: true,
AutodetectManualMerge: true,
AllowRebase: false,
DefaultAllowMaintainerEdit: true,
}), r.DefaultAuthentication)
@@ -517,7 +519,8 @@ func importRepos(packages []string) {
AllowSquash: false,
AllowFastForwardOnly: true,
AllowRebaseUpdate: false,
AllowManualMerge: false,
AllowManualMerge: true,
AutodetectManualMerge: true,
DefaultMergeStyle: "fast-forward-only",
AllowRebase: false,
DefaultAllowMaintainerEdit: true,
@@ -535,14 +538,15 @@ func importRepos(packages []string) {
}
remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg, "remote", "show"), "\n")
if !slices.Contains(remotes, "devel") {
git.GitExecOrPanic(pkg, "remote", "add", "devel", repo.SSHURL)
if !slices.Contains(remotes, "develorigin") {
git.GitExecOrPanic(pkg, "remote", "add", "develorigin", repo.SSHURL)
}
if slices.Contains(remotes, "origin") {
git.GitExecOrPanic(pkg, "lfs", "fetch", "--all")
git.GitExecOrPanic(pkg, "lfs", "push", "devel", "--all")
git.GitExecOrPanic(pkg, "lfs", "push", "develorigin", "--all")
}
git.GitExecOrPanic(pkg, "push", "devel", "main", "-f")
git.GitExecOrPanic(pkg, "push", "develorigin", "main", "-f")
git.GitExec(pkg, "push", "develorigin", "--delete", "factory", "devel")
_, err := client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(pkg).WithBody(&models.EditRepoOption{
DefaultBranch: "main",

View File

@@ -0,0 +1,15 @@
Kernel:firmware
Kernel:kdump
devel:languages:clojure
devel:languages:erlang
devel:languages:erlang:Factory
devel:languages:hare
devel:languages:javascript
devel:languages:lua
network:dhcp
network:im:whatsapp
network:messaging:xmpp
systemsmanagement:cockpit
systemsmanagement:wbem
X11:lxde

2
go.mod
View File

@@ -8,8 +8,8 @@ require (
github.com/go-openapi/strfmt v0.23.0
github.com/go-openapi/swag v0.23.0
github.com/go-openapi/validate v0.24.0
github.com/mattn/go-sqlite3 v1.14.22
github.com/rabbitmq/amqp091-go v1.10.0
github.com/tailscale/hujson v0.0.0-20250226034555-ec1d1c113d33
go.uber.org/mock v0.5.0
)

4
go.sum
View File

@@ -40,8 +40,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
@@ -58,6 +56,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tailscale/hujson v0.0.0-20250226034555-ec1d1c113d33 h1:idh63uw+gsG05HwjZsAENCG4KZfyvjK03bpjxa5qRRk=
github.com/tailscale/hujson v0.0.0-20250226034555-ec1d1c113d33/go.mod h1:EbW0wDK/qEUYI0A5bqq0C2kF8JTQwWONmGDBbzsxxHo=
go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=
go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=

View File

@@ -7,6 +7,7 @@ import (
"regexp"
"slices"
"strconv"
"strings"
"time"
"src.opensuse.org/autogits/common"
@@ -14,21 +15,43 @@ import (
)
var configs common.AutogitConfigs
var reviewRx *regexp.Regexp
var acceptRx *regexp.Regexp
var rejectRx *regexp.Regexp
var groupName string
func InitRegex(groupName string) {
reviewRx = regexp.MustCompile("^" + groupName + "\\s*:\\s*LGTM")
rejectRx = regexp.MustCompile("^" + groupName + "\\s*:")
acceptRx = regexp.MustCompile("\\s*:\\s*LGTM")
rejectRx = regexp.MustCompile("\\s*:\\s*")
}
func ParseReviewLine(reviewText string) (bool, string) {
line := strings.TrimSpace(reviewText)
glen := len(groupName)
if len(line) < glen || line[0:glen] != groupName {
return false, line
}
return true, line[glen:]
}
func ReviewAccepted(reviewText string) bool {
return reviewRx.MatchString(reviewText)
for _, line := range common.SplitStringNoEmpty(reviewText, "\n") {
if matched, reviewLine := ParseReviewLine(line); matched {
return acceptRx.MatchString(reviewLine)
}
}
return false
}
func ReviewRejected(reviewText string) bool {
return rejectRx.MatchString(reviewText)
for _, line := range common.SplitStringNoEmpty(reviewText, "\n") {
if matched, reviewLine := ParseReviewLine(line); matched {
if rejectRx.MatchString(reviewLine) {
return !acceptRx.MatchString(reviewLine)
}
}
}
return false
}
func ProcessNotifications(notification *models.NotificationThread, gitea common.Gitea) {
@@ -38,9 +61,13 @@ func ProcessNotifications(notification *models.NotificationThread, gitea common.
}
}()
rx := regexp.MustCompile(`^https://src\.(?:open)?suse\.(?:org|de)/api/v\d+/repos/(?<org>[a-zA-Z0-9]+)/(?<project>[_a-zA-Z0-9]+)/issues/(?<num>[0-9]+)$`)
rx := regexp.MustCompile(`^/?api/v\d+/repos/(?<org>[_a-zA-Z0-9-]+)/(?<project>[_a-zA-Z0-9-]+)/issues/(?<num>[0-9]+)$`)
subject := notification.Subject
match := rx.FindStringSubmatch(subject.URL)
u, err := url.Parse(notification.Subject.URL)
if err != nil {
log.Panicln("Invalid format of notification: %s", subject.URL)
}
match := rx.FindStringSubmatch(u.Path)
if match == nil {
log.Panicf("** Unexpected format of notification: %s", subject.URL)
}
@@ -180,22 +207,20 @@ func main() {
return
}
_, err = url.Parse("amqps://" + *rabbitMqHost)
u, err := url.Parse("amqps://" + *rabbitMqHost)
if err != nil {
log.Panicln("Cannot parse RabbitMQ host:", err)
}
/*
common.ListenDefinitions{
RabbitURL: u,
GitAuthor: groupName,
Orgs: []string{"#"},
Handlers: map[string]common.RequestProcessor{
common.RequestType_PRReviewRequest: ReviewRequest,
common.RequestType_PRReviewRejected: ProcessReview,
common.RequestType_PRReviewAccepted: ProcessReview,
},
}
*/
configUpdates := &common.ListenDefinitions {
RabbitURL: u,
Orgs: []string{},
Handlers: map[string]common.RequestProcessor {
common.RequestType_Push: &ConfigUpdatePush{},
},
}
go configUpdates.ProcessRabbitMQEvents()
for {
PeriodReviewCheck(gitea)
time.Sleep(time.Duration(*interval * int64(time.Minute)))

View File

@@ -1 +1,50 @@
package main
import (
"fmt"
"slices"
"src.opensuse.org/autogits/common"
)
type ConfigUpdatePush struct{
}
func (s *ConfigUpdatePush) ProcessFunc(req *common.Request) error {
if req.Type != common.RequestType_Push {
return fmt.Errorf("Unhandled, ignored request type: %s", req.Type)
}
data := req.Data.(*common.PushWebhookEvent)
org := data.Repository.Owner.Username
repo := data.Repository.Name
const branch_ref = "refs/heads/"
if data.Ref[:len(branch_ref)] != branch_ref {
return fmt.Errorf("No branch updated. Ref: %s", data.Ref)
}
branch := data.Ref[len(branch_ref):]
c := configs.GetPrjGitConfig(org, repo, branch)
if c == nil {
return nil
}
if o, p, b := c.GetPrjGit(); o != org || p != repo || b != branch {
return nil
}
modified_config := false
for _, commit := range data.Commits {
modified_config = modified_config ||
slices.Contains(commit.Modified, common.ProjectConfigFile) ||
slices.Contains(commit.Added, common.ProjectConfigFile) ||
slices.Contains(commit.Removed, common.ProjectConfigFile)
}
s.config_modified <-
return nil
}

File diff suppressed because it is too large Load Diff

View File

@@ -5,6 +5,7 @@ import (
"strings"
"testing"
"src.opensuse.org/autogits/common"
"src.opensuse.org/autogits/common/gitea-generated/models"
)
@@ -47,25 +48,32 @@ func TestObsAPIHostFromWebHost(t *testing.T) {
func TestPRtoObsProjectMapping(t *testing.T) {
tests := []struct {
name string
pr string // org/repo/prNo
config common.StagingConfig
name string
pr string // org/repo/prNo
expectedProject string
}{
{
name: "Regular project",
pr: "foobar/Repo/10",
expectedProject: "home:foo:foobar:Repo:PullRequest:10",
expectedProject: "home:foo:foobar:Repo:PR:10",
},
{
name: "underscore repo name",
pr: "foobar/_FooBar/10",
expectedProject: "home:foo:foobar:XFooBar:PullRequest:10",
expectedProject: "home:foo:foobar:XFooBar:PR:10",
},
{
name: "Underscore repo and project",
pr: "_some_thing/_FooBar/11",
expectedProject: "home:foo:Xsome_thing:XFooBar:PullRequest:11",
expectedProject: "home:foo:Xsome_thing:XFooBar:PR:11",
},
{
config: common.StagingConfig{StagingProject: "staging:project:Pull_Request"},
name: "with staging set",
pr: "_some_thing/_PrjX/14",
expectedProject: "staging:project:Pull_Request:14",
},
}
@@ -86,7 +94,7 @@ func TestPRtoObsProjectMapping(t *testing.T) {
Index: n,
}
p := getObsProjectAssociatedWithPr("home:foo", &pr)
p := GetObsProjectAssociatedWithPr(&test.config, "home:foo", &pr)
if p != test.expectedProject {
t.Error("invalid project:", p, "Expected:", test.expectedProject)
}

View File

@@ -1,188 +0,0 @@
package main
/*
* This file is part of Autogits.
*
* Copyright © 2024 SUSE LLC
*
* Autogits is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 2 of the License, or (at your option) any later
* version.
*
* Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Foobar. If not, see <https://www.gnu.org/licenses/>.
*/
import (
"crypto/tls"
"encoding/json"
"log"
"strings"
"sync"
"time"
rabbitmq "github.com/rabbitmq/amqp091-go"
"src.opensuse.org/autogits/common"
"src.opensuse.org/autogits/common/gitea-generated/models"
)
type BuildNotification struct {
BuildSuccess bool
Project, Package, Repository, Arch, Release, Rev, Buildtype, Workerid string
Starttime, Endtime, Readytime string
}
var obsNotifications map[string]*BuildNotification // Project/Package/Repositry/Arch as key
var notificationMutex sync.RWMutex
var notificationChannels map[string][]chan *BuildNotification
func getProjectBuildStatus(project string) []*BuildNotification {
notificationMutex.RLock()
defer notificationMutex.RUnlock()
data := make([]*BuildNotification, 0, 4)
for _, val := range obsNotifications {
if val.Package == project {
data = append(data, val)
}
}
return data
}
func addProjectWatcher(meta *common.ProjectMeta, pr *models.PullReview) {
}
func addObsNotificationToCache(notification *BuildNotification) {
key := strings.Join(
[]string{
notification.Project,
notification.Package,
notification.Repository,
notification.Arch,
},
"/",
)
notificationMutex.Lock()
obsNotifications[key] = notification
chns, ok := notificationChannels[notification.Project]
notificationMutex.Unlock()
if ok {
for _, ch := range chns {
ch <- notification
}
}
}
func processObsMessage(msg *rabbitmq.Delivery) {
key := strings.SplitN(msg.RoutingKey, ".", 4)
if len(key) != 4 || len(key[3]) < 7 || key[3][:6] != "build_" {
return
}
buildSuccess := false
switch key[3][6:] {
case "success", "unchanged":
buildSuccess = true
case "fail":
buildSuccess = false
default:
log.Printf("unknown build_ logging message: %s\n", msg.RoutingKey)
return
}
notification := &BuildNotification{
BuildSuccess: buildSuccess,
}
err := json.Unmarshal(msg.Body, notification)
if err != nil {
log.Printf("Cannot unmarshall json object: %s\n", msg.Body)
return
}
log.Printf("%v\n", notification)
addObsNotificationToCache(notification)
}
func ProcessingObsMessages(host, username, password, queueName string) {
if obsNotifications == nil {
obsNotifications = make(map[string]*BuildNotification)
// notificationChannels = make(map[string]chan *BuildNotification)
}
auth := ""
if len(username) > 0 && len(password) > 0 {
auth = username + ":" + password + "@"
}
connection, err := rabbitmq.DialTLS("amqps://"+auth+host, &tls.Config{
ServerName: host,
})
failOnError(err, "Cannot connect to rabbit.opensuse.org")
defer connection.Close()
ch, err := connection.Channel()
failOnError(err, "Cannot create a channel")
defer ch.Close()
err = ch.ExchangeDeclarePassive("pubsub", "topic", true, false, false, false, nil)
failOnError(err, "Cannot find pubsub exchange")
var q rabbitmq.Queue
if len(queueName) == 0 {
q, err = ch.QueueDeclare("", false, true, true, false, nil)
} else {
q, err = ch.QueueDeclarePassive(queueName, true, false, true, false, nil)
if err != nil {
log.Printf("queue not found .. trying to create it: %v\n", err)
if ch.IsClosed() {
ch, err = connection.Channel()
failOnError(err, "Channel cannot be re-opened")
}
q, err = ch.QueueDeclare(queueName, true, false, true, false, nil)
if err != nil {
log.Printf("can't create persistent queue ... falling back to temporaty queue: %v\n", err)
if ch.IsClosed() {
ch, err = connection.Channel()
failOnError(err, "Channel cannot be re-opened")
}
q, err = ch.QueueDeclare("", false, true, true, false, nil)
}
}
}
failOnError(err, "Cannot declare queue")
log.Printf("queue: %s:%d", q.Name, q.Consumers)
err = ch.QueueBind(q.Name, "*.obs.package.*", "pubsub", false, nil)
failOnError(err, "Cannot bind queue to exchange")
msgs, err := ch.Consume(q.Name, "", true, true, false, false, nil)
failOnError(err, "Cannot start consumer")
log.Printf("queue: %s:%d", q.Name, q.Consumers)
for {
msg, ok := <-msgs
if !ok {
log.Printf("channel/connection closed?\n")
if connection.IsClosed() {
// reconnect
log.Printf("reconnecting...")
time.Sleep(5 * time.Second)
go ProcessingObsMessages(host, username, password, queueName)
}
return
}
processObsMessage(&msg)
}
}

View File

@@ -3,6 +3,11 @@ OBS Status Service
Reports build status of OBS service as an easily to produce SVG
Requests for individual build results:
/obs:project/package/repo/arch
Requests for project results
/obs:project
Areas of Responsibility
-----------------------

View File

@@ -19,100 +19,152 @@ package main
*/
import (
"bytes"
"flag"
"fmt"
"log"
"net/http"
"os"
"time"
"src.opensuse.org/autogits/common"
)
const (
ListenAddr = "[::1]:8003"
AppName = "obs-status-service"
AppName = "obs-status-service"
)
type BuildStatusCacheItem struct {
CacheTime time.Time
Result []*common.BuildResult
}
var obs *common.ObsClient
var buildStatusCache map[string]BuildStatusCacheItem
var debug bool
/*
func CacheBuildStatus(prj, pkg string) ([]common.BuildResult, error) {
list, err := obs.BuildStatus(prj, pkg)
if err != nil {
return nil, err
}
return
func LogDebug(v ...any) {
if debug {
log.Println(v...)
}
*/
func PackageBuildStatus(prj, pkg string) (common.ObsBuildStatusDetail, error) {
return common.ObsBuildStatusDetail{
Code: "succeeded",
Description: "stuff",
Success: true,
Finished: true,
}, nil
}
/*
func PackageStatusSvg(buildStatus []common.ObsBuildStatusDetail) []byte {
return
func ProjectStatusSummarySvg(project string) []byte {
res := GetCurrentStatus(project)
if res == nil {
return nil
}
*/
func PackageStatusSummarySvg(buildStatus common.ObsBuildStatusDetail) []byte {
fillColor := "orange"
textColor := "grey"
pkgs := res.GetPackageList()
maxLen := 0
for _, p := range pkgs {
maxLen = max(maxLen, len(p))
}
width := float32(len(res.Result))*1.5 + float32(maxLen)*0.8
height := 1.5*float32(maxLen) + 30
ret := bytes.Buffer{}
ret.WriteString(`<svg version="2.0" width="`)
ret.WriteString(fmt.Sprint(width))
ret.WriteString(`em" height="`)
ret.WriteString(fmt.Sprint(height))
ret.WriteString(`em" xmlns="http://www.w3.org/2000/svg">`)
ret.WriteString(`<defs>
<g id="f"> <!-- failed -->
<rect width="1em" height="1em" fill="#800" />
</g>
<g id="s"> <!--succeeded-->
<rect width="1em" height="1em" fill="#080" />
</g>
<g id="buidling"> <!--building-->
<rect width="1em" height="1em" fill="#880" />
</g>
</defs>`)
ret.WriteString(`<use href="#f" x="1em" y="2em"/>`)
ret.WriteString(`</svg>`)
return ret.Bytes()
}
func PackageStatusSummarySvg(status common.PackageBuildStatus) []byte {
buildStatus, ok := common.ObsBuildStatusDetails[status.Code]
if !ok {
buildStatus = common.ObsBuildStatusDetails["error"]
}
fillColor := "#480" // orange
textColor := "#888"
if buildStatus.Finished {
textColor = "black"
textColor = "#fff"
if buildStatus.Success {
fillColor = "green"
fillColor = "#080"
} else {
fillColor = "red"
fillColor = "#800"
}
}
return []byte(`
<svg version="1.1" width="200" height="20" xmlns="http://www.w3.org/2000/svg">
<rect width="100%" height="100%" fill="` + fillColor + `" />
<text x="20" y="25" font-size="60" text-anchor="middle" fill="` + textColor + `">` + buildStatus.Code + `</text>
log.Println(status, " -> ", buildStatus)
return []byte(`<svg version="2.0" width="8em" height="1.5em" xmlns="http://www.w3.org/2000/svg">
<rect width="100%" height="100%" fill="` + fillColor + `"/>
<text x="4em" y="1.1em" text-anchor="middle" fill="` + textColor + `">` + buildStatus.Code + `</text>
</svg>`)
}
func main() {
common.RequireObsSecretToken()
go ProcessingObsMessages("rabbit.opensuse.org", "opensuse", "opensuse", "pubsub")
cert := flag.String("cert-file", "", "TLS certificates file")
key := flag.String("key-file", "", "Private key for the TLS certificate")
listen := flag.String("listen", "[::1]:8080", "Listening string")
disableTls := flag.Bool("no-tls", false, "Disable TLS")
obsHost := flag.String("obs-host", "api.opensuse.org", "OBS API endpoint for package status information")
flag.BoolVar(&debug, "debug", false, "Enable debug logging")
flag.Parse()
obsHost := os.Getenv("OBS_HOSTNAME")
if len(obsHost) == 0 {
log.Fatal("OBS_HOSTNAME env required.")
common.PanicOnError(common.RequireObsSecretToken())
var err error
if obs, err = common.NewObsClient(*obsHost); err != nil {
log.Fatal(err)
}
/*
if obs, err := common.NewObsClient(obsHost); err != nil {
log.Fatal(err)
}
*/
http.HandleFunc("GET /{ObsProject}", func(res http.ResponseWriter, req *http.Request) {
http.HandleFunc("GET /{Project}", func(res http.ResponseWriter, req *http.Request) {
res.WriteHeader(http.StatusBadRequest)
})
http.HandleFunc("GET /{ObsProject}/{Package}", func(res http.ResponseWriter, req *http.Request) {
obsPrj := req.PathValue("ObsProject")
obsPkg := req.PathValue("ObsPackage")
http.HandleFunc("GET /{Project}/{Package}", func(res http.ResponseWriter, req *http.Request) {
/*
obsPrj := req.PathValue("Project")
obsPkg := req.PathValue("Package")
status, _ := PackageBuildStatus(obsPrj, obsPkg)
svg := PackageStatusSummarySvg(status)
status, _ := PackageBuildStatus(obsPrj, obsPkg)
svg := PackageStatusSummarySvg(status)
*/
res.Header().Add("content-type", "image/svg+xml")
res.Header().Add("size", fmt.Sprint(len(svg)))
res.Write(svg)
//res.Header().Add("size", fmt.Sprint(len(svg)))
//res.Write(svg)
})
http.HandleFunc("GET /{Project}/{Package}/{Repository}/{Arch}", func(res http.ResponseWriter, req *http.Request) {
prj := req.PathValue("Project")
pkg := req.PathValue("Package")
repo := req.PathValue("Repository")
arch := req.PathValue("Arch")
res.Header().Add("content-type", "image/svg+xml")
prjStatus := GetCurrentStatus(prj)
if prjStatus == nil {
return
}
for _, r := range prjStatus.Result {
if r.Arch == arch && r.Repository == repo {
for _, status := range r.Status {
if status.Package == pkg {
res.Write(PackageStatusSummarySvg(status))
return
}
}
}
}
})
log.Fatal(http.ListenAndServe(ListenAddr, nil))
go ProcessUpdates()
if *disableTls {
log.Fatal(http.ListenAndServe(*listen, nil))
} else {
log.Fatal(http.ListenAndServeTLS(*listen, *cert, *key, nil))
}
}

View File

@@ -1,219 +0,0 @@
package main
/*
* This file is part of Autogits.
*
* Copyright © 2024 SUSE LLC
*
* Autogits is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 2 of the License, or (at your option) any later
* version.
*
* Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Foobar. If not, see <https://www.gnu.org/licenses/>.
*/
import (
"crypto/tls"
"encoding/json"
"log"
"os"
"strings"
"sync"
"time"
_ "github.com/mattn/go-sqlite3"
rabbitmq "github.com/rabbitmq/amqp091-go"
"src.opensuse.org/autogits/common"
"src.opensuse.org/autogits/common/gitea-generated/models"
)
type BuildNotification struct {
BuildSuccess bool
Project, Package, Repository, Arch, Release, Rev, Buildtype, Workerid string
Starttime, Endtime, Readytime string
}
var out *os.File
var obsNotifications map[string]*BuildNotification // Project/Package/Repositry/Arch as key
var notificationMutex sync.RWMutex
var notificationChannels map[string][]chan *BuildNotification
func getProjectBuildStatus(project string) []*BuildNotification {
notificationMutex.RLock()
defer notificationMutex.RUnlock()
data := make([]*BuildNotification, 0, 4)
for _, val := range obsNotifications {
if val.Package == project {
data = append(data, val)
}
}
return data
}
func addProjectWatcher(meta *common.ProjectMeta, pr *models.PullReview) {
}
func addObsNotificationToCache(notification *BuildNotification) {
key := strings.Join(
[]string{
notification.Project,
notification.Package,
notification.Repository,
notification.Arch,
},
"/",
)
notificationMutex.Lock()
obsNotifications[key] = notification
chns, ok := notificationChannels[notification.Project]
notificationMutex.Unlock()
if ok {
for _, ch := range chns {
ch <- notification
}
}
}
func processObsMessage(msg *rabbitmq.Delivery) {
out.Write([]byte(msg.Timestamp.String()))
out.Write([]byte("\n"))
out.Write([]byte(msg.RoutingKey))
out.Write([]byte("\n"))
out.Write(msg.Body)
out.Write([]byte("\n--------------------------\n"))
return
key := strings.SplitN(msg.RoutingKey, ".", 4)
if len(key) != 4 || len(key[3]) < 7 || key[3][:6] != "build_" {
return
}
buildSuccess := false
switch key[3][6:] {
case "success", "unchanged":
buildSuccess = true
case "fail":
buildSuccess = false
default:
log.Printf("unknown build_ logging message: %s\n", msg.RoutingKey)
return
}
notification := &BuildNotification{
BuildSuccess: buildSuccess,
}
err := json.Unmarshal(msg.Body, notification)
if err != nil {
log.Printf("Cannot unmarshall json object: %s\n", msg.Body)
return
}
log.Printf("%v\n", notification)
addObsNotificationToCache(notification)
}
func failOnError(err error, msg string) {
if err != nil {
log.Panicf("%s: %s", err, msg)
}
}
func ProcessingObsMessages(host, username, password, queueName string) {
defer func() {
if r := recover(); r != nil {
log.Print("recovering... reconnecting...\n")
time.Sleep(5 * time.Second)
go ProcessingObsMessages(host, username, password, queueName)
}
}()
var err error
out, err = os.OpenFile("messages.txt", os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
log.Printf("Cannot open message.txt: %v", err)
return
}
if obsNotifications == nil {
obsNotifications = make(map[string]*BuildNotification)
// notificationChannels = make(map[string]chan *BuildNotification)
}
auth := ""
if len(username) > 0 && len(password) > 0 {
auth = username + ":" + password + "@"
}
connection, err := rabbitmq.DialTLS("amqps://"+auth+host, &tls.Config{
ServerName: host,
})
failOnError(err, "Cannot connect to rabbit.opensuse.org")
defer connection.Close()
ch, err := connection.Channel()
failOnError(err, "Cannot create a channel")
defer ch.Close()
err = ch.ExchangeDeclarePassive("pubsub", "topic", true, false, false, false, nil)
failOnError(err, "Cannot find pubsub exchange")
var q rabbitmq.Queue
if len(queueName) == 0 {
q, err = ch.QueueDeclare("", false, true, true, false, nil)
} else {
q, err = ch.QueueDeclarePassive(queueName, true, false, true, false, nil)
if err != nil {
log.Printf("queue not found .. trying to create it: %v\n", err)
if ch.IsClosed() {
ch, err = connection.Channel()
failOnError(err, "Channel cannot be re-opened")
}
q, err = ch.QueueDeclare(queueName, true, false, true, false, nil)
if err != nil {
log.Printf("can't create persistent queue ... falling back to temporaty queue: %v\n", err)
if ch.IsClosed() {
ch, err = connection.Channel()
failOnError(err, "Channel cannot be re-opened")
}
q, err = ch.QueueDeclare("", false, true, true, false, nil)
}
}
}
failOnError(err, "Cannot declare queue")
log.Printf("queue: %s:%d", q.Name, q.Consumers)
err = ch.QueueBind(q.Name, "*.obs.*.*", "pubsub", false, nil)
failOnError(err, "Cannot bind queue to exchange")
msgs, err := ch.Consume(q.Name, "", true, true, false, false, nil)
failOnError(err, "Cannot start consumer")
log.Printf("queue: %s:%d", q.Name, q.Consumers)
for {
msg, ok := <-msgs
if !ok {
log.Printf("channel/connection closed?\n")
if connection.IsClosed() {
// reconnect
log.Printf("reconnecting...")
time.Sleep(5 * time.Second)
go ProcessingObsMessages(host, username, password, queueName)
}
return
}
processObsMessage(&msg)
}
}

View File

@@ -0,0 +1,82 @@
package main
import (
"log"
"slices"
"sync"
"time"
"src.opensuse.org/autogits/common"
)
var WatchedRepos []string
var mutex sync.Mutex
var StatusUpdateCh chan StatusUpdateMsg = make(chan StatusUpdateMsg)
var statusMutex sync.RWMutex
var CurrentStatus map[string]*common.BuildResultList = make(map[string]*common.BuildResultList)
type StatusUpdateMsg struct {
ObsProject string
Result *common.BuildResultList
}
func GetCurrentStatus(project string) *common.BuildResultList {
statusMutex.RLock()
defer statusMutex.RUnlock()
if ret, found := CurrentStatus[project]; found {
return ret
} else {
go WatchObsProject(obs, project)
return nil
}
}
func ProcessUpdates() {
for {
msg := <-StatusUpdateCh
statusMutex.Lock()
CurrentStatus[msg.ObsProject] = msg.Result
drainedChannel:
for {
select {
case msg = <-StatusUpdateCh:
CurrentStatus[msg.ObsProject] = msg.Result
default:
statusMutex.Unlock()
break drainedChannel
}
}
}
}
func WatchObsProject(obs common.ObsStatusFetcherWithState, ObsProject string) {
old_state := ""
mutex.Lock()
if pos, found := slices.BinarySearch(WatchedRepos, ObsProject); found {
mutex.Unlock()
return
} else {
WatchedRepos = slices.Insert(WatchedRepos, pos, ObsProject)
mutex.Unlock()
}
LogDebug("+ watching", ObsProject)
opts := common.BuildResultOptions{}
for {
state, err := obs.BuildStatusWithState(ObsProject, &opts)
if err != nil {
log.Println(" *** Error fetching build for", ObsProject, err)
time.Sleep(time.Minute)
} else {
opts.OldState = state.State
LogDebug(" --> update", ObsProject, " => ", old_state)
StatusUpdateCh <- StatusUpdateMsg{ObsProject: ObsProject, Result: state}
}
}
}

View File

@@ -0,0 +1,34 @@
package main
import (
"testing"
"go.uber.org/mock/gomock"
"src.opensuse.org/autogits/common"
mock_common "src.opensuse.org/autogits/common/mock"
)
func TestWatchObsProject(t *testing.T) {
tests := []struct {
name string
res common.BuildResultList
}{
{
name: "two requests",
res: common.BuildResultList{
State: "success",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctl := gomock.NewController(t)
obs := mock_common.NewMockObsStatusFetcherWithState(ctl)
obs.EXPECT().BuildStatusWithState("test:foo", "").Return(&test.res, nil)
WatchObsProject(obs, "test:foo")
})
}
}

View File

@@ -19,7 +19,6 @@ package main
*/
import (
"errors"
"flag"
"fmt"
"io/fs"
@@ -78,7 +77,7 @@ func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error {
}
for _, config := range configs {
if config.GitProjectName == action.Repository.Name {
if org, repo, _ := config.GetPrjGit(); org == action.Repository.Owner.Username && repo == action.Repository.Name {
log.Println("+ ignoring repo event for PrjGit repository", config.GitProjectName)
return nil
}
@@ -93,53 +92,55 @@ func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error {
}
func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error {
prjgit := config.GitProjectName
ghi := common.GitHandlerGeneratorImpl{}
git, err := ghi.CreateGitHandler(GitAuthor, GitEmail, AppName)
gitOrg, gitPrj, gitBranch := config.GetPrjGit()
git, err := gh.CreateGitHandler(config.Organization)
common.PanicOnError(err)
if !DebugMode {
defer git.Close()
}
defer git.Close()
if len(config.Branch) == 0 {
config.Branch = action.Repository.Default_Branch
}
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, action.Organization.Username, prjgit)
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
if err != nil {
return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
return fmt.Errorf("Error accessing/creating prjgit: %s/%s#%d err: %w", gitOrg, gitPrj, gitBranch, err)
}
if _, err := fs.Stat(os.DirFS(git.GetPath()), config.GitProjectName); errors.Is(err, os.ErrNotExist) {
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, prjgit))
}
remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL)
common.PanicOnError(err)
switch action.Action {
case "created":
if action.Repository.Object_Format_Name != "sha256" {
return fmt.Errorf(" - '%s' repo is not sha256. Ignoring.", action.Repository.Name)
}
common.PanicOnError(git.GitExec(prjgit, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(prjgit, action.Repository.Name), "branch", "--show-current"))
common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all")
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current"))
if branch != config.Branch {
if err := git.GitExec(path.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
if err := git.GitExec(path.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
}
common.PanicOnError(git.GitExec(path.Join(prjgit, action.Repository.Name), "checkout", config.Branch))
common.PanicOnError(git.GitExec(path.Join(gitPrj, action.Repository.Name), "checkout", config.Branch))
}
common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package inclusion via Direct Workflow"))
if !noop {
common.PanicOnError(git.GitExec(gitPrj, "push"))
}
common.PanicOnError(git.GitExec(prjgit, "commit", "-m", "Automatic package inclusion via Direct Workflow"))
common.PanicOnError(git.GitExec(prjgit, "push"))
case "deleted":
if stat, err := os.Stat(filepath.Join(git.GetPath(), prjgit, action.Repository.Name)); err != nil || !stat.IsDir() {
if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
if DebugMode {
log.Println("delete event for", action.Repository.Name, "-- not in project. Ignoring")
}
return nil
}
common.PanicOnError(git.GitExec(prjgit, "rm", action.Repository.Name))
common.PanicOnError(git.GitExec(prjgit, "commit", "-m", "Automatic package removal via Direct Workflow"))
common.PanicOnError(git.GitExec(prjgit, "push"))
common.PanicOnError(git.GitExec(gitPrj, "rm", action.Repository.Name))
common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package removal via Direct Workflow"))
if !noop {
git.GitExecOrPanic(gitPrj, "push", remoteName)
}
default:
return fmt.Errorf("%s: %s", "Unknown action type", action.Action)
@@ -160,7 +161,7 @@ func (*PushActionProcessor) ProcessFunc(request *common.Request) error {
}
for _, config := range configs {
if config.GitProjectName == action.Repository.Name {
if gitOrg, gitPrj, _ := config.GetPrjGit(); gitOrg == action.Repository.Owner.Username && gitPrj == action.Repository.Name {
log.Println("+ ignoring push to PrjGit repository", config.GitProjectName)
return nil
}
@@ -175,47 +176,53 @@ func (*PushActionProcessor) ProcessFunc(request *common.Request) error {
}
func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) error {
prjgit := config.GitProjectName
ghi := common.GitHandlerGeneratorImpl{}
git, err := ghi.CreateGitHandler(GitAuthor, GitEmail, AppName)
gitOrg, gitPrj, gitBranch := config.GetPrjGit()
git, err := gh.CreateGitHandler(config.Organization)
common.PanicOnError(err)
if !DebugMode {
defer git.Close()
}
defer git.Close()
log.Printf("push to: %s/%s for %s/%s#%s", action.Repository.Owner.Username, action.Repository.Name, gitOrg, gitPrj, gitBranch)
if len(config.Branch) == 0 {
config.Branch = action.Repository.Default_Branch
log.Println(" + default branch", action.Repository.Default_Branch)
}
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, action.Repository.Owner.Username, prjgit)
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
if err != nil {
return fmt.Errorf("Error accessing/creating prjgit: %s err: %w", prjgit, err)
return fmt.Errorf("Error accessing/creating prjgit: %s/%s err: %w", gitOrg, gitPrj, err)
}
if _, err := fs.Stat(os.DirFS(git.GetPath()), config.GitProjectName); errors.Is(err, os.ErrNotExist) {
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGitRepo.SSHURL, prjgit))
remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL)
common.PanicOnError(err)
headCommitId, err := git.GitRemoteHead(gitPrj, remoteName, gitBranch)
common.PanicOnError(err)
commit, ok := git.GitSubmoduleCommitId(gitPrj, action.Repository.Name, headCommitId)
for ok && action.Head_Commit.Id == commit {
log.Println(" -- nothing to do, commit already in ProjectGit")
return nil
}
if stat, err := os.Stat(filepath.Join(git.GetPath(), prjgit, action.Repository.Name)); err != nil || !stat.IsDir() {
if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
if DebugMode {
log.Println("Pushed to package that is not part of the project. Ignoring:", err)
}
return nil
}
common.PanicOnError(git.GitExec(prjgit, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name))
if err := git.GitExec(filepath.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name)
defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all")
if err := git.GitExec(filepath.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "--force", remoteName, config.Branch+":"+config.Branch); err != nil {
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
}
id, err := git.GitBranchHead(filepath.Join(prjgit, action.Repository.Name), config.Branch)
id, err := git.GitRemoteHead(filepath.Join(gitPrj, action.Repository.Name), remoteName, config.Branch)
common.PanicOnError(err)
for _, commitId := range action.Commits {
if commitId.Id == id {
common.PanicOnError(git.GitExec(filepath.Join(prjgit, action.Repository.Name), "fetch", "--depth", "1", "origin", id))
common.PanicOnError(git.GitExec(filepath.Join(prjgit, action.Repository.Name), "checkout", id))
common.PanicOnError(git.GitExec(prjgit, "commit", "-a", "-m", "Automatic update via push via Direct Workflow"))
common.PanicOnError(git.GitExec(prjgit, "push"))
return nil
if action.Head_Commit.Id == id {
git.GitExecOrPanic(filepath.Join(gitPrj, action.Repository.Name), "checkout", id)
git.GitExecOrPanic(gitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow")
if !noop {
git.GitExecOrPanic(gitPrj, "push", remoteName)
}
return nil
}
log.Println("push of refs not on the configured branch", config.Branch, ". ignoring.")
@@ -233,22 +240,24 @@ func verifyProjectState(git common.Git, org string, config *common.AutogitConfig
}
}()
repo, err := gitea.CreateRepositoryIfNotExist(git, org, config.GitProjectName)
gitOrg, gitPrj, gitBranch := config.GetPrjGit()
repo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
if err != nil {
return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", org, config.GitProjectName, err)
return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", gitOrg, gitPrj, err)
}
if _, err := fs.Stat(os.DirFS(git.GetPath()), config.GitProjectName); errors.Is(err, os.ErrNotExist) {
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", repo.SSHURL, config.GitProjectName))
}
remoteName, err := git.GitClone(gitPrj, gitBranch, repo.SSHURL)
common.PanicOnError(err)
defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all")
log.Println(" * Getting submodule list")
sub, err := git.GitSubmoduleList(config.GitProjectName, "HEAD")
sub, err := git.GitSubmoduleList(gitPrj, "HEAD")
common.PanicOnError(err)
log.Println(" * Getting package links")
var pkgLinks []*PackageRebaseLink
if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), config.GitProjectName)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 {
if data, err := os.ReadFile(path.Join(git.GetPath(), config.GitProjectName, common.PrjLinksFile)); err == nil {
if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), gitPrj)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 {
if data, err := os.ReadFile(path.Join(git.GetPath(), gitPrj, common.PrjLinksFile)); err == nil {
pkgLinks, err = parseProjectLinks(data)
if err != nil {
log.Println("Cannot parse project links file:", err.Error())
@@ -267,27 +276,25 @@ func verifyProjectState(git common.Git, org string, config *common.AutogitConfig
next_package:
for filename, commitId := range sub {
// ignore project gits
for _, c := range configs {
if c.GitProjectName == filename {
log.Println(" prjgit as package? ignoring project git:", filename)
continue next_package
//for _, c := range configs {
if gitPrj == filename {
log.Println(" prjgit as package? ignoring project git:", filename)
continue next_package
}
//}
log.Printf(" verifying package: %s -> %s(%s)", commitId, filename, config.Branch)
commits, err := gitea.GetRecentCommits(org, filename, config.Branch, 10)
if len(commits) == 0 {
if repo, err := gitea.GetRepository(org, filename); repo == nil && err == nil {
git.GitExecOrPanic(gitPrj, "rm", filename)
isGitUpdated = true
}
}
log.Println(" verifying package:", filename, commitId, config.Branch)
commits, err := gitea.GetRecentCommits(org, filename, config.Branch, 10)
if err != nil {
// assumption that package does not exist, remove from project
// https://github.com/go-gitea/gitea/issues/31976
if err := git.GitExec(config.GitProjectName, "rm", filename); err != nil {
return fmt.Errorf("Failed to remove deleted submodule. Err: %w", err)
}
isGitUpdated = true
log.Println(" -> failed to fetch recent commits for package:", filename, " Err:", err)
continue
}
// if err != nil {
// return fmt.Errorf("Failed to fetch recent commits for package: '%s'. Err: %w", filename, err)
// }
idx := 1000
for i, c := range commits {
@@ -304,8 +311,8 @@ next_package:
log.Println(" -> linked package")
// so, we need to rebase here. Can't really optimize, so clone entire package tree and remote
pkgPath := path.Join(config.GitProjectName, filename)
git.GitExecOrPanic(config.GitProjectName, "submodule", "update", "--init", "--checkout", filename)
pkgPath := path.Join(gitPrj, filename)
git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--checkout", filename)
git.GitExecOrPanic(pkgPath, "fetch", "origin", commits[0].SHA)
git.GitExecOrPanic(pkgPath, "tag", "NOW")
git.GitExecOrPanic(pkgPath, "fetch", "origin")
@@ -315,7 +322,9 @@ next_package:
nCommits := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgPath, "rev-list", "^NOW", "HEAD"), "\n"))
if nCommits > 0 {
git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+config.Branch)
if !noop {
git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+config.Branch)
}
isGitUpdated = true
}
@@ -328,13 +337,14 @@ next_package:
// up-to-date
continue
} else if idx < len(commits) { // update
common.PanicOnError(git.GitExec(config.GitProjectName, "submodule", "update", "--init", "--depth", "1", "--checkout", filename))
common.PanicOnError(git.GitExec(filepath.Join(config.GitProjectName, filename), "fetch", "--depth", "1", "origin", commits[0].SHA))
common.PanicOnError(git.GitExec(filepath.Join(config.GitProjectName, filename), "checkout", commits[0].SHA))
common.PanicOnError(git.GitExec(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", filename))
common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "fetch", "--depth", "1", "origin", commits[0].SHA))
common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "checkout", commits[0].SHA))
log.Println(" -> updated to", commits[0].SHA)
isGitUpdated = true
} else {
// probably need `merge-base` or `rev-list` here instead, or the project updated already
return fmt.Errorf("Cannot find SHA of last matching update for package: '%s'. idx: %d", filename, idx)
log.Println(" *** Cannot find SHA of last matching update for package:", filename, " Ignoring")
}
}
}
@@ -366,12 +376,12 @@ next_repo:
continue next_repo
}
for _, c := range configs {
if c.Organization == org && c.GitProjectName == r.Name {
// ignore project gits
continue next_repo
}
// for _, c := range configs {
if gitPrj == r.Name {
// ignore project gits
continue next_repo
}
// }
for repo := range sub {
if repo == r.Name {
@@ -391,15 +401,15 @@ next_repo:
}
// add repository to git project
common.PanicOnError(git.GitExec(config.GitProjectName, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name))
common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name))
if len(config.Branch) > 0 {
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(config.GitProjectName, r.Name), "branch", "--show-current"))
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current"))
if branch != config.Branch {
if err := git.GitExec(path.Join(config.GitProjectName, r.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
if err := git.GitExec(path.Join(gitPrj, r.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", config.Branch, repo.Owner.UserName, r.Name)
}
common.PanicOnError(git.GitExec(path.Join(config.GitProjectName, r.Name), "checkout", config.Branch))
common.PanicOnError(git.GitExec(path.Join(gitPrj, r.Name), "checkout", config.Branch))
}
}
@@ -407,41 +417,50 @@ next_repo:
}
if isGitUpdated {
common.PanicOnError(git.GitExec(config.GitProjectName, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC"))
common.PanicOnError(git.GitExec(config.GitProjectName, "push"))
common.PanicOnError(git.GitExec(gitPrj, "commit", "-a", "-m", "Automatic update via push via Direct Workflow -- SYNC"))
if !noop {
git.GitExecOrPanic(gitPrj, "push", remoteName)
}
}
if DebugMode {
log.Println("Verification finished for ", org, ", config", config.GitProjectName)
log.Println("Verification finished for ", org, ", prjgit:", config.GitProjectName)
}
return nil
}
var checkOnStart bool
var noop bool
var checkInterval time.Duration
func checkOrg(org string, configs []*common.AutogitConfig) {
git, err := gh.CreateGitHandler(org)
if err != nil {
log.Println("Faield to allocate GitHandler:", err)
return
}
defer git.Close()
for _, config := range configs {
log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName)
if err := verifyProjectState(git, org, config, configs); err != nil {
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
} else {
log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName)
}
}
}
func checkRepos() {
for org, configs := range configuredRepos {
for _, config := range configs {
if checkInterval > 0 {
sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))
log.Println(" - sleep interval", sleepInterval, "until next check")
time.Sleep(sleepInterval)
}
log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName)
ghi := common.GitHandlerGeneratorImpl{}
git, err := ghi.CreateGitHandler(GitAuthor, GitEmail, AppName)
if err != nil {
log.Println("Faield to allocate GitHandler:", err)
return
}
if err := verifyProjectState(git, org, config, configs); err != nil {
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
}
log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName)
if checkInterval > 0 {
sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))
log.Println(" - sleep interval", sleepInterval, "until next check")
time.Sleep(sleepInterval)
}
checkOrg(org, configs)
}
}
@@ -461,6 +480,7 @@ func consistencyCheckProcess() {
}
var DebugMode bool
var gh common.GitHandlerGenerator
func updateConfiguration(configFilename string, orgs *[]string) {
configFile, err := common.ReadConfigFile(configFilename)
@@ -493,8 +513,10 @@ func main() {
giteaUrl := flag.String("gitea-url", "https://src.opensuse.org", "Gitea instance")
rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information")
flag.BoolVar(&noop, "no-op", false, "No-op mode. Do not push changes to remote repo.")
flag.BoolVar(&checkOnStart, "check-on-start", false, "Check all repositories for consistency on start, without delays")
checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
basePath := flag.String("repo-path", "", "Repository path. Default is temporary directory")
flag.Parse()
if err := common.RequireGiteaSecretToken(); err != nil {
@@ -505,6 +527,18 @@ func main() {
}
var defs common.ListenDefinitions
var err error
if len(*basePath) == 0 {
*basePath, err = os.MkdirTemp(os.TempDir(), AppName)
if err != nil {
log.Fatal(err)
}
}
gh, err = common.AllocateGitWorkTree(*basePath, GitAuthor, GitEmail)
if err != nil {
log.Fatal(err)
}
// handle reconfiguration
signalChannel := make(chan os.Signal, 1)

View File

@@ -20,8 +20,8 @@ package main
import (
"flag"
"log"
"net/url"
"os"
"slices"
"time"
@@ -35,71 +35,71 @@ const (
GitEmail = "adam+autogits-pr@zombino.com"
)
/*
func fetchPrGit(h *common.RequestHandler, pr *models.PullRequest) error {
// clone PR head and base and return path
if h.HasError() {
return h.Error
}
if _, err := os.Stat(path.Join(h.GitPath, pr.Head.Sha)); os.IsNotExist(err) {
h.GitExec("", "clone", "--depth", "1", pr.Head.Repo.CloneURL, pr.Head.Sha)
h.GitExec(pr.Head.Sha, "fetch", "--depth", "1", "origin", pr.Head.Sha, pr.Base.Sha)
} else if err != nil {
h.Error = err
}
return h.Error
}*/
var DebugMode bool
var ListPROnly bool
var PRID int64
var CurrentUser *models.User
var GitHandler common.GitHandlerGenerator
func main() {
if err := common.RequireGiteaSecretToken(); err != nil {
log.Fatal(err)
}
if err := common.RequireRabbitSecrets(); err != nil {
log.Fatal(err)
}
workflowConfig := flag.String("config", "", "Repository and workflow definition file")
giteaUrl := flag.String("gitea-url", "https://src.opensuse.org", "Gitea instance")
rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information")
debugMode := flag.Bool("debug", false, "Extra debugging information")
checkOnStart := flag.Bool("check-on-start", false, "Check all repositories for consistency on start, without delays")
checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
flag.BoolVar(&ListPROnly, "list-prs-only", false, "Only lists PRs without acting on them")
flag.Int64Var(&PRID, "id", -1, "Process only the specific ID and ignore the rest. Use for debugging")
basePath := flag.String("repo-path", "", "Repository path. Default is temporary directory")
flag.Parse()
common.SetLoggingLevel(common.LogLevelInfo)
if *debugMode {
common.SetLoggingLevel(common.LogLevelDebug)
}
if err := common.RequireGiteaSecretToken(); err != nil {
common.LogError("No Gitea secrets:", err)
return
}
if err := common.RequireRabbitSecrets(); err != nil {
common.LogError("No RabbitMQ secret:", err)
return
}
common.LogDebug("Parsing config:", *workflowConfig)
if len(*workflowConfig) == 0 {
log.Fatalln("No configuratio file specified. Aborting")
common.LogError("No configuratio file specified. Aborting")
return
}
gitea := common.AllocateGiteaTransport(*giteaUrl)
config, err := common.ReadConfigFile(*workflowConfig)
if err != nil {
log.Fatal(err)
common.LogError("Cannot read config files:", err)
return
}
configs, err := common.ResolveWorkflowConfigs(gitea, config)
configs, err := common.ResolveWorkflowConfigs(gitea, config)
if err != nil {
log.Fatal(err)
common.LogError("Cannot resolve config files:", err)
return
}
req := new(RequestProcessor)
req.configuredRepos = make(map[string][]*common.AutogitConfig)
req.git = &common.GitHandlerGeneratorImpl{}
if len(*basePath) == 0 {
*basePath, err = os.MkdirTemp(os.TempDir(), AppName)
common.PanicOnError(err)
defer os.RemoveAll(*basePath)
}
GitHandler, err = common.AllocateGitWorkTree(*basePath, GitAuthor, GitEmail)
common.PanicOnError(err)
orgs := make([]string, 0, 1)
for _, c := range configs {
if slices.Contains(c.Workflows, "pr") {
if DebugMode {
log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName)
}
common.LogDebug(" + adding org:", c.Organization, " branch:", c.Branch, " prjgit:", c.GitProjectName)
configs := req.configuredRepos[c.Organization]
if configs == nil {
configs = make([]*common.AutogitConfig, 0, 1)
@@ -112,9 +112,10 @@ func main() {
}
if CurrentUser, err = gitea.GetCurrentUser(); err != nil {
log.Fatal(err)
common.LogError("Failed to fetch current gitea user:", err)
return
}
log.Println("Running with token from", CurrentUser.UserName)
common.LogInfo("Running with token from", CurrentUser.UserName)
req.Synced = &PullRequestSynced{
gitea: gitea,
@@ -132,16 +133,17 @@ func main() {
checker := CreateDefaultStateChecker(*checkOnStart, req, gitea, time.Duration(*checkIntervalHours)*time.Hour)
go checker.ConsistencyCheckProcess()
var defs common.ListenDefinitions
listenDefs := common.ListenDefinitions{
Orgs: orgs,
GitAuthor: GitAuthor,
Handlers: map[string]common.RequestProcessor{
common.RequestType_PR: req,
common.RequestType_PRSync: req,
common.RequestType_PRReviewAccepted: req,
common.RequestType_PRReviewRejected: req,
},
}
listenDefs.RabbitURL, _ = url.Parse(*rabbitUrl)
defs.GitAuthor = GitAuthor
defs.RabbitURL, _ = url.Parse(*rabbitUrl)
defs.Handlers = make(map[string]common.RequestProcessor)
defs.Handlers[common.RequestType_PR] = req
defs.Handlers[common.RequestType_PRSync] = req
defs.Handlers[common.RequestType_PRReviewAccepted] = req
defs.Handlers[common.RequestType_PRReviewRejected] = req
log.Fatal(defs.ProcessRabbitMQEvents())
common.PanicOnError(listenDefs.ProcessRabbitMQEvents())
}

View File

@@ -16,7 +16,6 @@ type RequestProcessor struct {
Opened, Synced, Closed, Review PullRequestProcessor
configuredRepos map[string][]*common.AutogitConfig
git common.GitHandlerGenerator
}
func (w *RequestProcessor) ProcessFunc(request *common.Request) error {
@@ -44,10 +43,11 @@ func (w *RequestProcessor) ProcessFunc(request *common.Request) error {
return fmt.Errorf("Cannot find config for branch '%s'", req.Pull_Request.Base.Ref)
}
git, err := w.git.CreateGitHandler(GitAuthor, GitEmail, AppName)
git, err := GitHandler.CreateGitHandler(config.Organization)
if err != nil {
return fmt.Errorf("Error allocating GitHandler. Err: %w", err)
}
defer git.Close()
switch req.Action {
case "opened", "reopened":

View File

@@ -1,8 +1,6 @@
package main
import (
"log"
"src.opensuse.org/autogits/common"
)
@@ -15,8 +13,7 @@ func (*PullRequestClosed) Process(req *common.PullRequestWebhookEvent, git commo
return nil
}
log.Println("request was:", req.Pull_Request.State)
common.LogInfo(req.Pull_Request.Url, "is", req.Pull_Request.State)
return nil
/*
req := h.Data.(*common.PullRequestAction)
@@ -35,4 +32,3 @@ func (*PullRequestClosed) Process(req *common.PullRequestWebhookEvent, git commo
return nil
*/
}

View File

@@ -2,8 +2,6 @@ package main
import (
"fmt"
"log"
"src.opensuse.org/autogits/common"
)
@@ -11,14 +9,17 @@ type PullRequestOpened struct {
gitea common.Gitea
}
func (o *PullRequestOpened) Process(req *common.PullRequestWebhookEvent, git common.Git, config *common.AutogitConfig) error {
// requests against project are not handled here
if req.Repository.Name == config.GitProjectName {
return nil
}
func (o *PullRequestOpened) CreateOrUpdatePrjGitPR(req *common.PullRequestWebhookEvent, git common.Git, config *common.AutogitConfig) error {
// create PrjGit branch for buidling the pull request
branchName := prGitBranchNameForPR(req)
// TODO: fix this for config.Organization
org, prj, _ := config.GetPrjGit()
prjGit, err := o.gitea.CreateRepositoryIfNotExist(git, org, prj)
common.PanicOnErrorWithMsg(err, "Error creating a prjgitrepo: "+err.Error())
remoteName, err := git.GitClone(common.DefaultGitPrj, config.Branch, prjGit.SSHURL)
common.PanicOnError(err)
commitMsg := fmt.Sprintf(`auto-created for %s
This commit was autocreated by %s
@@ -32,29 +33,16 @@ referencing
req.Pull_Request.Number,
)
// TODO: fix this for config.Organization
prjGit, err := o.gitea.CreateRepositoryIfNotExist(git, config.Organization, config.GitProjectName)
if err != nil {
return err
}
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", prjGit.SSHURL, common.DefaultGitPrj))
err = git.GitExec(common.DefaultGitPrj, "fetch", "origin", branchName+":"+branchName)
if err != nil {
common.PanicOnError(git.GitExec(common.DefaultGitPrj, "checkout", "-B", branchName, prjGit.DefaultBranch))
} else {
common.PanicOnError(git.GitExec(common.DefaultGitPrj, "checkout", branchName))
}
subList, err := git.GitSubmoduleList(common.DefaultGitPrj, "HEAD")
common.PanicOnError(err)
if id := subList[req.Repository.Name]; id != req.Pull_Request.Head.Sha {
updateSubmoduleInPR(req, git)
common.PanicOnError(git.GitExec(common.DefaultGitPrj, "commit", "-a", "-m", commitMsg))
common.PanicOnError(git.GitExec(common.DefaultGitPrj, "push", "origin", "+HEAD:"+branchName))
common.PanicOnError(git.GitExec(common.DefaultGitPrj, "push", remoteName, "+HEAD:"+branchName))
}
PR, err := o.gitea.CreatePullRequestIfNotExist(prjGit, branchName, prjGit.DefaultBranch,
_, err = o.gitea.CreatePullRequestIfNotExist(prjGit, branchName, prjGit.DefaultBranch,
fmt.Sprintf("Forwarded PR: %s", req.Repository.Name),
fmt.Sprintf(`This is a forwarded pull request by %s
referencing the following pull request:
@@ -63,8 +51,16 @@ referencing the following pull request:
GitAuthor, req.Repository.Owner.Username, req.Repository.Name, req.Pull_Request.Number),
)
if err != nil {
return err
return err
}
func (o *PullRequestOpened) Process(req *common.PullRequestWebhookEvent, git common.Git, config *common.AutogitConfig) error {
// requests against project are not handled here
common.LogInfo("processing opened PR:", req.Pull_Request.Url)
if org, repo, _ := config.GetPrjGit(); req.Repository.Owner.Username != org || req.Repository.Name != repo {
if err := o.CreateOrUpdatePrjGitPR(req, git, config); err != nil {
return err
}
}
prset, err := common.FetchPRSet(o.gitea, req.Repository.Owner.Username, req.Repository.Name, req.Number, config)
@@ -73,7 +69,11 @@ referencing the following pull request:
}
// request build review
log.Println("num of current reviewers:", len(PR.RequestedReviewers))
PR, err := prset.GetPrjGitPR()
if err != nil {
return nil
}
common.LogDebug(" num of reviewers:", len(PR.RequestedReviewers))
maintainers, err := common.FetchProjectMaintainershipData(o.gitea, config.Organization, config.GitProjectName, config.Branch)
if err != nil {
return err

View File

@@ -18,7 +18,7 @@ func (o *PullRequestReviewed) Process(req *common.PullRequestWebhookEvent, git c
}
if prset.IsApproved(o.gitea, maintainers) {
prset.Merge(GitAuthor, GitEmail)
return prset.Merge(GitHandler)
}
return nil

View File

@@ -3,7 +3,6 @@ package main
import (
"bufio"
"fmt"
"log"
"path"
"strings"
@@ -62,17 +61,17 @@ func (o *PullRequestSynced) Process(req *common.PullRequestWebhookEvent, git com
// nothing changed, still in sync
if commitId == req.Pull_Request.Head.Sha {
log.Println("commitID already match - nothing to do")
common.LogDebug("commitID already match - nothing to do")
return nil
}
log.Printf("different ids: '%s' vs. '%s'\n", req.Pull_Request.Head.Sha, commitId)
common.LogDebug("Sync repo update. Old", commitId, " New", req.Pull_Request.Head.Sha)
commitMsg := fmt.Sprintf(`Sync PR
Update to %s`, req.Pull_Request.Head.Sha)
log.Println("will create new commit msg:", commitMsg)
common.LogDebug("Creating new commit msg:", commitMsg)
// we need to update prjgit PR with the new head hash
branchName := prGitBranchNameForPR(req)

View File

@@ -181,8 +181,6 @@ func TestSyncPR(t *testing.T) {
setupGitForTests(t, git)
git.DebugLogger = true
DebugMode = true
// mock.EXPECT().GetAssociatedPrjGitPR(event).Return(PrjGitPR, nil)
mock.EXPECT().GetPullRequest(config.Organization, "testRepo", event.Pull_Request.Number).Return(modelPR, nil)
mock.EXPECT().GetPullRequest(config.Organization, "prj", int64(24)).Return(PrjGitPR, nil)

View File

@@ -101,7 +101,6 @@ func TestPRProcessor(t *testing.T) {
req := &RequestProcessor{
configuredRepos: testConfiguration,
git: &common.GitHandlerGeneratorImpl{},
}
test.req(req, mock)
@@ -120,7 +119,6 @@ func TestPRProcessor(t *testing.T) {
req := &RequestProcessor{
configuredRepos: testConfiguration,
git: &common.GitHandlerGeneratorImpl{},
}
t.Run("Edit PR handling", func(t *testing.T) {
@@ -197,17 +195,12 @@ func TestPRProcessor(t *testing.T) {
ctl := gomock.NewController(t)
gitHandler := mock_common.NewMockGitHandlerGenerator(ctl)
gitHandler.EXPECT().CreateGitHandler(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("some error"))
origHandler := req.git
req.git = gitHandler
gitHandler.EXPECT().CreateGitHandler(gomock.Any()).Return(nil, fmt.Errorf("some error"))
err := req.ProcessFunc(&common.Request{
Data: event,
})
req.git = origHandler
if err == nil {
t.Error(logBuf.String())
}

View File

@@ -3,7 +3,6 @@ package main
import (
"errors"
"fmt"
"log"
"math/rand"
"path"
"runtime/debug"
@@ -11,12 +10,13 @@ import (
"time"
"src.opensuse.org/autogits/common"
"src.opensuse.org/autogits/common/gitea-generated/models"
)
//go:generate mockgen -source=repo_check.go -destination=mock/repo_check.go -typed
type StateChecker interface {
VerifyProjectState(orgName string, configs []*common.AutogitConfig, idx int) error
VerifyProjectState(configs *common.AutogitConfig) error
CheckRepos() error
ConsistencyCheckProcess() error
}
@@ -27,14 +27,12 @@ type DefaultStateChecker struct {
checkInterval time.Duration
gitea common.Gitea
git common.GitHandlerGenerator
processor *RequestProcessor
i StateChecker
}
func CreateDefaultStateChecker(checkOnStart bool, processor *RequestProcessor, gitea common.Gitea, interval time.Duration) *DefaultStateChecker {
var s = &DefaultStateChecker{
git: &common.GitHandlerGeneratorImpl{},
gitea: gitea,
checkInterval: interval,
checkOnStart: checkOnStart,
@@ -44,80 +42,86 @@ func CreateDefaultStateChecker(checkOnStart bool, processor *RequestProcessor, g
return s
}
func (s *DefaultStateChecker) VerifyProjectState(org string, configs []*common.AutogitConfig, idx int) error {
func (s *DefaultStateChecker) ProcessPR(git common.Git, pr *models.PullRequest, config *common.AutogitConfig) error {
var event common.PullRequestWebhookEvent
event.Pull_Request = common.PullRequestFromModel(pr)
event.Action = string(pr.State)
event.Number = pr.Index
event.Repository = common.RepositoryFromModel(pr.Base.Repo)
event.Sender = *common.UserFromModel(pr.User)
event.Requested_reviewer = nil
var err error
switch pr.State {
case "open":
err = s.processor.Opened.Process(&event, git, config)
case "closed":
err = s.processor.Closed.Process(&event, git, config)
default:
return fmt.Errorf("Unhandled pull request state: '%s'. %s/%s/%d", pr.State, config.Organization, "", pr.Index)
}
common.LogError(" * processor error returned:", err)
return nil
}
func (s *DefaultStateChecker) VerifyProjectState(config *common.AutogitConfig) error {
defer func() {
if r := recover(); r != nil {
log.Println("panic caught")
common.LogError("panic caught")
if err, ok := r.(error); !ok {
log.Println(err)
common.LogError(err)
}
log.Println(string(debug.Stack()))
common.LogError(string(debug.Stack()))
}
}()
git, err := s.git.CreateGitHandler(GitAuthor, GitEmail, AppName)
prjGitOrg, prjGitRepo, prjGitBranch := config.GetPrjGit()
common.LogInfo(" checking", prjGitOrg+"/"+prjGitRepo+"#"+prjGitBranch)
git, err := GitHandler.CreateGitHandler(prjGitOrg)
if err != nil {
return fmt.Errorf("Cannot create git handler: %w", err)
}
defer git.Close()
config := configs[idx]
repo, err := s.gitea.CreateRepositoryIfNotExist(git, org, config.GitProjectName)
repo, err := s.gitea.CreateRepositoryIfNotExist(git, prjGitOrg, prjGitRepo)
if err != nil {
return fmt.Errorf("Error fetching or creating '%s/%s' -- aborting verifyProjectState(). Err: %w", org, config.GitProjectName, err)
return fmt.Errorf("Error fetching or creating '%s/%s#%s' -- aborting verifyProjectState(). Err: %w", prjGitBranch, prjGitRepo, prjGitBranch, err)
}
common.PanicOnError(git.GitExec("", "clone", "--depth", "1", repo.SSHURL, config.GitProjectName))
log.Println("getting submodule list")
_, err = git.GitClone(config.GitProjectName, prjGitBranch, repo.SSHURL)
common.PanicOnError(err)
prs, err := s.gitea.GetRecentPullRequests(prjGitOrg, prjGitRepo, prjGitBranch)
if err != nil {
return fmt.Errorf("Error fetching PrjGit Prs for %s/%s#%s: %w", prjGitOrg, prjGitRepo, prjGitBranch, err)
}
for _, pr := range prs {
s.ProcessPR(git, pr, config)
}
common.LogDebug(" - # of PRs to check in PrjGit:", len(prs))
submodules, err := git.GitSubmoduleList(config.GitProjectName, "HEAD")
nextSubmodule:
for sub, commitID := range submodules {
log.Println(" + checking", sub, commitID)
common.LogDebug(" + checking", sub, commitID)
submoduleName := sub
if n := strings.LastIndex(sub, "/"); n != -1 {
submoduleName = sub[n+1:]
}
// check if open PR have PR against project
prs, err := s.gitea.GetRecentPullRequests(config.Organization, submoduleName)
prs, err := s.gitea.GetRecentPullRequests(config.Organization, submoduleName, config.Branch)
if err != nil {
return fmt.Errorf("Error fetching pull requests for %s/%s. Err: %w", config.Organization, submoduleName, err)
}
if DebugMode {
log.Println(" - # of PRs to check:", len(prs))
return fmt.Errorf("Error fetching pull requests for %s/%s#%s. Err: %w", config.Organization, submoduleName, config.Branch, err)
}
common.LogDebug(" - # of PRs to check:", len(prs))
for _, pr := range prs {
var event common.PullRequestWebhookEvent
event.Pull_Request = common.PullRequestFromModel(pr)
event.Action = string(pr.State)
event.Number = pr.Index
event.Repository = common.RepositoryFromModel(pr.Base.Repo)
event.Sender = *common.UserFromModel(pr.User)
event.Requested_reviewer = nil
git, err := s.git.CreateGitHandler(GitAuthor, GitEmail, AppName)
if err != nil {
return fmt.Errorf("Error allocating GitHandler. Err: %w", err)
}
if !DebugMode {
defer git.Close()
}
switch pr.State {
case "open":
err = s.processor.Opened.Process(&event, git, config)
case "closed":
err = s.processor.Closed.Process(&event, git, config)
default:
return fmt.Errorf("Unhandled pull request state: '%s'. %s/%s/%d", pr.State, config.Organization, submoduleName, pr.Index)
}
if err != nil {
log.Println(" * processor error returned:", err)
}
s.ProcessPR(git, pr, config)
}
// check if the commited changes are syned with branches
@@ -130,7 +134,7 @@ nextSubmodule:
if commit.SHA == commitID {
if idx != 0 {
// commit in past ...
log.Println(" W -", submoduleName, " is behind the branch by", idx, "This should not happen in PR workflow alone")
common.LogError(" -", submoduleName, " is behind the branch by", idx, "This should not happen in PR workflow alone")
}
continue nextSubmodule
}
@@ -142,9 +146,7 @@ nextSubmodule:
newCommits := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(subDir, "rev-list", "^origin/"+config.Branch, commitID), "\n")
if len(newCommits) >= 1 {
if DebugMode {
log.Println(" - updating branch", config.Branch, "to new head", commitID, " - len:", len(newCommits))
}
common.LogDebug(" - updating branch", config.Branch, "to new head", commitID, " - len:", len(newCommits))
git.GitExecOrPanic(subDir, "checkout", "-B", config.Branch, commitID)
url := git.GitExecWithOutputOrPanic(subDir, "remote", "get-url", "origin", "--push")
sshUrl, err := common.TranslateHttpsToSshUrl(strings.TrimSpace(url))
@@ -164,19 +166,19 @@ func (s *DefaultStateChecker) CheckRepos() error {
errorList := make([]error, 0, 10)
for org, configs := range s.processor.configuredRepos {
for configIdx, config := range configs {
for _, config := range configs {
if s.checkInterval > 0 {
sleepInterval := (s.checkInterval - s.checkInterval/2) + time.Duration(rand.Int63n(int64(s.checkInterval)))
log.Println(" - sleep interval", sleepInterval, "until next check")
common.LogInfo(" - sleep interval", sleepInterval, "until next check")
time.Sleep(sleepInterval)
}
log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName)
if err := s.i.VerifyProjectState(org, configs, configIdx); err != nil {
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
common.LogInfo(" ++ starting verification, org:", org, "config:", config.GitProjectName)
if err := s.i.VerifyProjectState(config); err != nil {
common.LogError(" *** verification failed, org:", org, err)
errorList = append(errorList, err)
}
log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName)
common.LogInfo(" ++ verification complete, org:", org, "config:", config.GitProjectName)
}
}
@@ -187,9 +189,9 @@ func (s *DefaultStateChecker) ConsistencyCheckProcess() error {
if s.checkOnStart {
savedCheckInterval := s.checkInterval
s.checkInterval = 0
log.Println("== Startup consistency check begin...")
common.LogInfo("== Startup consistency check begin...")
s.i.CheckRepos()
log.Println("== Startup consistency check done...")
common.LogInfo("== Startup consistency check done...")
s.checkInterval = savedCheckInterval
}

View File

@@ -89,14 +89,13 @@ func TestRepoCheck(t *testing.T) {
"repo3_org": []*common.AutogitConfig{config3},
},
}
r := configs.configuredRepos
c := CreateDefaultStateChecker(true, configs, gitea, 100)
c.i = state
state.EXPECT().VerifyProjectState("repo1_org", r["repo1_org"], 0)
state.EXPECT().VerifyProjectState("repo2_org", r["repo2_org"], 0)
state.EXPECT().VerifyProjectState("repo3_org", r["repo3_org"], 0)
state.EXPECT().VerifyProjectState(configs.configuredRepos["repo1_org"][0])
state.EXPECT().VerifyProjectState(configs.configuredRepos["repo2_org"][0])
state.EXPECT().VerifyProjectState(configs.configuredRepos["repo3_org"][0])
if err := c.CheckRepos(); err != nil {
t.Error(err)
@@ -123,10 +122,9 @@ func TestRepoCheck(t *testing.T) {
c := CreateDefaultStateChecker(true, configs, gitea, 100)
c.i = state
c.git = git
err := errors.New("test error")
state.EXPECT().VerifyProjectState("repo1_org", gomock.Any(), 0).Return(err)
state.EXPECT().VerifyProjectState(configs.configuredRepos["repo1_org"][0]).Return(err)
r := c.CheckRepos()
@@ -184,7 +182,7 @@ func TestVerifyProjectState(t *testing.T) {
gitea.EXPECT().CreateRepositoryIfNotExist(gomock.Any(), gomock.Any(), config1.GitProjectName).Return(&models.Repository{
SSHURL: "./prj",
}, nil)
gitea.EXPECT().GetRecentPullRequests(org, "testRepo")
gitea.EXPECT().GetRecentPullRequests(org, "testRepo", "testing")
gitea.EXPECT().GetRecentCommits(org, "testRepo", "testing", gomock.Any())
c := CreateDefaultStateChecker(false, configs, gitea, 0)
@@ -192,7 +190,7 @@ func TestVerifyProjectState(t *testing.T) {
git: git,
}
err := c.VerifyProjectState("repo1_org", configs.configuredRepos[org], 0)
err := c.VerifyProjectState(configs.configuredRepos[org][0])
if err != nil {
t.Error(err)
@@ -231,7 +229,7 @@ func TestVerifyProjectState(t *testing.T) {
SSHURL: "./prj",
}, nil)
gitea.EXPECT().GetRecentPullRequests(org, "testRepo").Return([]*models.PullRequest{
gitea.EXPECT().GetRecentPullRequests(org, "testRepo", "testing").Return([]*models.PullRequest{
&models.PullRequest{
ID: 1234,
URL: "url here",
@@ -274,7 +272,7 @@ func TestVerifyProjectState(t *testing.T) {
process.EXPECT().Process(gomock.Any(), gomock.Any(), gomock.Any())
c.processor.Opened = process
err := c.VerifyProjectState("repo1_org", configs.configuredRepos[org], 0)
err := c.VerifyProjectState(configs.configuredRepos[org][0])
if err != nil {
t.Error(err)