11 Commits

Author SHA256 Message Date
f2089f99fc staging: use helper function to SetCommitStatus 2025-09-09 12:55:14 +02:00
10ea3a8f8f obs-staging-bot: Fix setting of commit status 2025-09-09 12:46:43 +02:00
9faa6ead49 Log errors on SetCommitStatus 2025-09-09 12:46:21 +02:00
29cce5741a staging: typo fix 2025-09-09 12:46:11 +02:00
804e542c3f Decline too large staging projects
In most cases anyway an error in pull request.
2025-09-09 12:41:07 +02:00
72899162b0 status: need to fetch repositories during sync
We need to fetch repositories so that we can have package
data. We only need to fetch one set of results per project,
not all repos.
2025-09-03 16:42:01 +02:00
168a419bbe status: allow for package search endpoint
OBS has issues searching for packages in scmsynced projects.
Since we have a list of all the repositories, we can allow
for a search endpoint here.

/search?q=term1&q=term2...

results is JSON

[
   project1/pkgA,
   project2/pkgB
]
2025-09-03 14:35:15 +02:00
6a71641295 common: take care of empty result sets
In case of empty result pages, we should ignore the X-Total-Count
header.

Fixes: 5addde0a71
2025-09-03 12:21:07 +02:00
5addde0a71 common: use X-Total-Count in multi-page results 2025-09-03 01:00:33 +02:00
90ea1c9463 common: remove duplicate 2025-09-02 20:50:23 +02:00
a4fb3e6151 PR: Don't clobber other's PrjGit description
If we did not create the PRjGit PR, don't touch the title
and description

Closes: #68
2025-09-02 19:47:47 +02:00
10 changed files with 481 additions and 573 deletions

View File

@@ -1,296 +0,0 @@
package common
import (
"errors"
"fmt"
"io"
)
const (
GitStatus_Untracked = 0
GitStatus_Modified = 1
GitStatus_Ignored = 2
GitStatus_Unmerged = 3 // States[0..3] -- Stage1, Stage2, Stage3 of merge objects
GitStatus_Renamed = 4 // orig name in States[0]
)
type GitStatusData struct {
Path string
Status int
States [3]string
/*
<sub> A 4 character field describing the submodule state.
"N..." when the entry is not a submodule.
"S<c><m><u>" when the entry is a submodule.
<c> is "C" if the commit changed; otherwise ".".
<m> is "M" if it has tracked changes; otherwise ".".
<u> is "U" if there are untracked changes; otherwise ".".
*/
SubmoduleChanges string
}
func parseGit_HexString(data io.ByteReader) (string, error) {
str := make([]byte, 0, 32)
for {
c, err := data.ReadByte()
if err != nil {
return "", err
}
switch {
case c == 0 || c == ' ':
return string(str), nil
case c >= 'a' && c <= 'f':
case c >= 'A' && c <= 'F':
case c >= '0' && c <= '9':
default:
return "", errors.New("Invalid character in hex string:" + string(c))
}
str = append(str, c)
}
}
func parseGit_String(data io.ByteReader) (string, error) {
str := make([]byte, 0, 100)
for {
c, err := data.ReadByte()
if err != nil {
return "", errors.New("Unexpected EOF. Expected NUL string term")
}
if c == 0 || c == ' ' {
return string(str), nil
}
str = append(str, c)
}
}
func parseGit_StringWithSpace(data io.ByteReader) (string, error) {
str := make([]byte, 0, 100)
for {
c, err := data.ReadByte()
if err != nil {
return "", errors.New("Unexpected EOF. Expected NUL string term")
}
if c == 0 {
return string(str), nil
}
str = append(str, c)
}
}
func skipGitStatusEntry(data io.ByteReader, skipSpaceLen int) error {
for skipSpaceLen > 0 {
c, err := data.ReadByte()
if err != nil {
return err
}
if c == ' ' {
skipSpaceLen--
}
}
return nil
}
func parseSingleStatusEntry(data io.ByteReader) (*GitStatusData, error) {
ret := GitStatusData{}
statusType, err := data.ReadByte()
if err != nil {
return nil, nil
}
switch statusType {
case '1':
var err error
if err = skipGitStatusEntry(data, 8); err != nil {
return nil, err
}
ret.Status = GitStatus_Modified
ret.Path, err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
case '2':
var err error
if err = skipGitStatusEntry(data, 9); err != nil {
return nil, err
}
ret.Status = GitStatus_Renamed
ret.Path, err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
ret.States[0], err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
case '?':
var err error
if err = skipGitStatusEntry(data, 1); err != nil {
return nil, err
}
ret.Status = GitStatus_Untracked
ret.Path, err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
case '!':
var err error
if err = skipGitStatusEntry(data, 1); err != nil {
return nil, err
}
ret.Status = GitStatus_Ignored
ret.Path, err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
case 'u':
var err error
if err = skipGitStatusEntry(data, 2); err != nil {
return nil, err
}
if ret.SubmoduleChanges, err = parseGit_String(data); err != nil {
return nil, err
}
if err = skipGitStatusEntry(data, 4); err != nil {
return nil, err
}
if ret.States[0], err = parseGit_HexString(data); err != nil {
return nil, err
}
if ret.States[1], err = parseGit_HexString(data); err != nil {
return nil, err
}
if ret.States[2], err = parseGit_HexString(data); err != nil {
return nil, err
}
ret.Status = GitStatus_Unmerged
ret.Path, err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
default:
return nil, errors.New("Invalid status type" + string(statusType))
}
return &ret, nil
}
func parseGitStatusData(data io.ByteReader) (Data, error) {
ret := make([]GitStatusData, 0, 10)
for {
data, err := parseSingleStatusEntry(data)
if err != nil {
return nil, err
} else if data == nil {
break
}
ret = append(ret, *data)
}
return ret, nil
}
type Data interface{}
type CommitStatus int
const (
Add CommitStatus = iota
Rm
Copy
Modify
Rename
TypeChange
Unmerged
Unknown
)
type GitDiffRawData struct {
SrcMode, DstMode string
SrcCommit, DstCommit string
Status CommitStatus
Src, Dst string
}
func parseGit_DiffIndexStatus(data io.ByteReader, d *GitDiffRawData) error {
b, err := data.ReadByte()
if err != nil {
return err
}
switch b {
case 'A':
d.Status = Add
case 'C':
d.Status = Copy
case 'D':
d.Status = Rm
case 'M':
d.Status = Modify
case 'R':
d.Status = Rename
case 'T':
d.Status = TypeChange
case 'U':
d.Status = Unmerged
case 'X':
return fmt.Errorf("Unexpected unknown change type. This is a git bug")
}
_, err = parseGit_StringWithSpace(data)
if err != nil {
return err
}
return nil
}
func parseSingleGitDiffIndexRawData(data io.ByteReader) (*GitDiffRawData, error) {
var ret GitDiffRawData
b, err := data.ReadByte()
if err != nil {
return nil, err
}
if b != ':' {
return nil, fmt.Errorf("Expected ':' but got '%s'", string(b))
}
if ret.SrcMode, err = parseGit_String(data); err != nil {
return nil, err
}
if ret.DstMode, err = parseGit_String(data); err != nil {
return nil, err
}
if ret.Src, err = parseGit_String(data); err != nil {
return nil, err
}
if ret.Dst, err = parseGit_String(data); err != nil {
return nil, err
}
if err = parseGit_DiffIndexStatus(data, &ret); err != nil {
return nil, err
}
ret.Dst = ret.Src
switch ret.Status {
case Copy, Rename:
if ret.Src, err = parseGit_StringWithSpace(data); err != nil {
return nil, err
}
}
return &ret, nil
}
func parseGitDiffIndexRawData(data io.ByteReader) (Data, error) {
ret := make([]GitDiffRawData, 0, 10)
for {
data, err := parseSingleGitDiffIndexRawData(data)
if err != nil {
return nil, err
} else if data == nil {
break
}
ret = append(ret, *data)
}
return ret, nil
}

View File

@@ -19,7 +19,9 @@ package common
*/
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"os"
@@ -38,10 +40,6 @@ type GitSubmoduleLister interface {
GitSubmoduleCommitId(cwd, packageName, commitId string) (subCommitId string, valid bool)
}
type GitSubmoduleFileConflictResolver interface {
GitResolveSubmoduleFileConflict(cwd string) error
}
type GitStatusLister interface {
GitStatus(cwd string) ([]GitStatusData, error)
}
@@ -72,7 +70,6 @@ type Git interface {
GitExecQuietOrPanic(cwd string, params ...string)
GitDiffLister
GitSubmoduleFileConflictResolver
}
type GitHandlerImpl struct {
@@ -250,7 +247,7 @@ func (e *GitHandlerImpl) GitClone(repo, branch, remoteUrl string) (string, error
e.GitExecOrPanic(repo, "fetch", "--prune", remoteName, remoteBranch)
}
/*
/*
refsBytes, err := os.ReadFile(path.Join(e.GitPath, repo, ".git/refs/remotes", remoteName, "HEAD"))
if err != nil {
LogError("Cannot read HEAD of remote", remoteName)
@@ -269,7 +266,7 @@ func (e *GitHandlerImpl) GitClone(repo, branch, remoteUrl string) (string, error
LogDebug("remoteRef", remoteRef)
LogDebug("branch", branch)
}
*/
*/
args := []string{"fetch", "--prune", remoteName, branch}
if strings.TrimSpace(e.GitExecWithOutputOrPanic(repo, "rev-parse", "--is-shallow-repository")) == "true" {
args = slices.Insert(args, 1, "--unshallow")
@@ -926,10 +923,193 @@ func (e *GitHandlerImpl) GitSubmoduleCommitId(cwd, packageName, commitId string)
return subCommitId, len(subCommitId) > 0
}
func (e *GitHandlerImpl) GitExecWithDataParse(cwd string, dataprocessor func(io.ByteReader) (Data, error), gitcmd string, args ...string) (Data, error) {
LogDebug("getting", gitcmd)
args = append([]string{gitcmd}, args...)
cmd := exec.Command("/usr/bin/git", args...)
const (
GitStatus_Untracked = 0
GitStatus_Modified = 1
GitStatus_Ignored = 2
GitStatus_Unmerged = 3 // States[0..3] -- Stage1, Stage2, Stage3 of merge objects
GitStatus_Renamed = 4 // orig name in States[0]
)
type GitStatusData struct {
Path string
Status int
States [3]string
/*
<sub> A 4 character field describing the submodule state.
"N..." when the entry is not a submodule.
"S<c><m><u>" when the entry is a submodule.
<c> is "C" if the commit changed; otherwise ".".
<m> is "M" if it has tracked changes; otherwise ".".
<u> is "U" if there are untracked changes; otherwise ".".
*/
SubmoduleChanges string
}
func parseGitStatusHexString(data io.ByteReader) (string, error) {
str := make([]byte, 0, 32)
for {
c, err := data.ReadByte()
if err != nil {
return "", err
}
switch {
case c == 0 || c == ' ':
return string(str), nil
case c >= 'a' && c <= 'f':
case c >= 'A' && c <= 'F':
case c >= '0' && c <= '9':
default:
return "", errors.New("Invalid character in hex string:" + string(c))
}
str = append(str, c)
}
}
func parseGitStatusString(data io.ByteReader) (string, error) {
str := make([]byte, 0, 100)
for {
c, err := data.ReadByte()
if err != nil {
return "", errors.New("Unexpected EOF. Expected NUL string term")
}
if c == 0 || c == ' ' {
return string(str), nil
}
str = append(str, c)
}
}
func parseGitStatusStringWithSpace(data io.ByteReader) (string, error) {
str := make([]byte, 0, 100)
for {
c, err := data.ReadByte()
if err != nil {
return "", errors.New("Unexpected EOF. Expected NUL string term")
}
if c == 0 {
return string(str), nil
}
str = append(str, c)
}
}
func skipGitStatusEntry(data io.ByteReader, skipSpaceLen int) error {
for skipSpaceLen > 0 {
c, err := data.ReadByte()
if err != nil {
return err
}
if c == ' ' {
skipSpaceLen--
}
}
return nil
}
func parseSingleStatusEntry(data io.ByteReader) (*GitStatusData, error) {
ret := GitStatusData{}
statusType, err := data.ReadByte()
if err != nil {
return nil, nil
}
switch statusType {
case '1':
var err error
if err = skipGitStatusEntry(data, 8); err != nil {
return nil, err
}
ret.Status = GitStatus_Modified
ret.Path, err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
case '2':
var err error
if err = skipGitStatusEntry(data, 9); err != nil {
return nil, err
}
ret.Status = GitStatus_Renamed
ret.Path, err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
ret.States[0], err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
case '?':
var err error
if err = skipGitStatusEntry(data, 1); err != nil {
return nil, err
}
ret.Status = GitStatus_Untracked
ret.Path, err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
case '!':
var err error
if err = skipGitStatusEntry(data, 1); err != nil {
return nil, err
}
ret.Status = GitStatus_Ignored
ret.Path, err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
case 'u':
var err error
if err = skipGitStatusEntry(data, 2); err != nil {
return nil, err
}
if ret.SubmoduleChanges, err = parseGitStatusString(data); err != nil {
return nil, err
}
if err = skipGitStatusEntry(data, 4); err != nil {
return nil, err
}
if ret.States[0], err = parseGitStatusHexString(data); err != nil {
return nil, err
}
if ret.States[1], err = parseGitStatusHexString(data); err != nil {
return nil, err
}
if ret.States[2], err = parseGitStatusHexString(data); err != nil {
return nil, err
}
ret.Status = GitStatus_Unmerged
ret.Path, err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
default:
return nil, errors.New("Invalid status type" + string(statusType))
}
return &ret, nil
}
func parseGitStatusData(data io.ByteReader) ([]GitStatusData, error) {
ret := make([]GitStatusData, 0, 10)
for {
data, err := parseSingleStatusEntry(data)
if err != nil {
return nil, err
} else if data == nil {
break
}
ret = append(ret, *data)
}
return ret, nil
}
func (e *GitHandlerImpl) GitStatus(cwd string) (ret []GitStatusData, err error) {
LogDebug("getting git-status()")
cmd := exec.Command("/usr/bin/git", "status", "--porcelain=2", "-z")
cmd.Env = []string{
"GIT_CEILING_DIRECTORIES=" + e.GitPath,
"GIT_LFS_SKIP_SMUDGE=1",
@@ -946,12 +1126,7 @@ func (e *GitHandlerImpl) GitExecWithDataParse(cwd string, dataprocessor func(io.
LogError("Error running command", cmd.Args, err)
}
return dataprocessor(bytes.NewReader(out))
}
func (e *GitHandlerImpl) GitStatus(cwd string) (ret []GitStatusData, err error) {
data, err := e.GitExecWithDataParse(cwd, parseGitStatusData, "status", "--porcelain=2", "-z")
return data.([]GitStatusData), err
return parseGitStatusData(bufio.NewReader(bytes.NewReader(out)))
}
func (e *GitHandlerImpl) GitDiff(cwd, base, head string) (string, error) {
@@ -976,94 +1151,3 @@ func (e *GitHandlerImpl) GitDiff(cwd, base, head string) (string, error) {
return string(out), nil
}
func (e *GitHandlerImpl) GitDiffIndex(cwd, commit string) ([]GitDiffRawData, error) {
data, err := e.GitExecWithDataParse("diff-index", parseGitDiffIndexRawData, cwd, "diff-index", "-z", "--raw", "--full-index", "--submodule=short", "HEAD")
return data.([]GitDiffRawData), err
}
func (git *GitHandlerImpl) GitResolveSubmoduleFileConflict(cwd string) error {
status, err := git.GitStatus(cwd)
if err != nil {
return fmt.Errorf("Status failed: %w", err)
}
// we can only resolve conflicts with .gitmodules
for _, s := range status {
if s.Status == GitStatus_Unmerged {
if s.Path != ".gitmodules" {
return err
}
submodules, err := git.GitSubmoduleList(cwd, "HEAD")
if err != nil {
return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
}
// We need to adjust the `submodules` list by the pending changes to the index
s1, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[0])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
s2, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[1])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
s3, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[2])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
subs1, err := ParseSubmodulesFile(strings.NewReader(s1))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
subs2, err := ParseSubmodulesFile(strings.NewReader(s2))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
subs3, err := ParseSubmodulesFile(strings.NewReader(s3))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
for r := range submodules {
LogError(r)
}
// merge from subs3 (target), subs1 (orig), subs2 (2-nd base that is missing from target base)
// this will update submodules
mergedSubs := slices.Concat(subs1, subs2, subs3)
var filteredSubs []Submodule = make([]Submodule, 0, max(len(subs1), len(subs2), len(subs3)))
nextSub:
for subName := range submodules {
for i := range mergedSubs {
if path.Base(mergedSubs[i].Path) == subName {
filteredSubs = append(filteredSubs, mergedSubs[i])
continue nextSub
}
}
return fmt.Errorf("Cannot find submodule for path: %s", subName)
}
out, err := os.Create(path.Join(git.GetPath(), cwd, ".gitmodules"))
if err != nil {
return fmt.Errorf("Can't open .gitmodules for writing: %w", err)
}
if err = WriteSubmodules(filteredSubs, out); err != nil {
return fmt.Errorf("Can't write .gitmodules: %w", err)
}
if out.Close(); err != nil {
return fmt.Errorf("Can't close .gitmodules: %w", err)
}
git.GitExecOrPanic(cwd, "add", ".gitmodules")
git.GitExecOrPanic(cwd, "-c", "core.editor=true", "merge", "--continue")
}
}
return nil
}

View File

@@ -24,7 +24,6 @@ import (
"os"
"os/exec"
"path"
"runtime/debug"
"slices"
"strings"
"testing"
@@ -94,93 +93,6 @@ func TestGitClone(t *testing.T) {
}
}
func TestSubmoduleConflictResolution(t *testing.T) {
tests := []struct {
name string
checkout, merge string
result string
}{
{
name: "adding two submodules",
checkout: "base_add_b1",
merge: "base_add_b2",
result: `[submodule "pkgA"]
path = pkgA
url = ../pkgA
[submodule "pkgB"]
path = pkgB
url = ../pkgB
[submodule "pkgC"]
path = pkgC
url = ../pkgC
[submodule "pkgB1"]
path = pkgB1
url = ../pkgB1
[submodule "pkgB2"]
path = pkgB2
url = ../pkgB2
`,
},
}
d, err := os.MkdirTemp(os.TempDir(), "submoduletests")
if err != nil {
t.Fatal(err)
}
cwd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
cmd := exec.Command(cwd + "/test_repo_setup.sh")
cmd.Dir = d
_, err = cmd.Output()
if err != nil {
t.Fatal(err)
}
gh, err := AllocateGitWorkTree(d, "test", "foo@example.com")
if err != nil {
t.Fatal(err)
}
success := true
noErrorOrFail := func(t *testing.T, err error) {
if err != nil {
t.Fatal(string(debug.Stack()), err)
}
}
for _, test := range tests {
success = t.Run(test.name, func(t *testing.T) {
git, err := gh.ReadExistingPath("prjgit")
if err != nil {
t.Fatal(err)
}
noErrorOrFail(t, git.GitExec("", "checkout", "-B", "test", "main"))
noErrorOrFail(t, git.GitExec("", "merge", "base_add_b1"))
err = git.GitExec("", "merge", "base_add_b2")
if err == nil {
t.Fatal("expected a conflict")
}
err = git.GitResolveSubmoduleFileConflict("")
if err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(git.GetPath() + "/.gitmodules")
if err != nil {
t.Fatal("Cannot read .gitmodules.", err)
}
if string(data) != test.result {
t.Error("Expected", test.result, "but have", string(data))
}
}) && success
}
if success {
os.RemoveAll(d)
}
}
func TestGitMsgParsing(t *testing.T) {
t.Run("tree message with size 56", func(t *testing.T) {
const hdr = "f40888ea4515fe2e8eea617a16f5f50a45f652d894de3ad181d58de3aafb8f98 tree 56\x00"

View File

@@ -24,11 +24,13 @@ import (
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"slices"
"strconv"
"time"
transport "github.com/go-openapi/runtime/client"
@@ -182,7 +184,6 @@ type Gitea interface {
GiteaCommitStatusGetter
GiteaCommitStatusSetter
GiteaSetRepoOptions
GiteaTimelineFetcher
GetNotifications(Type string, since *time.Time) ([]*models.NotificationThread, error)
GetDoneNotifications(Type string, page int64) ([]*models.NotificationThread, error)
@@ -199,7 +200,32 @@ type Gitea interface {
GetCurrentUser() (*models.User, error)
}
type GiteaHeaderInterceptor struct {
Length int
http.RoundTripper
}
func (i *GiteaHeaderInterceptor) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := i.RoundTripper.RoundTrip(req)
if err != nil {
return nil, err
}
count_header := resp.Header["X-Total-Count"]
if len(count_header) == 1 {
i.Length, err = strconv.Atoi(resp.Header["X-Total-Count"][0])
if err != nil {
LogError("Converting X-Total-Count response header error", err)
i.Length = -1
return nil, err
}
} else {
i.Length = -1
}
return resp, nil
}
type GiteaTransport struct {
headers *GiteaHeaderInterceptor
transport *transport.Runtime
client *apiclient.GiteaAPI
}
@@ -212,7 +238,9 @@ func AllocateGiteaTransport(giteaUrl string) Gitea {
log.Panicln("Failed to parse gitea url:", err)
}
r.headers = &GiteaHeaderInterceptor{RoundTripper: http.DefaultTransport}
r.transport = transport.New(url.Host, apiclient.DefaultBasePath, [](string){url.Scheme})
r.transport.Transport = r.headers
r.transport.DefaultAuthentication = transport.BearerToken(giteaToken)
r.client = apiclient.New(r.transport, nil)
@@ -287,10 +315,9 @@ func (gitea *GiteaTransport) ManualMergePR(org, repo string, num int64, commitid
}
func (gitea *GiteaTransport) GetPullRequests(org, repo string) ([]*models.PullRequest, error) {
var page, limit int64
var page int64
prs := make([]*models.PullRequest, 0)
limit = 20
state := "open"
for {
@@ -302,16 +329,18 @@ func (gitea *GiteaTransport) GetPullRequests(org, repo string) ([]*models.PullRe
WithOwner(org).
WithRepo(repo).
WithState(&state).
WithPage(&page).
WithLimit(&limit),
WithPage(&page),
gitea.transport.DefaultAuthentication)
if err != nil {
return nil, fmt.Errorf("cannot fetch PR list for %s / %s : %w", org, repo, err)
}
if len(req.Payload) == 0 {
break
}
prs = slices.Concat(prs, req.Payload)
if len(req.Payload) < int(limit) {
if len(prs) >= gitea.headers.Length {
break
}
}
@@ -320,21 +349,23 @@ func (gitea *GiteaTransport) GetPullRequests(org, repo string) ([]*models.PullRe
}
func (gitea *GiteaTransport) GetCommitStatus(org, repo, hash string) ([]*models.CommitStatus, error) {
page := int64(1)
limit := int64(10)
var page int64
var res []*models.CommitStatus
for {
page++
r, err := gitea.client.Repository.RepoListStatuses(
repository.NewRepoListStatusesParams().WithDefaults().WithOwner(org).WithRepo(repo).WithSha(hash).WithPage(&page).WithLimit(&limit),
repository.NewRepoListStatusesParams().WithDefaults().WithOwner(org).WithRepo(repo).WithSha(hash).WithPage(&page),
gitea.transport.DefaultAuthentication)
if err != nil {
return res, err
}
if len(r.Payload) == 0 {
break
}
res = append(res, r.Payload...)
if len(r.Payload) < int(limit) {
if len(res) >= gitea.headers.Length {
break
}
}
@@ -377,19 +408,18 @@ func (gitea *GiteaTransport) GetRepository(org, pkg string) (*models.Repository,
}
func (gitea *GiteaTransport) GetPullRequestReviews(org, project string, PRnum int64) ([]*models.PullReview, error) {
limit := int64(20)
var page int64
var allReviews []*models.PullReview
for {
page++
reviews, err := gitea.client.Repository.RepoListPullReviews(
repository.NewRepoListPullReviewsParams().
WithDefaults().
WithOwner(org).
WithRepo(project).
WithIndex(PRnum).
WithPage(&page).
WithLimit(&limit),
WithPage(&page),
gitea.transport.DefaultAuthentication,
)
@@ -397,11 +427,13 @@ func (gitea *GiteaTransport) GetPullRequestReviews(org, project string, PRnum in
return nil, err
}
allReviews = slices.Concat(allReviews, reviews.Payload)
if len(reviews.Payload) < int(limit) {
if len(reviews.Payload) == 0 {
break
}
allReviews = slices.Concat(allReviews, reviews.Payload)
if len(allReviews) >= gitea.headers.Length {
break
}
page++
}
return allReviews, nil
@@ -469,7 +501,6 @@ const (
)
func (gitea *GiteaTransport) GetNotifications(Type string, since *time.Time) ([]*models.NotificationThread, error) {
bigLimit := int64(20)
ret := make([]*models.NotificationThread, 0, 100)
for page := int64(1); ; page++ {
@@ -477,7 +508,6 @@ func (gitea *GiteaTransport) GetNotifications(Type string, since *time.Time) ([]
WithDefaults().
WithSubjectType([]string{Type}).
WithStatusTypes([]string{"unread"}).
WithLimit(&bigLimit).
WithPage(&page)
if since != nil {
@@ -490,8 +520,11 @@ func (gitea *GiteaTransport) GetNotifications(Type string, since *time.Time) ([]
return nil, err
}
if len(list.Payload) == 0 {
break
}
ret = slices.Concat(ret, list.Payload)
if len(list.Payload) < int(bigLimit) {
if len(ret) >= gitea.headers.Length {
break
}
}
@@ -500,7 +533,6 @@ func (gitea *GiteaTransport) GetNotifications(Type string, since *time.Time) ([]
}
func (gitea *GiteaTransport) GetDoneNotifications(Type string, page int64) ([]*models.NotificationThread, error) {
limit := int64(20)
t := true
if page <= 0 {
@@ -511,7 +543,6 @@ func (gitea *GiteaTransport) GetDoneNotifications(Type string, page int64) ([]*m
WithAll(&t).
WithSubjectType([]string{Type}).
WithStatusTypes([]string{"read"}).
WithLimit(&limit).
WithPage(&page),
gitea.transport.DefaultAuthentication)
if err != nil {
@@ -564,9 +595,12 @@ func (gitea *GiteaTransport) GetOrganizationRepositories(orgName string) ([]*mod
if len(ret.Payload) == 0 {
break
}
repos = append(repos, ret.Payload...)
page++
if len(repos) >= gitea.headers.Length {
break
}
}
return repos, nil
@@ -780,15 +814,18 @@ func (gitea *GiteaTransport) GetTimeline(org, repo string, idx int64) ([]*models
resCount = len(res.Payload)
LogDebug("page:", page, "len:", resCount)
if resCount == 0 {
break
}
page++
for _, d := range res.Payload {
if d != nil {
retData = append(retData, d)
}
retData = append(retData, res.Payload...)
if len(retData) >= gitea.headers.Length {
break
}
}
LogDebug("total results:", len(retData))
retData = slices.DeleteFunc(retData, func(a *models.TimelineComment) bool { return a == nil })
slices.SortFunc(retData, func(a, b *models.TimelineComment) int {
return time.Time(b.Created).Compare(time.Time(a.Created))
})

View File

@@ -4,6 +4,8 @@ import (
"bufio"
"errors"
"fmt"
"os"
"path"
"slices"
"strings"
@@ -405,8 +407,80 @@ func (rs *PRSet) Merge(gitea GiteaReviewUnrequester, git Git) error {
err = git.GitExec(DefaultGitPrj, "merge", "--no-ff", "-m", msg, prjgit.Head.Sha)
if err != nil {
if resolveError := git.GitResolveSubmoduleFileConflict(DefaultGitPrj); resolveError != nil {
return fmt.Errorf("Merge failed. (%w): %w", err, resolveError)
status, statusErr := git.GitStatus(DefaultGitPrj)
if statusErr != nil {
return fmt.Errorf("Failed to merge: %w . Status also failed: %w", err, statusErr)
}
// we can only resolve conflicts with .gitmodules
for _, s := range status {
if s.Status == GitStatus_Unmerged {
panic("Can't handle conflicts yet")
if s.Path != ".gitmodules" {
return err
}
submodules, err := git.GitSubmoduleList(DefaultGitPrj, "MERGE_HEAD")
if err != nil {
return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
}
s1, err := git.GitExecWithOutput(DefaultGitPrj, "cat-file", "blob", s.States[0])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
s2, err := git.GitExecWithOutput(DefaultGitPrj, "cat-file", "blob", s.States[1])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
s3, err := git.GitExecWithOutput(DefaultGitPrj, "cat-file", "blob", s.States[2])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
subs1, err := ParseSubmodulesFile(strings.NewReader(s1))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
subs2, err := ParseSubmodulesFile(strings.NewReader(s2))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
subs3, err := ParseSubmodulesFile(strings.NewReader(s3))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
// merge from subs3 (target), subs1 (orig), subs2 (2-nd base that is missing from target base)
// this will update submodules
mergedSubs := slices.Concat(subs1, subs2, subs3)
var filteredSubs []Submodule = make([]Submodule, 0, max(len(subs1), len(subs2), len(subs3)))
nextSub:
for subName := range submodules {
for i := range mergedSubs {
if path.Base(mergedSubs[i].Path) == subName {
filteredSubs = append(filteredSubs, mergedSubs[i])
continue nextSub
}
}
return fmt.Errorf("Cannot find submodule for path: %s", subName)
}
out, err := os.Create(path.Join(git.GetPath(), DefaultGitPrj, ".gitmodules"))
if err != nil {
return fmt.Errorf("Can't open .gitmodules for writing: %w", err)
}
if err = WriteSubmodules(filteredSubs, out); err != nil {
return fmt.Errorf("Can't write .gitmodules: %w", err)
}
if out.Close(); err != nil {
return fmt.Errorf("Can't close .gitmodules: %w", err)
}
git.GitExecOrPanic(DefaultGitPrj, "add", ".gitmodules")
git.GitExecOrPanic(DefaultGitPrj, "-c", "core.editor=true", "merge", "--continue")
}
}
}

View File

@@ -15,9 +15,8 @@ import (
"src.opensuse.org/autogits/common/gitea-generated/models"
mock_common "src.opensuse.org/autogits/common/mock"
)
/*
func TestCockpit(t *testing.T) {
func TestCockpit(t *testing.T) {
common.SetLoggingLevel(common.LogLevelDebug)
gitea := common.AllocateGiteaTransport("https://src.opensuse.org")
tl, err := gitea.GetTimeline("cockpit", "cockpit", 29)
@@ -31,7 +30,7 @@ import (
}
t.Error(r)
}
}
*/
func reviewsToTimeline(reviews []*models.PullReview) []*models.TimelineComment {
timeline := make([]*models.TimelineComment, len(reviews))
@@ -940,7 +939,6 @@ func TestPRMerge(t *testing.T) {
pr: &models.PullRequest{
Base: &models.PRBranchInfo{
Sha: "e8b0de43d757c96a9d2c7101f4bff404e322f53a1fa4041fb85d646110c38ad4", // "base_add_b1"
Name: "master",
Repo: &models.Repository{
Name: "prj",
Owner: &models.User{
@@ -961,7 +959,6 @@ func TestPRMerge(t *testing.T) {
pr: &models.PullRequest{
Base: &models.PRBranchInfo{
Sha: "4fbd1026b2d7462ebe9229a49100c11f1ad6555520a21ba515122d8bc41328a8",
Name: "master",
Repo: &models.Repository{
Name: "prj",
Owner: &models.User{
@@ -981,7 +978,6 @@ func TestPRMerge(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
ctl := gomock.NewController(t)
mock := mock_common.NewMockGiteaPRTimelineFetcher(ctl)
reviewUnrequestMock := mock_common.NewMockGiteaReviewUnrequester(ctl)
reviewUnrequestMock.EXPECT().UnrequestReview(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)

View File

@@ -322,10 +322,13 @@ func GenerateObsPrjMeta(git common.Git, gitea common.Gitea, pr *models.PullReque
urlPkg = append(urlPkg, "onlybuild="+url.QueryEscape(pkg))
}
meta.ScmSync = pr.Head.Repo.CloneURL + "?" + strings.Join(urlPkg, "&") + "#" + pr.Head.Sha
if len(meta.ScmSync) >= 65535 {
return nil, errors.New("Reached max amount of package changes per request")
}
meta.Title = fmt.Sprintf("PR#%d to %s", pr.Index, pr.Base.Name)
// QE wants it published ... also we should not hardcode it here, since
// it is configurable via the :PullRequest project
// meta.PublicFlags = common.Flags{Contents: "<disable/>"}
// QE wants it published ... also we should not hardcode it here, since
// it is configurable via the :PullRequest project
// meta.PublicFlags = common.Flags{Contents: "<disable/>"}
meta.Groups = nil
meta.Persons = nil
@@ -633,6 +636,14 @@ func CleanupPullNotification(gitea common.Gitea, thread *models.NotificationThre
return false // cleaned up now, but the cleanup was not aleady done
}
func SetStatus(gitea common.Gitea, org, repo, hash string, status *models.CommitStatus) error {
_, err := gitea.SetCommitStatus(org, repo, hash, status)
if err != nil {
common.LogError(err)
}
return err
}
func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, error) {
dir, err := os.MkdirTemp(os.TempDir(), BotName)
common.PanicOnError(err)
@@ -837,6 +848,22 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
TargetURL: ObsWebHost + "/project/show/" + stagingProject,
}
if err != nil {
msg := "Unable to setup stage project " + stagingConfig.ObsProject
status.Status = common.CommitStatus_Fail
common.LogError(msg)
if !IsDryRun {
SetStatus(gitea, org, repo, pr.Head.Sha, status)
_, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, msg)
if err != nil {
common.LogError(err)
} else {
return true, nil
}
}
return false, nil
}
msg := "Changed source updated for build"
if change == RequestModificationProjectCreated {
msg = "Build is started in " + ObsWebHost + "/project/show/" +
@@ -845,8 +872,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
if len(stagingConfig.QA) > 0 {
msg = msg + "\nAdditional QA builds: \n"
}
gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
SetStatus(gitea, org, repo, pr.Head.Sha, status)
for _, setup := range stagingConfig.QA {
CreateQASubProject(stagingConfig, git, gitea, pr,
stagingProject,
@@ -870,32 +896,34 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e
}
buildStatus := ProcessBuildStatus(stagingResult, baseResult)
done := false
switch buildStatus {
case BuildStatusSummarySuccess:
status.Status = common.CommitStatus_Success
done = true
if !IsDryRun {
_, err := gitea.AddReviewComment(pr, common.ReviewStateApproved, "Build successful")
if err != nil {
common.LogError(err)
} else {
return true, nil
}
}
case BuildStatusSummaryFailed:
status.Status = common.CommitStatus_Fail
done = true
if !IsDryRun {
_, err := gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Build failed")
if err != nil {
common.LogError(err)
} else {
return true, nil
}
}
}
common.LogInfo("Build status:", buildStatus)
gitea.SetCommitStatus(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Head.Sha, status)
// waiting for build results -- nothing to do
if !IsDryRun {
if err = SetStatus(gitea, org, repo, pr.Head.Sha, status); err != nil {
return false, err
}
}
return done, nil
} else if err == NonActionableReviewError || err == NoReviewsFoundError {
return true, nil

View File

@@ -20,6 +20,7 @@ package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
@@ -268,6 +269,33 @@ func main() {
res.Write(BuildStatusSvg(nil, &common.PackageBuildStatus{Package: pkg, Code: "unknown"}))
})
http.HandleFunc("GET /search", func(res http.ResponseWriter, req *http.Request) {
common.LogInfo("GET /serach?" + req.URL.RawQuery)
queries := req.URL.Query()
if !queries.Has("q") {
res.WriteHeader(400)
return
}
names := queries["q"]
if len(names) != 1 {
res.WriteHeader(400)
return
}
packages := FindPackages(names[0])
data, err := json.MarshalIndent(packages, "", " ")
if err != nil {
res.WriteHeader(500)
common.LogError("Error in marshalling data.", err)
return
}
res.Write(data)
res.Header().Add("content-type", "application/json")
res.WriteHeader(200)
})
http.HandleFunc("GET /buildlog/{Project}/{Package}/{Repository}/{Arch}", func(res http.ResponseWriter, req *http.Request) {
prj := req.PathValue("Project")
pkg := req.PathValue("Package")

View File

@@ -29,13 +29,15 @@ func UpdateResults(r *common.BuildResult) {
RepoStatusLock.Lock()
defer RepoStatusLock.Unlock()
updateResultsWithoutLocking(r)
}
func updateResultsWithoutLocking(r *common.BuildResult) {
key := "result." + r.Project + "/" + r.Repository + "/" + r.Arch
common.LogDebug(" + Updating", key)
data, err := redisClient.HGetAll(context.Background(), key).Result()
if err != nil {
common.LogError("Failed fetching build results for", key, err)
}
common.LogDebug(" + Update size", len(data))
reset_time := time.Date(1000, 1, 1, 1, 1, 1, 1, time.Local)
for _, pkg := range r.Status {
@@ -110,6 +112,27 @@ func FindRepoResults(project, repo string) []*common.BuildResult {
return ret
}
func FindPackages(pkg string) []string {
RepoStatusLock.RLock()
defer RepoStatusLock.RUnlock()
data := make([]string, 0, 100)
for _, repo := range RepoStatus {
for _, status := range repo.Status {
if pkg == status.Package {
entry := repo.Project + "/" + pkg
if idx, found := slices.BinarySearch(data, entry); !found {
data = slices.Insert(data, idx, entry)
if len(data) >= 100 {
return data
}
}
}
}
}
return data
}
func FindAndUpdateProjectResults(project string) []*common.BuildResult {
res := FindProjectResults(project)
wg := &sync.WaitGroup{}
@@ -161,6 +184,8 @@ func RescanRepositories() error {
RepoStatusLock.Unlock()
var count int
projectsLooked := make([]string, 0, 10000)
for {
var data []string
data, cursor, err = redisClient.ScanType(ctx, cursor, "", 1000, "hash").Result()
@@ -169,6 +194,7 @@ func RescanRepositories() error {
return err
}
wg := &sync.WaitGroup{}
RepoStatusLock.Lock()
for _, repo := range data {
r := strings.Split(repo, "/")
@@ -180,14 +206,28 @@ func RescanRepositories() error {
Repository: r[1],
Arch: r[2],
}
if pos, found := slices.BinarySearchFunc(RepoStatus, d, common.BuildResultComp); found {
var pos int
var found bool
if pos, found = slices.BinarySearchFunc(RepoStatus, d, common.BuildResultComp); found {
RepoStatus[pos].Dirty = true
} else {
d.Dirty = true
RepoStatus = slices.Insert(RepoStatus, pos, d)
count++
}
// fetch all keys, one per non-maintenance/non-home: projects, for package search
if idx, found := slices.BinarySearch(projectsLooked, d.Project); !found && !strings.Contains(d.Project, ":Maintenance:") && (len(d.Project) < 5 || d.Project[0:5] != "home:") {
projectsLooked = slices.Insert(projectsLooked, idx, d.Project)
wg.Add(1)
go func(r *common.BuildResult) {
updateResultsWithoutLocking(r)
wg.Done()
}(RepoStatus[pos])
}
}
wg.Wait()
RepoStatusLock.Unlock()
if cursor == 0 {

View File

@@ -317,10 +317,15 @@ func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
}
PrjGitTitle, PrjGitBody := PrjGitDescription(prset)
if PrjGitPR.PR.User.UserName == CurrentUser.UserName {
if PrjGitPR.PR.Title != PrjGitTitle || PrjGitPR.PR.Body != PrjGitBody {
common.LogDebug("New title:", PrjGitTitle)
common.LogDebug(PrjGitBody)
}
} else {
// TODO: find our first comment in timeline
}
if !common.IsDryRun {
if headCommit != newHeadCommit {