SHA256
1
0

13 Commits

Author SHA256 Message Date
c25d3be44e direct: add systemd unit file 2025-11-05 13:24:54 +01:00
8db558891a direct: remove config.Branch clobbering
use our own copy of branch instead of writing it in the config.
This should fix handling of default branches where the default
branch differs between repositories.
2025-11-04 18:00:21 +01:00
0e06ba5993 common: classifying rm branches on name
Branches with suffixes

  -rm
  -removed
  -deleted

are now classified as removed. This is important in case project
config refers to default branch names which must exist so we need
to be able to classify such branches to either use them or ignore
them
2025-11-04 18:00:21 +01:00
736769d630 direct: add a repo with branch but no submodule 2025-11-04 18:00:21 +01:00
93c970d0dd direct: move logging to common.Log* function 2025-11-04 18:00:21 +01:00
5544a65947 obs-staging-bot: Expand possible branch of QA repos
That way a source merge of any product is not triggering rebuilds in
pull request QA sub projects. We may need a config option here to
enable/disable this.
2025-11-03 17:54:57 +01:00
918723d57b Merge commit '55846562c1d9dcb395e545f7c8e0bcb74c47b85693f4e955ef488530781b9bf2'
PR!88
2025-11-03 17:49:45 +01:00
a418b48809 pr: process PR on comments, not issue changes 2025-10-31 13:07:18 +01:00
55846562c1 Add simple readme for gitea_status_proxy 2025-10-31 10:33:58 +01:00
95c7770cad Change log level for auth errors 2025-10-31 10:33:58 +01:00
1b900e3202 Properly proxy json input directly to gitea 2025-10-31 10:33:51 +01:00
d083acfd1c Be more verbose about authentication errors 2025-10-30 13:09:17 +01:00
244160e20e Update authorization headers
For gitea API AuthorizationHeaderToken tokens must be prepended with "token" followed by a space, also fix content type
2025-10-30 13:09:17 +01:00
18 changed files with 611 additions and 837 deletions

2
.gitignore vendored
View File

@@ -1,4 +1,2 @@
*.osc
*.conf
utils/gitmodules-automerge/gitmodules-automerge
utils/hujson/hujson

View File

@@ -96,7 +96,6 @@ Provides: /usr/bin/hujson
%description utils
HuJSON to JSON parser, using stdin -> stdout pipe
gitmodules-automerge fixes conflicts in .gitmodules conflicts during merge
%package workflow-direct
@@ -128,9 +127,6 @@ go build \
go build \
-C utils/hujson \
-buildmode=pie
go build \
-C utils/gitmodules-automerge \
-buildmode=pie
go build \
-C gitea-events-rabbitmq-publisher \
-buildmode=pie
@@ -177,9 +173,9 @@ install -D -m0644 systemd/obs-staging-bot.service
install -D -m0755 obs-status-service/obs-status-service %{buildroot}%{_bindir}/obs-status-service
install -D -m0644 systemd/obs-status-service.service %{buildroot}%{_unitdir}/obs-status-service.service
install -D -m0755 workflow-direct/workflow-direct %{buildroot}%{_bindir}/workflow-direct
install -D -m0644 systemd/workflow-direct@.service %{buildroot}%{_unitdir}/workflow-direct@.service
install -D -m0755 workflow-pr/workflow-pr %{buildroot}%{_bindir}/workflow-pr
install -D -m0755 utils/hujson/hujson %{buildroot}%{_bindir}/hujson
install -D -m0755 utils/gitmodules-automerge/gitmodules-automerge %{buildroot}%{_bindir}/gitmodules-automerge
%pre gitea-events-rabbitmq-publisher
%service_add_pre gitea-events-rabbitmq-publisher.service
@@ -261,12 +257,12 @@ install -D -m0755 utils/gitmodules-automerge/gitmodules-automerge
%files utils
%license COPYING
%{_bindir}/hujson
%{_bindir}/gitmodules-automerge
%files workflow-direct
%license COPYING
%doc workflow-direct/README.md
%{_bindir}/workflow-direct
%{_unitdir}/workflow-direct@.service
%files workflow-pr
%license COPYING

View File

@@ -1,296 +0,0 @@
package common
import (
"errors"
"fmt"
"io"
)
const (
GitStatus_Untracked = 0
GitStatus_Modified = 1
GitStatus_Ignored = 2
GitStatus_Unmerged = 3 // States[0..3] -- Stage1, Stage2, Stage3 of merge objects
GitStatus_Renamed = 4 // orig name in States[0]
)
type GitStatusData struct {
Path string
Status int
States [3]string
/*
<sub> A 4 character field describing the submodule state.
"N..." when the entry is not a submodule.
"S<c><m><u>" when the entry is a submodule.
<c> is "C" if the commit changed; otherwise ".".
<m> is "M" if it has tracked changes; otherwise ".".
<u> is "U" if there are untracked changes; otherwise ".".
*/
SubmoduleChanges string
}
func parseGit_HexString(data io.ByteReader) (string, error) {
str := make([]byte, 0, 32)
for {
c, err := data.ReadByte()
if err != nil {
return "", err
}
switch {
case c == 0 || c == ' ':
return string(str), nil
case c >= 'a' && c <= 'f':
case c >= 'A' && c <= 'F':
case c >= '0' && c <= '9':
default:
return "", errors.New("Invalid character in hex string:" + string(c))
}
str = append(str, c)
}
}
func parseGit_String(data io.ByteReader) (string, error) {
str := make([]byte, 0, 100)
for {
c, err := data.ReadByte()
if err != nil {
return "", errors.New("Unexpected EOF. Expected NUL string term")
}
if c == 0 || c == ' ' {
return string(str), nil
}
str = append(str, c)
}
}
func parseGit_StringWithSpace(data io.ByteReader) (string, error) {
str := make([]byte, 0, 100)
for {
c, err := data.ReadByte()
if err != nil {
return "", errors.New("Unexpected EOF. Expected NUL string term")
}
if c == 0 {
return string(str), nil
}
str = append(str, c)
}
}
func skipGitStatusEntry(data io.ByteReader, skipSpaceLen int) error {
for skipSpaceLen > 0 {
c, err := data.ReadByte()
if err != nil {
return err
}
if c == ' ' {
skipSpaceLen--
}
}
return nil
}
func parseSingleStatusEntry(data io.ByteReader) (*GitStatusData, error) {
ret := GitStatusData{}
statusType, err := data.ReadByte()
if err != nil {
return nil, nil
}
switch statusType {
case '1':
var err error
if err = skipGitStatusEntry(data, 8); err != nil {
return nil, err
}
ret.Status = GitStatus_Modified
ret.Path, err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
case '2':
var err error
if err = skipGitStatusEntry(data, 9); err != nil {
return nil, err
}
ret.Status = GitStatus_Renamed
ret.Path, err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
ret.States[0], err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
case '?':
var err error
if err = skipGitStatusEntry(data, 1); err != nil {
return nil, err
}
ret.Status = GitStatus_Untracked
ret.Path, err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
case '!':
var err error
if err = skipGitStatusEntry(data, 1); err != nil {
return nil, err
}
ret.Status = GitStatus_Ignored
ret.Path, err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
case 'u':
var err error
if err = skipGitStatusEntry(data, 2); err != nil {
return nil, err
}
if ret.SubmoduleChanges, err = parseGit_String(data); err != nil {
return nil, err
}
if err = skipGitStatusEntry(data, 4); err != nil {
return nil, err
}
if ret.States[0], err = parseGit_HexString(data); err != nil {
return nil, err
}
if ret.States[1], err = parseGit_HexString(data); err != nil {
return nil, err
}
if ret.States[2], err = parseGit_HexString(data); err != nil {
return nil, err
}
ret.Status = GitStatus_Unmerged
ret.Path, err = parseGit_StringWithSpace(data)
if err != nil {
return nil, err
}
default:
return nil, errors.New("Invalid status type" + string(statusType))
}
return &ret, nil
}
func parseGitStatusData(data io.ByteReader) (Data, error) {
ret := make([]GitStatusData, 0, 10)
for {
data, err := parseSingleStatusEntry(data)
if err != nil {
return nil, err
} else if data == nil {
break
}
ret = append(ret, *data)
}
return ret, nil
}
type Data interface{}
type CommitStatus int
const (
Add CommitStatus = iota
Rm
Copy
Modify
Rename
TypeChange
Unmerged
Unknown
)
type GitDiffRawData struct {
SrcMode, DstMode string
SrcCommit, DstCommit string
Status CommitStatus
Src, Dst string
}
func parseGit_DiffIndexStatus(data io.ByteReader, d *GitDiffRawData) error {
b, err := data.ReadByte()
if err != nil {
return err
}
switch b {
case 'A':
d.Status = Add
case 'C':
d.Status = Copy
case 'D':
d.Status = Rm
case 'M':
d.Status = Modify
case 'R':
d.Status = Rename
case 'T':
d.Status = TypeChange
case 'U':
d.Status = Unmerged
case 'X':
return fmt.Errorf("Unexpected unknown change type. This is a git bug")
}
_, err = parseGit_StringWithSpace(data)
if err != nil {
return err
}
return nil
}
func parseSingleGitDiffIndexRawData(data io.ByteReader) (*GitDiffRawData, error) {
var ret GitDiffRawData
b, err := data.ReadByte()
if err != nil {
return nil, err
}
if b != ':' {
return nil, fmt.Errorf("Expected ':' but got '%s'", string(b))
}
if ret.SrcMode, err = parseGit_String(data); err != nil {
return nil, err
}
if ret.DstMode, err = parseGit_String(data); err != nil {
return nil, err
}
if ret.Src, err = parseGit_String(data); err != nil {
return nil, err
}
if ret.Dst, err = parseGit_String(data); err != nil {
return nil, err
}
if err = parseGit_DiffIndexStatus(data, &ret); err != nil {
return nil, err
}
ret.Dst = ret.Src
switch ret.Status {
case Copy, Rename:
if ret.Src, err = parseGit_StringWithSpace(data); err != nil {
return nil, err
}
}
return &ret, nil
}
func parseGitDiffIndexRawData(data io.ByteReader) (Data, error) {
ret := make([]GitDiffRawData, 0, 10)
for {
data, err := parseSingleGitDiffIndexRawData(data)
if err != nil {
return nil, err
} else if data == nil {
break
}
ret = append(ret, *data)
}
return ret, nil
}

View File

@@ -19,7 +19,9 @@ package common
*/
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"os"
@@ -42,11 +44,6 @@ type GitDirectoryLister interface {
GitDirectoryList(gitPath, commitId string) (dirlist map[string]string, err error)
}
type GitSubmoduleFileConflictResolver interface {
GitResolveConflicts(cwd, MergeBase, Head, MergeHead string) error
GitResolveSubmoduleFileConflict(s GitStatusData, cwd, mergeBase, head, mergeHead string) error
}
type GitStatusLister interface {
GitStatus(cwd string) ([]GitStatusData, error)
}
@@ -78,7 +75,6 @@ type Git interface {
GitExecQuietOrPanic(cwd string, params ...string)
GitDiffLister
GitSubmoduleFileConflictResolver
}
type GitHandlerImpl struct {
@@ -357,20 +353,13 @@ func (e *GitHandlerImpl) GitExecWithOutput(cwd string, params ...string) (string
cmd.Env = []string{
"GIT_CEILING_DIRECTORIES=" + e.GitPath,
"GIT_CONFIG_GLOBAL=/dev/null",
"GIT_AUTHOR_NAME=" + e.GitCommiter,
"GIT_COMMITTER_NAME=" + e.GitCommiter,
"EMAIL=not@exist@src.opensuse.org",
"GIT_LFS_SKIP_SMUDGE=1",
"GIT_LFS_SKIP_PUSH=1",
"GIT_SSH_COMMAND=/usr/bin/ssh -o StrictHostKeyChecking=yes",
}
if len(e.GitEmail) > 0 {
cmd.Env = append(cmd.Env, "EMAIL="+e.GitEmail)
}
if len(e.GitCommiter) > 0 {
cmd.Env = append(cmd.Env,
"GIT_AUTHOR_NAME="+e.GitCommiter,
"GIT_COMMITTER_NAME="+e.GitCommiter,
)
}
if len(ExtraGitParams) > 0 {
cmd.Env = append(cmd.Env, ExtraGitParams...)
}
@@ -1013,10 +1002,193 @@ func (e *GitHandlerImpl) GitSubmoduleCommitId(cwd, packageName, commitId string)
return subCommitId, len(subCommitId) > 0
}
func (e *GitHandlerImpl) GitExecWithDataParse(cwd string, dataprocessor func(io.ByteReader) (Data, error), gitcmd string, args ...string) (Data, error) {
LogDebug("getting", gitcmd)
args = append([]string{gitcmd}, args...)
cmd := exec.Command("/usr/bin/git", args...)
const (
GitStatus_Untracked = 0
GitStatus_Modified = 1
GitStatus_Ignored = 2
GitStatus_Unmerged = 3 // States[0..3] -- Stage1, Stage2, Stage3 of merge objects
GitStatus_Renamed = 4 // orig name in States[0]
)
type GitStatusData struct {
Path string
Status int
States [3]string
/*
<sub> A 4 character field describing the submodule state.
"N..." when the entry is not a submodule.
"S<c><m><u>" when the entry is a submodule.
<c> is "C" if the commit changed; otherwise ".".
<m> is "M" if it has tracked changes; otherwise ".".
<u> is "U" if there are untracked changes; otherwise ".".
*/
SubmoduleChanges string
}
func parseGitStatusHexString(data io.ByteReader) (string, error) {
str := make([]byte, 0, 32)
for {
c, err := data.ReadByte()
if err != nil {
return "", err
}
switch {
case c == 0 || c == ' ':
return string(str), nil
case c >= 'a' && c <= 'f':
case c >= 'A' && c <= 'F':
case c >= '0' && c <= '9':
default:
return "", errors.New("Invalid character in hex string:" + string(c))
}
str = append(str, c)
}
}
func parseGitStatusString(data io.ByteReader) (string, error) {
str := make([]byte, 0, 100)
for {
c, err := data.ReadByte()
if err != nil {
return "", errors.New("Unexpected EOF. Expected NUL string term")
}
if c == 0 || c == ' ' {
return string(str), nil
}
str = append(str, c)
}
}
func parseGitStatusStringWithSpace(data io.ByteReader) (string, error) {
str := make([]byte, 0, 100)
for {
c, err := data.ReadByte()
if err != nil {
return "", errors.New("Unexpected EOF. Expected NUL string term")
}
if c == 0 {
return string(str), nil
}
str = append(str, c)
}
}
func skipGitStatusEntry(data io.ByteReader, skipSpaceLen int) error {
for skipSpaceLen > 0 {
c, err := data.ReadByte()
if err != nil {
return err
}
if c == ' ' {
skipSpaceLen--
}
}
return nil
}
func parseSingleStatusEntry(data io.ByteReader) (*GitStatusData, error) {
ret := GitStatusData{}
statusType, err := data.ReadByte()
if err != nil {
return nil, nil
}
switch statusType {
case '1':
var err error
if err = skipGitStatusEntry(data, 8); err != nil {
return nil, err
}
ret.Status = GitStatus_Modified
ret.Path, err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
case '2':
var err error
if err = skipGitStatusEntry(data, 9); err != nil {
return nil, err
}
ret.Status = GitStatus_Renamed
ret.Path, err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
ret.States[0], err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
case '?':
var err error
if err = skipGitStatusEntry(data, 1); err != nil {
return nil, err
}
ret.Status = GitStatus_Untracked
ret.Path, err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
case '!':
var err error
if err = skipGitStatusEntry(data, 1); err != nil {
return nil, err
}
ret.Status = GitStatus_Ignored
ret.Path, err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
case 'u':
var err error
if err = skipGitStatusEntry(data, 2); err != nil {
return nil, err
}
if ret.SubmoduleChanges, err = parseGitStatusString(data); err != nil {
return nil, err
}
if err = skipGitStatusEntry(data, 4); err != nil {
return nil, err
}
if ret.States[0], err = parseGitStatusHexString(data); err != nil {
return nil, err
}
if ret.States[1], err = parseGitStatusHexString(data); err != nil {
return nil, err
}
if ret.States[2], err = parseGitStatusHexString(data); err != nil {
return nil, err
}
ret.Status = GitStatus_Unmerged
ret.Path, err = parseGitStatusStringWithSpace(data)
if err != nil {
return nil, err
}
default:
return nil, errors.New("Invalid status type" + string(statusType))
}
return &ret, nil
}
func parseGitStatusData(data io.ByteReader) ([]GitStatusData, error) {
ret := make([]GitStatusData, 0, 10)
for {
data, err := parseSingleStatusEntry(data)
if err != nil {
return nil, err
} else if data == nil {
break
}
ret = append(ret, *data)
}
return ret, nil
}
func (e *GitHandlerImpl) GitStatus(cwd string) (ret []GitStatusData, err error) {
LogDebug("getting git-status()")
cmd := exec.Command("/usr/bin/git", "status", "--porcelain=2", "-z")
cmd.Env = []string{
"GIT_CEILING_DIRECTORIES=" + e.GitPath,
"GIT_LFS_SKIP_SMUDGE=1",
@@ -1033,12 +1205,7 @@ func (e *GitHandlerImpl) GitExecWithDataParse(cwd string, dataprocessor func(io.
LogError("Error running command", cmd.Args, err)
}
return dataprocessor(bytes.NewReader(out))
}
func (e *GitHandlerImpl) GitStatus(cwd string) (ret []GitStatusData, err error) {
data, err := e.GitExecWithDataParse(cwd, parseGitStatusData, "status", "--porcelain=2", "-z")
return data.([]GitStatusData), err
return parseGitStatusData(bufio.NewReader(bytes.NewReader(out)))
}
func (e *GitHandlerImpl) GitDiff(cwd, base, head string) (string, error) {
@@ -1063,122 +1230,3 @@ func (e *GitHandlerImpl) GitDiff(cwd, base, head string) (string, error) {
return string(out), nil
}
func (e *GitHandlerImpl) GitDiffIndex(cwd, commit string) ([]GitDiffRawData, error) {
data, err := e.GitExecWithDataParse("diff-index", parseGitDiffIndexRawData, cwd, "diff-index", "-z", "--raw", "--full-index", "--submodule=short", "HEAD")
return data.([]GitDiffRawData), err
}
func (git *GitHandlerImpl) GitResolveConflicts(cwd, mergeBase, head, mergeHead string) error {
status, err := git.GitStatus(cwd)
if err != nil {
return fmt.Errorf("Status failed: %w", err)
}
// we can only resolve conflicts with .gitmodules
for _, s := range status {
if s.Status == GitStatus_Unmerged && s.Path == ".gitmodules" {
if err := git.GitResolveSubmoduleFileConflict(s, cwd, mergeBase, head, mergeHead); err != nil {
return err
}
} else if s.Status == GitStatus_Unmerged {
return fmt.Errorf("Cannot automatically resolve conflict: %s", s.Path)
}
}
return git.GitExec(cwd, "-c", "core.editor=true", "merge", "--continue")
}
func (git *GitHandlerImpl) GitResolveSubmoduleFileConflict(s GitStatusData, cwd, mergeBase, head, mergeHead string) error {
submodules1, err := git.GitSubmoduleList(cwd, mergeBase)
if err != nil {
return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
}
/*
submodules2, err := git.GitSubmoduleList(cwd, head)
if err != nil {
return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
}
*/
submodules3, err := git.GitSubmoduleList(cwd, mergeHead)
if err != nil {
return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
}
// find modified submodules in the mergeHead
modifiedSubmodules := make([]string, 0, 10)
removedSubmodules := make([]string, 0, 10)
addedSubmodules := make([]string, 0, 10)
for submodulePath, oldHash := range submodules1 {
if newHash, found := submodules3[submodulePath]; found && newHash != oldHash {
modifiedSubmodules = append(modifiedSubmodules, submodulePath)
} else if !found {
removedSubmodules = append(removedSubmodules, submodulePath)
}
}
for submodulePath, _ := range submodules3 {
if _, found := submodules1[submodulePath]; !found {
addedSubmodules = append(addedSubmodules, submodulePath)
}
}
// We need to adjust the `submodules` list by the pending changes to the index
s1, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[0])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
s2, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[1])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
s3, err := git.GitExecWithOutput(cwd, "cat-file", "blob", s.States[2])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
_, err = ParseSubmodulesFile(strings.NewReader(s1))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
subs2, err := ParseSubmodulesFile(strings.NewReader(s2))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
subs3, err := ParseSubmodulesFile(strings.NewReader(s3))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
overrideSet := make([]Submodule, 0, len(addedSubmodules)+len(modifiedSubmodules))
for i := range subs3 {
if slices.Contains(addedSubmodules, subs3[i].Path) || slices.Contains(modifiedSubmodules, subs3[i].Path) {
overrideSet = append(overrideSet, subs3[i])
}
}
// merge from subs1 (merge-base), subs2 (changes in base since merge-base HEAD), subs3 (merge_source MERGE_HEAD)
// this will update submodules
SubmoduleCompare := func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) }
CompactCompare := func(a, b Submodule) bool { return a.Path == b.Path }
// remove submodules that are removed in the PR
subs2 = slices.DeleteFunc(subs2, func(a Submodule) bool { return slices.Contains(removedSubmodules, a.Path) })
mergedSubs := slices.Concat(overrideSet, subs2)
slices.SortStableFunc(mergedSubs, SubmoduleCompare)
filteredSubs := slices.CompactFunc(mergedSubs, CompactCompare)
out, err := os.Create(path.Join(git.GetPath(), cwd, ".gitmodules"))
if err != nil {
return fmt.Errorf("Can't open .gitmodules for writing: %w", err)
}
if err = WriteSubmodules(filteredSubs, out); err != nil {
return fmt.Errorf("Can't write .gitmodules: %w", err)
}
if out.Close(); err != nil {
return fmt.Errorf("Can't close .gitmodules: %w", err)
}
git.GitExecOrPanic(cwd, "add", ".gitmodules")
return nil
}

View File

@@ -24,7 +24,6 @@ import (
"os"
"os/exec"
"path"
"runtime/debug"
"slices"
"strings"
"testing"
@@ -94,145 +93,6 @@ func TestGitClone(t *testing.T) {
}
}
func TestSubmoduleConflictResolution(t *testing.T) {
tests := []struct {
name string
checkout, merge string
result string
merge_fail bool
}{
{
name: "adding two submodules",
checkout: "base_add_b1",
merge: "base_add_b2",
result: `[submodule "pkgA"]
path = pkgA
url = ../pkgA
[submodule "pkgB"]
path = pkgB
url = ../pkgB
[submodule "pkgB1"]
path = pkgB1
url = ../pkgB1
[submodule "pkgB2"]
path = pkgB2
url = ../pkgB2
[submodule "pkgC"]
path = pkgC
url = ../pkgC
`,
},
{
name: "remove one module and add another",
checkout: "base_rm_c",
merge: "base_add_b2",
result: `[submodule "pkgA"]
path = pkgA
url = ../pkgA
[submodule "pkgB"]
path = pkgB
url = ../pkgB
[submodule "pkgB2"]
path = pkgB2
url = ../pkgB2
`,
},
{
name: "add one and remove another",
checkout: "base_add_b2",
merge: "base_rm_c",
result: `[submodule "pkgA"]
path = pkgA
url = ../pkgA
[submodule "pkgB"]
path = pkgB
url = ../pkgB
[submodule "pkgB2"]
path = pkgB2
url = ../pkgB2
`,
},
{
name: "rm modified submodule",
checkout: "base_modify_c",
merge: "base_rm_c",
merge_fail: true,
},
{
name: "modified removed submodule",
checkout: "base_rm_c",
merge: "base_modify_c",
merge_fail: true,
},
}
d, err := os.MkdirTemp(os.TempDir(), "submoduletests")
if err != nil {
t.Fatal(err)
}
cwd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
cmd := exec.Command(cwd + "/test_repo_setup.sh")
cmd.Dir = d
_, err = cmd.Output()
if err != nil {
t.Fatal(err)
}
gh, err := AllocateGitWorkTree(d, "test", "foo@example.com")
if err != nil {
t.Fatal(err)
}
success := true
noErrorOrFail := func(t *testing.T, err error) {
if err != nil {
t.Fatal(string(debug.Stack()), err)
}
}
for _, test := range tests {
success = t.Run(test.name, func(t *testing.T) {
git, err := gh.ReadExistingPath("prjgit")
defer git.Close()
if err != nil {
t.Fatal(err)
}
noErrorOrFail(t, git.GitExec("", "reset", "--hard"))
noErrorOrFail(t, git.GitExec("", "checkout", "-B", "test", test.checkout))
// noErrorOrFail(t, git.GitExec("", "merge", test.checkout))
err = git.GitExec("", "merge", test.merge)
if err == nil {
t.Fatal("expected a conflict")
}
err = git.GitResolveConflicts("", "main", test.checkout, test.merge)
if err != nil {
if test.merge_fail {
return // success
}
t.Fatal(err)
}
if test.merge_fail {
t.Fatal("Expected fail but succeeded?")
}
data, err := os.ReadFile(git.GetPath() + "/.gitmodules")
if err != nil {
t.Fatal("Cannot read .gitmodules.", err)
}
if string(data) != test.result {
t.Error("Expected", len(test.result), test.result, "but have", len(data), string(data))
}
}) && success
}
if success {
os.RemoveAll(d)
}
}
func TestGitMsgParsing(t *testing.T) {
t.Run("tree message with size 56", func(t *testing.T) {
const hdr = "f40888ea4515fe2e8eea617a16f5f50a45f652d894de3ad181d58de3aafb8f98 tree 56\x00"
@@ -724,15 +584,12 @@ func TestGitStatusParse(t *testing.T) {
if err != nil {
t.Fatal(err)
}
res := r.([]GitStatusData)
if len(res) != len(test.res) {
t.Fatal("len(r):", len(res), "is not expected", len(test.res))
if len(r) != len(test.res) {
t.Fatal("len(r):", len(r), "is not expected", len(test.res))
}
for _, expected := range test.res {
if !slices.Contains(res, expected) {
if !slices.Contains(r, expected) {
t.Fatal("result", r, "doesn't contains expected", expected)
}
}

View File

@@ -4,6 +4,8 @@ import (
"bufio"
"errors"
"fmt"
"os"
"path"
"slices"
"strings"
@@ -431,8 +433,80 @@ func (rs *PRSet) Merge(gitea GiteaReviewUnrequester, git Git) error {
err = git.GitExec(DefaultGitPrj, "merge", "--no-ff", "-m", msg, prjgit.Head.Sha)
if err != nil {
if resolveError := git.GitResolveConflicts(DefaultGitPrj, prjgit.MergeBase, prjgit.Base.Sha, prjgit.Head.Sha); resolveError != nil {
return fmt.Errorf("Merge failed. (%w): %w", err, resolveError)
status, statusErr := git.GitStatus(DefaultGitPrj)
if statusErr != nil {
return fmt.Errorf("Failed to merge: %w . Status also failed: %w", err, statusErr)
}
// we can only resolve conflicts with .gitmodules
for _, s := range status {
if s.Status == GitStatus_Unmerged {
panic("Can't handle conflicts yet")
if s.Path != ".gitmodules" {
return err
}
submodules, err := git.GitSubmoduleList(DefaultGitPrj, "MERGE_HEAD")
if err != nil {
return fmt.Errorf("Failed to fetch submodules during merge resolution: %w", err)
}
s1, err := git.GitExecWithOutput(DefaultGitPrj, "cat-file", "blob", s.States[0])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
s2, err := git.GitExecWithOutput(DefaultGitPrj, "cat-file", "blob", s.States[1])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
s3, err := git.GitExecWithOutput(DefaultGitPrj, "cat-file", "blob", s.States[2])
if err != nil {
return fmt.Errorf("Failed fetching data during .gitmodules merge resoulution: %w", err)
}
subs1, err := ParseSubmodulesFile(strings.NewReader(s1))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
subs2, err := ParseSubmodulesFile(strings.NewReader(s2))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
subs3, err := ParseSubmodulesFile(strings.NewReader(s3))
if err != nil {
return fmt.Errorf("Failed parsing submodule file [%s] in merge: %w", s.States[0], err)
}
// merge from subs3 (target), subs1 (orig), subs2 (2-nd base that is missing from target base)
// this will update submodules
mergedSubs := slices.Concat(subs1, subs2, subs3)
var filteredSubs []Submodule = make([]Submodule, 0, max(len(subs1), len(subs2), len(subs3)))
nextSub:
for subName := range submodules {
for i := range mergedSubs {
if path.Base(mergedSubs[i].Path) == subName {
filteredSubs = append(filteredSubs, mergedSubs[i])
continue nextSub
}
}
return fmt.Errorf("Cannot find submodule for path: %s", subName)
}
out, err := os.Create(path.Join(git.GetPath(), DefaultGitPrj, ".gitmodules"))
if err != nil {
return fmt.Errorf("Can't open .gitmodules for writing: %w", err)
}
if err = WriteSubmodules(filteredSubs, out); err != nil {
return fmt.Errorf("Can't write .gitmodules: %w", err)
}
if out.Close(); err != nil {
return fmt.Errorf("Can't close .gitmodules: %w", err)
}
git.GitExecOrPanic(DefaultGitPrj, "add", ".gitmodules")
git.GitExecOrPanic(DefaultGitPrj, "-c", "core.editor=true", "merge", "--continue")
}
}
}

View File

@@ -15,23 +15,22 @@ import (
"src.opensuse.org/autogits/common/gitea-generated/models"
mock_common "src.opensuse.org/autogits/common/mock"
)
/*
func TestCockpit(t *testing.T) {
common.SetLoggingLevel(common.LogLevelDebug)
gitea := common.AllocateGiteaTransport("https://src.opensuse.org")
tl, err := gitea.GetTimeline("cockpit", "cockpit", 29)
if err != nil {
t.Fatal("Fail to timeline", err)
}
t.Log(tl)
r, err := common.FetchGiteaReviews(gitea, []string{}, "cockpit", "cockpit", 29)
if err != nil {
t.Fatal("Error:", err)
}
t.Error(r)
func TestCockpit(t *testing.T) {
common.SetLoggingLevel(common.LogLevelDebug)
gitea := common.AllocateGiteaTransport("https://src.opensuse.org")
tl, err := gitea.GetTimeline("cockpit", "cockpit", 29)
if err != nil {
t.Fatal("Fail to timeline", err)
}
t.Log(tl)
r, err := common.FetchGiteaReviews(gitea, []string{}, "cockpit", "cockpit", 29)
if err != nil {
t.Fatal("Error:", err)
}
t.Error(r)
}
*/
func reviewsToTimeline(reviews []*models.PullReview) []*models.TimelineComment {
timeline := make([]*models.TimelineComment, len(reviews))
@@ -941,7 +940,6 @@ func TestPRMerge(t *testing.T) {
pr: &models.PullRequest{
Base: &models.PRBranchInfo{
Sha: "e8b0de43d757c96a9d2c7101f4bff404e322f53a1fa4041fb85d646110c38ad4", // "base_add_b1"
Name: "master",
Repo: &models.Repository{
Name: "prj",
Owner: &models.User{
@@ -962,7 +960,6 @@ func TestPRMerge(t *testing.T) {
pr: &models.PullRequest{
Base: &models.PRBranchInfo{
Sha: "4fbd1026b2d7462ebe9229a49100c11f1ad6555520a21ba515122d8bc41328a8",
Name: "master",
Repo: &models.Repository{
Name: "prj",
Owner: &models.User{
@@ -982,7 +979,6 @@ func TestPRMerge(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
ctl := gomock.NewController(t)
mock := mock_common.NewMockGiteaPRTimelineFetcher(ctl)
reviewUnrequestMock := mock_common.NewMockGiteaReviewUnrequester(ctl)
reviewUnrequestMock.EXPECT().UnrequestReview(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil)

View File

@@ -40,24 +40,10 @@ create_prjgit_sample() {
git submodule -q add ../pkgB2 pkgB2
git commit -q -m "pkgB2 added"
git checkout -b base_rm_c main
git checkout main
git clean -ffxd
git rm pkgC
git commit -q -m 'pkgC removed'
git checkout -b base_modify_c main
git submodule update --init pkgC
pushd pkgC
echo "mofieid" >> README.md
git commit -q -m "modified" README.md
popd
git commit pkgC -m "modifiedC"
git submodule deinit -f pkgC
# git checkout main
# git clean -ffxd
# git submodule -q add -f ../pkgB1 pkgB1
# git commit -q -m "main adding pkgB1"
git submodule -q add -f ../pkgB1 pkgB1
git commit -q -m "main adding pkgB1"
popd
}

View File

@@ -168,9 +168,10 @@ func FetchDevelProjects() (DevelProjects, error) {
}
var DevelProjectNotFound = errors.New("Devel project not found")
func (d DevelProjects) GetDevelProject(pkg string) (string, error) {
for _, item := range d {
if item.Package == pkg {
if item.Package == pkg {
return item.Project, nil
}
}
@@ -178,3 +179,33 @@ func (d DevelProjects) GetDevelProject(pkg string) (string, error) {
return "", DevelProjectNotFound
}
var removedBranchNameSuffixes []string = []string{
"-rm",
"-removed",
"-deleted",
}
func findRemovedBranchSuffix(branchName string) string {
branchName = strings.ToLower(branchName)
for _, suffix := range removedBranchNameSuffixes {
if len(suffix) < len(branchName) && strings.HasSuffix(branchName, suffix) {
return suffix
}
}
return ""
}
func IsRemovedBranch(branchName string) bool {
return len(findRemovedBranchSuffix(branchName)) > 0
}
func TrimRemovedBranchSuffix(branchName string) string {
suffix := findRemovedBranchSuffix(branchName)
if len(suffix) > 0 {
return branchName[0 : len(branchName)-len(suffix)]
}
return branchName
}

View File

@@ -165,3 +165,58 @@ func TestRemoteName(t *testing.T) {
})
}
}
func TestRemovedBranchName(t *testing.T) {
tests := []struct {
name string
branchName string
isRemoved bool
regularName string
}{
{
name: "Empty branch",
},
{
name: "Removed suffix only",
branchName: "-rm",
isRemoved: false,
regularName: "-rm",
},
{
name: "Capital suffix",
branchName: "Foo-Rm",
isRemoved: true,
regularName: "Foo",
},
{
name: "Other suffixes",
isRemoved: true,
branchName: "Goo-Rm-DeleteD",
regularName: "Goo-Rm",
},
{
name: "Other suffixes",
isRemoved: true,
branchName: "main-REMOVED",
regularName: "main",
},
{
name: "Not removed separator",
isRemoved: false,
branchName: "main;REMOVED",
regularName: "main;REMOVED",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if r := common.IsRemovedBranch(test.branchName); r != test.isRemoved {
t.Error("Expecting isRemoved:", test.isRemoved, "but received", r)
}
if tn := common.TrimRemovedBranchSuffix(test.branchName); tn != test.regularName {
t.Error("Expected stripped branch name to be:", test.regularName, "but have:", tn)
}
})
}
}

View File

@@ -14,15 +14,11 @@ import (
"src.opensuse.org/autogits/common"
)
type Status struct {
Context string `json:"context"`
State string `json:"state"`
TargetUrl string `json:"target_url"`
}
type StatusInput struct {
State string `json:"state"`
TargetUrl string `json:"target_url"`
Description string `json:"description"`
Context string `json:"context"`
State string `json:"state"`
TargetUrl string `json:"target_url"`
}
func main() {
@@ -59,23 +55,26 @@ func StatusProxy(w http.ResponseWriter, r *http.Request) {
config, ok := r.Context().Value(configKey).(*Config)
if !ok {
common.LogError("Config missing from context")
common.LogDebug("Config missing from context")
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
header := r.Header.Get("Authorization")
if header == "" {
common.LogDebug("Authorization header not found")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
token_arr := strings.Split(header, " ")
if len(token_arr) != 2 {
common.LogDebug("Authorization header malformed")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
if !strings.EqualFold(token_arr[0], "Bearer") {
if !strings.EqualFold(token_arr[0], "token") {
common.LogDebug("Token not found in Authorization header")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
@@ -83,6 +82,7 @@ func StatusProxy(w http.ResponseWriter, r *http.Request) {
token := token_arr[1]
if !slices.Contains(config.Keys, token) {
common.LogDebug("Provided token is not known")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
@@ -104,13 +104,8 @@ func StatusProxy(w http.ResponseWriter, r *http.Request) {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
status := Status{
Context: "Build in obs",
State: statusinput.State,
TargetUrl: statusinput.TargetUrl,
}
status_payload, err := json.Marshal(status)
status_payload, err := json.Marshal(statusinput)
if err != nil {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
@@ -131,8 +126,8 @@ func StatusProxy(w http.ResponseWriter, r *http.Request) {
return
}
req.Header.Add("Content-Type", "Content-Type")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ForgeToken))
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("token %s", ForgeToken))
resp, err := client.Do(req)

View File

@@ -0,0 +1,48 @@
# gitea_status_proxy
Allows bots without code owner permission to set Gitea's commit status
## Basic usage
To beging, you need the json config and a Gitea token with permissions to the repository you want to write to.
Keys should be randomly generated, i.e by using openssl: `openssl rand -base64 48`
Generate a json config file, with the key generated from running the command above, save as example.json:
```
{
"forge_url": "https://src.opensuse.org/api/v1",
"keys": ["$YOUR_TOKEN_GOES_HERE"]
}
```
### start the proxy:
```
GITEA_TOKEN=YOURTOKEN ./gitea_status_proxy -config example.json
2025/10/30 12:53:18 [I] server up and listening on :3000
```
Now the proxy should be able to accept requests under: `localhost:3000/repos/{owner}/{repo}/statuses/{sha}`, the token to be used when authenticating to the proxy must be in the `keys` list of the configuration json file (example.json above)
### example:
On a separate terminal, you can use curl to post a status to the proxy, if the GITEA_TOKEN has permissions on the target
repository, it will result in a new status being set for the given commit
```
curl -X 'POST' \
'localhost:3000/repos/szarate/test-actions-gitea/statuses/cd5847c92fb65a628bdd6015f96ee7e569e1ad6e4fc487acc149b52e788262f9' \
-H 'accept: application/json' \
-H 'Authorization: token $YOUR_TOKEN_GOES_HERE' \
-H 'Content-Type: application/json' \
-d '{
"context": "Proxy test",
"description": "Status posted from the proxy",
"state": "success",
"target_url": "https://src.opensuse.org"
}'
```
After this you should be able to the results in the pull request, e.g from above: https://src.opensuse.org/szarate/test-actions-gitea/pulls/1

View File

@@ -386,6 +386,28 @@ func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, git
}
// patch baseMeta to become the new project
templateMeta.Name = stagingProject + ":" + subProjectName
// freeze tag for now
if len(templateMeta.ScmSync) > 0 {
repository, err := url.Parse(templateMeta.ScmSync)
if err != nil {
panic(err)
}
common.LogDebug("getting data for ", repository.EscapedPath())
split := strings.Split(repository.EscapedPath(), "/")
org, repo := split[1], split[2]
common.LogDebug("getting commit for ", org, " repo ", repo, " fragment ", repository.Fragment)
branch, err := gitea.GetCommit(org, repo, repository.Fragment)
if err != nil {
panic(err)
}
// set expanded commit url
repository.Fragment = branch.SHA
templateMeta.ScmSync = repository.String()
common.LogDebug("Setting scmsync url to ", templateMeta.ScmSync)
}
// Cleanup ReleaseTarget and modify affected path entries
for idx, r := range templateMeta.Repositories {
templateMeta.Repositories[idx].ReleaseTargets = nil
@@ -1044,6 +1066,7 @@ func main() {
ObsWebHost = ObsWebHostFromApiHost(*obsApiHost)
}
common.LogDebug("OBS Gitea Host:", GiteaUrl)
common.LogDebug("OBS Web Host:", ObsWebHost)
common.LogDebug("OBS API Host:", *obsApiHost)

View File

@@ -0,0 +1,15 @@
[Unit]
Description=WorkflowDirect git bot for %i
After=network-online.target
[Service]
Type=exec
ExecStart=/usr/bin/workflow-direct
EnvironmentFile=-/etc/default/%i/workflow-direct.env
DynamicUser=yes
NoNewPrivileges=yes
ProtectSystem=strict
[Install]
WantedBy=multi-user.target

View File

@@ -1,9 +0,0 @@
Purpose
-------
Automatically resolve git configlicts in .gitmodules if there's a conflict
due to a merge.
It uses HEAD and MERGE_HEAD to calculate merge base and pass it to the
conflict resolution

View File

@@ -1,42 +0,0 @@
package main
import (
"os"
"strings"
"src.opensuse.org/autogits/common"
)
func main() {
cwd, err := os.Getwd()
if err != nil {
common.LogError(err)
return
}
gh, err := common.AllocateGitWorkTree(cwd, "", "")
if err != nil {
common.LogError(err)
return
}
git, err := gh.ReadExistingPath("")
if err != nil {
common.LogError(err)
return
}
MergeBase := strings.TrimSpace(git.GitExecWithOutputOrPanic("", "merge-base", "HEAD", "MERGE_HEAD"))
status, err := git.GitStatus("")
if err != nil {
common.LogError(err)
return
}
for _, s := range status {
if s.Path == ".gitmodules" && s.Status == common.GitStatus_Unmerged {
if err := git.GitResolveSubmoduleFileConflict(s, "", MergeBase, "HEAD", "MERGE_HEAD"); err != nil {
common.LogError(err)
}
}
}
}

View File

@@ -22,7 +22,6 @@ import (
"flag"
"fmt"
"io/fs"
"log"
"math/rand"
"net/url"
"os"
@@ -40,7 +39,7 @@ import (
const (
AppName = "direct_workflow"
GitAuthor = "AutoGits prjgit-updater"
GitEmail = "adam+autogits-direct@zombino.com"
GitEmail = "autogits-direct@noreply@src.opensuse.org"
)
var configuredRepos map[string][]*common.AutogitConfig
@@ -53,18 +52,6 @@ func isConfiguredOrg(org *common.Organization) bool {
return found
}
func concatenateErrors(err1, err2 error) error {
if err1 == nil {
return err2
}
if err2 == nil {
return err1
}
return fmt.Errorf("%w\n%w", err1, err2)
}
type RepositoryActionProcessor struct{}
func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error {
@@ -72,38 +59,43 @@ func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error {
configs, configFound := configuredRepos[action.Organization.Username]
if !configFound {
log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username)
common.LogInfo("Repository event for", action.Organization.Username, ". Not configured. Ignoring.", action.Organization.Username)
return nil
}
for _, config := range configs {
if org, repo, _ := config.GetPrjGit(); org == action.Repository.Owner.Username && repo == action.Repository.Name {
log.Println("+ ignoring repo event for PrjGit repository", config.GitProjectName)
common.LogError("+ ignoring repo event for PrjGit repository", config.GitProjectName)
return nil
}
}
var err error
for _, config := range configs {
err = concatenateErrors(err, processConfiguredRepositoryAction(action, config))
processConfiguredRepositoryAction(action, config)
}
return err
return nil
}
func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error {
func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) {
gitOrg, gitPrj, gitBranch := config.GetPrjGit()
git, err := gh.CreateGitHandler(config.Organization)
common.PanicOnError(err)
defer git.Close()
if len(config.Branch) == 0 {
config.Branch = action.Repository.Default_Branch
configBranch := config.Branch
if len(configBranch) == 0 {
configBranch = action.Repository.Default_Branch
if common.IsRemovedBranch(configBranch) {
common.LogDebug(" - default branch has deleted suffix. Skipping")
return
}
}
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
if err != nil {
return fmt.Errorf("Error accessing/creating prjgit: %s/%s#%s err: %w", gitOrg, gitPrj, gitBranch, err)
common.LogError("Error accessing/creating prjgit:", gitOrg, gitPrj, gitBranch, err)
return
}
remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL)
@@ -112,29 +104,29 @@ func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, co
switch action.Action {
case "created":
if action.Repository.Object_Format_Name != "sha256" {
return fmt.Errorf(" - '%s' repo is not sha256. Ignoring.", action.Repository.Name)
common.LogError(" - '%s' repo is not sha256. Ignoring.", action.Repository.Name)
return
}
common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all")
defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all")
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current"))
if branch != config.Branch {
if err := git.GitExec(path.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
if branch != configBranch {
if err := git.GitExec(path.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", configBranch+":"+configBranch); err != nil {
common.LogError("error fetching branch", configBranch, ". ignoring as non-existent.", err) // no branch? so ignore repo here
return
}
common.PanicOnError(git.GitExec(path.Join(gitPrj, action.Repository.Name), "checkout", config.Branch))
common.PanicOnError(git.GitExec(path.Join(gitPrj, action.Repository.Name), "checkout", configBranch))
}
common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package inclusion via Direct Workflow"))
common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Auto-inclusion "+action.Repository.Name))
if !noop {
common.PanicOnError(git.GitExec(gitPrj, "push"))
}
case "deleted":
if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
if DebugMode {
log.Println("delete event for", action.Repository.Name, "-- not in project. Ignoring")
}
return nil
common.LogDebug("delete event for", action.Repository.Name, "-- not in project. Ignoring")
return
}
common.PanicOnError(git.GitExec(gitPrj, "rm", action.Repository.Name))
common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package removal via Direct Workflow"))
@@ -143,10 +135,9 @@ func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, co
}
default:
return fmt.Errorf("%s: %s", "Unknown action type", action.Action)
common.LogError("Unknown action type:", action.Action)
return
}
return nil
}
type PushActionProcessor struct{}
@@ -156,40 +147,44 @@ func (*PushActionProcessor) ProcessFunc(request *common.Request) error {
configs, configFound := configuredRepos[action.Repository.Owner.Username]
if !configFound {
log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Repository.Owner.Username)
common.LogDebug("Repository event for", action.Repository.Owner.Username, ". Not configured. Ignoring.")
return nil
}
for _, config := range configs {
if gitOrg, gitPrj, _ := config.GetPrjGit(); gitOrg == action.Repository.Owner.Username && gitPrj == action.Repository.Name {
log.Println("+ ignoring push to PrjGit repository", config.GitProjectName)
common.LogInfo("+ ignoring push to PrjGit repository", config.GitProjectName)
return nil
}
}
var err error
for _, config := range configs {
err = concatenateErrors(err, processConfiguredPushAction(action, config))
processConfiguredPushAction(action, config)
}
return err
return nil
}
func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) error {
func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) {
gitOrg, gitPrj, gitBranch := config.GetPrjGit()
git, err := gh.CreateGitHandler(config.Organization)
common.PanicOnError(err)
defer git.Close()
log.Printf("push to: %s/%s for %s/%s#%s", action.Repository.Owner.Username, action.Repository.Name, gitOrg, gitPrj, gitBranch)
if len(config.Branch) == 0 {
config.Branch = action.Repository.Default_Branch
log.Println(" + default branch", action.Repository.Default_Branch)
common.LogDebug("push to:", action.Repository.Owner.Username, action.Repository.Name, "for:", gitOrg, gitPrj, gitBranch)
branch := config.Branch
if len(branch) == 0 {
if common.IsRemovedBranch(branch) {
common.LogDebug(" + default branch has removed suffix:", branch, "Skipping.")
return
}
branch = action.Repository.Default_Branch
common.LogDebug(" + using default branch", branch)
}
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
if err != nil {
return fmt.Errorf("Error accessing/creating prjgit: %s/%s err: %w", gitOrg, gitPrj, err)
common.LogError("Error accessing/creating prjgit:", gitOrg, gitPrj, err)
return
}
remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL)
@@ -198,23 +193,25 @@ func processConfiguredPushAction(action *common.PushWebhookEvent, config *common
common.PanicOnError(err)
commit, ok := git.GitSubmoduleCommitId(gitPrj, action.Repository.Name, headCommitId)
for ok && action.Head_Commit.Id == commit {
log.Println(" -- nothing to do, commit already in ProjectGit")
return nil
common.LogDebug(" -- nothing to do, commit already in ProjectGit")
return
}
if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
if DebugMode {
log.Println("Pushed to package that is not part of the project. Ignoring:", err)
}
return nil
if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil {
git.GitExecOrPanic(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name)
common.LogDebug("Pushed to package that is not part of the project. Re-adding...", err)
} else if !stat.IsDir() {
common.LogError("Pushed to a package that is not a submodule but exists in the project. Ignoring.")
return
}
git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name)
defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all")
defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all")
if err := git.GitExec(filepath.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "--force", remoteName, config.Branch+":"+config.Branch); err != nil {
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here
if err := git.GitExec(filepath.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "--force", remoteName, branch+":"+branch); err != nil {
common.LogError("Error fetching branch:", branch, "Ignoring as non-existent.", err)
return
}
id, err := git.GitRemoteHead(filepath.Join(gitPrj, action.Repository.Name), remoteName, config.Branch)
id, err := git.GitRemoteHead(filepath.Join(gitPrj, action.Repository.Name), remoteName, branch)
common.PanicOnError(err)
if action.Head_Commit.Id == id {
git.GitExecOrPanic(filepath.Join(gitPrj, action.Repository.Name), "checkout", id)
@@ -222,11 +219,10 @@ func processConfiguredPushAction(action *common.PushWebhookEvent, config *common
if !noop {
git.GitExecOrPanic(gitPrj, "push", remoteName)
}
return nil
return
}
log.Println("push of refs not on the configured branch", config.Branch, ". ignoring.")
return nil
common.LogDebug("push of refs not on the configured branch", branch, ". ignoring.")
}
func verifyProjectState(git common.Git, org string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) {
@@ -248,51 +244,61 @@ func verifyProjectState(git common.Git, org string, config *common.AutogitConfig
remoteName, err := git.GitClone(gitPrj, gitBranch, repo.SSHURL)
common.PanicOnError(err)
defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all")
defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all")
log.Println(" * Getting submodule list")
common.LogDebug(" * Getting submodule list")
sub, err := git.GitSubmoduleList(gitPrj, "HEAD")
common.PanicOnError(err)
log.Println(" * Getting package links")
common.LogDebug(" * Getting package links")
var pkgLinks []*PackageRebaseLink
if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), gitPrj)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 {
if data, err := os.ReadFile(path.Join(git.GetPath(), gitPrj, common.PrjLinksFile)); err == nil {
pkgLinks, err = parseProjectLinks(data)
if err != nil {
log.Println("Cannot parse project links file:", err.Error())
common.LogError("Cannot parse project links file:", err.Error())
pkgLinks = nil
} else {
ResolveLinks(org, pkgLinks, gitea)
}
}
} else {
log.Println(" - No package links defined")
common.LogInfo(" - No package links defined")
}
/* Check existing submodule that they are updated */
isGitUpdated := false
next_package:
for filename, commitId := range sub {
// ignore project gits
//for _, c := range configs {
if gitPrj == filename {
log.Println(" prjgit as package? ignoring project git:", filename)
common.LogDebug(" prjgit as package? ignoring project git:", filename)
continue next_package
}
//}
log.Printf(" verifying package: %s -> %s(%s)", commitId, filename, config.Branch)
commits, err := gitea.GetRecentCommits(org, filename, config.Branch, 10)
if len(commits) == 0 {
if repo, err := gitea.GetRepository(org, filename); repo == nil && err == nil {
branch := config.Branch
common.LogDebug(" verifying package: %s -> %s(%s)", commitId, filename, branch)
if repo, err := gitea.GetRepository(org, filename); repo == nil && err == nil {
common.LogDebug(" repository removed...")
git.GitExecOrPanic(gitPrj, "rm", filename)
isGitUpdated = true
continue
}
if len(branch) == 0 {
branch = repo.DefaultBranch
if common.IsRemovedBranch(branch) {
common.LogDebug(" Default branch for", filename, "is excluded")
git.GitExecOrPanic(gitPrj, "rm", filename)
isGitUpdated = true
continue
}
}
commits, err := gitea.GetRecentCommits(org, filename, branch, 10)
if err != nil {
log.Println(" -> failed to fetch recent commits for package:", filename, " Err:", err)
common.LogDebug(" -> failed to fetch recent commits for package:", filename, " Err:", err)
continue
}
@@ -309,7 +315,7 @@ next_package:
if l.Pkg == filename {
link = l
log.Println(" -> linked package")
common.LogDebug(" -> linked package")
// so, we need to rebase here. Can't really optimize, so clone entire package tree and remote
pkgPath := path.Join(gitPrj, filename)
git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--checkout", filename)
@@ -323,7 +329,7 @@ next_package:
nCommits := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgPath, "rev-list", "^NOW", "HEAD"), "\n"))
if nCommits > 0 {
if !noop {
git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+config.Branch)
git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+branch)
}
isGitUpdated = true
}
@@ -340,42 +346,27 @@ next_package:
common.PanicOnError(git.GitExec(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", filename))
common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "fetch", "--depth", "1", "origin", commits[0].SHA))
common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "checkout", commits[0].SHA))
log.Println(" -> updated to", commits[0].SHA)
common.LogDebug(" -> updated to", commits[0].SHA)
isGitUpdated = true
} else {
// probably need `merge-base` or `rev-list` here instead, or the project updated already
log.Println(" *** Cannot find SHA of last matching update for package:", filename, " Ignoring")
common.LogInfo(" *** Cannot find SHA of last matching update for package:", filename, " Ignoring")
}
}
}
// find all missing repositories, and add them
if DebugMode {
log.Println("checking for missing repositories...")
}
common.LogDebug("checking for missing repositories...")
repos, err := gitea.GetOrganizationRepositories(org)
if err != nil {
return err
}
if DebugMode {
log.Println(" nRepos:", len(repos))
}
common.LogDebug(" nRepos:", len(repos))
/* Check repositories in org to make sure they are included in project git */
next_repo:
for _, r := range repos {
if DebugMode {
log.Println(" -- checking", r.Name)
}
if r.ObjectFormatName != "sha256" {
if DebugMode {
log.Println(" + ", r.ObjectFormatName, ". Needs to be sha256. Ignoring")
}
continue next_repo
}
// for _, c := range configs {
if gitPrj == r.Name {
// ignore project gits
@@ -390,27 +381,32 @@ next_repo:
}
}
if DebugMode {
log.Println(" -- checking repository:", r.Name)
}
common.LogDebug(" -- checking repository:", r.Name)
if _, err := gitea.GetRecentCommits(org, r.Name, config.Branch, 1); err != nil {
branch := config.Branch
if len(branch) == 0 {
branch = r.DefaultBranch
if common.IsRemovedBranch(branch) {
continue
}
}
if commits, err := gitea.GetRecentCommits(org, r.Name, branch, 1); err != nil || len(commits) == 0 {
// assumption that package does not exist, so not part of project
// https://github.com/go-gitea/gitea/issues/31976
// or, we do not have commits here
continue
}
// add repository to git project
common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name))
if len(config.Branch) > 0 {
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current"))
if branch != config.Branch {
if err := git.GitExec(path.Join(gitPrj, r.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil {
return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", config.Branch, repo.Owner.UserName, r.Name)
}
common.PanicOnError(git.GitExec(path.Join(gitPrj, r.Name), "checkout", config.Branch))
curBranch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current"))
if branch != curBranch {
if err := git.GitExec(path.Join(gitPrj, r.Name), "fetch", "--depth", "1", "origin", branch+":"+branch); err != nil {
return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", branch, repo.Owner.UserName, r.Name)
}
common.PanicOnError(git.GitExec(path.Join(gitPrj, r.Name), "checkout", branch))
}
isGitUpdated = true
@@ -423,10 +419,7 @@ next_repo:
}
}
if DebugMode {
log.Println("Verification finished for ", org, ", prjgit:", config.GitProjectName)
}
common.LogInfo("Verification finished for ", org, ", prjgit:", config.GitProjectName)
return nil
}
@@ -437,17 +430,17 @@ var checkInterval time.Duration
func checkOrg(org string, configs []*common.AutogitConfig) {
git, err := gh.CreateGitHandler(org)
if err != nil {
log.Println("Faield to allocate GitHandler:", err)
common.LogError("Failed to allocate GitHandler:", err)
return
}
defer git.Close()
for _, config := range configs {
log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName)
common.LogInfo(" ++ starting verification, org:", org, "config:", config.GitProjectName)
if err := verifyProjectState(git, org, config, configs); err != nil {
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err)
common.LogError(" *** verification failed, org:", org, err)
} else {
log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName)
common.LogError(" ++ verification complete, org:", org, config.GitProjectName)
}
}
}
@@ -456,7 +449,7 @@ func checkRepos() {
for org, configs := range configuredRepos {
if checkInterval > 0 {
sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))
log.Println(" - sleep interval", sleepInterval, "until next check")
common.LogInfo(" - sleep interval", sleepInterval, "until next check")
time.Sleep(sleepInterval)
}
@@ -468,9 +461,9 @@ func consistencyCheckProcess() {
if checkOnStart {
savedCheckInterval := checkInterval
checkInterval = 0
log.Println("== Startup consistency check begin...")
common.LogInfo("== Startup consistency check begin...")
checkRepos()
log.Println("== Startup consistency check done...")
common.LogInfo("== Startup consistency check done...")
checkInterval = savedCheckInterval
}
@@ -485,7 +478,8 @@ var gh common.GitHandlerGenerator
func updateConfiguration(configFilename string, orgs *[]string) {
configFile, err := common.ReadConfigFile(configFilename)
if err != nil {
log.Fatal(err)
common.LogError(err)
os.Exit(4)
}
configs, _ := common.ResolveWorkflowConfigs(gitea, configFile)
@@ -493,9 +487,7 @@ func updateConfiguration(configFilename string, orgs *[]string) {
*orgs = make([]string, 0, 1)
for _, c := range configs {
if slices.Contains(c.Workflows, "direct") {
if DebugMode {
log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName)
}
common.LogDebug(" + adding org:", c.Organization, ", branch:", c.Branch, ", prjgit:", c.GitProjectName)
configs := configuredRepos[c.Organization]
if configs == nil {
configs = make([]*common.AutogitConfig, 0, 1)
@@ -520,10 +512,12 @@ func main() {
flag.Parse()
if err := common.RequireGiteaSecretToken(); err != nil {
log.Fatal(err)
common.LogError(err)
os.Exit(1)
}
if err := common.RequireRabbitSecrets(); err != nil {
log.Fatal(err)
common.LogError(err)
os.Exit(1)
}
defs := &common.RabbitMQGiteaEventsProcessor{}
@@ -532,12 +526,14 @@ func main() {
if len(*basePath) == 0 {
*basePath, err = os.MkdirTemp(os.TempDir(), AppName)
if err != nil {
log.Fatal(err)
common.LogError(err)
os.Exit(1)
}
}
gh, err = common.AllocateGitWorkTree(*basePath, GitAuthor, GitEmail)
if err != nil {
log.Fatal(err)
common.LogError(err)
os.Exit(1)
}
// handle reconfiguration
@@ -552,10 +548,10 @@ func main() {
}
if sig != syscall.SIGHUP {
log.Println("Unexpected signal received:", sig)
common.LogError("Unexpected signal received:", sig)
continue
}
log.Println("*** Reconfiguring ***")
common.LogError("*** Reconfiguring ***")
updateConfiguration(*configFilename, &defs.Orgs)
defs.Connection().UpdateTopics(defs)
}
@@ -567,23 +563,25 @@ func main() {
gitea = common.AllocateGiteaTransport(*giteaUrl)
CurrentUser, err := gitea.GetCurrentUser()
if err != nil {
log.Fatalln("Cannot fetch current user:", err)
common.LogError("Cannot fetch current user:", err)
os.Exit(2)
}
log.Println("Current User:", CurrentUser.UserName)
common.LogInfo("Current User:", CurrentUser.UserName)
updateConfiguration(*configFilename, &defs.Orgs)
defs.Connection().RabbitURL, err = url.Parse(*rabbitUrl)
if err != nil {
log.Panicf("cannot parse server URL. Err: %#v\n", err)
common.LogError("cannot parse server URL. Err:", err)
os.Exit(3)
}
go consistencyCheckProcess()
log.Println("defs:", *defs)
common.LogInfo("defs:", *defs)
defs.Handlers = make(map[string]common.RequestProcessor)
defs.Handlers[common.RequestType_Push] = &PushActionProcessor{}
defs.Handlers[common.RequestType_Repository] = &RepositoryActionProcessor{}
log.Fatal(common.ProcessRabbitMQEvents(defs))
common.LogError(common.ProcessRabbitMQEvents(defs))
}

View File

@@ -269,6 +269,7 @@ func (pr *PRProcessor) RebaseAndSkipSubmoduleCommits(prset *common.PRSet, branch
}
var updatePrjGitError_requeue error = errors.New("Commits do not match. Requeing after 5 seconds.")
func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
_, _, PrjGitBranch := prset.Config.GetPrjGit()
PrjGitPR, err := prset.GetPrjGitPR()
@@ -572,7 +573,7 @@ func (pr *PRProcessor) Process(req *models.PullRequest) error {
type RequestProcessor struct {
configuredRepos map[string][]*common.AutogitConfig
recursive int
recursive int
}
func ProcesPullRequest(pr *models.PullRequest, configs []*common.AutogitConfig) error {
@@ -613,7 +614,7 @@ func (w *RequestProcessor) ProcessFunc(request *common.Request) (err error) {
common.LogError("Cannot find PR for issue:", req.Pull_Request.Base.Repo.Owner.Username, req.Pull_Request.Base.Repo.Name, req.Pull_Request.Number)
return err
}
} else if req, ok := request.Data.(*common.IssueWebhookEvent); ok {
} else if req, ok := request.Data.(*common.IssueCommentWebhookEvent); ok {
pr, err = Gitea.GetPullRequest(req.Repository.Owner.Username, req.Repository.Name, int64(req.Issue.Number))
if err != nil {
common.LogError("Cannot find PR for issue:", req.Repository.Owner.Username, req.Repository.Name, int64(req.Issue.Number))