SHA256
1
0

13 Commits

Author SHA256 Message Date
c25d3be44e direct: add systemd unit file 2025-11-05 13:24:54 +01:00
8db558891a direct: remove config.Branch clobbering
use our own copy of branch instead of writing it in the config.
This should fix handling of default branches where the default
branch differs between repositories.
2025-11-04 18:00:21 +01:00
0e06ba5993 common: classifying rm branches on name
Branches with suffixes

  -rm
  -removed
  -deleted

are now classified as removed. This is important in case project
config refers to default branch names which must exist so we need
to be able to classify such branches to either use them or ignore
them
2025-11-04 18:00:21 +01:00
736769d630 direct: add a repo with branch but no submodule 2025-11-04 18:00:21 +01:00
93c970d0dd direct: move logging to common.Log* function 2025-11-04 18:00:21 +01:00
5544a65947 obs-staging-bot: Expand possible branch of QA repos
That way a source merge of any product is not triggering rebuilds in
pull request QA sub projects. We may need a config option here to
enable/disable this.
2025-11-03 17:54:57 +01:00
918723d57b Merge commit '55846562c1d9dcb395e545f7c8e0bcb74c47b85693f4e955ef488530781b9bf2'
PR!88
2025-11-03 17:49:45 +01:00
a418b48809 pr: process PR on comments, not issue changes 2025-10-31 13:07:18 +01:00
55846562c1 Add simple readme for gitea_status_proxy 2025-10-31 10:33:58 +01:00
95c7770cad Change log level for auth errors 2025-10-31 10:33:58 +01:00
1b900e3202 Properly proxy json input directly to gitea 2025-10-31 10:33:51 +01:00
d083acfd1c Be more verbose about authentication errors 2025-10-30 13:09:17 +01:00
244160e20e Update authorization headers
For gitea API AuthorizationHeaderToken tokens must be prepended with "token" followed by a space, also fix content type
2025-10-30 13:09:17 +01:00
9 changed files with 322 additions and 154 deletions

View File

@@ -173,6 +173,7 @@ install -D -m0644 systemd/obs-staging-bot.service
install -D -m0755 obs-status-service/obs-status-service %{buildroot}%{_bindir}/obs-status-service install -D -m0755 obs-status-service/obs-status-service %{buildroot}%{_bindir}/obs-status-service
install -D -m0644 systemd/obs-status-service.service %{buildroot}%{_unitdir}/obs-status-service.service install -D -m0644 systemd/obs-status-service.service %{buildroot}%{_unitdir}/obs-status-service.service
install -D -m0755 workflow-direct/workflow-direct %{buildroot}%{_bindir}/workflow-direct install -D -m0755 workflow-direct/workflow-direct %{buildroot}%{_bindir}/workflow-direct
install -D -m0644 systemd/workflow-direct@.service %{buildroot}%{_unitdir}/workflow-direct@.service
install -D -m0755 workflow-pr/workflow-pr %{buildroot}%{_bindir}/workflow-pr install -D -m0755 workflow-pr/workflow-pr %{buildroot}%{_bindir}/workflow-pr
install -D -m0755 utils/hujson/hujson %{buildroot}%{_bindir}/hujson install -D -m0755 utils/hujson/hujson %{buildroot}%{_bindir}/hujson
@@ -261,6 +262,7 @@ install -D -m0755 utils/hujson/hujson
%license COPYING %license COPYING
%doc workflow-direct/README.md %doc workflow-direct/README.md
%{_bindir}/workflow-direct %{_bindir}/workflow-direct
%{_unitdir}/workflow-direct@.service
%files workflow-pr %files workflow-pr
%license COPYING %license COPYING

View File

@@ -168,6 +168,7 @@ func FetchDevelProjects() (DevelProjects, error) {
} }
var DevelProjectNotFound = errors.New("Devel project not found") var DevelProjectNotFound = errors.New("Devel project not found")
func (d DevelProjects) GetDevelProject(pkg string) (string, error) { func (d DevelProjects) GetDevelProject(pkg string) (string, error) {
for _, item := range d { for _, item := range d {
if item.Package == pkg { if item.Package == pkg {
@@ -178,3 +179,33 @@ func (d DevelProjects) GetDevelProject(pkg string) (string, error) {
return "", DevelProjectNotFound return "", DevelProjectNotFound
} }
var removedBranchNameSuffixes []string = []string{
"-rm",
"-removed",
"-deleted",
}
func findRemovedBranchSuffix(branchName string) string {
branchName = strings.ToLower(branchName)
for _, suffix := range removedBranchNameSuffixes {
if len(suffix) < len(branchName) && strings.HasSuffix(branchName, suffix) {
return suffix
}
}
return ""
}
func IsRemovedBranch(branchName string) bool {
return len(findRemovedBranchSuffix(branchName)) > 0
}
func TrimRemovedBranchSuffix(branchName string) string {
suffix := findRemovedBranchSuffix(branchName)
if len(suffix) > 0 {
return branchName[0 : len(branchName)-len(suffix)]
}
return branchName
}

View File

@@ -165,3 +165,58 @@ func TestRemoteName(t *testing.T) {
}) })
} }
} }
func TestRemovedBranchName(t *testing.T) {
tests := []struct {
name string
branchName string
isRemoved bool
regularName string
}{
{
name: "Empty branch",
},
{
name: "Removed suffix only",
branchName: "-rm",
isRemoved: false,
regularName: "-rm",
},
{
name: "Capital suffix",
branchName: "Foo-Rm",
isRemoved: true,
regularName: "Foo",
},
{
name: "Other suffixes",
isRemoved: true,
branchName: "Goo-Rm-DeleteD",
regularName: "Goo-Rm",
},
{
name: "Other suffixes",
isRemoved: true,
branchName: "main-REMOVED",
regularName: "main",
},
{
name: "Not removed separator",
isRemoved: false,
branchName: "main;REMOVED",
regularName: "main;REMOVED",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if r := common.IsRemovedBranch(test.branchName); r != test.isRemoved {
t.Error("Expecting isRemoved:", test.isRemoved, "but received", r)
}
if tn := common.TrimRemovedBranchSuffix(test.branchName); tn != test.regularName {
t.Error("Expected stripped branch name to be:", test.regularName, "but have:", tn)
}
})
}
}

View File

@@ -14,13 +14,9 @@ import (
"src.opensuse.org/autogits/common" "src.opensuse.org/autogits/common"
) )
type Status struct {
Context string `json:"context"`
State string `json:"state"`
TargetUrl string `json:"target_url"`
}
type StatusInput struct { type StatusInput struct {
Description string `json:"description"`
Context string `json:"context"`
State string `json:"state"` State string `json:"state"`
TargetUrl string `json:"target_url"` TargetUrl string `json:"target_url"`
} }
@@ -59,23 +55,26 @@ func StatusProxy(w http.ResponseWriter, r *http.Request) {
config, ok := r.Context().Value(configKey).(*Config) config, ok := r.Context().Value(configKey).(*Config)
if !ok { if !ok {
common.LogError("Config missing from context") common.LogDebug("Config missing from context")
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return return
} }
header := r.Header.Get("Authorization") header := r.Header.Get("Authorization")
if header == "" { if header == "" {
common.LogDebug("Authorization header not found")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return return
} }
token_arr := strings.Split(header, " ") token_arr := strings.Split(header, " ")
if len(token_arr) != 2 { if len(token_arr) != 2 {
common.LogDebug("Authorization header malformed")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return return
} }
if !strings.EqualFold(token_arr[0], "Bearer") { if !strings.EqualFold(token_arr[0], "token") {
common.LogDebug("Token not found in Authorization header")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return return
} }
@@ -83,6 +82,7 @@ func StatusProxy(w http.ResponseWriter, r *http.Request) {
token := token_arr[1] token := token_arr[1]
if !slices.Contains(config.Keys, token) { if !slices.Contains(config.Keys, token) {
common.LogDebug("Provided token is not known")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return return
} }
@@ -104,13 +104,8 @@ func StatusProxy(w http.ResponseWriter, r *http.Request) {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return return
} }
status := Status{
Context: "Build in obs",
State: statusinput.State,
TargetUrl: statusinput.TargetUrl,
}
status_payload, err := json.Marshal(status) status_payload, err := json.Marshal(statusinput)
if err != nil { if err != nil {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
@@ -131,8 +126,8 @@ func StatusProxy(w http.ResponseWriter, r *http.Request) {
return return
} }
req.Header.Add("Content-Type", "Content-Type") req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ForgeToken)) req.Header.Add("Authorization", fmt.Sprintf("token %s", ForgeToken))
resp, err := client.Do(req) resp, err := client.Do(req)

View File

@@ -0,0 +1,48 @@
# gitea_status_proxy
Allows bots without code owner permission to set Gitea's commit status
## Basic usage
To beging, you need the json config and a Gitea token with permissions to the repository you want to write to.
Keys should be randomly generated, i.e by using openssl: `openssl rand -base64 48`
Generate a json config file, with the key generated from running the command above, save as example.json:
```
{
"forge_url": "https://src.opensuse.org/api/v1",
"keys": ["$YOUR_TOKEN_GOES_HERE"]
}
```
### start the proxy:
```
GITEA_TOKEN=YOURTOKEN ./gitea_status_proxy -config example.json
2025/10/30 12:53:18 [I] server up and listening on :3000
```
Now the proxy should be able to accept requests under: `localhost:3000/repos/{owner}/{repo}/statuses/{sha}`, the token to be used when authenticating to the proxy must be in the `keys` list of the configuration json file (example.json above)
### example:
On a separate terminal, you can use curl to post a status to the proxy, if the GITEA_TOKEN has permissions on the target
repository, it will result in a new status being set for the given commit
```
curl -X 'POST' \
'localhost:3000/repos/szarate/test-actions-gitea/statuses/cd5847c92fb65a628bdd6015f96ee7e569e1ad6e4fc487acc149b52e788262f9' \
-H 'accept: application/json' \
-H 'Authorization: token $YOUR_TOKEN_GOES_HERE' \
-H 'Content-Type: application/json' \
-d '{
"context": "Proxy test",
"description": "Status posted from the proxy",
"state": "success",
"target_url": "https://src.opensuse.org"
}'
```
After this you should be able to the results in the pull request, e.g from above: https://src.opensuse.org/szarate/test-actions-gitea/pulls/1

View File

@@ -386,6 +386,28 @@ func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, git
} }
// patch baseMeta to become the new project // patch baseMeta to become the new project
templateMeta.Name = stagingProject + ":" + subProjectName templateMeta.Name = stagingProject + ":" + subProjectName
// freeze tag for now
if len(templateMeta.ScmSync) > 0 {
repository, err := url.Parse(templateMeta.ScmSync)
if err != nil {
panic(err)
}
common.LogDebug("getting data for ", repository.EscapedPath())
split := strings.Split(repository.EscapedPath(), "/")
org, repo := split[1], split[2]
common.LogDebug("getting commit for ", org, " repo ", repo, " fragment ", repository.Fragment)
branch, err := gitea.GetCommit(org, repo, repository.Fragment)
if err != nil {
panic(err)
}
// set expanded commit url
repository.Fragment = branch.SHA
templateMeta.ScmSync = repository.String()
common.LogDebug("Setting scmsync url to ", templateMeta.ScmSync)
}
// Cleanup ReleaseTarget and modify affected path entries // Cleanup ReleaseTarget and modify affected path entries
for idx, r := range templateMeta.Repositories { for idx, r := range templateMeta.Repositories {
templateMeta.Repositories[idx].ReleaseTargets = nil templateMeta.Repositories[idx].ReleaseTargets = nil
@@ -1044,6 +1066,7 @@ func main() {
ObsWebHost = ObsWebHostFromApiHost(*obsApiHost) ObsWebHost = ObsWebHostFromApiHost(*obsApiHost)
} }
common.LogDebug("OBS Gitea Host:", GiteaUrl)
common.LogDebug("OBS Web Host:", ObsWebHost) common.LogDebug("OBS Web Host:", ObsWebHost)
common.LogDebug("OBS API Host:", *obsApiHost) common.LogDebug("OBS API Host:", *obsApiHost)

View File

@@ -0,0 +1,15 @@
[Unit]
Description=WorkflowDirect git bot for %i
After=network-online.target
[Service]
Type=exec
ExecStart=/usr/bin/workflow-direct
EnvironmentFile=-/etc/default/%i/workflow-direct.env
DynamicUser=yes
NoNewPrivileges=yes
ProtectSystem=strict
[Install]
WantedBy=multi-user.target

View File

@@ -22,7 +22,6 @@ import (
"flag" "flag"
"fmt" "fmt"
"io/fs" "io/fs"
"log"
"math/rand" "math/rand"
"net/url" "net/url"
"os" "os"
@@ -40,7 +39,7 @@ import (
const ( const (
AppName = "direct_workflow" AppName = "direct_workflow"
GitAuthor = "AutoGits prjgit-updater" GitAuthor = "AutoGits prjgit-updater"
GitEmail = "adam+autogits-direct@zombino.com" GitEmail = "autogits-direct@noreply@src.opensuse.org"
) )
var configuredRepos map[string][]*common.AutogitConfig var configuredRepos map[string][]*common.AutogitConfig
@@ -53,18 +52,6 @@ func isConfiguredOrg(org *common.Organization) bool {
return found return found
} }
func concatenateErrors(err1, err2 error) error {
if err1 == nil {
return err2
}
if err2 == nil {
return err1
}
return fmt.Errorf("%w\n%w", err1, err2)
}
type RepositoryActionProcessor struct{} type RepositoryActionProcessor struct{}
func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error { func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error {
@@ -72,38 +59,43 @@ func (*RepositoryActionProcessor) ProcessFunc(request *common.Request) error {
configs, configFound := configuredRepos[action.Organization.Username] configs, configFound := configuredRepos[action.Organization.Username]
if !configFound { if !configFound {
log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Organization.Username) common.LogInfo("Repository event for", action.Organization.Username, ". Not configured. Ignoring.", action.Organization.Username)
return nil return nil
} }
for _, config := range configs { for _, config := range configs {
if org, repo, _ := config.GetPrjGit(); org == action.Repository.Owner.Username && repo == action.Repository.Name { if org, repo, _ := config.GetPrjGit(); org == action.Repository.Owner.Username && repo == action.Repository.Name {
log.Println("+ ignoring repo event for PrjGit repository", config.GitProjectName) common.LogError("+ ignoring repo event for PrjGit repository", config.GitProjectName)
return nil return nil
} }
} }
var err error
for _, config := range configs { for _, config := range configs {
err = concatenateErrors(err, processConfiguredRepositoryAction(action, config)) processConfiguredRepositoryAction(action, config)
} }
return err return nil
} }
func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) error { func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, config *common.AutogitConfig) {
gitOrg, gitPrj, gitBranch := config.GetPrjGit() gitOrg, gitPrj, gitBranch := config.GetPrjGit()
git, err := gh.CreateGitHandler(config.Organization) git, err := gh.CreateGitHandler(config.Organization)
common.PanicOnError(err) common.PanicOnError(err)
defer git.Close() defer git.Close()
if len(config.Branch) == 0 { configBranch := config.Branch
config.Branch = action.Repository.Default_Branch if len(configBranch) == 0 {
configBranch = action.Repository.Default_Branch
if common.IsRemovedBranch(configBranch) {
common.LogDebug(" - default branch has deleted suffix. Skipping")
return
}
} }
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj) prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
if err != nil { if err != nil {
return fmt.Errorf("Error accessing/creating prjgit: %s/%s#%s err: %w", gitOrg, gitPrj, gitBranch, err) common.LogError("Error accessing/creating prjgit:", gitOrg, gitPrj, gitBranch, err)
return
} }
remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL) remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL)
@@ -112,29 +104,29 @@ func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, co
switch action.Action { switch action.Action {
case "created": case "created":
if action.Repository.Object_Format_Name != "sha256" { if action.Repository.Object_Format_Name != "sha256" {
return fmt.Errorf(" - '%s' repo is not sha256. Ignoring.", action.Repository.Name) common.LogError(" - '%s' repo is not sha256. Ignoring.", action.Repository.Name)
return
} }
common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name)) common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name))
defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all") defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all")
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current")) branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, action.Repository.Name), "branch", "--show-current"))
if branch != config.Branch { if branch != configBranch {
if err := git.GitExec(path.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil { if err := git.GitExec(path.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "origin", configBranch+":"+configBranch); err != nil {
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here common.LogError("error fetching branch", configBranch, ". ignoring as non-existent.", err) // no branch? so ignore repo here
return
} }
common.PanicOnError(git.GitExec(path.Join(gitPrj, action.Repository.Name), "checkout", config.Branch)) common.PanicOnError(git.GitExec(path.Join(gitPrj, action.Repository.Name), "checkout", configBranch))
} }
common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package inclusion via Direct Workflow")) common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Auto-inclusion "+action.Repository.Name))
if !noop { if !noop {
common.PanicOnError(git.GitExec(gitPrj, "push")) common.PanicOnError(git.GitExec(gitPrj, "push"))
} }
case "deleted": case "deleted":
if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() { if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() {
if DebugMode { common.LogDebug("delete event for", action.Repository.Name, "-- not in project. Ignoring")
log.Println("delete event for", action.Repository.Name, "-- not in project. Ignoring") return
}
return nil
} }
common.PanicOnError(git.GitExec(gitPrj, "rm", action.Repository.Name)) common.PanicOnError(git.GitExec(gitPrj, "rm", action.Repository.Name))
common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package removal via Direct Workflow")) common.PanicOnError(git.GitExec(gitPrj, "commit", "-m", "Automatic package removal via Direct Workflow"))
@@ -143,10 +135,9 @@ func processConfiguredRepositoryAction(action *common.RepositoryWebhookEvent, co
} }
default: default:
return fmt.Errorf("%s: %s", "Unknown action type", action.Action) common.LogError("Unknown action type:", action.Action)
return
} }
return nil
} }
type PushActionProcessor struct{} type PushActionProcessor struct{}
@@ -156,40 +147,44 @@ func (*PushActionProcessor) ProcessFunc(request *common.Request) error {
configs, configFound := configuredRepos[action.Repository.Owner.Username] configs, configFound := configuredRepos[action.Repository.Owner.Username]
if !configFound { if !configFound {
log.Printf("Repository event for %s. Not configured. Ignoring.\n", action.Repository.Owner.Username) common.LogDebug("Repository event for", action.Repository.Owner.Username, ". Not configured. Ignoring.")
return nil return nil
} }
for _, config := range configs { for _, config := range configs {
if gitOrg, gitPrj, _ := config.GetPrjGit(); gitOrg == action.Repository.Owner.Username && gitPrj == action.Repository.Name { if gitOrg, gitPrj, _ := config.GetPrjGit(); gitOrg == action.Repository.Owner.Username && gitPrj == action.Repository.Name {
log.Println("+ ignoring push to PrjGit repository", config.GitProjectName) common.LogInfo("+ ignoring push to PrjGit repository", config.GitProjectName)
return nil return nil
} }
} }
var err error
for _, config := range configs { for _, config := range configs {
err = concatenateErrors(err, processConfiguredPushAction(action, config)) processConfiguredPushAction(action, config)
}
return nil
} }
return err func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) {
}
func processConfiguredPushAction(action *common.PushWebhookEvent, config *common.AutogitConfig) error {
gitOrg, gitPrj, gitBranch := config.GetPrjGit() gitOrg, gitPrj, gitBranch := config.GetPrjGit()
git, err := gh.CreateGitHandler(config.Organization) git, err := gh.CreateGitHandler(config.Organization)
common.PanicOnError(err) common.PanicOnError(err)
defer git.Close() defer git.Close()
log.Printf("push to: %s/%s for %s/%s#%s", action.Repository.Owner.Username, action.Repository.Name, gitOrg, gitPrj, gitBranch) common.LogDebug("push to:", action.Repository.Owner.Username, action.Repository.Name, "for:", gitOrg, gitPrj, gitBranch)
if len(config.Branch) == 0 { branch := config.Branch
config.Branch = action.Repository.Default_Branch if len(branch) == 0 {
log.Println(" + default branch", action.Repository.Default_Branch) if common.IsRemovedBranch(branch) {
common.LogDebug(" + default branch has removed suffix:", branch, "Skipping.")
return
}
branch = action.Repository.Default_Branch
common.LogDebug(" + using default branch", branch)
} }
prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj) prjGitRepo, err := gitea.CreateRepositoryIfNotExist(git, gitOrg, gitPrj)
if err != nil { if err != nil {
return fmt.Errorf("Error accessing/creating prjgit: %s/%s err: %w", gitOrg, gitPrj, err) common.LogError("Error accessing/creating prjgit:", gitOrg, gitPrj, err)
return
} }
remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL) remoteName, err := git.GitClone(gitPrj, gitBranch, prjGitRepo.SSHURL)
@@ -198,23 +193,25 @@ func processConfiguredPushAction(action *common.PushWebhookEvent, config *common
common.PanicOnError(err) common.PanicOnError(err)
commit, ok := git.GitSubmoduleCommitId(gitPrj, action.Repository.Name, headCommitId) commit, ok := git.GitSubmoduleCommitId(gitPrj, action.Repository.Name, headCommitId)
for ok && action.Head_Commit.Id == commit { for ok && action.Head_Commit.Id == commit {
log.Println(" -- nothing to do, commit already in ProjectGit") common.LogDebug(" -- nothing to do, commit already in ProjectGit")
return nil return
} }
if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil || !stat.IsDir() { if stat, err := os.Stat(filepath.Join(git.GetPath(), gitPrj, action.Repository.Name)); err != nil {
if DebugMode { git.GitExecOrPanic(gitPrj, "submodule", "--quiet", "add", "--depth", "1", action.Repository.Clone_Url, action.Repository.Name)
log.Println("Pushed to package that is not part of the project. Ignoring:", err) common.LogDebug("Pushed to package that is not part of the project. Re-adding...", err)
} } else if !stat.IsDir() {
return nil common.LogError("Pushed to a package that is not a submodule but exists in the project. Ignoring.")
return
} }
git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name) git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", action.Repository.Name)
defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all") defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all")
if err := git.GitExec(filepath.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "--force", remoteName, config.Branch+":"+config.Branch); err != nil { if err := git.GitExec(filepath.Join(gitPrj, action.Repository.Name), "fetch", "--depth", "1", "--force", remoteName, branch+":"+branch); err != nil {
return fmt.Errorf("error fetching branch %s. ignoring as non-existent. err: %w", config.Branch, err) // no branch? so ignore repo here common.LogError("Error fetching branch:", branch, "Ignoring as non-existent.", err)
return
} }
id, err := git.GitRemoteHead(filepath.Join(gitPrj, action.Repository.Name), remoteName, config.Branch) id, err := git.GitRemoteHead(filepath.Join(gitPrj, action.Repository.Name), remoteName, branch)
common.PanicOnError(err) common.PanicOnError(err)
if action.Head_Commit.Id == id { if action.Head_Commit.Id == id {
git.GitExecOrPanic(filepath.Join(gitPrj, action.Repository.Name), "checkout", id) git.GitExecOrPanic(filepath.Join(gitPrj, action.Repository.Name), "checkout", id)
@@ -222,11 +219,10 @@ func processConfiguredPushAction(action *common.PushWebhookEvent, config *common
if !noop { if !noop {
git.GitExecOrPanic(gitPrj, "push", remoteName) git.GitExecOrPanic(gitPrj, "push", remoteName)
} }
return nil return
} }
log.Println("push of refs not on the configured branch", config.Branch, ". ignoring.") common.LogDebug("push of refs not on the configured branch", branch, ". ignoring.")
return nil
} }
func verifyProjectState(git common.Git, org string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) { func verifyProjectState(git common.Git, org string, config *common.AutogitConfig, configs []*common.AutogitConfig) (err error) {
@@ -248,51 +244,61 @@ func verifyProjectState(git common.Git, org string, config *common.AutogitConfig
remoteName, err := git.GitClone(gitPrj, gitBranch, repo.SSHURL) remoteName, err := git.GitClone(gitPrj, gitBranch, repo.SSHURL)
common.PanicOnError(err) common.PanicOnError(err)
defer git.GitExecOrPanic(gitPrj, "submodule", "deinit", "--all") defer git.GitExecQuietOrPanic(gitPrj, "submodule", "deinit", "--all")
log.Println(" * Getting submodule list") common.LogDebug(" * Getting submodule list")
sub, err := git.GitSubmoduleList(gitPrj, "HEAD") sub, err := git.GitSubmoduleList(gitPrj, "HEAD")
common.PanicOnError(err) common.PanicOnError(err)
log.Println(" * Getting package links") common.LogDebug(" * Getting package links")
var pkgLinks []*PackageRebaseLink var pkgLinks []*PackageRebaseLink
if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), gitPrj)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 { if f, err := fs.Stat(os.DirFS(path.Join(git.GetPath(), gitPrj)), common.PrjLinksFile); err == nil && (f.Mode()&fs.ModeType == 0) && f.Size() < 1000000 {
if data, err := os.ReadFile(path.Join(git.GetPath(), gitPrj, common.PrjLinksFile)); err == nil { if data, err := os.ReadFile(path.Join(git.GetPath(), gitPrj, common.PrjLinksFile)); err == nil {
pkgLinks, err = parseProjectLinks(data) pkgLinks, err = parseProjectLinks(data)
if err != nil { if err != nil {
log.Println("Cannot parse project links file:", err.Error()) common.LogError("Cannot parse project links file:", err.Error())
pkgLinks = nil pkgLinks = nil
} else { } else {
ResolveLinks(org, pkgLinks, gitea) ResolveLinks(org, pkgLinks, gitea)
} }
} }
} else { } else {
log.Println(" - No package links defined") common.LogInfo(" - No package links defined")
} }
/* Check existing submodule that they are updated */ /* Check existing submodule that they are updated */
isGitUpdated := false isGitUpdated := false
next_package: next_package:
for filename, commitId := range sub { for filename, commitId := range sub {
// ignore project gits // ignore project gits
//for _, c := range configs { //for _, c := range configs {
if gitPrj == filename { if gitPrj == filename {
log.Println(" prjgit as package? ignoring project git:", filename) common.LogDebug(" prjgit as package? ignoring project git:", filename)
continue next_package continue next_package
} }
//} //}
log.Printf(" verifying package: %s -> %s(%s)", commitId, filename, config.Branch) branch := config.Branch
commits, err := gitea.GetRecentCommits(org, filename, config.Branch, 10) common.LogDebug(" verifying package: %s -> %s(%s)", commitId, filename, branch)
if len(commits) == 0 {
if repo, err := gitea.GetRepository(org, filename); repo == nil && err == nil { if repo, err := gitea.GetRepository(org, filename); repo == nil && err == nil {
common.LogDebug(" repository removed...")
git.GitExecOrPanic(gitPrj, "rm", filename) git.GitExecOrPanic(gitPrj, "rm", filename)
isGitUpdated = true isGitUpdated = true
continue
}
if len(branch) == 0 {
branch = repo.DefaultBranch
if common.IsRemovedBranch(branch) {
common.LogDebug(" Default branch for", filename, "is excluded")
git.GitExecOrPanic(gitPrj, "rm", filename)
isGitUpdated = true
continue
} }
} }
commits, err := gitea.GetRecentCommits(org, filename, branch, 10)
if err != nil { if err != nil {
log.Println(" -> failed to fetch recent commits for package:", filename, " Err:", err) common.LogDebug(" -> failed to fetch recent commits for package:", filename, " Err:", err)
continue continue
} }
@@ -309,7 +315,7 @@ next_package:
if l.Pkg == filename { if l.Pkg == filename {
link = l link = l
log.Println(" -> linked package") common.LogDebug(" -> linked package")
// so, we need to rebase here. Can't really optimize, so clone entire package tree and remote // so, we need to rebase here. Can't really optimize, so clone entire package tree and remote
pkgPath := path.Join(gitPrj, filename) pkgPath := path.Join(gitPrj, filename)
git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--checkout", filename) git.GitExecOrPanic(gitPrj, "submodule", "update", "--init", "--checkout", filename)
@@ -323,7 +329,7 @@ next_package:
nCommits := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgPath, "rev-list", "^NOW", "HEAD"), "\n")) nCommits := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgPath, "rev-list", "^NOW", "HEAD"), "\n"))
if nCommits > 0 { if nCommits > 0 {
if !noop { if !noop {
git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+config.Branch) git.GitExecOrPanic(pkgPath, "push", "-f", "origin", "HEAD:"+branch)
} }
isGitUpdated = true isGitUpdated = true
} }
@@ -340,42 +346,27 @@ next_package:
common.PanicOnError(git.GitExec(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", filename)) common.PanicOnError(git.GitExec(gitPrj, "submodule", "update", "--init", "--depth", "1", "--checkout", filename))
common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "fetch", "--depth", "1", "origin", commits[0].SHA)) common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "fetch", "--depth", "1", "origin", commits[0].SHA))
common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "checkout", commits[0].SHA)) common.PanicOnError(git.GitExec(filepath.Join(gitPrj, filename), "checkout", commits[0].SHA))
log.Println(" -> updated to", commits[0].SHA) common.LogDebug(" -> updated to", commits[0].SHA)
isGitUpdated = true isGitUpdated = true
} else { } else {
// probably need `merge-base` or `rev-list` here instead, or the project updated already // probably need `merge-base` or `rev-list` here instead, or the project updated already
log.Println(" *** Cannot find SHA of last matching update for package:", filename, " Ignoring") common.LogInfo(" *** Cannot find SHA of last matching update for package:", filename, " Ignoring")
} }
} }
} }
// find all missing repositories, and add them // find all missing repositories, and add them
if DebugMode { common.LogDebug("checking for missing repositories...")
log.Println("checking for missing repositories...")
}
repos, err := gitea.GetOrganizationRepositories(org) repos, err := gitea.GetOrganizationRepositories(org)
if err != nil { if err != nil {
return err return err
} }
if DebugMode { common.LogDebug(" nRepos:", len(repos))
log.Println(" nRepos:", len(repos))
}
/* Check repositories in org to make sure they are included in project git */ /* Check repositories in org to make sure they are included in project git */
next_repo: next_repo:
for _, r := range repos { for _, r := range repos {
if DebugMode {
log.Println(" -- checking", r.Name)
}
if r.ObjectFormatName != "sha256" {
if DebugMode {
log.Println(" + ", r.ObjectFormatName, ". Needs to be sha256. Ignoring")
}
continue next_repo
}
// for _, c := range configs { // for _, c := range configs {
if gitPrj == r.Name { if gitPrj == r.Name {
// ignore project gits // ignore project gits
@@ -390,27 +381,32 @@ next_repo:
} }
} }
if DebugMode { common.LogDebug(" -- checking repository:", r.Name)
log.Println(" -- checking repository:", r.Name)
}
if _, err := gitea.GetRecentCommits(org, r.Name, config.Branch, 1); err != nil { branch := config.Branch
if len(branch) == 0 {
branch = r.DefaultBranch
if common.IsRemovedBranch(branch) {
continue
}
}
if commits, err := gitea.GetRecentCommits(org, r.Name, branch, 1); err != nil || len(commits) == 0 {
// assumption that package does not exist, so not part of project // assumption that package does not exist, so not part of project
// https://github.com/go-gitea/gitea/issues/31976 // https://github.com/go-gitea/gitea/issues/31976
// or, we do not have commits here
continue continue
} }
// add repository to git project // add repository to git project
common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name)) common.PanicOnError(git.GitExec(gitPrj, "submodule", "--quiet", "add", "--depth", "1", r.CloneURL, r.Name))
if len(config.Branch) > 0 { curBranch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current"))
branch := strings.TrimSpace(git.GitExecWithOutputOrPanic(path.Join(gitPrj, r.Name), "branch", "--show-current")) if branch != curBranch {
if branch != config.Branch { if err := git.GitExec(path.Join(gitPrj, r.Name), "fetch", "--depth", "1", "origin", branch+":"+branch); err != nil {
if err := git.GitExec(path.Join(gitPrj, r.Name), "fetch", "--depth", "1", "origin", config.Branch+":"+config.Branch); err != nil { return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", branch, repo.Owner.UserName, r.Name)
return fmt.Errorf("Fetching branch %s for %s/%s failed. Ignoring.", config.Branch, repo.Owner.UserName, r.Name)
}
common.PanicOnError(git.GitExec(path.Join(gitPrj, r.Name), "checkout", config.Branch))
} }
common.PanicOnError(git.GitExec(path.Join(gitPrj, r.Name), "checkout", branch))
} }
isGitUpdated = true isGitUpdated = true
@@ -423,10 +419,7 @@ next_repo:
} }
} }
if DebugMode { common.LogInfo("Verification finished for ", org, ", prjgit:", config.GitProjectName)
log.Println("Verification finished for ", org, ", prjgit:", config.GitProjectName)
}
return nil return nil
} }
@@ -437,17 +430,17 @@ var checkInterval time.Duration
func checkOrg(org string, configs []*common.AutogitConfig) { func checkOrg(org string, configs []*common.AutogitConfig) {
git, err := gh.CreateGitHandler(org) git, err := gh.CreateGitHandler(org)
if err != nil { if err != nil {
log.Println("Faield to allocate GitHandler:", err) common.LogError("Failed to allocate GitHandler:", err)
return return
} }
defer git.Close() defer git.Close()
for _, config := range configs { for _, config := range configs {
log.Printf(" ++ starting verification, org: `%s` config: `%s`\n", org, config.GitProjectName) common.LogInfo(" ++ starting verification, org:", org, "config:", config.GitProjectName)
if err := verifyProjectState(git, org, config, configs); err != nil { if err := verifyProjectState(git, org, config, configs); err != nil {
log.Printf(" *** verification failed, org: `%s`, err: %#v\n", org, err) common.LogError(" *** verification failed, org:", org, err)
} else { } else {
log.Printf(" ++ verification complete, org: `%s` config: `%s`\n", org, config.GitProjectName) common.LogError(" ++ verification complete, org:", org, config.GitProjectName)
} }
} }
} }
@@ -456,7 +449,7 @@ func checkRepos() {
for org, configs := range configuredRepos { for org, configs := range configuredRepos {
if checkInterval > 0 { if checkInterval > 0 {
sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval))) sleepInterval := checkInterval - checkInterval/2 + time.Duration(rand.Int63n(int64(checkInterval)))
log.Println(" - sleep interval", sleepInterval, "until next check") common.LogInfo(" - sleep interval", sleepInterval, "until next check")
time.Sleep(sleepInterval) time.Sleep(sleepInterval)
} }
@@ -468,9 +461,9 @@ func consistencyCheckProcess() {
if checkOnStart { if checkOnStart {
savedCheckInterval := checkInterval savedCheckInterval := checkInterval
checkInterval = 0 checkInterval = 0
log.Println("== Startup consistency check begin...") common.LogInfo("== Startup consistency check begin...")
checkRepos() checkRepos()
log.Println("== Startup consistency check done...") common.LogInfo("== Startup consistency check done...")
checkInterval = savedCheckInterval checkInterval = savedCheckInterval
} }
@@ -485,7 +478,8 @@ var gh common.GitHandlerGenerator
func updateConfiguration(configFilename string, orgs *[]string) { func updateConfiguration(configFilename string, orgs *[]string) {
configFile, err := common.ReadConfigFile(configFilename) configFile, err := common.ReadConfigFile(configFilename)
if err != nil { if err != nil {
log.Fatal(err) common.LogError(err)
os.Exit(4)
} }
configs, _ := common.ResolveWorkflowConfigs(gitea, configFile) configs, _ := common.ResolveWorkflowConfigs(gitea, configFile)
@@ -493,9 +487,7 @@ func updateConfiguration(configFilename string, orgs *[]string) {
*orgs = make([]string, 0, 1) *orgs = make([]string, 0, 1)
for _, c := range configs { for _, c := range configs {
if slices.Contains(c.Workflows, "direct") { if slices.Contains(c.Workflows, "direct") {
if DebugMode { common.LogDebug(" + adding org:", c.Organization, ", branch:", c.Branch, ", prjgit:", c.GitProjectName)
log.Printf(" + adding org: '%s', branch: '%s', prjgit: '%s'\n", c.Organization, c.Branch, c.GitProjectName)
}
configs := configuredRepos[c.Organization] configs := configuredRepos[c.Organization]
if configs == nil { if configs == nil {
configs = make([]*common.AutogitConfig, 0, 1) configs = make([]*common.AutogitConfig, 0, 1)
@@ -520,10 +512,12 @@ func main() {
flag.Parse() flag.Parse()
if err := common.RequireGiteaSecretToken(); err != nil { if err := common.RequireGiteaSecretToken(); err != nil {
log.Fatal(err) common.LogError(err)
os.Exit(1)
} }
if err := common.RequireRabbitSecrets(); err != nil { if err := common.RequireRabbitSecrets(); err != nil {
log.Fatal(err) common.LogError(err)
os.Exit(1)
} }
defs := &common.RabbitMQGiteaEventsProcessor{} defs := &common.RabbitMQGiteaEventsProcessor{}
@@ -532,12 +526,14 @@ func main() {
if len(*basePath) == 0 { if len(*basePath) == 0 {
*basePath, err = os.MkdirTemp(os.TempDir(), AppName) *basePath, err = os.MkdirTemp(os.TempDir(), AppName)
if err != nil { if err != nil {
log.Fatal(err) common.LogError(err)
os.Exit(1)
} }
} }
gh, err = common.AllocateGitWorkTree(*basePath, GitAuthor, GitEmail) gh, err = common.AllocateGitWorkTree(*basePath, GitAuthor, GitEmail)
if err != nil { if err != nil {
log.Fatal(err) common.LogError(err)
os.Exit(1)
} }
// handle reconfiguration // handle reconfiguration
@@ -552,10 +548,10 @@ func main() {
} }
if sig != syscall.SIGHUP { if sig != syscall.SIGHUP {
log.Println("Unexpected signal received:", sig) common.LogError("Unexpected signal received:", sig)
continue continue
} }
log.Println("*** Reconfiguring ***") common.LogError("*** Reconfiguring ***")
updateConfiguration(*configFilename, &defs.Orgs) updateConfiguration(*configFilename, &defs.Orgs)
defs.Connection().UpdateTopics(defs) defs.Connection().UpdateTopics(defs)
} }
@@ -567,23 +563,25 @@ func main() {
gitea = common.AllocateGiteaTransport(*giteaUrl) gitea = common.AllocateGiteaTransport(*giteaUrl)
CurrentUser, err := gitea.GetCurrentUser() CurrentUser, err := gitea.GetCurrentUser()
if err != nil { if err != nil {
log.Fatalln("Cannot fetch current user:", err) common.LogError("Cannot fetch current user:", err)
os.Exit(2)
} }
log.Println("Current User:", CurrentUser.UserName) common.LogInfo("Current User:", CurrentUser.UserName)
updateConfiguration(*configFilename, &defs.Orgs) updateConfiguration(*configFilename, &defs.Orgs)
defs.Connection().RabbitURL, err = url.Parse(*rabbitUrl) defs.Connection().RabbitURL, err = url.Parse(*rabbitUrl)
if err != nil { if err != nil {
log.Panicf("cannot parse server URL. Err: %#v\n", err) common.LogError("cannot parse server URL. Err:", err)
os.Exit(3)
} }
go consistencyCheckProcess() go consistencyCheckProcess()
log.Println("defs:", *defs) common.LogInfo("defs:", *defs)
defs.Handlers = make(map[string]common.RequestProcessor) defs.Handlers = make(map[string]common.RequestProcessor)
defs.Handlers[common.RequestType_Push] = &PushActionProcessor{} defs.Handlers[common.RequestType_Push] = &PushActionProcessor{}
defs.Handlers[common.RequestType_Repository] = &RepositoryActionProcessor{} defs.Handlers[common.RequestType_Repository] = &RepositoryActionProcessor{}
log.Fatal(common.ProcessRabbitMQEvents(defs)) common.LogError(common.ProcessRabbitMQEvents(defs))
} }

View File

@@ -269,6 +269,7 @@ func (pr *PRProcessor) RebaseAndSkipSubmoduleCommits(prset *common.PRSet, branch
} }
var updatePrjGitError_requeue error = errors.New("Commits do not match. Requeing after 5 seconds.") var updatePrjGitError_requeue error = errors.New("Commits do not match. Requeing after 5 seconds.")
func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error { func (pr *PRProcessor) UpdatePrjGitPR(prset *common.PRSet) error {
_, _, PrjGitBranch := prset.Config.GetPrjGit() _, _, PrjGitBranch := prset.Config.GetPrjGit()
PrjGitPR, err := prset.GetPrjGitPR() PrjGitPR, err := prset.GetPrjGitPR()
@@ -613,7 +614,7 @@ func (w *RequestProcessor) ProcessFunc(request *common.Request) (err error) {
common.LogError("Cannot find PR for issue:", req.Pull_Request.Base.Repo.Owner.Username, req.Pull_Request.Base.Repo.Name, req.Pull_Request.Number) common.LogError("Cannot find PR for issue:", req.Pull_Request.Base.Repo.Owner.Username, req.Pull_Request.Base.Repo.Name, req.Pull_Request.Number)
return err return err
} }
} else if req, ok := request.Data.(*common.IssueWebhookEvent); ok { } else if req, ok := request.Data.(*common.IssueCommentWebhookEvent); ok {
pr, err = Gitea.GetPullRequest(req.Repository.Owner.Username, req.Repository.Name, int64(req.Issue.Number)) pr, err = Gitea.GetPullRequest(req.Repository.Owner.Username, req.Repository.Name, int64(req.Issue.Number))
if err != nil { if err != nil {
common.LogError("Cannot find PR for issue:", req.Repository.Owner.Username, req.Repository.Name, int64(req.Issue.Number)) common.LogError("Cannot find PR for issue:", req.Repository.Owner.Username, req.Repository.Name, int64(req.Issue.Number))