adamm/autogits
SHA256
2
0
Fork 1
Code Issues 8 Projects 2 Activity

fixes

Browse Source
This commit is contained in:
Adam Majer 2025-03-03 15:03:43 +01:00
parent 1cacb914b4
commit 72c2967d1f
4 changed files with 213 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
bots-common/git_utils.go
View File

@ -45,7 +45,7 @@ type GitStatusLister interface {
} }
type Git interface { type Git interface {
GitParseCommits(cwd string, commitIDs []string) (parsedCommits []commit, err error) GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error)
GitCatFile(cwd, commitId, filename string) (data []byte, err error) GitCatFile(cwd, commitId, filename string) (data []byte, err error)
GetPath() string GetPath() string
@ -285,18 +285,18 @@ func (c *ChanIO) Read(data []byte) (idx int, err error) {
return return
} }
type gitMsg struct { type GitMsg struct {
hash string hash string
itemType string itemType string
size int size int
} }
type commit struct { type GitCommit struct {
Tree string Tree string
Msg string Msg string
} }
type tree_entry struct { type GitTreeEntry struct {
name string name string
mode int mode int
hash string hash string
@ -304,23 +304,23 @@ type tree_entry struct {
size int size int
} }
type tree struct { type GitTree struct {
items []tree_entry items []GitTreeEntry
} }
func (t *tree_entry) isSubmodule() bool { func (t *GitTreeEntry) isSubmodule() bool {
return (t.mode & 0170000) == 0160000 return (t.mode & 0170000) == 0160000
} }
func (t *tree_entry) isTree() bool { func (t *GitTreeEntry) isTree() bool {
return (t.mode & 0170000) == 0040000 return (t.mode & 0170000) == 0040000
} }
func (t *tree_entry) isBlob() bool { func (t *GitTreeEntry) isBlob() bool {
return !t.isTree() && !t.isSubmodule() return !t.isTree() && !t.isSubmodule()
} }
func parseGitMsg(data <-chan byte) (gitMsg, error) { func parseGitMsg(data <-chan byte) (GitMsg, error) {
var id []byte = make([]byte, 64) var id []byte = make([]byte, 64)
var msgType []byte = make([]byte, 16) var msgType []byte = make([]byte, 16)
var size int var size int
@ -331,7 +331,7 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) {
id[pos] = c id[pos] = c
pos++ pos++
} else { } else {
return gitMsg{}, fmt.Errorf("Invalid character during object hash parse '%c' at %d", c, pos) return GitMsg{}, fmt.Errorf("Invalid character during object hash parse '%c' at %d", c, pos)
} }
} }
id = id[:pos] id = id[:pos]
@ -343,7 +343,7 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) {
msgType[pos] = c msgType[pos] = c
pos++ pos++
} else { } else {
return gitMsg{}, fmt.Errorf("Invalid character during object type parse '%c' at %d", c, pos) return GitMsg{}, fmt.Errorf("Invalid character during object type parse '%c' at %d", c, pos)
} }
} }
msgType = msgType[:pos] msgType = msgType[:pos]
@ -353,26 +353,26 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) {
break break
case "missing": case "missing":
if c != '\x00' { if c != '\x00' {
return gitMsg{}, fmt.Errorf("Missing format weird") return GitMsg{}, fmt.Errorf("Missing format weird")
} }
return gitMsg{ return GitMsg{
hash: string(id[:]), hash: string(id[:]),
itemType: "missing", itemType: "missing",
size: 0, size: 0,
}, fmt.Errorf("Object not found: '%s'", string(id)) }, fmt.Errorf("Object not found: '%s'", string(id))
default: default:
return gitMsg{}, fmt.Errorf("Invalid object type: '%s'", string(msgType)) return GitMsg{}, fmt.Errorf("Invalid object type: '%s'", string(msgType))
} }
for c = <-data; c != '\000'; c = <-data { for c = <-data; c != '\000'; c = <-data {
if c >= '0' && c <= '9' { if c >= '0' && c <= '9' {
size = size*10 + (int(c) - '0') size = size*10 + (int(c) - '0')
} else { } else {
return gitMsg{}, fmt.Errorf("Invalid character during object size parse: '%c'", c) return GitMsg{}, fmt.Errorf("Invalid character during object size parse: '%c'", c)
} }
} }
return gitMsg{ return GitMsg{
hash: string(id[:]), hash: string(id[:]),
itemType: string(msgType), itemType: string(msgType),
size: size, size: size,
@ -412,20 +412,20 @@ func parseGitCommitMsg(data <-chan byte, l int) (string, error) {
return string(msg), nil return string(msg), nil
} }
func parseGitCommit(data <-chan byte) (commit, error) { func parseGitCommit(data <-chan byte) (GitCommit, error) {
hdr, err := parseGitMsg(data) hdr, err := parseGitMsg(data)
if err != nil { if err != nil {
return commit{}, err return GitCommit{}, err
} else if hdr.itemType != "commit" { } else if hdr.itemType != "commit" {
return commit{}, fmt.Errorf("expected commit but parsed %s", hdr.itemType) return GitCommit{}, fmt.Errorf("expected commit but parsed %s", hdr.itemType)
} }
var c commit var c GitCommit
l := hdr.size l := hdr.size
for { for {
hdr, err := parseGitCommitHdr(data) hdr, err := parseGitCommitHdr(data)
if err != nil { if err != nil {
return commit{}, nil return GitCommit{}, nil
} }
if len(hdr[0])+len(hdr[1]) == 0 { // hdr end marker if len(hdr[0])+len(hdr[1]) == 0 { // hdr end marker
@ -445,8 +445,8 @@ func parseGitCommit(data <-chan byte) (commit, error) {
return c, err return c, err
} }
func parseTreeEntry(data <-chan byte, hashLen int) (tree_entry, error) { func parseTreeEntry(data <-chan byte, hashLen int) (GitTreeEntry, error) {
var e tree_entry var e GitTreeEntry
for c := <-data; c != ' '; c = <-data { for c := <-data; c != ' '; c = <-data {
e.mode = e.mode*8 + int(c-'0') e.mode = e.mode*8 + int(c-'0')
@ -475,20 +475,20 @@ func parseTreeEntry(data <-chan byte, hashLen int) (tree_entry, error) {
return e, nil return e, nil
} }
func parseGitTree(data <-chan byte) (tree, error) { func parseGitTree(data <-chan byte) (GitTree, error) {
hdr, err := parseGitMsg(data) hdr, err := parseGitMsg(data)
if err != nil { if err != nil {
return tree{}, err return GitTree{}, err
} }
// max capacity to length of hash // max capacity to length of hash
t := tree{items: make([]tree_entry, 0, hdr.size/len(hdr.hash))} t := GitTree{items: make([]GitTreeEntry, 0, hdr.size/len(hdr.hash))}
parsedLen := 0 parsedLen := 0
for parsedLen < hdr.size { for parsedLen < hdr.size {
entry, err := parseTreeEntry(data, len(hdr.hash)/2) entry, err := parseTreeEntry(data, len(hdr.hash)/2)
if err != nil { if err != nil {
return tree{}, nil return GitTree{}, nil
} }
t.items = append(t.items, entry) t.items = append(t.items, entry)
@ -525,12 +525,12 @@ func parseGitBlob(data <-chan byte) ([]byte, error) {
return d, nil return d, nil
} }
func (e *GitHandlerImpl) GitParseCommits(cwd string, commitIDs []string) (parsedCommits []commit, err error) { func (e *GitHandlerImpl) GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error) {
var done sync.Mutex var done sync.Mutex
done.Lock() done.Lock()
data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)} data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
parsedCommits = make([]commit, 0, len(commitIDs)) parsedCommits = make([]GitCommit, 0, len(commitIDs))
go func() { go func() {
defer done.Unlock() defer done.Unlock()
@ -653,7 +653,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
data_out.Write([]byte(commitId)) data_out.Write([]byte(commitId))
data_out.ch <- '\x00' data_out.ch <- '\x00'
var c commit var c GitCommit
c, err = parseGitCommit(data_in.ch) c, err = parseGitCommit(data_in.ch)
if err != nil { if err != nil {
err = fmt.Errorf("Error parsing git commit. Err: %w", err) err = fmt.Errorf("Error parsing git commit. Err: %w", err)
@ -661,7 +661,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
} }
data_out.Write([]byte(c.Tree)) data_out.Write([]byte(c.Tree))
data_out.ch <- '\x00' data_out.ch <- '\x00'
var tree tree var tree GitTree
tree, err = parseGitTree(data_in.ch) tree, err = parseGitTree(data_in.ch)
if err != nil { if err != nil {

72
bots-common/maintainership.go
View File

@ -2,6 +2,8 @@ package common
import ( import (
"encoding/json" "encoding/json"
"fmt"
"io"
"slices" "slices"
"src.opensuse.org/autogits/common/gitea-generated/client/repository" "src.opensuse.org/autogits/common/gitea-generated/client/repository"
@ -21,16 +23,16 @@ const ProjectKey = ""
const ProjectFileKey = "_project" const ProjectFileKey = "_project"
type MaintainershipMap struct { type MaintainershipMap struct {
data map[string][]string Data map[string][]string
is_dir bool IsDir bool
fetchPackage func(string) ([]byte, error) FetchPackage func(string) ([]byte, error)
} }
func parseMaintainershipData(data []byte) (*MaintainershipMap, error) { func parseMaintainershipData(data []byte) (*MaintainershipMap, error) {
maintainers := &MaintainershipMap{ maintainers := &MaintainershipMap{
data: make(map[string][]string), Data: make(map[string][]string),
} }
if err := json.Unmarshal(data, &maintainers.data); err != nil { if err := json.Unmarshal(data, &maintainers.Data); err != nil {
return nil, err return nil, err
} }
@ -58,8 +60,8 @@ func FetchProjectMaintainershipData(gitea GiteaMaintainershipReader, org, prjGit
m, err := parseMaintainershipData(data) m, err := parseMaintainershipData(data)
if m != nil { if m != nil {
m.is_dir = dir m.IsDir = dir
m.fetchPackage = func(pkg string) ([]byte, error) { m.FetchPackage = func(pkg string) ([]byte, error) {
return gitea.FetchMaintainershipDirFile(org, prjGit, branch, pkg) return gitea.FetchMaintainershipDirFile(org, prjGit, branch, pkg)
} }
} }
@ -71,7 +73,7 @@ func (data *MaintainershipMap) ListProjectMaintainers() []string {
return nil return nil
} }
m, found := data.data[ProjectKey] m, found := data.Data[ProjectKey]
if !found { if !found {
return nil return nil
} }
@ -97,13 +99,13 @@ func (data *MaintainershipMap) ListPackageMaintainers(pkg string) []string {
return nil return nil
} }
pkgMaintainers, found := data.data[pkg] pkgMaintainers, found := data.Data[pkg]
if !found && data.is_dir { if !found && data.IsDir {
pkgData, err := data.fetchPackage(pkg) pkgData, err := data.FetchPackage(pkg)
if err == nil { if err == nil {
pkgMaintainers = parsePkgDirData(pkg, pkgData) pkgMaintainers = parsePkgDirData(pkg, pkgData)
if len(pkgMaintainers) > 0 { if len(pkgMaintainers) > 0 {
data.data[pkg] = pkgMaintainers data.Data[pkg] = pkgMaintainers
} }
} }
} }
@ -123,15 +125,15 @@ prjMaintainer:
} }
func (data *MaintainershipMap) IsApproved(pkg string, reviews []*models.PullReview) bool { func (data *MaintainershipMap) IsApproved(pkg string, reviews []*models.PullReview) bool {
reviewers, found := data.data[pkg] reviewers, found := data.Data[pkg]
if !found { if !found {
if pkg != ProjectKey && data.is_dir { if pkg != ProjectKey && data.IsDir {
r, err := data.fetchPackage(pkg) r, err := data.FetchPackage(pkg)
if err != nil { if err != nil {
return false return false
} }
reviewers = parsePkgDirData(pkg, r) reviewers = parsePkgDirData(pkg, r)
data.data[pkg] = reviewers data.Data[pkg] = reviewers
} else { } else {
return true return true
} }
@ -149,3 +151,41 @@ func (data *MaintainershipMap) IsApproved(pkg string, reviews []*models.PullRevi
} }
return false return false
} }
func (data *MaintainershipMap) WriteMaintainershipFile(writer io.StringWriter) error {
if data.IsDir {
return fmt.Errorf("Not implemented")
}
writer.WriteString("{\n")
if data, ok := data.Data[""]; ok {
slices.Sort(data)
str, _ := json.Marshal(data)
writer.WriteString(fmt.Sprintf(" \"\": %s\n", string(str)))
}
keys := make([]string, len(data.Data))
i := 0
for pkg := range data.Data {
if pkg == "" {
continue
}
keys[i] = pkg
i++
}
if len(keys) >= i {
keys = slices.Delete(keys, i, len(keys))
}
slices.Sort(keys)
for _, pkg := range(keys) {
maintainers := data.Data[pkg]
slices.Sort(maintainers)
pkgStr, _ := json.Marshal(pkg)
maintainersStr, _ := json.Marshal(maintainers)
writer.WriteString(fmt.Sprintf(" %s: %s\n", pkgStr, maintainersStr))
}
writer.WriteString("}\n")
return nil
}

44
bots-common/maintainership_test.go
View File

@ -1,6 +1,7 @@
package common_test package common_test
import ( import (
"bytes"
"errors" "errors"
"slices" "slices"
"testing" "testing"
@ -184,5 +185,46 @@ func TestMaintainership(t *testing.T) {
} }
} }
func TestMaintainershipDir(t *testing.T) { func TestMaintainershipFileWrite(t *testing.T) {
tests := []struct {
name string
is_dir bool
maintainers map[string][]string
expected_output string
expected_error error
}{
{
name: "empty dataset",
expected_output: "{\n}\n",
},
{
name: "2 project maintainers and 2 single package maintainers",
maintainers: map[string][]string{
"": {"two", "one"},
"pkg1": {"three"},
"foo": {"four", "byte"},
},
expected_output: "{\n \"\": [\"one\",\"two\"]\n \"foo\": [\"byte\",\"four\"]\n \"pkg1\": [\"three\"]\n}\n",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
b := bytes.Buffer{}
data := common.MaintainershipMap{
Data: test.maintainers,
IsDir: test.is_dir,
}
if err := data.WriteMaintainershipFile(&b); err != test.expected_error {
t.Fatal("unexpected error:", err, "Expecting:", test.expected_error)
}
output := b.String()
if test.expected_output != output {
t.Fatal("unexpected output:", output, "Expecting:", test.expected_output)
}
})
}
} }

46
devel-importer/main.go
View File

@ -173,7 +173,7 @@ func importRepos(packages []string) {
for _, repo := range factoryRepos { for _, repo := range factoryRepos {
oldPackageNames = append(oldPackageNames, repo.Name) oldPackageNames = append(oldPackageNames, repo.Name)
} }
/*
// fork packags from pool // fork packags from pool
for _, pkg := range oldPackageNames { for _, pkg := range oldPackageNames {
log.Println(" + package:", pkg) log.Println(" + package:", pkg)
@ -187,7 +187,7 @@ func importRepos(packages []string) {
log.Println("Error returned by importer.", err) log.Println("Error returned by importer.", err)
} }
} }
*/
log.Println("adding remotes...") log.Println("adding remotes...")
for i := range factoryRepos { for i := range factoryRepos {
pkg := factoryRepos[i] pkg := factoryRepos[i]
@ -251,7 +251,7 @@ func importRepos(packages []string) {
for _, pkgName := range oldPackageNames { for _, pkgName := range oldPackageNames {
log.Println("fetching git:", pkgName) log.Println("fetching git:", pkgName)
remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "remote", "show", "-n"), "\n") remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "remote", "show", "-n"), "\n")
/*
params := []string{"fetch", "--multiple"} params := []string{"fetch", "--multiple"}
params = append(params, remotes...) params = append(params, remotes...)
out := git.GitExecWithOutputOrPanic(pkgName, params...) out := git.GitExecWithOutputOrPanic(pkgName, params...)
@ -263,7 +263,6 @@ func importRepos(packages []string) {
log.Println(" --- scmsync already, so we are done") log.Println(" --- scmsync already, so we are done")
continue continue
} }
*/
// check if devel is ahead or behind factory and use that as reference // check if devel is ahead or behind factory and use that as reference
import_branch := "factory" import_branch := "factory"
@ -315,10 +314,11 @@ func importRepos(packages []string) {
} }
if !found { if !found {
log.Panicln("Cannot find same tree for pkg", pkgName) log.Println("*** WARNING: Cannot find same tree for pkg", pkgName, "Will use current import instead")
git.GitExecOrPanic(pkgName, "checkout", "-B", "main", import_branch)
} }
} else { } else {
git.GitExecOrPanic(pkgName, "git", "checkout", "-B", "main", import_branch) git.GitExecOrPanic(pkgName, "checkout", "-B", "main", import_branch)
} }
} }
@ -329,7 +329,6 @@ func importRepos(packages []string) {
log.Panicln(err2) log.Panicln(err2)
} }
} else { } else {
/*
cmd := exec.Command("./git-importer", "-p", prj, "-r", git.GetPath(), pkg) cmd := exec.Command("./git-importer", "-p", prj, "-r", git.GetPath(), pkg)
if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 { if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 {
cmd.Env = slices.Delete(cmd.Env, idx, idx+1) cmd.Env = slices.Delete(cmd.Env, idx, idx+1)
@ -339,7 +338,6 @@ func importRepos(packages []string) {
if err != nil { if err != nil {
log.Panicln("Error returned by importer.", err) log.Panicln("Error returned by importer.", err)
} }
*/
} }
// mark newer branch as main // mark newer branch as main
@ -389,9 +387,8 @@ func importRepos(packages []string) {
git.GitExecOrPanic(pkg.Name, "remote", "add", "devel", repo.SSHURL) git.GitExecOrPanic(pkg.Name, "remote", "add", "devel", repo.SSHURL)
// git.GitExecOrPanic(pkg.Name, "fetch", "devel") // git.GitExecOrPanic(pkg.Name, "fetch", "devel")
} }
git.GitExecOrPanic(pkg.Name, "branch", "main", "-f", "devel/main") git.GitExecOrPanic(pkg.Name, "push", "devel", "main")
// git.GitExecOrPanic(pkg.Name, "push", "devel", "main") // git.GitExecOrPanic(pkg.Name, "checkout", "-B", "main", "devel/main")
/*
_, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(&models.EditRepoOption{ _, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(&models.EditRepoOption{
DefaultBranch: "main", DefaultBranch: "main",
DefaultMergeStyle: "fast-forward-only", DefaultMergeStyle: "fast-forward-only",
@ -412,7 +409,6 @@ func importRepos(packages []string) {
if err != nil { if err != nil {
log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err) log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err)
} }
*/
} }
for _, pkg := range develProjectPackages { for _, pkg := range develProjectPackages {
@ -458,7 +454,7 @@ func importRepos(packages []string) {
if !slices.Contains(strings.Split(remotes, "\n"), "devel") { if !slices.Contains(strings.Split(remotes, "\n"), "devel") {
git.GitExecOrPanic(pkg, "remote", "add", "devel", repo.SSHURL) git.GitExecOrPanic(pkg, "remote", "add", "devel", repo.SSHURL)
} }
// git.GitExecOrPanic(pkg, "push", "devel", "main") git.GitExecOrPanic(pkg, "push", "devel", "main")
_, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(pkg).WithBody(&models.EditRepoOption{ _, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(pkg).WithBody(&models.EditRepoOption{
DefaultBranch: "main", DefaultBranch: "main",
@ -547,7 +543,7 @@ func syncOrgTeams(groupName string, teamMembers []common.PersonRepoMeta) []strin
} }
func syncOrgOwners(uids []common.PersonRepoMeta) []string { func syncOrgOwners(uids []common.PersonRepoMeta) []string {
return syncOrgTeams("Owners", uids) return syncOrgTeams("Owners", append(uids, common.PersonRepoMeta{UserID: "autogits-devel"}))
} }
func syncPackageCollaborators(pkg string, uids []common.PersonRepoMeta) []string { func syncPackageCollaborators(pkg string, uids []common.PersonRepoMeta) []string {
@ -589,6 +585,10 @@ func syncPackageCollaborators(pkg string, uids []common.PersonRepoMeta) []string
} }
func syncMaintainersToGitea(pkgs []string) { func syncMaintainersToGitea(pkgs []string) {
maintainers := common.MaintainershipMap{
Data: map[string][]string{},
}
prjMeta, err := obs.GetProjectMeta(prj) prjMeta, err := obs.GetProjectMeta(prj)
if err != nil { if err != nil {
log.Panicln("failed to get project meta", prj, err) log.Panicln("failed to get project meta", prj, err)
@ -606,6 +606,11 @@ func syncMaintainersToGitea(pkgs []string) {
log.Println("syncing", group.GroupID, teamMembers.Persons) log.Println("syncing", group.GroupID, teamMembers.Persons)
} }
missingDevs = append(missingDevs, syncOrgTeams(group.GroupID, teamMembers.Persons.Persons)...) missingDevs = append(missingDevs, syncOrgTeams(group.GroupID, teamMembers.Persons.Persons)...)
devs := []string{}
for _, m := range teamMembers.Persons.Persons {
devs = append(devs, m.UserID)
}
maintainers.Data[""] = devs
} }
missingDevs = append(missingDevs, syncOrgOwners(prjMeta.Persons)...) missingDevs = append(missingDevs, syncOrgOwners(prjMeta.Persons)...)
@ -615,9 +620,22 @@ func syncMaintainersToGitea(pkgs []string) {
log.Panicln("failed to get package meta", prj, pkg, err) log.Panicln("failed to get package meta", prj, pkg, err)
} }
missingDevs = append(missingDevs, syncPackageCollaborators(pkg, pkgMeta.Persons)...) missingDevs = append(missingDevs, syncPackageCollaborators(pkg, pkgMeta.Persons)...)
devs := []string{}
for _, m := range pkgMeta.Persons {
devs = append(devs, m.UserID)
}
maintainers.Data[pkg] = devs
} }
slices.Sort(missingDevs) slices.Sort(missingDevs)
file, err := os.Create(common.MaintainershipFile)
if err != nil {
log.Println(" *** Cannot create maintainership file:", err)
} else {
maintainers.WriteMaintainershipFile(file)
file.Close()
}
log.Println("Users without Gitea accounts:", slices.Compact(missingDevs)) log.Println("Users without Gitea accounts:", slices.Compact(missingDevs))
} }