adamm/autogits
SHA256
2
0
Fork 1
Code Issues 8 Projects 2 Activity

fixes

Browse Source
This commit is contained in:
Adam Majer 2025-03-03 15:03:43 +01:00
parent 1cacb914b4
commit 72c2967d1f
4 changed files with 213 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
bots-common/git_utils.go
View File

@ -45,7 +45,7 @@ type GitStatusLister interface {
}
type Git interface {
GitParseCommits(cwd string, commitIDs []string) (parsedCommits []commit, err error)
GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error)
GitCatFile(cwd, commitId, filename string) (data []byte, err error)
GetPath() string
@ -285,18 +285,18 @@ func (c *ChanIO) Read(data []byte) (idx int, err error) {
return
}
type gitMsg struct {
type GitMsg struct {
hash string
itemType string
size int
}
type commit struct {
type GitCommit struct {
Tree string
Msg string
}
type tree_entry struct {
type GitTreeEntry struct {
name string
mode int
hash string
@ -304,23 +304,23 @@ type tree_entry struct {
size int
}
type tree struct {
items []tree_entry
type GitTree struct {
items []GitTreeEntry
}
func (t *tree_entry) isSubmodule() bool {
func (t *GitTreeEntry) isSubmodule() bool {
return (t.mode & 0170000) == 0160000
}
func (t *tree_entry) isTree() bool {
func (t *GitTreeEntry) isTree() bool {
return (t.mode & 0170000) == 0040000
}
func (t *tree_entry) isBlob() bool {
func (t *GitTreeEntry) isBlob() bool {
return !t.isTree() && !t.isSubmodule()
}
func parseGitMsg(data <-chan byte) (gitMsg, error) {
func parseGitMsg(data <-chan byte) (GitMsg, error) {
var id []byte = make([]byte, 64)
var msgType []byte = make([]byte, 16)
var size int
@ -331,7 +331,7 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) {
id[pos] = c
pos++
} else {
return gitMsg{}, fmt.Errorf("Invalid character during object hash parse '%c' at %d", c, pos)
return GitMsg{}, fmt.Errorf("Invalid character during object hash parse '%c' at %d", c, pos)
}
}
id = id[:pos]
@ -343,7 +343,7 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) {
msgType[pos] = c
pos++
} else {
return gitMsg{}, fmt.Errorf("Invalid character during object type parse '%c' at %d", c, pos)
return GitMsg{}, fmt.Errorf("Invalid character during object type parse '%c' at %d", c, pos)
}
}
msgType = msgType[:pos]
@ -353,26 +353,26 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) {
break
case "missing":
if c != '\x00' {
return gitMsg{}, fmt.Errorf("Missing format weird")
return GitMsg{}, fmt.Errorf("Missing format weird")
}
return gitMsg{
return GitMsg{
hash: string(id[:]),
itemType: "missing",
size: 0,
}, fmt.Errorf("Object not found: '%s'", string(id))
default:
return gitMsg{}, fmt.Errorf("Invalid object type: '%s'", string(msgType))
return GitMsg{}, fmt.Errorf("Invalid object type: '%s'", string(msgType))
}
for c = <-data; c != '\000'; c = <-data {
if c >= '0' && c <= '9' {
size = size*10 + (int(c) - '0')
} else {
return gitMsg{}, fmt.Errorf("Invalid character during object size parse: '%c'", c)
return GitMsg{}, fmt.Errorf("Invalid character during object size parse: '%c'", c)
}
}
return gitMsg{
return GitMsg{
hash: string(id[:]),
itemType: string(msgType),
size: size,
@ -412,20 +412,20 @@ func parseGitCommitMsg(data <-chan byte, l int) (string, error) {
return string(msg), nil
}
func parseGitCommit(data <-chan byte) (commit, error) {
func parseGitCommit(data <-chan byte) (GitCommit, error) {
hdr, err := parseGitMsg(data)
if err != nil {
return commit{}, err
return GitCommit{}, err
} else if hdr.itemType != "commit" {
return commit{}, fmt.Errorf("expected commit but parsed %s", hdr.itemType)
return GitCommit{}, fmt.Errorf("expected commit but parsed %s", hdr.itemType)
}
var c commit
var c GitCommit
l := hdr.size
for {
hdr, err := parseGitCommitHdr(data)
if err != nil {
return commit{}, nil
return GitCommit{}, nil
}
if len(hdr[0])+len(hdr[1]) == 0 { // hdr end marker
@ -445,8 +445,8 @@ func parseGitCommit(data <-chan byte) (commit, error) {
return c, err
}
func parseTreeEntry(data <-chan byte, hashLen int) (tree_entry, error) {
var e tree_entry
func parseTreeEntry(data <-chan byte, hashLen int) (GitTreeEntry, error) {
var e GitTreeEntry
for c := <-data; c != ' '; c = <-data {
e.mode = e.mode*8 + int(c-'0')
@ -475,20 +475,20 @@ func parseTreeEntry(data <-chan byte, hashLen int) (tree_entry, error) {
return e, nil
}
func parseGitTree(data <-chan byte) (tree, error) {
func parseGitTree(data <-chan byte) (GitTree, error) {
hdr, err := parseGitMsg(data)
if err != nil {
return tree{}, err
return GitTree{}, err
}
// max capacity to length of hash
t := tree{items: make([]tree_entry, 0, hdr.size/len(hdr.hash))}
t := GitTree{items: make([]GitTreeEntry, 0, hdr.size/len(hdr.hash))}
parsedLen := 0
for parsedLen < hdr.size {
entry, err := parseTreeEntry(data, len(hdr.hash)/2)
if err != nil {
return tree{}, nil
return GitTree{}, nil
}
t.items = append(t.items, entry)
@ -525,12 +525,12 @@ func parseGitBlob(data <-chan byte) ([]byte, error) {
return d, nil
}
func (e *GitHandlerImpl) GitParseCommits(cwd string, commitIDs []string) (parsedCommits []commit, err error) {
func (e *GitHandlerImpl) GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error) {
var done sync.Mutex
done.Lock()
data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)}
parsedCommits = make([]commit, 0, len(commitIDs))
parsedCommits = make([]GitCommit, 0, len(commitIDs))
go func() {
defer done.Unlock()
@ -653,7 +653,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
data_out.Write([]byte(commitId))
data_out.ch <- '\x00'
var c commit
var c GitCommit
c, err = parseGitCommit(data_in.ch)
if err != nil {
err = fmt.Errorf("Error parsing git commit. Err: %w", err)
@ -661,7 +661,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi
}
data_out.Write([]byte(c.Tree))
data_out.ch <- '\x00'
var tree tree
var tree GitTree
tree, err = parseGitTree(data_in.ch)
if err != nil {

72
bots-common/maintainership.go
View File

@ -2,6 +2,8 @@ package common
import (
"encoding/json"
"fmt"
"io"
"slices"
"src.opensuse.org/autogits/common/gitea-generated/client/repository"
@ -21,16 +23,16 @@ const ProjectKey = ""
const ProjectFileKey = "_project"
type MaintainershipMap struct {
data map[string][]string
is_dir bool
fetchPackage func(string) ([]byte, error)
Data map[string][]string
IsDir bool
FetchPackage func(string) ([]byte, error)
}
func parseMaintainershipData(data []byte) (*MaintainershipMap, error) {
maintainers := &MaintainershipMap{
data: make(map[string][]string),
Data: make(map[string][]string),
}
if err := json.Unmarshal(data, &maintainers.data); err != nil {
if err := json.Unmarshal(data, &maintainers.Data); err != nil {
return nil, err
}
@ -58,8 +60,8 @@ func FetchProjectMaintainershipData(gitea GiteaMaintainershipReader, org, prjGit
m, err := parseMaintainershipData(data)
if m != nil {
m.is_dir = dir
m.fetchPackage = func(pkg string) ([]byte, error) {
m.IsDir = dir
m.FetchPackage = func(pkg string) ([]byte, error) {
return gitea.FetchMaintainershipDirFile(org, prjGit, branch, pkg)
}
}
@ -71,7 +73,7 @@ func (data *MaintainershipMap) ListProjectMaintainers() []string {
return nil
}
m, found := data.data[ProjectKey]
m, found := data.Data[ProjectKey]
if !found {
return nil
}
@ -97,13 +99,13 @@ func (data *MaintainershipMap) ListPackageMaintainers(pkg string) []string {
return nil
}
pkgMaintainers, found := data.data[pkg]
if !found && data.is_dir {
pkgData, err := data.fetchPackage(pkg)
pkgMaintainers, found := data.Data[pkg]
if !found && data.IsDir {
pkgData, err := data.FetchPackage(pkg)
if err == nil {
pkgMaintainers = parsePkgDirData(pkg, pkgData)
if len(pkgMaintainers) > 0 {
data.data[pkg] = pkgMaintainers
data.Data[pkg] = pkgMaintainers
}
}
}
@ -123,15 +125,15 @@ prjMaintainer:
}
func (data *MaintainershipMap) IsApproved(pkg string, reviews []*models.PullReview) bool {
reviewers, found := data.data[pkg]
reviewers, found := data.Data[pkg]
if !found {
if pkg != ProjectKey && data.is_dir {
r, err := data.fetchPackage(pkg)
if pkg != ProjectKey && data.IsDir {
r, err := data.FetchPackage(pkg)
if err != nil {
return false
}
reviewers = parsePkgDirData(pkg, r)
data.data[pkg] = reviewers
data.Data[pkg] = reviewers
} else {
return true
}
@ -149,3 +151,41 @@ func (data *MaintainershipMap) IsApproved(pkg string, reviews []*models.PullRevi
}
return false
}
func (data *MaintainershipMap) WriteMaintainershipFile(writer io.StringWriter) error {
if data.IsDir {
return fmt.Errorf("Not implemented")
}
writer.WriteString("{\n")
if data, ok := data.Data[""]; ok {
slices.Sort(data)
str, _ := json.Marshal(data)
writer.WriteString(fmt.Sprintf(" \"\": %s\n", string(str)))
}
keys := make([]string, len(data.Data))
i := 0
for pkg := range data.Data {
if pkg == "" {
continue
}
keys[i] = pkg
i++
}
if len(keys) >= i {
keys = slices.Delete(keys, i, len(keys))
}
slices.Sort(keys)
for _, pkg := range(keys) {
maintainers := data.Data[pkg]
slices.Sort(maintainers)
pkgStr, _ := json.Marshal(pkg)
maintainersStr, _ := json.Marshal(maintainers)
writer.WriteString(fmt.Sprintf(" %s: %s\n", pkgStr, maintainersStr))
}
writer.WriteString("}\n")
return nil
}

44
bots-common/maintainership_test.go
View File

@ -1,6 +1,7 @@
package common_test
import (
"bytes"
"errors"
"slices"
"testing"
@ -184,5 +185,46 @@ func TestMaintainership(t *testing.T) {
}
}
func TestMaintainershipDir(t *testing.T) {
func TestMaintainershipFileWrite(t *testing.T) {
tests := []struct {
name string
is_dir bool
maintainers map[string][]string
expected_output string
expected_error error
}{
{
name: "empty dataset",
expected_output: "{\n}\n",
},
{
name: "2 project maintainers and 2 single package maintainers",
maintainers: map[string][]string{
"": {"two", "one"},
"pkg1": {"three"},
"foo": {"four", "byte"},
},
expected_output: "{\n \"\": [\"one\",\"two\"]\n \"foo\": [\"byte\",\"four\"]\n \"pkg1\": [\"three\"]\n}\n",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
b := bytes.Buffer{}
data := common.MaintainershipMap{
Data: test.maintainers,
IsDir: test.is_dir,
}
if err := data.WriteMaintainershipFile(&b); err != test.expected_error {
t.Fatal("unexpected error:", err, "Expecting:", test.expected_error)
}
output := b.String()
if test.expected_output != output {
t.Fatal("unexpected output:", output, "Expecting:", test.expected_output)
}
})
}
}

146
devel-importer/main.go
View File

@ -173,21 +173,21 @@ func importRepos(packages []string) {
for _, repo := range factoryRepos {
oldPackageNames = append(oldPackageNames, repo.Name)
}
/*
// fork packags from pool
for _, pkg := range oldPackageNames {
log.Println(" + package:", pkg)
cmd := exec.Command("./git-importer", "-r", git.GetPath(), pkg)
if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 {
cmd.Env = slices.Delete(cmd.Env, idx, idx+1)
}
out, err := cmd.CombinedOutput()
log.Println(string(out))
if err != nil {
log.Println("Error returned by importer.", err)
}
// fork packags from pool
for _, pkg := range oldPackageNames {
log.Println(" + package:", pkg)
cmd := exec.Command("./git-importer", "-r", git.GetPath(), pkg)
if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 {
cmd.Env = slices.Delete(cmd.Env, idx, idx+1)
}
*/
out, err := cmd.CombinedOutput()
log.Println(string(out))
if err != nil {
log.Println("Error returned by importer.", err)
}
}
log.Println("adding remotes...")
for i := range factoryRepos {
pkg := factoryRepos[i]
@ -251,19 +251,18 @@ func importRepos(packages []string) {
for _, pkgName := range oldPackageNames {
log.Println("fetching git:", pkgName)
remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "remote", "show", "-n"), "\n")
/*
params := []string{"fetch", "--multiple"}
params = append(params, remotes...)
out := git.GitExecWithOutputOrPanic(pkgName, params...)
if len(strings.TrimSpace(out)) > 1 {
log.Println(out)
}
if slices.Contains(remotes, "origin") {
log.Println(" --- scmsync already, so we are done")
continue
}
*/
params := []string{"fetch", "--multiple"}
params = append(params, remotes...)
out := git.GitExecWithOutputOrPanic(pkgName, params...)
if len(strings.TrimSpace(out)) > 1 {
log.Println(out)
}
if slices.Contains(remotes, "origin") {
log.Println(" --- scmsync already, so we are done")
continue
}
// check if devel is ahead or behind factory and use that as reference
import_branch := "factory"
@ -315,10 +314,11 @@ func importRepos(packages []string) {
}
if !found {
log.Panicln("Cannot find same tree for pkg", pkgName)
log.Println("*** WARNING: Cannot find same tree for pkg", pkgName, "Will use current import instead")
git.GitExecOrPanic(pkgName, "checkout", "-B", "main", import_branch)
}
} else {
git.GitExecOrPanic(pkgName, "git", "checkout", "-B", "main", import_branch)
git.GitExecOrPanic(pkgName, "checkout", "-B", "main", import_branch)
}
}
@ -329,17 +329,15 @@ func importRepos(packages []string) {
log.Panicln(err2)
}
} else {
/*
cmd := exec.Command("./git-importer", "-p", prj, "-r", git.GetPath(), pkg)
if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 {
cmd.Env = slices.Delete(cmd.Env, idx, idx+1)
}
out, err := cmd.CombinedOutput()
log.Println(string(out))
if err != nil {
log.Panicln("Error returned by importer.", err)
}
*/
cmd := exec.Command("./git-importer", "-p", prj, "-r", git.GetPath(), pkg)
if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 {
cmd.Env = slices.Delete(cmd.Env, idx, idx+1)
}
out, err := cmd.CombinedOutput()
log.Println(string(out))
if err != nil {
log.Panicln("Error returned by importer.", err)
}
}
// mark newer branch as main
@ -389,30 +387,28 @@ func importRepos(packages []string) {
git.GitExecOrPanic(pkg.Name, "remote", "add", "devel", repo.SSHURL)
// git.GitExecOrPanic(pkg.Name, "fetch", "devel")
}
git.GitExecOrPanic(pkg.Name, "branch", "main", "-f", "devel/main")
// git.GitExecOrPanic(pkg.Name, "push", "devel", "main")
/*
_, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(&models.EditRepoOption{
DefaultBranch: "main",
DefaultMergeStyle: "fast-forward-only",
HasPullRequests: true,
HasPackages: false,
HasReleases: false,
HasActions: false,
AllowMerge: true,
AllowRebaseMerge: false,
AllowSquash: false,
AllowFastForwardOnly: true,
AllowRebaseUpdate: false,
AllowManualMerge: false,
AllowRebase: false,
DefaultAllowMaintainerEdit: true,
}), r.DefaultAuthentication)
git.GitExecOrPanic(pkg.Name, "push", "devel", "main")
// git.GitExecOrPanic(pkg.Name, "checkout", "-B", "main", "devel/main")
_, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(&models.EditRepoOption{
DefaultBranch: "main",
DefaultMergeStyle: "fast-forward-only",
HasPullRequests: true,
HasPackages: false,
HasReleases: false,
HasActions: false,
AllowMerge: true,
AllowRebaseMerge: false,
AllowSquash: false,
AllowFastForwardOnly: true,
AllowRebaseUpdate: false,
AllowManualMerge: false,
AllowRebase: false,
DefaultAllowMaintainerEdit: true,
}), r.DefaultAuthentication)
if err != nil {
log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err)
}
*/
if err != nil {
log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err)
}
}
for _, pkg := range develProjectPackages {
@ -458,7 +454,7 @@ func importRepos(packages []string) {
if !slices.Contains(strings.Split(remotes, "\n"), "devel") {
git.GitExecOrPanic(pkg, "remote", "add", "devel", repo.SSHURL)
}
// git.GitExecOrPanic(pkg, "push", "devel", "main")
git.GitExecOrPanic(pkg, "push", "devel", "main")
_, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(pkg).WithBody(&models.EditRepoOption{
DefaultBranch: "main",
@ -547,7 +543,7 @@ func syncOrgTeams(groupName string, teamMembers []common.PersonRepoMeta) []strin
}
func syncOrgOwners(uids []common.PersonRepoMeta) []string {
return syncOrgTeams("Owners", uids)
return syncOrgTeams("Owners", append(uids, common.PersonRepoMeta{UserID: "autogits-devel"}))
}
func syncPackageCollaborators(pkg string, uids []common.PersonRepoMeta) []string {
@ -589,6 +585,10 @@ func syncPackageCollaborators(pkg string, uids []common.PersonRepoMeta) []string
}
func syncMaintainersToGitea(pkgs []string) {
maintainers := common.MaintainershipMap{
Data: map[string][]string{},
}
prjMeta, err := obs.GetProjectMeta(prj)
if err != nil {
log.Panicln("failed to get project meta", prj, err)
@ -606,6 +606,11 @@ func syncMaintainersToGitea(pkgs []string) {
log.Println("syncing", group.GroupID, teamMembers.Persons)
}
missingDevs = append(missingDevs, syncOrgTeams(group.GroupID, teamMembers.Persons.Persons)...)
devs := []string{}
for _, m := range teamMembers.Persons.Persons {
devs = append(devs, m.UserID)
}
maintainers.Data[""] = devs
}
missingDevs = append(missingDevs, syncOrgOwners(prjMeta.Persons)...)
@ -615,9 +620,22 @@ func syncMaintainersToGitea(pkgs []string) {
log.Panicln("failed to get package meta", prj, pkg, err)
}
missingDevs = append(missingDevs, syncPackageCollaborators(pkg, pkgMeta.Persons)...)
devs := []string{}
for _, m := range pkgMeta.Persons {
devs = append(devs, m.UserID)
}
maintainers.Data[pkg] = devs
}
slices.Sort(missingDevs)
file, err := os.Create(common.MaintainershipFile)
if err != nil {
log.Println(" *** Cannot create maintainership file:", err)
} else {
maintainers.WriteMaintainershipFile(file)
file.Close()
}
log.Println("Users without Gitea accounts:", slices.Compact(missingDevs))
}