diff --git a/bots-common/git_utils.go b/bots-common/git_utils.go index 96975e2..3b6bd61 100644 --- a/bots-common/git_utils.go +++ b/bots-common/git_utils.go @@ -45,7 +45,7 @@ type GitStatusLister interface { } type Git interface { - GitParseCommits(cwd string, commitIDs []string) (parsedCommits []commit, err error) + GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error) GitCatFile(cwd, commitId, filename string) (data []byte, err error) GetPath() string @@ -285,18 +285,18 @@ func (c *ChanIO) Read(data []byte) (idx int, err error) { return } -type gitMsg struct { +type GitMsg struct { hash string itemType string size int } -type commit struct { +type GitCommit struct { Tree string Msg string } -type tree_entry struct { +type GitTreeEntry struct { name string mode int hash string @@ -304,23 +304,23 @@ type tree_entry struct { size int } -type tree struct { - items []tree_entry +type GitTree struct { + items []GitTreeEntry } -func (t *tree_entry) isSubmodule() bool { +func (t *GitTreeEntry) isSubmodule() bool { return (t.mode & 0170000) == 0160000 } -func (t *tree_entry) isTree() bool { +func (t *GitTreeEntry) isTree() bool { return (t.mode & 0170000) == 0040000 } -func (t *tree_entry) isBlob() bool { +func (t *GitTreeEntry) isBlob() bool { return !t.isTree() && !t.isSubmodule() } -func parseGitMsg(data <-chan byte) (gitMsg, error) { +func parseGitMsg(data <-chan byte) (GitMsg, error) { var id []byte = make([]byte, 64) var msgType []byte = make([]byte, 16) var size int @@ -331,7 +331,7 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) { id[pos] = c pos++ } else { - return gitMsg{}, fmt.Errorf("Invalid character during object hash parse '%c' at %d", c, pos) + return GitMsg{}, fmt.Errorf("Invalid character during object hash parse '%c' at %d", c, pos) } } id = id[:pos] @@ -343,7 +343,7 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) { msgType[pos] = c pos++ } else { - return gitMsg{}, fmt.Errorf("Invalid character during object type parse '%c' at %d", c, pos) + return GitMsg{}, fmt.Errorf("Invalid character during object type parse '%c' at %d", c, pos) } } msgType = msgType[:pos] @@ -353,26 +353,26 @@ func parseGitMsg(data <-chan byte) (gitMsg, error) { break case "missing": if c != '\x00' { - return gitMsg{}, fmt.Errorf("Missing format weird") + return GitMsg{}, fmt.Errorf("Missing format weird") } - return gitMsg{ + return GitMsg{ hash: string(id[:]), itemType: "missing", size: 0, }, fmt.Errorf("Object not found: '%s'", string(id)) default: - return gitMsg{}, fmt.Errorf("Invalid object type: '%s'", string(msgType)) + return GitMsg{}, fmt.Errorf("Invalid object type: '%s'", string(msgType)) } for c = <-data; c != '\000'; c = <-data { if c >= '0' && c <= '9' { size = size*10 + (int(c) - '0') } else { - return gitMsg{}, fmt.Errorf("Invalid character during object size parse: '%c'", c) + return GitMsg{}, fmt.Errorf("Invalid character during object size parse: '%c'", c) } } - return gitMsg{ + return GitMsg{ hash: string(id[:]), itemType: string(msgType), size: size, @@ -412,20 +412,20 @@ func parseGitCommitMsg(data <-chan byte, l int) (string, error) { return string(msg), nil } -func parseGitCommit(data <-chan byte) (commit, error) { +func parseGitCommit(data <-chan byte) (GitCommit, error) { hdr, err := parseGitMsg(data) if err != nil { - return commit{}, err + return GitCommit{}, err } else if hdr.itemType != "commit" { - return commit{}, fmt.Errorf("expected commit but parsed %s", hdr.itemType) + return GitCommit{}, fmt.Errorf("expected commit but parsed %s", hdr.itemType) } - var c commit + var c GitCommit l := hdr.size for { hdr, err := parseGitCommitHdr(data) if err != nil { - return commit{}, nil + return GitCommit{}, nil } if len(hdr[0])+len(hdr[1]) == 0 { // hdr end marker @@ -445,8 +445,8 @@ func parseGitCommit(data <-chan byte) (commit, error) { return c, err } -func parseTreeEntry(data <-chan byte, hashLen int) (tree_entry, error) { - var e tree_entry +func parseTreeEntry(data <-chan byte, hashLen int) (GitTreeEntry, error) { + var e GitTreeEntry for c := <-data; c != ' '; c = <-data { e.mode = e.mode*8 + int(c-'0') @@ -475,20 +475,20 @@ func parseTreeEntry(data <-chan byte, hashLen int) (tree_entry, error) { return e, nil } -func parseGitTree(data <-chan byte) (tree, error) { +func parseGitTree(data <-chan byte) (GitTree, error) { hdr, err := parseGitMsg(data) if err != nil { - return tree{}, err + return GitTree{}, err } // max capacity to length of hash - t := tree{items: make([]tree_entry, 0, hdr.size/len(hdr.hash))} + t := GitTree{items: make([]GitTreeEntry, 0, hdr.size/len(hdr.hash))} parsedLen := 0 for parsedLen < hdr.size { entry, err := parseTreeEntry(data, len(hdr.hash)/2) if err != nil { - return tree{}, nil + return GitTree{}, nil } t.items = append(t.items, entry) @@ -525,12 +525,12 @@ func parseGitBlob(data <-chan byte) ([]byte, error) { return d, nil } -func (e *GitHandlerImpl) GitParseCommits(cwd string, commitIDs []string) (parsedCommits []commit, err error) { +func (e *GitHandlerImpl) GitParseCommits(cwd string, commitIDs []string) (parsedCommits []GitCommit, err error) { var done sync.Mutex done.Lock() data_in, data_out := ChanIO{make(chan byte, 256)}, ChanIO{make(chan byte, 70)} - parsedCommits = make([]commit, 0, len(commitIDs)) + parsedCommits = make([]GitCommit, 0, len(commitIDs)) go func() { defer done.Unlock() @@ -653,7 +653,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi data_out.Write([]byte(commitId)) data_out.ch <- '\x00' - var c commit + var c GitCommit c, err = parseGitCommit(data_in.ch) if err != nil { err = fmt.Errorf("Error parsing git commit. Err: %w", err) @@ -661,7 +661,7 @@ func (e *GitHandlerImpl) GitSubmoduleList(gitPath, commitId string) (submoduleLi } data_out.Write([]byte(c.Tree)) data_out.ch <- '\x00' - var tree tree + var tree GitTree tree, err = parseGitTree(data_in.ch) if err != nil { diff --git a/bots-common/maintainership.go b/bots-common/maintainership.go index c597d49..c378f36 100644 --- a/bots-common/maintainership.go +++ b/bots-common/maintainership.go @@ -2,6 +2,8 @@ package common import ( "encoding/json" + "fmt" + "io" "slices" "src.opensuse.org/autogits/common/gitea-generated/client/repository" @@ -21,16 +23,16 @@ const ProjectKey = "" const ProjectFileKey = "_project" type MaintainershipMap struct { - data map[string][]string - is_dir bool - fetchPackage func(string) ([]byte, error) + Data map[string][]string + IsDir bool + FetchPackage func(string) ([]byte, error) } func parseMaintainershipData(data []byte) (*MaintainershipMap, error) { maintainers := &MaintainershipMap{ - data: make(map[string][]string), + Data: make(map[string][]string), } - if err := json.Unmarshal(data, &maintainers.data); err != nil { + if err := json.Unmarshal(data, &maintainers.Data); err != nil { return nil, err } @@ -58,8 +60,8 @@ func FetchProjectMaintainershipData(gitea GiteaMaintainershipReader, org, prjGit m, err := parseMaintainershipData(data) if m != nil { - m.is_dir = dir - m.fetchPackage = func(pkg string) ([]byte, error) { + m.IsDir = dir + m.FetchPackage = func(pkg string) ([]byte, error) { return gitea.FetchMaintainershipDirFile(org, prjGit, branch, pkg) } } @@ -71,7 +73,7 @@ func (data *MaintainershipMap) ListProjectMaintainers() []string { return nil } - m, found := data.data[ProjectKey] + m, found := data.Data[ProjectKey] if !found { return nil } @@ -97,13 +99,13 @@ func (data *MaintainershipMap) ListPackageMaintainers(pkg string) []string { return nil } - pkgMaintainers, found := data.data[pkg] - if !found && data.is_dir { - pkgData, err := data.fetchPackage(pkg) + pkgMaintainers, found := data.Data[pkg] + if !found && data.IsDir { + pkgData, err := data.FetchPackage(pkg) if err == nil { pkgMaintainers = parsePkgDirData(pkg, pkgData) if len(pkgMaintainers) > 0 { - data.data[pkg] = pkgMaintainers + data.Data[pkg] = pkgMaintainers } } } @@ -123,15 +125,15 @@ prjMaintainer: } func (data *MaintainershipMap) IsApproved(pkg string, reviews []*models.PullReview) bool { - reviewers, found := data.data[pkg] + reviewers, found := data.Data[pkg] if !found { - if pkg != ProjectKey && data.is_dir { - r, err := data.fetchPackage(pkg) + if pkg != ProjectKey && data.IsDir { + r, err := data.FetchPackage(pkg) if err != nil { return false } reviewers = parsePkgDirData(pkg, r) - data.data[pkg] = reviewers + data.Data[pkg] = reviewers } else { return true } @@ -149,3 +151,41 @@ func (data *MaintainershipMap) IsApproved(pkg string, reviews []*models.PullRevi } return false } + +func (data *MaintainershipMap) WriteMaintainershipFile(writer io.StringWriter) error { + if data.IsDir { + return fmt.Errorf("Not implemented") + } + + writer.WriteString("{\n") + + if data, ok := data.Data[""]; ok { + slices.Sort(data) + str, _ := json.Marshal(data) + writer.WriteString(fmt.Sprintf(" \"\": %s\n", string(str))) + } + + keys := make([]string, len(data.Data)) + i := 0 + for pkg := range data.Data { + if pkg == "" { + continue + } + keys[i] = pkg + i++ + } + if len(keys) >= i { + keys = slices.Delete(keys, i, len(keys)) + } + slices.Sort(keys) + for _, pkg := range(keys) { + maintainers := data.Data[pkg] + slices.Sort(maintainers) + pkgStr, _ := json.Marshal(pkg) + maintainersStr, _ := json.Marshal(maintainers) + writer.WriteString(fmt.Sprintf(" %s: %s\n", pkgStr, maintainersStr)) + } + + writer.WriteString("}\n") + return nil +} diff --git a/bots-common/maintainership_test.go b/bots-common/maintainership_test.go index 11b9162..2e8177c 100644 --- a/bots-common/maintainership_test.go +++ b/bots-common/maintainership_test.go @@ -1,6 +1,7 @@ package common_test import ( + "bytes" "errors" "slices" "testing" @@ -184,5 +185,46 @@ func TestMaintainership(t *testing.T) { } } -func TestMaintainershipDir(t *testing.T) { +func TestMaintainershipFileWrite(t *testing.T) { + tests := []struct { + name string + is_dir bool + maintainers map[string][]string + expected_output string + expected_error error + }{ + { + name: "empty dataset", + expected_output: "{\n}\n", + }, + { + name: "2 project maintainers and 2 single package maintainers", + maintainers: map[string][]string{ + "": {"two", "one"}, + "pkg1": {"three"}, + "foo": {"four", "byte"}, + }, + expected_output: "{\n \"\": [\"one\",\"two\"]\n \"foo\": [\"byte\",\"four\"]\n \"pkg1\": [\"three\"]\n}\n", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + b := bytes.Buffer{} + data := common.MaintainershipMap{ + Data: test.maintainers, + IsDir: test.is_dir, + } + + if err := data.WriteMaintainershipFile(&b); err != test.expected_error { + t.Fatal("unexpected error:", err, "Expecting:", test.expected_error) + } + + output := b.String() + + if test.expected_output != output { + t.Fatal("unexpected output:", output, "Expecting:", test.expected_output) + } + }) + } } diff --git a/devel-importer/main.go b/devel-importer/main.go index d8c322d..728e4dc 100644 --- a/devel-importer/main.go +++ b/devel-importer/main.go @@ -173,21 +173,21 @@ func importRepos(packages []string) { for _, repo := range factoryRepos { oldPackageNames = append(oldPackageNames, repo.Name) } - /* - // fork packags from pool - for _, pkg := range oldPackageNames { - log.Println(" + package:", pkg) - cmd := exec.Command("./git-importer", "-r", git.GetPath(), pkg) - if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 { - cmd.Env = slices.Delete(cmd.Env, idx, idx+1) - } - out, err := cmd.CombinedOutput() - log.Println(string(out)) - if err != nil { - log.Println("Error returned by importer.", err) - } + + // fork packags from pool + for _, pkg := range oldPackageNames { + log.Println(" + package:", pkg) + cmd := exec.Command("./git-importer", "-r", git.GetPath(), pkg) + if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 { + cmd.Env = slices.Delete(cmd.Env, idx, idx+1) } - */ + out, err := cmd.CombinedOutput() + log.Println(string(out)) + if err != nil { + log.Println("Error returned by importer.", err) + } + } + log.Println("adding remotes...") for i := range factoryRepos { pkg := factoryRepos[i] @@ -251,19 +251,18 @@ func importRepos(packages []string) { for _, pkgName := range oldPackageNames { log.Println("fetching git:", pkgName) remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "remote", "show", "-n"), "\n") - /* - params := []string{"fetch", "--multiple"} - params = append(params, remotes...) - out := git.GitExecWithOutputOrPanic(pkgName, params...) - if len(strings.TrimSpace(out)) > 1 { - log.Println(out) - } - if slices.Contains(remotes, "origin") { - log.Println(" --- scmsync already, so we are done") - continue - } - */ + params := []string{"fetch", "--multiple"} + params = append(params, remotes...) + out := git.GitExecWithOutputOrPanic(pkgName, params...) + if len(strings.TrimSpace(out)) > 1 { + log.Println(out) + } + + if slices.Contains(remotes, "origin") { + log.Println(" --- scmsync already, so we are done") + continue + } // check if devel is ahead or behind factory and use that as reference import_branch := "factory" @@ -315,10 +314,11 @@ func importRepos(packages []string) { } if !found { - log.Panicln("Cannot find same tree for pkg", pkgName) + log.Println("*** WARNING: Cannot find same tree for pkg", pkgName, "Will use current import instead") + git.GitExecOrPanic(pkgName, "checkout", "-B", "main", import_branch) } } else { - git.GitExecOrPanic(pkgName, "git", "checkout", "-B", "main", import_branch) + git.GitExecOrPanic(pkgName, "checkout", "-B", "main", import_branch) } } @@ -329,17 +329,15 @@ func importRepos(packages []string) { log.Panicln(err2) } } else { - /* - cmd := exec.Command("./git-importer", "-p", prj, "-r", git.GetPath(), pkg) - if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 { - cmd.Env = slices.Delete(cmd.Env, idx, idx+1) - } - out, err := cmd.CombinedOutput() - log.Println(string(out)) - if err != nil { - log.Panicln("Error returned by importer.", err) - } - */ + cmd := exec.Command("./git-importer", "-p", prj, "-r", git.GetPath(), pkg) + if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 { + cmd.Env = slices.Delete(cmd.Env, idx, idx+1) + } + out, err := cmd.CombinedOutput() + log.Println(string(out)) + if err != nil { + log.Panicln("Error returned by importer.", err) + } } // mark newer branch as main @@ -389,30 +387,28 @@ func importRepos(packages []string) { git.GitExecOrPanic(pkg.Name, "remote", "add", "devel", repo.SSHURL) // git.GitExecOrPanic(pkg.Name, "fetch", "devel") } - git.GitExecOrPanic(pkg.Name, "branch", "main", "-f", "devel/main") - // git.GitExecOrPanic(pkg.Name, "push", "devel", "main") - /* - _, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(&models.EditRepoOption{ - DefaultBranch: "main", - DefaultMergeStyle: "fast-forward-only", - HasPullRequests: true, - HasPackages: false, - HasReleases: false, - HasActions: false, - AllowMerge: true, - AllowRebaseMerge: false, - AllowSquash: false, - AllowFastForwardOnly: true, - AllowRebaseUpdate: false, - AllowManualMerge: false, - AllowRebase: false, - DefaultAllowMaintainerEdit: true, - }), r.DefaultAuthentication) + git.GitExecOrPanic(pkg.Name, "push", "devel", "main") + // git.GitExecOrPanic(pkg.Name, "checkout", "-B", "main", "devel/main") + _, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(&models.EditRepoOption{ + DefaultBranch: "main", + DefaultMergeStyle: "fast-forward-only", + HasPullRequests: true, + HasPackages: false, + HasReleases: false, + HasActions: false, + AllowMerge: true, + AllowRebaseMerge: false, + AllowSquash: false, + AllowFastForwardOnly: true, + AllowRebaseUpdate: false, + AllowManualMerge: false, + AllowRebase: false, + DefaultAllowMaintainerEdit: true, + }), r.DefaultAuthentication) - if err != nil { - log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err) - } - */ + if err != nil { + log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err) + } } for _, pkg := range develProjectPackages { @@ -458,7 +454,7 @@ func importRepos(packages []string) { if !slices.Contains(strings.Split(remotes, "\n"), "devel") { git.GitExecOrPanic(pkg, "remote", "add", "devel", repo.SSHURL) } - // git.GitExecOrPanic(pkg, "push", "devel", "main") + git.GitExecOrPanic(pkg, "push", "devel", "main") _, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(pkg).WithBody(&models.EditRepoOption{ DefaultBranch: "main", @@ -547,7 +543,7 @@ func syncOrgTeams(groupName string, teamMembers []common.PersonRepoMeta) []strin } func syncOrgOwners(uids []common.PersonRepoMeta) []string { - return syncOrgTeams("Owners", uids) + return syncOrgTeams("Owners", append(uids, common.PersonRepoMeta{UserID: "autogits-devel"})) } func syncPackageCollaborators(pkg string, uids []common.PersonRepoMeta) []string { @@ -589,6 +585,10 @@ func syncPackageCollaborators(pkg string, uids []common.PersonRepoMeta) []string } func syncMaintainersToGitea(pkgs []string) { + maintainers := common.MaintainershipMap{ + Data: map[string][]string{}, + } + prjMeta, err := obs.GetProjectMeta(prj) if err != nil { log.Panicln("failed to get project meta", prj, err) @@ -606,6 +606,11 @@ func syncMaintainersToGitea(pkgs []string) { log.Println("syncing", group.GroupID, teamMembers.Persons) } missingDevs = append(missingDevs, syncOrgTeams(group.GroupID, teamMembers.Persons.Persons)...) + devs := []string{} + for _, m := range teamMembers.Persons.Persons { + devs = append(devs, m.UserID) + } + maintainers.Data[""] = devs } missingDevs = append(missingDevs, syncOrgOwners(prjMeta.Persons)...) @@ -615,9 +620,22 @@ func syncMaintainersToGitea(pkgs []string) { log.Panicln("failed to get package meta", prj, pkg, err) } missingDevs = append(missingDevs, syncPackageCollaborators(pkg, pkgMeta.Persons)...) + + devs := []string{} + for _, m := range pkgMeta.Persons { + devs = append(devs, m.UserID) + } + maintainers.Data[pkg] = devs } slices.Sort(missingDevs) + file, err := os.Create(common.MaintainershipFile) + if err != nil { + log.Println(" *** Cannot create maintainership file:", err) + } else { + maintainers.WriteMaintainershipFile(file) + file.Close() + } log.Println("Users without Gitea accounts:", slices.Compact(missingDevs)) }