package main /* * This file is part of Autogits. * * Copyright © 2024 SUSE LLC * * Autogits is free software: you can redistribute it and/or modify it under * the terms of the GNU General Public License as published by the Free Software * Foundation, either version 2 of the License, or (at your option) any later * version. * * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A * PARTICULAR PURPOSE. See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along with * Foobar. If not, see . */ import ( "bytes" "errors" "flag" "fmt" "log" "os" "os/exec" "path/filepath" "slices" "strings" transport "github.com/go-openapi/runtime/client" "src.opensuse.org/autogits/common" apiclient "src.opensuse.org/autogits/common/gitea-generated/client" "src.opensuse.org/autogits/common/gitea-generated/client/organization" "src.opensuse.org/autogits/common/gitea-generated/client/repository" "src.opensuse.org/autogits/common/gitea-generated/models" ) const commandLineHelp = ` SYNTAX devel-importer ` func printHelp(flags string) { os.Stdout.WriteString(commandLineHelp) os.Stdout.WriteString(flags) } func outputList(str string) []string { out := []string{} for _, l := range strings.Split(strings.TrimSpace(str), "\n") { l = strings.TrimSpace(l) if len(l) > 0 { out = append(out, l) } } return out } func runObsCommand(args ...string) ([]string, error) { cmd := exec.Command("osc", args...) out, err := cmd.Output() return outputList(string(out)), err } var DebugMode bool func projectMaintainer(obs *common.ObsClient, prj string) ([]string, []string) { // users, groups meta, err := obs.GetProjectMeta(prj) if err != nil { log.Panicln(err) } uids := []string{} gids := []string{} for _, p := range meta.Persons { if !slices.Contains(uids, p.UserID) { uids = append(uids, p.UserID) } } for _, g := range meta.Groups { if !slices.Contains(gids, g.GroupID) { gids = append(gids, g.GroupID) } } return uids, gids } func packageMaintainers(obs *common.ObsClient, prj, pkg string) ([]string, []string) { // users, groups meta, err := obs.GetPackageMeta(prj, pkg) if err != nil { log.Panicln(err, "FOR:", prj, "/", pkg) } uids := []string{} gids := []string{} for _, p := range meta.Persons { if !slices.Contains(uids, p.UserID) { uids = append(uids, p.UserID) } } for _, g := range meta.Groups { if !slices.Contains(gids, g.GroupID) { gids = append(gids, g.GroupID) } } return uids, gids } func listMaintainers(obs *common.ObsClient, prj string, pkgs []string) { users, groups := projectMaintainer(obs, prj) log.Println("Fetching maintainers for prj:", prj, users) for _, pkg := range pkgs { u, g := packageMaintainers(obs, prj, pkg) log.Println("maintainers for pkg:", pkg, u) users = append(users, u...) groups = append(groups, g...) } slices.Sort(users) slices.Sort(groups) users = slices.Compact(users) groups = slices.Compact(groups) log.Println("need to contact following:", strings.Join(users, ", ")) contact_email := []string{} for _, uid := range users { user, err := obs.GetUserMeta(uid) if err != nil { log.Panicln(err) } contact_email = append(contact_email, fmt.Sprintf("%s <%s>", user.Name, user.Email)) } log.Println(strings.Join(contact_email, ", ")) } func importRepos(packages []string) { factoryRepos := make([]*models.Repository, 0, len(packages)*2) develProjectPackages := make([]string, 0, len(packages)) for _, pkg := range packages { src_pkg_name := strings.Split(pkg, ":") repo, err := client.Repository.RepoGet( repository.NewRepoGetParams(). WithDefaults().WithOwner("pool").WithRepo(src_pkg_name[0]), r.DefaultAuthentication) if err != nil { if !errors.Is(err, &repository.RepoGetNotFound{}) { log.Panicln(err) } log.Println("Cannot find src package:", src_pkg_name) develProjectPackages = append(develProjectPackages, src_pkg_name[0]) } else { factoryRepos = append(factoryRepos, repo.Payload) } } log.Println("Num repos found:", len(factoryRepos)) oldPackageNames := make([]string, 0, len(factoryRepos)) for _, repo := range factoryRepos { oldPackageNames = append(oldPackageNames, repo.Name) } // fork packags from pool for _, pkg := range oldPackageNames { log.Println(" + package:", pkg) cmd := exec.Command("./git-importer", "-r", git.GetPath(), pkg) if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 { cmd.Env = slices.Delete(cmd.Env, idx, idx+1) } out, err := cmd.CombinedOutput() log.Println(string(out)) if err != nil { log.Println("Error returned by importer.", err) } } log.Println("adding remotes...") for i := range factoryRepos { pkg := factoryRepos[i] // verify that package was created by `git-importer`, or it's scmsync package and clone it fi, err := os.Stat(filepath.Join(git.GetPath(), pkg.Name)) if os.IsNotExist(err) { // scmsync? devel_project, err := runObsCommand("develproject", "openSUSE:Factory", pkg.Name) if err != nil || len(devel_project) != 1 { log.Panicln("devel project len:", len(devel_project), "err:", err) } d := strings.Split(devel_project[0], "/") if len(d) != 2 { log.Panicln("expected devel project/package. got:", d) } meta, _ := obs.GetPackageMeta(d[0], d[1]) if len(meta.ScmSync) > 0 { if err2 := git.CloneDevel("", pkg.Name, meta.ScmSync); err != nil { log.Panicln(err2) } } // try again, should now exist if fi, err = os.Stat(filepath.Join(git.GetPath(), pkg.Name)); err != nil { log.Panicln(err) } } else if err != nil { log.Panicln(err) } else { // verify that we do not have scmsync for imported packages meta, err := obs.GetPackageMeta(prj, pkg.Name) if err != nil { log.Panicln(err) } if len(meta.ScmSync) > 0 { log.Panicln("importing an scmsync package??:", prj, pkg.Name) } } if !fi.IsDir() { log.Panicln("Expected package file should be a directory. It's not.", fi) } // add remote repos out := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "show", "-n") switch pkg.Owner.UserName { case "pool": if !slices.Contains(strings.Split(out, "\n"), "factory") { out := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "add", "factory", pkg.CloneURL) if len(strings.TrimSpace(out)) > 1 { log.Println(out) } } default: log.Panicln(pkg.Owner.UserName) } } for _, pkgName := range oldPackageNames { log.Println("fetching git:", pkgName) remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "remote", "show", "-n"), "\n") params := []string{"fetch", "--multiple"} params = append(params, remotes...) out := git.GitExecWithOutputOrPanic(pkgName, params...) if len(strings.TrimSpace(out)) > 1 { log.Println(out) } if slices.Contains(remotes, "origin") { log.Println(" --- scmsync already, so we are done") continue } // check if devel is ahead or behind factory and use that as reference import_branch := "factory" if len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "rev-list", "^factory", "devel"), "\n")) > 0 { log.Println(" *** devel ahead. Swtiching branches.") import_branch = "devel" } // check that nothing is broken with the update if slices.Contains(remotes, "factory") { // check which branch is ahead branches := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "branch", "-r"), "\n") pool_branch := "factory" if slices.Contains(branches, "factory/devel") && slices.Contains(branches, "factory/factory") { if len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "rev-list", "^factory/factory", "factory/devel"), "\n")) > 0 { log.Println(" *** pool branch devel ahead. Switching branches.") pool_branch = "devel" } } else if slices.Contains(branches, "factory/devel") && !slices.Contains(branches, "factory/factory") { pool_branch = "devel" } else if !slices.Contains(branches, "factory/devel") && slices.Contains(branches, "factory/factory") { } else { log.Panicln("branches screwed up for pkg", pkgName, branches) } // find tree object in factory branch tree := strings.TrimSpace(git.GitExecWithOutputOrPanic(pkgName, "rev-list", "-1", "--format=%T", "--no-commit-header", "factory/"+import_branch)) log.Println("tree", tree) import_tree_commits := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "rev-list", "--format=%H %T", "--no-commit-header", import_branch), "\n") found := false for i := range import_tree_commits { commit_tree := strings.Split(import_tree_commits[i], " ") if len(commit_tree) != 2 { log.Panicln("wrong format?", commit_tree) } if commit_tree[1] == tree { found = true cherry_picks := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "rev-list", "--reverse", "--ancestry-path", commit_tree[0]+".."+import_branch), "\n") log.Println("cherry picks", cherry_picks) git.GitExecOrPanic(pkgName, "checkout", "-B", "main", pool_branch) for _, pick := range cherry_picks { git.GitExecOrPanic(pkgName, "cherry-pick", pick) } break } } if !found { log.Println("*** WARNING: Cannot find same tree for pkg", pkgName, "Will use current import instead") git.GitExecOrPanic(pkgName, "checkout", "-B", "main", import_branch) } } else { git.GitExecOrPanic(pkgName, "checkout", "-B", "main", import_branch) } } for _, pkg := range develProjectPackages { meta, _ := obs.GetPackageMeta(prj, pkg) if len(meta.ScmSync) > 0 { if err2 := git.CloneDevel("", pkg, meta.ScmSync); err2 != nil { log.Panicln(err2) } } else { cmd := exec.Command("./git-importer", "-p", prj, "-r", git.GetPath(), pkg) if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 { cmd.Env = slices.Delete(cmd.Env, idx, idx+1) } out, err := cmd.CombinedOutput() log.Println(string(out)) if err != nil { log.Panicln("Error returned by importer.", err) } } // mark newer branch as main branch := "factory" if len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg, "rev-list", "^factory", "devel"), "\n")) > 0 { log.Println(" *** pool branch 'devel' ahead. Switching branches.") branch = "devel" } git.GitExecOrPanic(pkg, "checkout", "-B", "main", branch) } slices.SortFunc(factoryRepos, func(a, b *models.Repository) int { if a.Name == b.Name { orgOrderNo := func(org string) int { switch org { case "pool": return 1 } return 0 // current devel to clone } return orgOrderNo(a.Owner.UserName) - orgOrderNo(b.Owner.UserName) } return strings.Compare(a.Name, b.Name) }) factoryRepos = slices.CompactFunc(factoryRepos, func(a, b *models.Repository) bool { return a.Name == b.Name }) for _, pkg := range factoryRepos { // update package fork, err := client.Repository.CreateFork(repository.NewCreateForkParams(). WithOwner(pkg.Owner.UserName). WithRepo(pkg.Name). WithBody(&models.CreateForkOption{ Organization: org, }), r.DefaultAuthentication) if err != nil { log.Panicln("Error while trying to create fork from pool", pkg.Name, err) } repo := fork.Payload // branchName := repo.DefaultBranch remotes := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "show") if !slices.Contains(strings.Split(remotes, "\n"), "devel") { git.GitExecOrPanic(pkg.Name, "remote", "add", "devel", repo.SSHURL) // git.GitExecOrPanic(pkg.Name, "fetch", "devel") } git.GitExecOrPanic(pkg.Name, "push", "devel", "main") // git.GitExecOrPanic(pkg.Name, "checkout", "-B", "main", "devel/main") _, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(&models.EditRepoOption{ DefaultBranch: "main", DefaultMergeStyle: "fast-forward-only", HasPullRequests: true, HasPackages: false, HasReleases: false, HasActions: false, AllowMerge: true, AllowRebaseMerge: false, AllowSquash: false, AllowFastForwardOnly: true, AllowRebaseUpdate: false, AllowManualMerge: false, AllowRebase: false, DefaultAllowMaintainerEdit: true, }), r.DefaultAuthentication) if err != nil { log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err) } } for _, pkg := range develProjectPackages { _, err := client.Organization.CreateOrgRepo(organization.NewCreateOrgRepoParams().WithOrg(org).WithBody( &models.CreateRepoOption{ ObjectFormatName: "sha256", AutoInit: false, Name: &pkg, DefaultBranch: "main", }), r.DefaultAuthentication, ) if err != nil { log.Panicln("Error creating new package repository:", pkg, err) } ret, err := client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(pkg).WithBody( &models.EditRepoOption{ HasPullRequests: true, HasPackages: false, HasReleases: false, HasActions: false, AllowMerge: true, AllowRebaseMerge: false, AllowSquash: false, AllowFastForwardOnly: true, AllowRebaseUpdate: false, AllowManualMerge: false, DefaultMergeStyle: "fast-forward-only", AllowRebase: false, DefaultAllowMaintainerEdit: true, }), r.DefaultAuthentication, ) if err != nil { log.Panicln("Failed to adjust repository:", pkg, err) } repo := ret.Payload remotes := git.GitExecWithOutputOrPanic(pkg, "remote", "show") if !slices.Contains(strings.Split(remotes, "\n"), "devel") { git.GitExecOrPanic(pkg, "remote", "add", "devel", repo.SSHURL) } git.GitExecOrPanic(pkg, "push", "devel", "main") _, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(pkg).WithBody(&models.EditRepoOption{ DefaultBranch: "main", DefaultMergeStyle: "fast-forward-only", }), r.DefaultAuthentication) if err != nil { log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err) } } } func syncOrgTeams(groupName string, teamMembers []common.PersonRepoMeta) []string { missing := []string{} if DebugMode { log.Println("syncOrgTeams", groupName, " -> ", teamMembers) } teamsRes, err := client.Organization.OrgListTeams(organization.NewOrgListTeamsParams().WithOrg(org), r.DefaultAuthentication) if err != nil { log.Panicln("failed to list org teams", org, err) } teams := teamsRes.Payload found := false teamID := int64(0) for _, team := range teams { if team.Name == groupName { teamID = team.ID membersRes, err := client.Organization.OrgListTeamMembers(organization.NewOrgListTeamMembersParams().WithID(team.ID), r.DefaultAuthentication) if err != nil { log.Panicln("failed get team members", team.Name, err) } for _, m := range membersRes.Payload { for i := 0; i < len(teamMembers); { if teamMembers[i].UserID == m.UserName || (teamMembers[i].Role != "maintainer" && teamMembers[i].Role != "") { teamMembers = slices.Delete(teamMembers, i, i+1) } else { i++ } } } found = true break } } if !found { team, err := client.Organization.OrgCreateTeam(organization.NewOrgCreateTeamParams().WithOrg(org).WithBody(&models.CreateTeamOption{ CanCreateOrgRepo: false, IncludesAllRepositories: true, Permission: "write", Name: &groupName, Units: []string{"repo.code"}, UnitsMap: map[string]string{"repo.code": "write"}, }), r.DefaultAuthentication) if err != nil { log.Panicln("can't create orgteam", org, err) } teamID = team.Payload.ID } if DebugMode && len(teamMembers) > 0 { log.Println("missing team members:", teamMembers) } for _, user := range teamMembers { _, err := client.Organization.OrgAddTeamMember(organization.NewOrgAddTeamMemberParams().WithID(teamID).WithUsername(user.UserID), r.DefaultAuthentication) if err != nil { if _, notFound := err.(*organization.OrgAddTeamMemberNotFound); !notFound { log.Panicln(err) } else { if DebugMode { log.Println("can't add user to group:", groupName, user.UserID) } missing = append(missing, user.UserID) } } } if DebugMode { log.Println("org team synced", groupName) } return missing } func syncOrgOwners(uids []common.PersonRepoMeta) []string { return syncOrgTeams("Owners", append(uids, common.PersonRepoMeta{UserID: "autogits-devel"})) } func syncPackageCollaborators(pkg string, uids []common.PersonRepoMeta) []string { missing := []string{} collab, err := client.Repository.RepoListCollaborators(repository.NewRepoListCollaboratorsParams().WithOwner(org).WithRepo(pkg), r.DefaultAuthentication) if err != nil { log.Panicln(err) } for _, u := range collab.Payload { for i := range uids { if uids[i].UserID == u.UserName { uids = slices.Delete(uids, i, i+1) break } } } if DebugMode && len(uids) > 0 { log.Println("missing collabs for", pkg, ":", uids) } for _, u := range uids { _, err := client.Repository.RepoAddCollaborator(repository.NewRepoAddCollaboratorParams().WithOwner(org).WithRepo(pkg).WithBody(&models.AddCollaboratorOption{ Permission: "write", }).WithCollaborator(u.UserID), r.DefaultAuthentication) if err != nil { if _, notFound := err.(*repository.RepoAddCollaboratorUnprocessableEntity); notFound { if DebugMode { log.Println("can't add user to collaborators:", u.UserID) } missing = append(missing, u.UserID) } else { log.Println(err) } } } return missing } func syncMaintainersToGitea(pkgs []string) { maintainers := common.MaintainershipMap{ Data: map[string][]string{}, } prjMeta, err := obs.GetProjectMeta(prj) if err != nil { log.Panicln("failed to get project meta", prj, err) } missingDevs := []string{} for _, group := range prjMeta.Groups { teamMembers, err := obs.GetGroupMeta(group.GroupID) if err != nil { log.Panicln("failed to get group", err) } if DebugMode { log.Println("syncing", group.GroupID, teamMembers.Persons) } missingDevs = append(missingDevs, syncOrgTeams(group.GroupID, teamMembers.Persons.Persons)...) devs := []string{} for _, m := range teamMembers.Persons.Persons { devs = append(devs, m.UserID) } maintainers.Data[""] = devs } missingDevs = append(missingDevs, syncOrgOwners(prjMeta.Persons)...) for _, pkg := range pkgs { pkgMeta, err := obs.GetPackageMeta(prj, pkg) if err != nil { log.Panicln("failed to get package meta", prj, pkg, err) } missingDevs = append(missingDevs, syncPackageCollaborators(pkg, pkgMeta.Persons)...) devs := []string{} for _, m := range pkgMeta.Persons { devs = append(devs, m.UserID) } maintainers.Data[pkg] = devs } slices.Sort(missingDevs) file, err := os.Create(common.MaintainershipFile) if err != nil { log.Println(" *** Cannot create maintainership file:", err) } else { maintainers.WriteMaintainershipFile(file) file.Close() } log.Println("Users without Gitea accounts:", slices.Compact(missingDevs)) } var client *apiclient.GiteaAPI var r *transport.Runtime var git common.Git var obs *common.ObsClient var prj, org string func main() { if err := common.RequireGiteaSecretToken(); err != nil { log.Panicln("Missing GITEA_TOKEN") } if err := common.RequireObsSecretToken(); err != nil { log.Panicln("Missing OBS_PASSWORD and/or OBS_USER") } flags := flag.NewFlagSet("devel-importer", flag.ContinueOnError) helpString := new(bytes.Buffer) flags.SetOutput(helpString) //workflowConfig := flag.String("config", "", "Repository and workflow definition file") giteaHost := flags.String("gitea", "src.opensuse.org", "Gitea instance") //rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance") flags.BoolVar(&DebugMode, "debug", false, "Extra debugging information") // revNew := flag.Int("nrevs", 20, "Number of new revisions in factory branch. Indicator of broken history import") purgeOnly := flags.Bool("purge-only", false, "Purges package repositories on Gitea. Use with caution") debugGitPath := flags.String("git-path", "", "Path for temporary git directory. Only used if DebugMode") getMaintainers := flags.Bool("maintainer-only", false, "Get maintainers only and exit") syncMaintainers := flags.Bool("sync-maintainers-only", false, "Sync maintainers to Gitea and exit") if help := flags.Parse(os.Args[1:]); help == flag.ErrHelp || flags.NArg() != 2 { printHelp(helpString.String()) return } r = transport.New(*giteaHost, apiclient.DefaultBasePath, [](string){"https"}) r.DefaultAuthentication = transport.BearerToken(common.GetGiteaToken()) // r.SetDebug(true) client = apiclient.New(r, nil) obs, _ = common.NewObsClient("api.opensuse.org") gh := common.GitHandlerGeneratorImpl{} var err error git, err = gh.CreateGitHandler("Autogits - Devel Importer", "not.exist", "devel-importer") if err != nil { log.Panicln("Failed to allocate git handler. Err:", err) } if DebugMode { if len(*debugGitPath) > 0 { git.Close() git, err = gh.ReadExistingPath("Autogits - Devel Importer", "not.exist", *debugGitPath) if err != nil { log.Panicln(err) } } log.Println(" - working directory:" + git.GetPath()) } else { defer git.Close() } prj = flags.Arg(0) org = flags.Arg(1) packages, err := runObsCommand("ls", prj) /* for _, pkg := range packages { if _, err := client.Organization.CreateOrgRepo(organization.NewCreateOrgRepoParams().WithOrg(org).WithBody( &models.CreateRepoOption{ ObjectFormatName: "sha256", AutoInit: false, Name: &pkg, DefaultBranch: "main", }), r.DefaultAuthentication, ); err != nil { log.Println(err) } } */ if *getMaintainers { listMaintainers(obs, prj, packages) return } if *syncMaintainers { syncMaintainersToGitea(packages) return } if err != nil { log.Printf("Cannot list packages for project '%s'. Err: %v\n", prj, err) return } slices.Sort(packages) for i := range packages { packages[i] = strings.Split(packages[i], ":")[0] } packages = slices.Compact(packages) log.Printf("%d packages: %s\n\n", len(packages), strings.Join(packages, " ")) if *purgeOnly { log.Println("Purging repositories...") for _, pkg := range packages { client.Repository.RepoDelete(repository.NewRepoDeleteParams().WithOwner(org).WithRepo(pkg), r.DefaultAuthentication) } os.Exit(10) } importRepos(packages) syncMaintainersToGitea(packages) }