package main /* * This file is part of Autogits. * * Copyright © 2024 SUSE LLC * * Autogits is free software: you can redistribute it and/or modify it under * the terms of the GNU General Public License as published by the Free Software * Foundation, either version 2 of the License, or (at your option) any later * version. * * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A * PARTICULAR PURPOSE. See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along with * Foobar. If not, see . */ import ( "bytes" "errors" "flag" "fmt" "log" "net/url" "os" "os/exec" "path" "path/filepath" "slices" "strings" transport "github.com/go-openapi/runtime/client" "src.opensuse.org/autogits/common" apiclient "src.opensuse.org/autogits/common/gitea-generated/client" "src.opensuse.org/autogits/common/gitea-generated/client/organization" "src.opensuse.org/autogits/common/gitea-generated/client/repository" "src.opensuse.org/autogits/common/gitea-generated/models" ) const commandLineHelp = ` SYNTAX devel-importer ` func printHelp(flags string) { os.Stdout.WriteString(commandLineHelp) os.Stdout.WriteString(flags) } func outputList(str string) []string { out := []string{} for _, l := range strings.Split(strings.TrimSpace(str), "\n") { l = strings.TrimSpace(l) if len(l) > 0 { out = append(out, l) } } return out } func runObsCommand(args ...string) ([]string, error) { cmd := exec.Command("osc", args...) out, err := cmd.Output() return outputList(string(out)), err } var DebugMode bool func giteaPackage(pkg string) string { return strings.ReplaceAll(pkg, "+", "_") } func projectMaintainer(obs *common.ObsClient, prj string) ([]string, []string) { // users, groups meta, err := obs.GetProjectMeta(prj) if err != nil { log.Panicln(err) } uids := []string{} gids := []string{} for _, p := range meta.Persons { if !slices.Contains(uids, p.UserID) { uids = append(uids, p.UserID) } } for _, g := range meta.Groups { if !slices.Contains(gids, g.GroupID) { gids = append(gids, g.GroupID) } } return uids, gids } func packageMaintainers(obs *common.ObsClient, prj, pkg string) ([]string, []string) { // users, groups meta, err := obs.GetPackageMeta(prj, pkg) if err != nil { log.Panicln(err, "FOR:", prj, "/", pkg) } uids := []string{} gids := []string{} for _, p := range meta.Persons { if !slices.Contains(uids, p.UserID) { uids = append(uids, p.UserID) } } for _, g := range meta.Groups { if !slices.Contains(gids, g.GroupID) { gids = append(gids, g.GroupID) } } return uids, gids } func listMaintainers(obs *common.ObsClient, prj string, pkgs []string) { users, groups := projectMaintainer(obs, prj) log.Println("Fetching maintainers for prj:", prj, users) for _, pkg := range pkgs { u, g := packageMaintainers(obs, prj, pkg) log.Println("maintainers for pkg:", pkg, u) users = append(users, u...) groups = append(groups, g...) } slices.Sort(users) slices.Sort(groups) users = slices.Compact(users) groups = slices.Compact(groups) log.Println("need to contact following:", strings.Join(users, ", ")) contact_email := []string{} for _, uid := range users { user, err := obs.GetUserMeta(uid) if err != nil { log.Panicln(err) } if user != nil { contact_email = append(contact_email, fmt.Sprintf("%s <%s>", user.Name, user.Email)) } } log.Println(strings.Join(contact_email, ", ")) } func gitImporter(prj, pkg string) error { params := []string{"-p", prj, "-r", git.GetPath()} if DebugMode { params = append(params, "-l", "debug") } params = append(params, pkg) common.LogDebug("git-importer", params) cmd := exec.Command("./git-importer", params...) if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 { cmd.Env = slices.Delete(cmd.Env, idx, idx+1) } out, err := cmd.CombinedOutput() log.Println(string(out)) if code := cmd.ProcessState.ExitCode(); code != 0 { return fmt.Errorf("Non-zero exit code from git-importer: %d %w", code, err) } return nil } func cloneDevel(git common.Git, gitDir, outName, urlString, remote string, fatal bool) error { if url, _ := url.Parse(urlString); url != nil { url.Fragment = "" urlString = url.String() } params := []string{"clone", "-o", remote} params = append(params, urlString, outName) if fatal { git.GitExecOrPanic(gitDir, params...) } else { git.GitExec(gitDir, params...) } return nil } func findMissingDevelBranch(git common.Git, pkg, project string) { d, err := git.GitBranchHead(pkg, "devel") if err != nil { if _, err = git.GitBranchHead(pkg, "factory"); err != nil { log.Println("factory is missing... so maybe repo is b0rked.") return } hash := common.SplitLines(git.GitExecWithOutputOrPanic(pkg, "log", "factory", "--all", "--grep=build.opensuse.org/package/show/"+project+"/"+pkg, "-1", "--pretty=format:%H")) if len(hash) > 0 { log.Println(" devel @", hash[0]) git.GitExecOrPanic(pkg, "branch", "devel", hash[0]) } else { git.GitExecOrPanic(pkg, "branch", "devel", "factory") } } else { log.Println(" devel already exists?", d) } } func importFactoryRepoAndCheckHistory(pkg string, meta *common.PackageMeta) (factoryRepo *models.Repository, retErr error) { if repo, err := client.Repository.RepoGet(repository.NewRepoGetParams().WithDefaults().WithOwner("pool").WithRepo(giteaPackage(pkg)), r.DefaultAuthentication); err != nil || repo.Payload.ObjectFormatName != "sha256" { if err != nil && !errors.Is(err, &repository.RepoGetNotFound{}) { log.Panicln(err) } log.Println("Cannot find src package:", pkg) return nil, nil } else { factoryRepo = repo.Payload CreatePoolFork(factoryRepo) } if _, err := os.Stat(filepath.Join(git.GetPath(), pkg)); os.IsNotExist(err) { common.LogDebug("Cloning factory...") cloneDevel(git, "", pkg, factoryRepo.CloneURL, "pool", true) // in case we have imported } else if err != nil { common.PanicOnError(err) } else { // we have already cloned it... so, fetch pool remote common.LogDebug("Fetching pool, as already should have the remote") if err = git.GitExec(pkg, "fetch", "pool"); err != nil { common.LogError(err) return factoryRepo, err } } roots := 0 if _, err := git.GitRemoteHead(pkg, "pool", "devel"); err == nil { factory_roots := strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "pool/factory", "--max-parents=0")) devel_roots := strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "pool/devel", "--max-parents=0")) roots = len(common.SplitLines(factory_roots)) if devel_roots != factory_roots || len(common.SplitLines(factory_roots)) != 1 { roots = 10 } } else if _, err := git.GitRemoteHead(pkg, "pool", "factory"); err == nil { items := strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "pool/factory", "--max-parents=0")) roots = len(common.SplitLines(items)) } else { common.LogInfo("No factory import ...") } if roots != 1 { common.LogError("Expected 1 root in factory, but found", roots) common.LogError("Ignoring current import") common.PanicOnError(os.RemoveAll(path.Join(git.GetPath(), pkg))) retErr = fmt.Errorf("Invalid factory repo -- treating as devel project only") return } devel_project, err := devel_projects.GetDevelProject(pkg) common.LogDebug("Devel project:", devel_project, err) if err == common.DevelProjectNotFound { // assume it's this project, maybe removed from factory devel_project = prj } common.LogDebug("finding missing branches in", pkg, devel_project) findMissingDevelBranch(git, pkg, devel_project) return } func SetRepoOptions(repo *models.Repository) { _, err := client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody( &models.EditRepoOption{ HasPullRequests: true, HasPackages: false, HasReleases: false, HasActions: false, AllowMerge: true, AllowRebaseMerge: false, AllowSquash: false, AllowFastForwardOnly: true, AllowRebaseUpdate: false, AllowManualMerge: true, AutodetectManualMerge: true, DefaultMergeStyle: "fast-forward-only", AllowRebase: false, DefaultAllowMaintainerEdit: true, DefaultBranch: "main", }), r.DefaultAuthentication, ) if err != nil { log.Panicln("Failed to adjust repository:", repo.Name, err) } } func CreatePoolFork(factoryRepo *models.Repository) *models.Repository { pkg := factoryRepo.Name log.Println("factory fork creator for develProjectPackage:", pkg) if repoData, err := client.Repository.RepoGet(repository.NewRepoGetParams().WithOwner(org).WithRepo(pkg), r.DefaultAuthentication); err != nil { // update package fork, err := client.Repository.CreateFork(repository.NewCreateForkParams(). WithOwner("pool"). WithRepo(factoryRepo.Name). WithBody(&models.CreateForkOption{ Organization: org, }), r.DefaultAuthentication) if err != nil { log.Panicln("Error while trying to create fork from 'pool'", pkg, "to", org, ":", err) } repo := fork.Payload return repo } else { return repoData.Payload } } func CreateDevelOnlyPackage(pkg string) *models.Repository { log.Println("repo creator for develProjectPackage:", pkg) if repoData, err := client.Repository.RepoGet(repository.NewRepoGetParams().WithOwner(org).WithRepo(giteaPackage(pkg)), r.DefaultAuthentication); err != nil { giteaPkg := giteaPackage(pkg) repoData, err := client.Organization.CreateOrgRepo(organization.NewCreateOrgRepoParams().WithOrg(org).WithBody( &models.CreateRepoOption{ ObjectFormatName: "sha256", AutoInit: false, Name: &giteaPkg, DefaultBranch: "main", }), r.DefaultAuthentication, ) if err != nil { log.Panicln("Error creating new package repository:", pkg, err) } repo := repoData.Payload return repo } else { return repoData.Payload } } func PushRepository(factoryRepo, develRepo *models.Repository, pkg string) (repo *models.Repository) { // branchName := repo.DefaultBranch if factoryRepo != nil { repo = CreatePoolFork(factoryRepo) /* devel for _, b := range branches { if len(b) > 12 && b[0:12] == "develorigin/" { b = b[12:] if b == "factory" || b == "devel" { git.GitExec(pkg, "push", "develorigin", "--delete", b) } } }*/ //branches := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg.Name, "branch", "-r"), "\n") /* factory for _, b := range branches { if len(b) > 12 && b[0:12] == "develorigin/" { b = b[12:] if b == "factory" || b == "devel" { git.GitExec(pkg.Name, "push", "develorigin", "--delete", b) } } } */ // git.GitExecOrPanic(pkg.ame, "checkout", "-B", "main", "devel/main") } else { repo = CreateDevelOnlyPackage(pkg) } remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg, "remote", "show"), "\n") if !slices.Contains(remotes, "develorigin") { git.GitExecOrPanic(pkg, "remote", "add", "develorigin", repo.SSHURL) // git.GitExecOrPanic(pkgName, "fetch", "devel") } if slices.Contains(remotes, "origin") { git.GitExecOrPanic(pkg, "lfs", "fetch", "--all") git.GitExecOrPanic(pkg, "lfs", "push", "develorigin", "--all") } git.GitExecOrPanic(pkg, "push", "develorigin", "main", "-f") SetRepoOptions(repo) git.GitExec(pkg, "push", "develorigin", "--delete", "factory") git.GitExec(pkg, "push", "develorigin", "--delete", "devel") git.GitExec(pkg, "push", "develorigin", "--delete", "leap-16.0") return repo } func importDevelRepoAndCheckHistory(pkg string, meta *common.PackageMeta) *models.Repository { repo := CreateDevelOnlyPackage(pkg) if _, err := os.Stat(filepath.Join(git.GetPath(), pkg)); os.IsNotExist(err) { cloneDevel(git, "", pkg, repo.SSHURL, "develorigin", false) // in case we have imported } if CloneScmsync(pkg, meta) { return repo } var p, dp string factory_branch, fhe := git.GitRemoteHead(pkg, "develorigin", "factory") if fhe == nil { p = strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "--max-parents=0", "--count", factory_branch)) } else { common.LogError(fhe) } devel_branch, dhe := git.GitRemoteHead(pkg, "develorigin", "devel") if dhe != nil { devel_project, err := devel_projects.GetDevelProject(pkg) common.LogDebug("Devel project:", devel_project, err) if err == common.DevelProjectNotFound { // assume it's this project, maybe removed from factory devel_project = prj } common.LogDebug("finding missing branches in", pkg, devel_project) findMissingDevelBranch(git, pkg, devel_project) devel_branch, dhe = git.GitBranchHead(pkg, "devel") } if dhe == nil { dp = strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "--max-parents=0", "--count", devel_branch)) } else { common.LogError(dhe) } // even if one parent for both, we need common ancestry, or we are comparing different things. mb, mb_err := git.GitExecWithOutput(pkg, "merge-base", factory_branch, devel_branch) mb = strings.TrimSpace(mb) if p != "1" || dp != "1" || mb_err != nil || mb != factory_branch || mb != devel_branch { common.LogInfo("Bad export found ... clearing", p, dp) common.LogInfo(" merge branch:", mb, factory_branch, devel_branch, mb_err) common.PanicOnError(os.RemoveAll(path.Join(git.GetPath(), pkg))) } if err := gitImporter("openSUSE:Factory", pkg); err != nil { common.PanicOnError(gitImporter(prj, pkg)) } if p := strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "--max-parents=0", "--count", "factory")); p != "1" { common.LogError("Failed to import package:", pkg) common.PanicOnError(fmt.Errorf("Expecting 1 root in after devel import, but have %s", p)) } if out, err := git.GitExecWithOutput(pkg, "show-ref", "--branches"); err != nil || len(common.SplitStringNoEmpty(out, "\n")) == 0 { common.LogError(" *** no branches in package. removing") return repo } return repo } func SetMainBranch(pkg string, meta *common.PackageMeta) { // scnsync, follow that and don't care common.LogDebug("Setting main branch...") remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg, "remote", "show"), "\n") if slices.Contains(remotes, "origin") { u, err := url.Parse(meta.ScmSync) common.PanicOnError(err) if len(u.Fragment) == 0 { u.Fragment = "HEAD" } if err := git.GitExec(pkg, "checkout", "-B", "main", u.Fragment); err != nil { git.GitExecOrPanic(pkg, "checkout", "-B", "main", "origin/"+u.Fragment) } return } // check if we have factory if _, err := git.GitBranchHead(pkg, "factory"); err != nil { if len(git.GitExecWithOutputOrPanic(pkg, "show-ref", "pool/factory")) > 20 { git.GitExecOrPanic(pkg, "branch", "factory", "pool/factory") } } // mark newer branch as main branch := "factory" if len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg, "rev-list", "^factory", "devel"), "\n")) > 0 { branch = "devel" } common.LogInfo("setting main to", branch) git.GitExecOrPanic(pkg, "checkout", "-B", "main", branch) } func ObsToRepoName(obspkg string) string { return strings.ReplaceAll(obspkg, "+", "_") } func ImportSha1Sync(pkg string, url *url.URL) { common.LogDebug("Converting SHA1", url.String()) branch := url.Fragment url.Fragment = "" p := path.Join(pkg, "sha1stuff") common.PanicOnError(os.RemoveAll(path.Join(git.GetPath(), p))) git.GitExecOrPanic(pkg, "clone", "--mirror", url.String(), "sha1stuff") git.GitExecOrPanic(p, "fetch", "origin", branch) gitexport := exec.Command("/usr/bin/git", "fast-export", "--signed-tags=strip", "--tag-of-filtered-object=drop", "--all") gitexport.Dir = path.Join(git.GetPath(), p) gitexportData, err := gitexport.Output() common.LogDebug("Got export data size:", len(gitexportData)) common.PanicOnError(err) gitimport := exec.Command("/usr/bin/git", "fast-import", "--allow-unsafe-features") gitimport.Dir = path.Join(git.GetPath(), pkg) gitimport.Stdin = bytes.NewReader(gitexportData) data, err := gitimport.CombinedOutput() common.LogError(string(data)) common.PanicOnError(err) common.PanicOnError(os.RemoveAll(path.Join(git.GetPath(), p))) git.GitExecOrPanic(pkg, "checkout", branch) } func LfsImport(pkg string) { git.GitExecOrPanic(pkg, "lfs", "migrate", "import", "--everything", "--include=*.7z,*.bsp,*.bz2,*.gem,*.gz,*.jar,*.lz,*.lzma,*.obscpio,*.oxt,*.pdf,*.png,*.rpm,*.tar,*.tbz,*.tbz2,*.tgz,*.ttf,*.txz,*.whl,*.xz,*.zip,*.zst") } func CloneScmsync(pkg string, meta *common.PackageMeta) bool { if len(meta.ScmSync) > 0 { u, _ := url.Parse(meta.ScmSync) if remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg, "remote", "show"), "\n"); !slices.Contains(remotes, "origin") { branch := u.Fragment if len(branch) == 0 { branch = "HEAD" } u.Fragment = "" git.GitExecOrPanic(pkg, "remote", "add", "origin", u.String()) u.Fragment = branch } if err := git.GitExec(pkg, "fetch", "origin"); err != nil && strings.Contains(err.Error(), "fatal: mismatched algorithms: client sha256; server sha1") { ImportSha1Sync(pkg, u) } else if err != nil { panic(err) } LfsImport(pkg) return true } return false } func importRepo(pkg string) (BrokenFactoryPackage, FailedImport bool) { BrokenFactoryPackage = false FailedImport = false var develRepo, factoryRepo *models.Repository src_pkg_name := strings.Split(pkg, ":") pkg = src_pkg_name[0] meta, err := obs.GetPackageMeta(prj, pkg) if err != nil { meta, err = obs.GetPackageMeta(prj, pkg) if err != nil { log.Println("Error fetching pkg meta for:", prj, pkg, err) } } if err != nil { common.PanicOnError(err) } if meta == nil { panic("package meta is nil...") } factoryRepo, err = importFactoryRepoAndCheckHistory(pkg, meta) if factoryRepo != nil && err != nil { BrokenFactoryPackage = true } if factoryRepo == nil && forceNonPoolPackages { log.Println(" IGNORING and will create these as non-pool packages!") } if factoryRepo == nil || BrokenFactoryPackage { develRepo = importDevelRepoAndCheckHistory(pkg, meta) } else { CloneScmsync(pkg, meta) } SetMainBranch(pkg, meta) PushRepository(factoryRepo, develRepo, pkg) return } func syncOrgTeams(groupName string, origTeam []common.PersonRepoMeta) []string { teamMembers := make([]common.PersonRepoMeta, len(origTeam)) copy(teamMembers, origTeam) missing := []string{} if DebugMode { log.Println("syncOrgTeams", groupName, " -> ", teamMembers) } teamsRes, err := client.Organization.OrgListTeams(organization.NewOrgListTeamsParams().WithOrg(org), r.DefaultAuthentication) if err != nil { log.Panicln("failed to list org teams", org, err) } teams := teamsRes.Payload found := false teamID := int64(0) for _, team := range teams { if team.Name == groupName { teamID = team.ID membersRes, err := client.Organization.OrgListTeamMembers(organization.NewOrgListTeamMembersParams().WithID(team.ID), r.DefaultAuthentication) if err != nil { log.Panicln("failed get team members", team.Name, err) } for _, m := range membersRes.Payload { for i := 0; i < len(teamMembers); { if teamMembers[i].UserID == m.UserName || (teamMembers[i].Role != "maintainer" && teamMembers[i].Role != "") { teamMembers = slices.Delete(teamMembers, i, i+1) } else { i++ } } } found = true break } } if !found { team, err := client.Organization.OrgCreateTeam(organization.NewOrgCreateTeamParams().WithOrg(org).WithBody(&models.CreateTeamOption{ CanCreateOrgRepo: false, IncludesAllRepositories: true, Permission: "write", Name: &groupName, Units: []string{"repo.code"}, UnitsMap: map[string]string{"repo.code": "write"}, }), r.DefaultAuthentication) if err != nil { log.Panicln("can't create orgteam", org, err) } teamID = team.Payload.ID } if DebugMode && len(teamMembers) > 0 { log.Println("missing team members:", teamMembers) } for _, user := range teamMembers { _, err := client.Organization.OrgAddTeamMember(organization.NewOrgAddTeamMemberParams().WithID(teamID).WithUsername(user.UserID), r.DefaultAuthentication) if err != nil { if _, notFound := err.(*organization.OrgAddTeamMemberNotFound); !notFound { log.Panicln(err) } else { if DebugMode { log.Println("can't add user to group:", groupName, user.UserID) } missing = append(missing, user.UserID) } } } if DebugMode { log.Println("org team synced", groupName) } return missing } func syncOrgOwners(uids []common.PersonRepoMeta) []string { return syncOrgTeams("Owners", append(uids, common.PersonRepoMeta{UserID: "autogits-devel"})) } func syncPackageCollaborators(pkg string, orig_uids []common.PersonRepoMeta) []string { missing := []string{} uids := make([]common.PersonRepoMeta, len(orig_uids)) copy(uids, orig_uids) collab, err := client.Repository.RepoListCollaborators(repository.NewRepoListCollaboratorsParams().WithOwner(org).WithRepo(giteaPackage(pkg)), r.DefaultAuthentication) if err != nil { if errors.Is(err, &repository.RepoListCollaboratorsNotFound{}) { return missing } log.Panicln(err) } for _, u := range collab.Payload { for i := range uids { if uids[i].UserID == u.UserName { uids = slices.Delete(uids, i, i+1) break } } } if DebugMode && len(uids) > 0 { log.Println("missing collabs for", pkg, ":", uids) } for _, u := range uids { _, err := client.Repository.RepoAddCollaborator(repository.NewRepoAddCollaboratorParams().WithOwner(org).WithRepo(giteaPackage(pkg)).WithBody(&models.AddCollaboratorOption{ Permission: "write", }).WithCollaborator(u.UserID), r.DefaultAuthentication) if err != nil { if _, notFound := err.(*repository.RepoAddCollaboratorUnprocessableEntity); notFound { if DebugMode { log.Println("can't add user to collaborators:", u.UserID) } missing = append(missing, u.UserID) } else { log.Println(err) } } } return missing } func syncMaintainersToGitea(pkgs []string) { maintainers := common.MaintainershipMap{ Data: map[string][]string{}, } prjMeta, err := obs.GetProjectMeta(prj) if err != nil { log.Panicln("failed to get project meta", prj, err) } missingDevs := []string{} devs := []string{} if len(prjMeta.ScmSync) > 0 { common.LogInfo("Project already in Git. Maintainers must have been already synced. Skipping...") return } for _, group := range prjMeta.Groups { if group.GroupID == "factory-maintainers" { log.Println("Ignoring factory-maintainers") continue } teamMembers, err := obs.GetGroupMeta(group.GroupID) if err != nil { log.Panicln("failed to get group", err) } if DebugMode { log.Println("syncing", group.GroupID, teamMembers.Persons) } missingDevs = append(missingDevs, syncOrgTeams(group.GroupID, teamMembers.Persons.Persons)...) for _, m := range teamMembers.Persons.Persons { devs = append(devs, m.UserID) } } for _, p := range prjMeta.Persons { devs = append(devs, p.UserID) } missingDevs = append(missingDevs, syncOrgOwners(prjMeta.Persons)...) maintainers.Data[""] = devs for _, pkg := range pkgs { pkgMeta, err := obs.GetPackageMeta(prj, pkg) if err != nil { log.Panicln("failed to get package meta", prj, pkg, err) } missingDevs = append(missingDevs, syncPackageCollaborators(pkg, pkgMeta.Persons)...) devs := []string{} for _, m := range pkgMeta.Persons { devs = append(devs, m.UserID) } maintainers.Data[pkg] = devs } createPrjGit() file, err := os.Create(path.Join(git.GetPath(), common.DefaultGitPrj, common.MaintainershipFile)) if err != nil { log.Println(" *** Cannot create maintainership file:", err) } else { maintainers.WriteMaintainershipFile(file) file.Close() status, err := git.GitStatus(common.DefaultGitPrj) if err != nil { log.Panicln(err) } for _, s := range status { if s.Path == common.MaintainershipFile && s.Status == common.GitStatus_Untracked { git.GitExecOrPanic(common.DefaultGitPrj, "add", common.MaintainershipFile) git.GitExecOrPanic(common.DefaultGitPrj, "commit", "-m", "Initial sync of maintainership with OBS") git.GitExecOrPanic(common.DefaultGitPrj, "push") } } if l := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(common.DefaultGitPrj, "status", "--porcelain=2"), "\n")); l > 0 { } } slices.Sort(missingDevs) log.Println("Users without Gitea accounts:", slices.Compact(missingDevs)) } func createPrjGit() { if _, err := os.Stat(path.Join(git.GetPath(), common.DefaultGitPrj)); errors.Is(err, os.ErrNotExist) { if err := git.GitExec("", "clone", "gitea@src.opensuse.org:"+org+"/_ObsPrj.git", common.DefaultGitPrj); err != nil { repoName := common.DefaultGitPrj _, err := client.Organization.CreateOrgRepo(organization.NewCreateOrgRepoParams().WithOrg(org).WithBody( &models.CreateRepoOption{ AutoInit: false, Name: &repoName, ObjectFormatName: "sha256", }), r.DefaultAuthentication) if err != nil { log.Panicln(err) } git.GitExecOrPanic("", "clone", "gitea@src.opensuse.org:"+org+"/_ObsPrj.git", common.DefaultGitPrj) config, err := obs.ProjectConfig(prj) if err != nil { log.Panicln(err) } file, err := os.Create(path.Join(git.GetPath(), common.DefaultGitPrj, "_config")) if err != nil { log.Panicln(err) } config = strings.TrimSpace(config) if len(config) > 1 { file.Write([]byte(config)) file.Close() git.GitExecOrPanic(common.DefaultGitPrj, "add", "_config") } file, err = os.Create(path.Join(git.GetPath(), common.DefaultGitPrj, "staging.config")) if err != nil { log.Panicln(err) } file.WriteString("{\n // Reference build project\n \"ObsProject\": \"" + prj + "\",\n}\n") file.Close() git.GitExecOrPanic(common.DefaultGitPrj, "add", "staging.config") if file, err = os.Create(path.Join(git.GetPath(), common.DefaultGitPrj, "workflow.config")); err != nil { log.Panicln(err) } file.WriteString("{\n \"Workflows\": [\"direct\", \"pr\"],\n \"Organization\": \"" + org + "\",\n}\n") file.Close() git.GitExecOrPanic(common.DefaultGitPrj, "add", "workflow.config") } } } func TrimMultibuildPackages(packages []string) []string { for i := 0; i < len(packages); { if strings.Contains(packages[i], ":") { packages = slices.Delete(packages, i, i+1) } else { i++ } } return packages } var client *apiclient.GiteaAPI var r *transport.Runtime var git common.Git var obs *common.ObsClient var prj, org string var forceBadPool bool var forceNonPoolPackages bool var devel_projects common.DevelProjects func main() { if err := common.RequireGiteaSecretToken(); err != nil { log.Panicln("Missing GITEA_TOKEN") } if err := common.RequireObsSecretToken(); err != nil { log.Panicln("Missing OBS_PASSWORD and/or OBS_USER") } flags := flag.NewFlagSet("devel-importer", flag.ContinueOnError) helpString := new(bytes.Buffer) flags.SetOutput(helpString) //workflowConfig := flag.String("config", "", "Repository and workflow definition file") giteaHost := flags.String("gitea", "src.opensuse.org", "Gitea instance") obsUrl := flags.String("obs-url", "https://api.opensuse.org", "OBS API Url") //rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance") flags.BoolVar(&DebugMode, "debug", false, "Extra debugging information") // revNew := flag.Int("nrevs", 20, "Number of new revisions in factory branch. Indicator of broken history import") purgeOnly := flags.Bool("purge-only", false, "Purges package repositories on Gitea. Use with caution") debugGitPath := flags.String("git-path", "", "Path for temporary git directory. Only used if DebugMode") getMaintainers := flags.Bool("maintainers-only", false, "Get maintainers only and exit") syncMaintainers := flags.Bool("sync-maintainers-only", false, "Sync maintainers to Gitea and exit") flags.BoolVar(&forceBadPool, "bad-pool", false, "Force packages if pool has no branches due to bad import") flags.BoolVar(&forceNonPoolPackages, "non-pool", false, "Allow packages that are not in pool to be created. WARNING: Can't add to factory later!") specificPackages := flags.String("packages", "", "Process specific package, separated by commas, ignoring the others") resumeAt := flags.String("resume", "", "Resume import at given pacakge") syncPool := flags.Bool("sync-pool-only", false, "Force updates pool based on currrently imported project") if help := flags.Parse(os.Args[1:]); help == flag.ErrHelp || flags.NArg() != 2 { printHelp(helpString.String()) return } if DebugMode { common.SetLoggingLevel(common.LogLevelDebug) } r = transport.New(*giteaHost, apiclient.DefaultBasePath, [](string){"https"}) r.DefaultAuthentication = transport.BearerToken(common.GetGiteaToken()) // r.SetDebug(true) client = apiclient.New(r, nil) obs, _ = common.NewObsClient(*obsUrl) var gh common.GitHandlerGenerator var err error devel_projects, err = common.FetchDevelProjects() if err != nil { log.Panic("Cannot load devel projects:", err) } log.Println("# devel projects loaded:", len(devel_projects)) if DebugMode { if len(*debugGitPath) > 0 { gh, err = common.AllocateGitWorkTree(*debugGitPath, "Autogits - Devel Importer", "not.exist") if err != nil { log.Panicln(err) } } } else { dir, _ := os.MkdirTemp(os.TempDir(), "devel-importer") gh, err = common.AllocateGitWorkTree(dir, "Autogits - Devel Importer", "not.exist") if err != nil { log.Panicln("Failed to allocate git handler", err) } } prj = flags.Arg(0) org = flags.Arg(1) packages, err := runObsCommand("ls", prj) packages = TrimMultibuildPackages(packages) git, err = gh.CreateGitHandler(org) if err != nil { log.Panicln("Cannot create git", err) } defer git.Close() log.Println(" - working directory:" + git.GetPath()) if *syncPool { factory_pkgs, err := runObsCommand("ls", "openSUSE:Factory") common.PanicOnError(err) common.LogInfo("Syncing pool only...") factory_pkgs = TrimMultibuildPackages(factory_pkgs) for _, pkg := range packages { if !slices.Contains(factory_pkgs, pkg) { continue } repo, err := client.Repository.RepoGet(repository.NewRepoGetParams().WithOwner(org).WithRepo(ObsToRepoName(pkg)), r.DefaultAuthentication) common.PanicOnError(err) if !slices.Contains(common.SplitLines(git.GitExecWithOutputOrPanic(pkg, "remote")), "pool") { git.GitExecOrPanic(pkg, "remote", "add", "pool", repo.Payload.SSHURL) } git.GitExecOrPanic(pkg, "fetch", "pool") } } /* for _, pkg := range packages { if _, err := client.Organization.CreateOrgRepo(organization.NewCreateOrgRepoParams().WithOrg(org).WithBody( &models.CreateRepoOption{ ObjectFormatName: "sha256", AutoInit: false, Name: &pkg, DefaultBranch: "main", }), r.DefaultAuthentication, ); err != nil { log.Println(err) } } */ if *getMaintainers { listMaintainers(obs, prj, packages) return } if *syncMaintainers { syncMaintainersToGitea(packages) return } if err != nil { log.Printf("Cannot list packages for project '%s'. Err: %v\n", prj, err) return } slices.Sort(packages) for i := range packages { packages[i] = strings.Split(packages[i], ":")[0] } packages = slices.Compact(packages) log.Printf("%d packages: %s\n\n", len(packages), strings.Join(packages, " ")) if *purgeOnly { log.Println("Purging repositories...") pkgs := packages if len(*specificPackages) > 0 { pkgs = common.SplitStringNoEmpty(*specificPackages, ",") } for _, pkg := range pkgs { client.Repository.RepoDelete(repository.NewRepoDeleteParams().WithOwner(org).WithRepo(giteaPackage(pkg)), r.DefaultAuthentication) } os.Exit(10) } if len(*specificPackages) != 0 { packages = common.SplitStringNoEmpty(*specificPackages, ",") } slices.Sort(packages) BrokenOBSPackage := []string{} FailedImport := []string{} for _, pkg := range packages { if len(*resumeAt) > 0 && strings.Compare(*resumeAt, pkg) > 0 { common.LogDebug(pkg, "skipped due to resuming at", *resumeAt) continue } b, f := importRepo(pkg) if b { BrokenOBSPackage = append(BrokenOBSPackage, pkg) } if f { FailedImport = append(FailedImport, pkg) } } syncMaintainersToGitea(packages) common.LogError("Have broken pool packages:", len(BrokenOBSPackage)) // common.LogError("Total pool packages:", len(factoryRepos)) common.LogError("Failed to import:", strings.Join(FailedImport, ",")) common.LogInfo("BROKEN Pool packages:", strings.Join(BrokenOBSPackage, "\n")) }