package main
/*
* This file is part of Autogits.
*
* Copyright © 2024 SUSE LLC
*
* Autogits is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 2 of the License, or (at your option) any later
* version.
*
* Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Foobar. If not, see .
*/
import (
"bytes"
"errors"
"flag"
"fmt"
"log"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"slices"
"strings"
transport "github.com/go-openapi/runtime/client"
"src.opensuse.org/autogits/common"
apiclient "src.opensuse.org/autogits/common/gitea-generated/client"
"src.opensuse.org/autogits/common/gitea-generated/client/organization"
"src.opensuse.org/autogits/common/gitea-generated/client/repository"
"src.opensuse.org/autogits/common/gitea-generated/models"
)
const commandLineHelp = `
SYNTAX
devel-importer
`
func printHelp(flags string) {
os.Stdout.WriteString(commandLineHelp)
os.Stdout.WriteString(flags)
}
func outputList(str string) []string {
out := []string{}
for _, l := range strings.Split(strings.TrimSpace(str), "\n") {
l = strings.TrimSpace(l)
if len(l) > 0 {
out = append(out, l)
}
}
return out
}
func runObsCommand(args ...string) ([]string, error) {
cmd := exec.Command("osc", args...)
out, err := cmd.Output()
return outputList(string(out)), err
}
var DebugMode bool
func giteaPackage(pkg string) string {
return strings.ReplaceAll(pkg, "+", "_")
}
func projectMaintainer(obs *common.ObsClient, prj string) ([]string, []string) { // users, groups
meta, err := obs.GetProjectMeta(prj)
if err != nil {
log.Panicln(err)
}
uids := []string{}
gids := []string{}
for _, p := range meta.Persons {
if !slices.Contains(uids, p.UserID) {
uids = append(uids, p.UserID)
}
}
for _, g := range meta.Groups {
if !slices.Contains(gids, g.GroupID) {
gids = append(gids, g.GroupID)
}
}
return uids, gids
}
func packageMaintainers(obs *common.ObsClient, prj, pkg string) ([]string, []string) { // users, groups
meta, err := obs.GetPackageMeta(prj, pkg)
if err != nil {
log.Panicln(err, "FOR:", prj, "/", pkg)
}
uids := []string{}
gids := []string{}
for _, p := range meta.Persons {
if !slices.Contains(uids, p.UserID) {
uids = append(uids, p.UserID)
}
}
for _, g := range meta.Groups {
if !slices.Contains(gids, g.GroupID) {
gids = append(gids, g.GroupID)
}
}
return uids, gids
}
func listMaintainers(obs *common.ObsClient, prj string, pkgs []string) {
users, groups := projectMaintainer(obs, prj)
log.Println("Fetching maintainers for prj:", prj, users)
for _, pkg := range pkgs {
u, g := packageMaintainers(obs, prj, pkg)
log.Println("maintainers for pkg:", pkg, u)
users = append(users, u...)
groups = append(groups, g...)
}
slices.Sort(users)
slices.Sort(groups)
users = slices.Compact(users)
groups = slices.Compact(groups)
log.Println("need to contact following:", strings.Join(users, ", "))
contact_email := []string{}
for _, uid := range users {
user, err := obs.GetUserMeta(uid)
if err != nil {
log.Panicln(err)
}
if user != nil {
contact_email = append(contact_email, fmt.Sprintf("%s <%s>", user.Name, user.Email))
}
}
log.Println(strings.Join(contact_email, ", "))
}
func gitImporter(prj, pkg string) error {
params := []string{"-p", prj, "-r", git.GetPath()}
if DebugMode {
params = append(params, "-l", "debug")
}
params = append(params, pkg)
common.LogDebug("git-importer", params)
cmd := exec.Command("./git-importer", params...)
if idx := slices.IndexFunc(cmd.Env, func(val string) bool { return val[0:12] == "GITEA_TOKEN=" }); idx != -1 {
cmd.Env = slices.Delete(cmd.Env, idx, idx+1)
}
out, err := cmd.CombinedOutput()
log.Println(string(out))
if code := cmd.ProcessState.ExitCode(); code != 0 {
return fmt.Errorf("Non-zero exit code from git-importer: %d %w", code, err)
}
return nil
}
func cloneDevel(git common.Git, gitDir, outName, urlString, remote string, fatal bool) error {
if url, _ := url.Parse(urlString); url != nil {
url.Fragment = ""
urlString = url.String()
}
params := []string{"clone", "-o", remote}
params = append(params, urlString, outName)
if fatal {
git.GitExecOrPanic(gitDir, params...)
} else {
git.GitExec(gitDir, params...)
}
return nil
}
func findMissingDevelBranch(git common.Git, pkg, project string) {
d, err := git.GitBranchHead(pkg, "devel")
if err != nil {
if _, err = git.GitBranchHead(pkg, "factory"); err != nil {
log.Println("factory is missing... so maybe repo is b0rked.")
return
}
hash := common.SplitLines(git.GitExecWithOutputOrPanic(pkg,
"log",
"factory",
"--all",
"--grep=build.opensuse.org/package/show/"+project+"/"+pkg,
"-1",
"--pretty=format:%H"))
if len(hash) > 0 {
log.Println(" devel @", hash[0])
git.GitExecOrPanic(pkg, "branch", "devel", hash[0])
} else {
git.GitExecOrPanic(pkg, "branch", "devel", "factory")
}
} else {
log.Println(" devel already exists?", d)
}
}
func importFactoryRepoAndCheckHistory(pkg string, meta *common.PackageMeta) (factoryRepo *models.Repository, retErr error) {
if repo, err := client.Repository.RepoGet(repository.NewRepoGetParams().WithDefaults().WithOwner("pool").WithRepo(giteaPackage(pkg)), r.DefaultAuthentication); err != nil || repo.Payload.ObjectFormatName != "sha256" {
if err != nil && !errors.Is(err, &repository.RepoGetNotFound{}) {
log.Panicln(err)
}
log.Println("Cannot find src package:", pkg)
return nil, nil
} else {
factoryRepo = repo.Payload
CreatePoolFork(factoryRepo)
}
if _, err := os.Stat(filepath.Join(git.GetPath(), pkg)); os.IsNotExist(err) {
common.LogDebug("Cloning factory...")
cloneDevel(git, "", pkg, factoryRepo.CloneURL, "pool", true) // in case we have imported
} else if err != nil {
common.PanicOnError(err)
} else {
// we have already cloned it... so, fetch pool remote
common.LogDebug("Fetching pool, as already should have the remote")
if err = git.GitExec(pkg, "fetch", "pool"); err != nil {
common.LogError(err)
return factoryRepo, err
}
}
roots := 0
if _, err := git.GitRemoteHead(pkg, "pool", "devel"); err == nil {
factory_roots := strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "pool/factory", "--max-parents=0"))
devel_roots := strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "pool/devel", "--max-parents=0"))
roots = len(common.SplitLines(factory_roots))
if devel_roots != factory_roots || len(common.SplitLines(factory_roots)) != 1 {
roots = 10
}
} else if _, err := git.GitRemoteHead(pkg, "pool", "factory"); err == nil {
items := strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "pool/factory", "--max-parents=0"))
roots = len(common.SplitLines(items))
} else {
common.LogInfo("No factory import ...")
}
if roots != 1 {
common.LogError("Expected 1 root in factory, but found", roots)
common.LogError("Ignoring current import")
common.PanicOnError(os.RemoveAll(path.Join(git.GetPath(), pkg)))
retErr = fmt.Errorf("Invalid factory repo -- treating as devel project only")
return
}
devel_project, err := devel_projects.GetDevelProject(pkg)
common.LogDebug("Devel project:", devel_project, err)
if err == common.DevelProjectNotFound {
// assume it's this project, maybe removed from factory
devel_project = prj
}
common.LogDebug("finding missing branches in", pkg, devel_project)
findMissingDevelBranch(git, pkg, devel_project)
return
}
func SetRepoOptions(repo *models.Repository) {
_, err := client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(
&models.EditRepoOption{
HasPullRequests: true,
HasPackages: false,
HasReleases: false,
HasActions: false,
AllowMerge: true,
AllowRebaseMerge: false,
AllowSquash: false,
AllowFastForwardOnly: true,
AllowRebaseUpdate: false,
AllowManualMerge: true,
AutodetectManualMerge: true,
DefaultMergeStyle: "fast-forward-only",
AllowRebase: false,
DefaultAllowMaintainerEdit: true,
DefaultBranch: "main",
}),
r.DefaultAuthentication,
)
if err != nil {
log.Panicln("Failed to adjust repository:", repo.Name, err)
}
}
func CreatePoolFork(factoryRepo *models.Repository) *models.Repository {
pkg := factoryRepo.Name
log.Println("factory fork creator for develProjectPackage:", pkg)
if repoData, err := client.Repository.RepoGet(repository.NewRepoGetParams().WithOwner(org).WithRepo(pkg), r.DefaultAuthentication); err != nil {
// update package
fork, err := client.Repository.CreateFork(repository.NewCreateForkParams().
WithOwner("pool").
WithRepo(factoryRepo.Name).
WithBody(&models.CreateForkOption{
Organization: org,
}), r.DefaultAuthentication)
if err != nil {
log.Panicln("Error while trying to create fork from 'pool'", pkg, "to", org, ":", err)
}
repo := fork.Payload
return repo
} else {
return repoData.Payload
}
}
func CreateDevelOnlyPackage(pkg string) *models.Repository {
log.Println("repo creator for develProjectPackage:", pkg)
if repoData, err := client.Repository.RepoGet(repository.NewRepoGetParams().WithOwner(org).WithRepo(giteaPackage(pkg)), r.DefaultAuthentication); err != nil {
giteaPkg := giteaPackage(pkg)
repoData, err := client.Organization.CreateOrgRepo(organization.NewCreateOrgRepoParams().WithOrg(org).WithBody(
&models.CreateRepoOption{
ObjectFormatName: "sha256",
AutoInit: false,
Name: &giteaPkg,
DefaultBranch: "main",
}),
r.DefaultAuthentication,
)
if err != nil {
log.Panicln("Error creating new package repository:", pkg, err)
}
repo := repoData.Payload
return repo
} else {
return repoData.Payload
}
}
func PushRepository(factoryRepo, develRepo *models.Repository, pkg string) (repo *models.Repository) {
// branchName := repo.DefaultBranch
if factoryRepo != nil {
repo = CreatePoolFork(factoryRepo)
/*
devel
for _, b := range branches {
if len(b) > 12 && b[0:12] == "develorigin/" {
b = b[12:]
if b == "factory" || b == "devel" {
git.GitExec(pkg, "push", "develorigin", "--delete", b)
}
}
}*/
//branches := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg.Name, "branch", "-r"), "\n")
/* factory
for _, b := range branches {
if len(b) > 12 && b[0:12] == "develorigin/" {
b = b[12:]
if b == "factory" || b == "devel" {
git.GitExec(pkg.Name, "push", "develorigin", "--delete", b)
}
}
}
*/
// git.GitExecOrPanic(pkg.ame, "checkout", "-B", "main", "devel/main")
} else {
repo = CreateDevelOnlyPackage(pkg)
}
remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg, "remote", "show"), "\n")
if !slices.Contains(remotes, "develorigin") {
git.GitExecOrPanic(pkg, "remote", "add", "develorigin", repo.SSHURL)
// git.GitExecOrPanic(pkgName, "fetch", "devel")
}
if slices.Contains(remotes, "origin") {
git.GitExecOrPanic(pkg, "lfs", "fetch", "--all")
git.GitExecOrPanic(pkg, "lfs", "push", "develorigin", "--all")
}
git.GitExecOrPanic(pkg, "push", "develorigin", "main", "-f")
SetRepoOptions(repo)
git.GitExec(pkg, "push", "develorigin", "--delete", "factory")
git.GitExec(pkg, "push", "develorigin", "--delete", "devel")
git.GitExec(pkg, "push", "develorigin", "--delete", "leap-16.0")
return repo
}
func importDevelRepoAndCheckHistory(pkg string, meta *common.PackageMeta) *models.Repository {
repo := CreateDevelOnlyPackage(pkg)
if _, err := os.Stat(filepath.Join(git.GetPath(), pkg)); os.IsNotExist(err) {
cloneDevel(git, "", pkg, repo.SSHURL, "develorigin", false) // in case we have imported
}
if CloneScmsync(pkg, meta) {
return repo
}
var p, dp string
factory_branch, fhe := git.GitRemoteHead(pkg, "develorigin", "factory")
if fhe == nil {
p = strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "--max-parents=0", "--count", factory_branch))
} else {
common.LogError(fhe)
}
devel_branch, dhe := git.GitRemoteHead(pkg, "develorigin", "devel")
if dhe != nil {
devel_project, err := devel_projects.GetDevelProject(pkg)
common.LogDebug("Devel project:", devel_project, err)
if err == common.DevelProjectNotFound {
// assume it's this project, maybe removed from factory
devel_project = prj
}
common.LogDebug("finding missing branches in", pkg, devel_project)
findMissingDevelBranch(git, pkg, devel_project)
devel_branch, dhe = git.GitBranchHead(pkg, "devel")
}
if dhe == nil {
dp = strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "--max-parents=0", "--count", devel_branch))
} else {
common.LogError(dhe)
}
// even if one parent for both, we need common ancestry, or we are comparing different things.
mb, mb_err := git.GitExecWithOutput(pkg, "merge-base", factory_branch, devel_branch)
mb = strings.TrimSpace(mb)
if p != "1" || dp != "1" || mb_err != nil || mb != factory_branch || mb != devel_branch {
common.LogInfo("Bad export found ... clearing", p, dp)
common.LogInfo(" merge branch:", mb, factory_branch, devel_branch, mb_err)
common.PanicOnError(os.RemoveAll(path.Join(git.GetPath(), pkg)))
}
if err := gitImporter("openSUSE:Factory", pkg); err != nil {
common.PanicOnError(gitImporter(prj, pkg))
}
if p := strings.TrimSpace(git.GitExecWithOutputOrPanic(pkg, "rev-list", "--max-parents=0", "--count", "factory")); p != "1" {
common.LogError("Failed to import package:", pkg)
common.PanicOnError(fmt.Errorf("Expecting 1 root in after devel import, but have %s", p))
}
if out, err := git.GitExecWithOutput(pkg, "show-ref", "--branches"); err != nil || len(common.SplitStringNoEmpty(out, "\n")) == 0 {
common.LogError(" *** no branches in package. removing")
return repo
}
return repo
}
func SetMainBranch(pkg string, meta *common.PackageMeta) {
// scnsync, follow that and don't care
common.LogDebug("Setting main branch...")
remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg, "remote", "show"), "\n")
if slices.Contains(remotes, "origin") {
u, err := url.Parse(meta.ScmSync)
common.PanicOnError(err)
if len(u.Fragment) == 0 {
u.Fragment = "HEAD"
}
if err := git.GitExec(pkg, "checkout", "-B", "main", u.Fragment); err != nil {
git.GitExecOrPanic(pkg, "checkout", "-B", "main", "origin/"+u.Fragment)
}
return
}
// check if we have factory
if _, err := git.GitBranchHead(pkg, "factory"); err != nil {
if len(git.GitExecWithOutputOrPanic(pkg, "show-ref", "pool/factory")) > 20 {
git.GitExecOrPanic(pkg, "branch", "factory", "pool/factory")
}
}
// mark newer branch as main
branch := "factory"
if len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg, "rev-list", "^factory", "devel"), "\n")) > 0 {
branch = "devel"
}
common.LogInfo("setting main to", branch)
git.GitExecOrPanic(pkg, "checkout", "-B", "main", branch)
}
func ObsToRepoName(obspkg string) string {
return strings.ReplaceAll(obspkg, "+", "_")
}
func ImportSha1Sync(pkg string, url *url.URL) {
common.LogDebug("Converting SHA1", url.String())
branch := url.Fragment
url.Fragment = ""
p := path.Join(pkg, "sha1stuff")
common.PanicOnError(os.RemoveAll(path.Join(git.GetPath(), p)))
git.GitExecOrPanic(pkg, "clone", "--mirror", url.String(), "sha1stuff")
git.GitExecOrPanic(p, "fetch", "origin", branch)
gitexport := exec.Command("/usr/bin/git", "fast-export", "--signed-tags=strip", "--tag-of-filtered-object=drop", "--all")
gitexport.Dir = path.Join(git.GetPath(), p)
gitexportData, err := gitexport.Output()
common.LogDebug("Got export data size:", len(gitexportData))
common.PanicOnError(err)
gitimport := exec.Command("/usr/bin/git", "fast-import", "--allow-unsafe-features")
gitimport.Dir = path.Join(git.GetPath(), pkg)
gitimport.Stdin = bytes.NewReader(gitexportData)
data, err := gitimport.CombinedOutput()
common.LogError(string(data))
common.PanicOnError(err)
common.PanicOnError(os.RemoveAll(path.Join(git.GetPath(), p)))
git.GitExecOrPanic(pkg, "checkout", branch)
}
func LfsImport(pkg string) {
git.GitExecOrPanic(pkg, "lfs", "migrate", "import", "--everything",
"--include=*.7z,*.bsp,*.bz2,*.gem,*.gz,*.jar,*.lz,*.lzma,*.obscpio,*.oxt,*.pdf,*.png,*.rpm,*.tar,*.tbz,*.tbz2,*.tgz,*.ttf,*.txz,*.whl,*.xz,*.zip,*.zst")
}
func CloneScmsync(pkg string, meta *common.PackageMeta) bool {
if len(meta.ScmSync) > 0 {
u, _ := url.Parse(meta.ScmSync)
if remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkg, "remote", "show"), "\n"); !slices.Contains(remotes, "origin") {
branch := u.Fragment
if len(branch) == 0 {
branch = "HEAD"
}
u.Fragment = ""
git.GitExecOrPanic(pkg, "remote", "add", "origin", u.String())
u.Fragment = branch
}
if err := git.GitExec(pkg, "fetch", "origin"); err != nil && strings.Contains(err.Error(), "fatal: mismatched algorithms: client sha256; server sha1") {
ImportSha1Sync(pkg, u)
} else if err != nil {
panic(err)
}
LfsImport(pkg)
return true
}
return false
}
func importRepo(pkg string) (BrokenFactoryPackage, FailedImport bool) {
BrokenFactoryPackage = false
FailedImport = false
var develRepo, factoryRepo *models.Repository
src_pkg_name := strings.Split(pkg, ":")
pkg = src_pkg_name[0]
meta, err := obs.GetPackageMeta(prj, pkg)
if err != nil {
meta, err = obs.GetPackageMeta(prj, pkg)
if err != nil {
log.Println("Error fetching pkg meta for:", prj, pkg, err)
}
}
if err != nil {
common.PanicOnError(err)
}
if meta == nil {
panic("package meta is nil...")
}
factoryRepo, err = importFactoryRepoAndCheckHistory(pkg, meta)
if factoryRepo != nil && err != nil {
BrokenFactoryPackage = true
}
if factoryRepo == nil && forceNonPoolPackages {
log.Println(" IGNORING and will create these as non-pool packages!")
}
if factoryRepo == nil || BrokenFactoryPackage {
develRepo = importDevelRepoAndCheckHistory(pkg, meta)
} else {
CloneScmsync(pkg, meta)
}
SetMainBranch(pkg, meta)
PushRepository(factoryRepo, develRepo, pkg)
return
}
func syncOrgTeams(groupName string, origTeam []common.PersonRepoMeta) []string {
teamMembers := make([]common.PersonRepoMeta, len(origTeam))
copy(teamMembers, origTeam)
missing := []string{}
if DebugMode {
log.Println("syncOrgTeams", groupName, " -> ", teamMembers)
}
teamsRes, err := client.Organization.OrgListTeams(organization.NewOrgListTeamsParams().WithOrg(org), r.DefaultAuthentication)
if err != nil {
log.Panicln("failed to list org teams", org, err)
}
teams := teamsRes.Payload
found := false
teamID := int64(0)
for _, team := range teams {
if team.Name == groupName {
teamID = team.ID
membersRes, err := client.Organization.OrgListTeamMembers(organization.NewOrgListTeamMembersParams().WithID(team.ID), r.DefaultAuthentication)
if err != nil {
log.Panicln("failed get team members", team.Name, err)
}
for _, m := range membersRes.Payload {
for i := 0; i < len(teamMembers); {
if teamMembers[i].UserID == m.UserName || (teamMembers[i].Role != "maintainer" && teamMembers[i].Role != "") {
teamMembers = slices.Delete(teamMembers, i, i+1)
} else {
i++
}
}
}
found = true
break
}
}
if !found {
team, err := client.Organization.OrgCreateTeam(organization.NewOrgCreateTeamParams().WithOrg(org).WithBody(&models.CreateTeamOption{
CanCreateOrgRepo: false,
IncludesAllRepositories: true,
Permission: "write",
Name: &groupName,
Units: []string{"repo.code"},
UnitsMap: map[string]string{"repo.code": "write"},
}), r.DefaultAuthentication)
if err != nil {
log.Panicln("can't create orgteam", org, err)
}
teamID = team.Payload.ID
}
if DebugMode && len(teamMembers) > 0 {
log.Println("missing team members:", teamMembers)
}
for _, user := range teamMembers {
_, err := client.Organization.OrgAddTeamMember(organization.NewOrgAddTeamMemberParams().WithID(teamID).WithUsername(user.UserID), r.DefaultAuthentication)
if err != nil {
if _, notFound := err.(*organization.OrgAddTeamMemberNotFound); !notFound {
log.Panicln(err)
} else {
if DebugMode {
log.Println("can't add user to group:", groupName, user.UserID)
}
missing = append(missing, user.UserID)
}
}
}
if DebugMode {
log.Println("org team synced", groupName)
}
return missing
}
func syncOrgOwners(uids []common.PersonRepoMeta) []string {
return syncOrgTeams("Owners", append(uids, common.PersonRepoMeta{UserID: "autogits-devel"}))
}
func syncPackageCollaborators(pkg string, orig_uids []common.PersonRepoMeta) []string {
missing := []string{}
uids := make([]common.PersonRepoMeta, len(orig_uids))
copy(uids, orig_uids)
collab, err := client.Repository.RepoListCollaborators(repository.NewRepoListCollaboratorsParams().WithOwner(org).WithRepo(giteaPackage(pkg)), r.DefaultAuthentication)
if err != nil {
if errors.Is(err, &repository.RepoListCollaboratorsNotFound{}) {
return missing
}
log.Panicln(err)
}
for _, u := range collab.Payload {
for i := range uids {
if uids[i].UserID == u.UserName {
uids = slices.Delete(uids, i, i+1)
break
}
}
}
if DebugMode && len(uids) > 0 {
log.Println("missing collabs for", pkg, ":", uids)
}
for _, u := range uids {
_, err := client.Repository.RepoAddCollaborator(repository.NewRepoAddCollaboratorParams().WithOwner(org).WithRepo(giteaPackage(pkg)).WithBody(&models.AddCollaboratorOption{
Permission: "write",
}).WithCollaborator(u.UserID), r.DefaultAuthentication)
if err != nil {
if _, notFound := err.(*repository.RepoAddCollaboratorUnprocessableEntity); notFound {
if DebugMode {
log.Println("can't add user to collaborators:", u.UserID)
}
missing = append(missing, u.UserID)
} else {
log.Println(err)
}
}
}
return missing
}
func syncMaintainersToGitea(pkgs []string) {
maintainers := common.MaintainershipMap{
Data: map[string][]string{},
}
prjMeta, err := obs.GetProjectMeta(prj)
if err != nil {
log.Panicln("failed to get project meta", prj, err)
}
missingDevs := []string{}
devs := []string{}
if len(prjMeta.ScmSync) > 0 {
common.LogInfo("Project already in Git. Maintainers must have been already synced. Skipping...")
return
}
for _, group := range prjMeta.Groups {
if group.GroupID == "factory-maintainers" {
log.Println("Ignoring factory-maintainers")
continue
}
teamMembers, err := obs.GetGroupMeta(group.GroupID)
if err != nil {
log.Panicln("failed to get group", err)
}
if DebugMode {
log.Println("syncing", group.GroupID, teamMembers.Persons)
}
missingDevs = append(missingDevs, syncOrgTeams(group.GroupID, teamMembers.Persons.Persons)...)
for _, m := range teamMembers.Persons.Persons {
devs = append(devs, m.UserID)
}
}
for _, p := range prjMeta.Persons {
devs = append(devs, p.UserID)
}
missingDevs = append(missingDevs, syncOrgOwners(prjMeta.Persons)...)
maintainers.Data[""] = devs
for _, pkg := range pkgs {
pkgMeta, err := obs.GetPackageMeta(prj, pkg)
if err != nil {
log.Panicln("failed to get package meta", prj, pkg, err)
}
missingDevs = append(missingDevs, syncPackageCollaborators(pkg, pkgMeta.Persons)...)
devs := []string{}
for _, m := range pkgMeta.Persons {
devs = append(devs, m.UserID)
}
maintainers.Data[pkg] = devs
}
createPrjGit()
file, err := os.Create(path.Join(git.GetPath(), common.DefaultGitPrj, common.MaintainershipFile))
if err != nil {
log.Println(" *** Cannot create maintainership file:", err)
} else {
maintainers.WriteMaintainershipFile(file)
file.Close()
status, err := git.GitStatus(common.DefaultGitPrj)
if err != nil {
log.Panicln(err)
}
for _, s := range status {
if s.Path == common.MaintainershipFile && s.Status == common.GitStatus_Untracked {
git.GitExecOrPanic(common.DefaultGitPrj, "add", common.MaintainershipFile)
git.GitExecOrPanic(common.DefaultGitPrj, "commit", "-m", "Initial sync of maintainership with OBS")
git.GitExecOrPanic(common.DefaultGitPrj, "push")
}
}
if l := len(common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(common.DefaultGitPrj, "status", "--porcelain=2"), "\n")); l > 0 {
}
}
slices.Sort(missingDevs)
log.Println("Users without Gitea accounts:", slices.Compact(missingDevs))
}
func createPrjGit() {
if _, err := os.Stat(path.Join(git.GetPath(), common.DefaultGitPrj)); errors.Is(err, os.ErrNotExist) {
if err := git.GitExec("", "clone", "gitea@src.opensuse.org:"+org+"/_ObsPrj.git", common.DefaultGitPrj); err != nil {
repoName := common.DefaultGitPrj
_, err := client.Organization.CreateOrgRepo(organization.NewCreateOrgRepoParams().WithOrg(org).WithBody(
&models.CreateRepoOption{
AutoInit: false,
Name: &repoName,
ObjectFormatName: "sha256",
}), r.DefaultAuthentication)
if err != nil {
log.Panicln(err)
}
git.GitExecOrPanic("", "clone", "gitea@src.opensuse.org:"+org+"/_ObsPrj.git", common.DefaultGitPrj)
config, err := obs.ProjectConfig(prj)
if err != nil {
log.Panicln(err)
}
file, err := os.Create(path.Join(git.GetPath(), common.DefaultGitPrj, "_config"))
if err != nil {
log.Panicln(err)
}
config = strings.TrimSpace(config)
if len(config) > 1 {
file.Write([]byte(config))
file.Close()
git.GitExecOrPanic(common.DefaultGitPrj, "add", "_config")
}
file, err = os.Create(path.Join(git.GetPath(), common.DefaultGitPrj, "staging.config"))
if err != nil {
log.Panicln(err)
}
file.WriteString("{\n // Reference build project\n \"ObsProject\": \"" + prj + "\",\n}\n")
file.Close()
git.GitExecOrPanic(common.DefaultGitPrj, "add", "staging.config")
if file, err = os.Create(path.Join(git.GetPath(), common.DefaultGitPrj, "workflow.config")); err != nil {
log.Panicln(err)
}
file.WriteString("{\n \"Workflows\": [\"direct\", \"pr\"],\n \"Organization\": \"" + org + "\",\n}\n")
file.Close()
git.GitExecOrPanic(common.DefaultGitPrj, "add", "workflow.config")
}
}
}
func TrimMultibuildPackages(packages []string) []string {
for i := 0; i < len(packages); {
if strings.Contains(packages[i], ":") {
packages = slices.Delete(packages, i, i+1)
} else {
i++
}
}
return packages
}
var client *apiclient.GiteaAPI
var r *transport.Runtime
var git common.Git
var obs *common.ObsClient
var prj, org string
var forceBadPool bool
var forceNonPoolPackages bool
var devel_projects common.DevelProjects
func main() {
if err := common.RequireGiteaSecretToken(); err != nil {
log.Panicln("Missing GITEA_TOKEN")
}
if err := common.RequireObsSecretToken(); err != nil {
log.Panicln("Missing OBS_PASSWORD and/or OBS_USER")
}
flags := flag.NewFlagSet("devel-importer", flag.ContinueOnError)
helpString := new(bytes.Buffer)
flags.SetOutput(helpString)
//workflowConfig := flag.String("config", "", "Repository and workflow definition file")
giteaHost := flags.String("gitea", "src.opensuse.org", "Gitea instance")
obsUrl := flags.String("obs-url", "https://api.opensuse.org", "OBS API Url")
//rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
flags.BoolVar(&DebugMode, "debug", false, "Extra debugging information")
// revNew := flag.Int("nrevs", 20, "Number of new revisions in factory branch. Indicator of broken history import")
purgeOnly := flags.Bool("purge-only", false, "Purges package repositories on Gitea. Use with caution")
debugGitPath := flags.String("git-path", "", "Path for temporary git directory. Only used if DebugMode")
getMaintainers := flags.Bool("maintainers-only", false, "Get maintainers only and exit")
syncMaintainers := flags.Bool("sync-maintainers-only", false, "Sync maintainers to Gitea and exit")
flags.BoolVar(&forceBadPool, "bad-pool", false, "Force packages if pool has no branches due to bad import")
flags.BoolVar(&forceNonPoolPackages, "non-pool", false, "Allow packages that are not in pool to be created. WARNING: Can't add to factory later!")
specificPackages := flags.String("packages", "", "Process specific package, separated by commas, ignoring the others")
resumeAt := flags.String("resume", "", "Resume import at given pacakge")
syncPool := flags.Bool("sync-pool-only", false, "Force updates pool based on currrently imported project")
if help := flags.Parse(os.Args[1:]); help == flag.ErrHelp || flags.NArg() != 2 {
printHelp(helpString.String())
return
}
if DebugMode {
common.SetLoggingLevel(common.LogLevelDebug)
}
r = transport.New(*giteaHost, apiclient.DefaultBasePath, [](string){"https"})
r.DefaultAuthentication = transport.BearerToken(common.GetGiteaToken())
// r.SetDebug(true)
client = apiclient.New(r, nil)
obs, _ = common.NewObsClient(*obsUrl)
var gh common.GitHandlerGenerator
var err error
devel_projects, err = common.FetchDevelProjects()
if err != nil {
log.Panic("Cannot load devel projects:", err)
}
log.Println("# devel projects loaded:", len(devel_projects))
if DebugMode {
if len(*debugGitPath) > 0 {
gh, err = common.AllocateGitWorkTree(*debugGitPath, "Autogits - Devel Importer", "not.exist")
if err != nil {
log.Panicln(err)
}
}
} else {
dir, _ := os.MkdirTemp(os.TempDir(), "devel-importer")
gh, err = common.AllocateGitWorkTree(dir, "Autogits - Devel Importer", "not.exist")
if err != nil {
log.Panicln("Failed to allocate git handler", err)
}
}
prj = flags.Arg(0)
org = flags.Arg(1)
packages, err := runObsCommand("ls", prj)
packages = TrimMultibuildPackages(packages)
git, err = gh.CreateGitHandler(org)
if err != nil {
log.Panicln("Cannot create git", err)
}
defer git.Close()
log.Println(" - working directory:" + git.GetPath())
if *syncPool {
factory_pkgs, err := runObsCommand("ls", "openSUSE:Factory")
common.PanicOnError(err)
common.LogInfo("Syncing pool only...")
factory_pkgs = TrimMultibuildPackages(factory_pkgs)
for _, pkg := range packages {
if !slices.Contains(factory_pkgs, pkg) {
continue
}
repo, err := client.Repository.RepoGet(repository.NewRepoGetParams().WithOwner(org).WithRepo(ObsToRepoName(pkg)), r.DefaultAuthentication)
common.PanicOnError(err)
if !slices.Contains(common.SplitLines(git.GitExecWithOutputOrPanic(pkg, "remote")), "pool") {
git.GitExecOrPanic(pkg, "remote", "add", "pool", repo.Payload.SSHURL)
}
git.GitExecOrPanic(pkg, "fetch", "pool")
}
}
/*
for _, pkg := range packages {
if _, err := client.Organization.CreateOrgRepo(organization.NewCreateOrgRepoParams().WithOrg(org).WithBody(
&models.CreateRepoOption{
ObjectFormatName: "sha256",
AutoInit: false,
Name: &pkg,
DefaultBranch: "main",
}),
r.DefaultAuthentication,
); err != nil {
log.Println(err)
}
}
*/
if *getMaintainers {
listMaintainers(obs, prj, packages)
return
}
if *syncMaintainers {
syncMaintainersToGitea(packages)
return
}
if err != nil {
log.Printf("Cannot list packages for project '%s'. Err: %v\n", prj, err)
return
}
slices.Sort(packages)
for i := range packages {
packages[i] = strings.Split(packages[i], ":")[0]
}
packages = slices.Compact(packages)
log.Printf("%d packages: %s\n\n", len(packages), strings.Join(packages, " "))
if *purgeOnly {
log.Println("Purging repositories...")
pkgs := packages
if len(*specificPackages) > 0 {
pkgs = common.SplitStringNoEmpty(*specificPackages, ",")
}
for _, pkg := range pkgs {
client.Repository.RepoDelete(repository.NewRepoDeleteParams().WithOwner(org).WithRepo(giteaPackage(pkg)), r.DefaultAuthentication)
}
os.Exit(10)
}
if len(*specificPackages) != 0 {
packages = common.SplitStringNoEmpty(*specificPackages, ",")
}
slices.Sort(packages)
BrokenOBSPackage := []string{}
FailedImport := []string{}
for _, pkg := range packages {
if len(*resumeAt) > 0 && strings.Compare(*resumeAt, pkg) > 0 {
common.LogDebug(pkg, "skipped due to resuming at", *resumeAt)
continue
}
b, f := importRepo(pkg)
if b {
BrokenOBSPackage = append(BrokenOBSPackage, pkg)
}
if f {
FailedImport = append(FailedImport, pkg)
}
}
syncMaintainersToGitea(packages)
common.LogError("Have broken pool packages:", len(BrokenOBSPackage))
// common.LogError("Total pool packages:", len(factoryRepos))
common.LogError("Failed to import:", strings.Join(FailedImport, ","))
common.LogInfo("BROKEN Pool packages:", strings.Join(BrokenOBSPackage, "\n"))
}