425 lines
13 KiB
Go
425 lines
13 KiB
Go
package main
|
|
|
|
/*
|
|
* This file is part of Autogits.
|
|
*
|
|
* Copyright © 2024 SUSE LLC
|
|
*
|
|
* Autogits is free software: you can redistribute it and/or modify it under
|
|
* the terms of the GNU General Public License as published by the Free Software
|
|
* Foundation, either version 2 of the License, or (at your option) any later
|
|
* version.
|
|
*
|
|
* Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
|
|
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along with
|
|
* Foobar. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
import (
|
|
"errors"
|
|
"flag"
|
|
"log"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
|
|
transport "github.com/go-openapi/runtime/client"
|
|
"src.opensuse.org/autogits/common"
|
|
apiclient "src.opensuse.org/autogits/common/gitea-generated/client"
|
|
"src.opensuse.org/autogits/common/gitea-generated/client/organization"
|
|
"src.opensuse.org/autogits/common/gitea-generated/client/repository"
|
|
"src.opensuse.org/autogits/common/gitea-generated/models"
|
|
)
|
|
|
|
const commandLineHelp = `
|
|
SYNTAX
|
|
devel-importer <obs-project> <gitea-org>`
|
|
|
|
func printHelp() {
|
|
log.Println(commandLineHelp)
|
|
}
|
|
|
|
func runObsCommand(args ...string) ([]byte, error) {
|
|
cmd := exec.Command("osc", args...)
|
|
return cmd.Output()
|
|
}
|
|
|
|
var DebugMode bool
|
|
|
|
func main() {
|
|
if err := common.RequireGiteaSecretToken(); err != nil {
|
|
log.Panicln("Missing GITEA_TOKEN")
|
|
}
|
|
|
|
if err := common.RequireObsSecretToken(); err != nil {
|
|
log.Panicln("Missing OBS_PASSWORD and/or OBS_USER")
|
|
}
|
|
|
|
//workflowConfig := flag.String("config", "", "Repository and workflow definition file")
|
|
giteaHost := flag.String("gitea", "src.opensuse.org", "Gitea instance")
|
|
//rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
|
|
flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information")
|
|
// revNew := flag.Int("nrevs", 20, "Number of new revisions in factory branch. Indicator of broken history import")
|
|
purgeOnly := flag.Bool("purge", false, "Purges package repositories. Use with caution")
|
|
debugGitPath := flag.String("git-path", "", "Path for temporary git directory. Only used if DebugMode")
|
|
flag.Parse()
|
|
|
|
git, err := common.CreateGitHandler("Autogits - Devel Importer", "not.exist", "devel-importer")
|
|
if err != nil {
|
|
log.Panicln("Failed to allocate git handler. Err:", err)
|
|
}
|
|
if DebugMode {
|
|
log.Println(" - working directory:" + git.GitPath)
|
|
if len(*debugGitPath) > 0 {
|
|
git.Close()
|
|
git.GitPath = *debugGitPath
|
|
}
|
|
} else {
|
|
defer git.Close()
|
|
}
|
|
|
|
if flag.NArg() != 2 {
|
|
printHelp()
|
|
os.Exit(1)
|
|
}
|
|
|
|
prj := flag.Arg(0)
|
|
org := flag.Arg(1)
|
|
packageList, err := runObsCommand("ls", prj)
|
|
|
|
if err != nil {
|
|
log.Printf("Cannot list packages for project '%s'. Err: %v\n", prj, err)
|
|
}
|
|
packages := strings.Split(strings.TrimSpace(string(packageList)), "\n")
|
|
slices.Sort(packages)
|
|
for i := range packages {
|
|
packages[i] = strings.Split(packages[i], ":")[0]
|
|
}
|
|
packages = slices.Compact(packages)
|
|
|
|
log.Printf("%d packages: %s\n\n", len(packages), strings.Join(packages, " "))
|
|
|
|
r := transport.New(*giteaHost, apiclient.DefaultBasePath, [](string){"https"})
|
|
r.DefaultAuthentication = transport.BearerToken(common.GetGiteaToken())
|
|
// r.SetDebug(true)
|
|
client := apiclient.New(r, nil)
|
|
|
|
if *purgeOnly {
|
|
log.Println("Purging repositories...")
|
|
for _, pkg := range packages {
|
|
client.Repository.RepoDelete(repository.NewRepoDeleteParams().WithOwner(org).WithRepo(pkg), r.DefaultAuthentication)
|
|
}
|
|
os.Exit(10)
|
|
}
|
|
|
|
factoryRepos := make([]*models.Repository, 0, len(packages)*2)
|
|
develProjectPackages := make([]string, 0, len(packages))
|
|
for _, pkg := range packages {
|
|
src_pkg_name := strings.Split(pkg, ":")
|
|
repo, err := client.Repository.RepoGet(
|
|
repository.NewRepoGetParams().
|
|
WithDefaults().WithOwner("pool").WithRepo(src_pkg_name[0]),
|
|
r.DefaultAuthentication)
|
|
|
|
rpm, rpmErr := client.Repository.RepoGet(
|
|
repository.NewRepoGetParams().WithDefaults().WithOwner("rpm").WithRepo(src_pkg_name[0]),
|
|
r.DefaultAuthentication)
|
|
|
|
if rpmErr == nil {
|
|
factoryRepos = append(factoryRepos, rpm.Payload)
|
|
} else {
|
|
if !errors.Is(rpmErr, &repository.RepoGetNotFound{}) {
|
|
log.Panicln(rpmErr)
|
|
}
|
|
|
|
log.Println("No RPM package?", src_pkg_name)
|
|
}
|
|
|
|
if err != nil {
|
|
if !errors.Is(err, &repository.RepoGetNotFound{}) {
|
|
log.Panicln(err)
|
|
}
|
|
|
|
log.Println("Cannot find src package:", src_pkg_name)
|
|
develProjectPackages = append(develProjectPackages, src_pkg_name[0])
|
|
} else {
|
|
factoryRepos = append(factoryRepos, repo.Payload)
|
|
}
|
|
|
|
}
|
|
log.Println("Num repos found:", len(factoryRepos))
|
|
|
|
oldPackageNames := make([]string, 0, len(factoryRepos))
|
|
for _, repo := range factoryRepos {
|
|
oldPackageNames = append(oldPackageNames, repo.Name)
|
|
}
|
|
slices.Sort(oldPackageNames)
|
|
oldPackageNames = slices.Compact(oldPackageNames)
|
|
copy(oldPackageNames[2:], oldPackageNames)
|
|
log.Println("Num of old packages:", len(oldPackageNames))
|
|
|
|
// fork packags from pool
|
|
cmd := exec.Command("./git-importer", append([]string{"-r", git.GitPath}, oldPackageNames...)...)
|
|
out, err := cmd.CombinedOutput()
|
|
log.Println(string(out))
|
|
if err != nil {
|
|
log.Println("Error returned by importer.", err)
|
|
}
|
|
|
|
reposOK := true
|
|
log.Println("adding remotes...")
|
|
for i := range factoryRepos {
|
|
pkg := factoryRepos[i]
|
|
|
|
// verify that package was created by `git-importer`, or it's scmsync package and clone it
|
|
fi, err := os.Stat(filepath.Join(git.GitPath, pkg.Name))
|
|
if os.IsNotExist(err) {
|
|
// scmsync?
|
|
devel_project, err := runObsCommand("develproject", "openSUSE:Factory", pkg.Name)
|
|
if err != nil {
|
|
log.Panicln(err)
|
|
}
|
|
d := strings.Split(strings.TrimSpace(string(devel_project)), "/")
|
|
if len(d) != 2 {
|
|
log.Panicln("expected devel project/package. got:", d)
|
|
}
|
|
obs, _ := common.NewObsClient("api.opensuse.org")
|
|
meta, _ := obs.GetPackageMeta(d[0], d[1])
|
|
if len(meta.ScmSync) > 0 {
|
|
if err2 := git.CloneDevel("", pkg.Name, meta.ScmSync); err != nil {
|
|
log.Panicln(err2)
|
|
}
|
|
}
|
|
|
|
// try again, should now exist
|
|
if fi, err = os.Stat(filepath.Join(git.GitPath, pkg.Name)); err != nil {
|
|
log.Panicln(err)
|
|
}
|
|
} else if err != nil {
|
|
log.Panicln(err)
|
|
}
|
|
|
|
if !fi.IsDir() {
|
|
log.Panicln("Expected package file should be a directory. It's not.", fi)
|
|
}
|
|
|
|
// add remote repos
|
|
out := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "show", "-n")
|
|
switch pkg.Owner.UserName {
|
|
case "pool":
|
|
if !slices.Contains(strings.Split(out, "\n"), "factory") {
|
|
out := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "add", "factory", pkg.CloneURL)
|
|
if len(strings.TrimSpace(out)) > 1 {
|
|
log.Println(out)
|
|
}
|
|
}
|
|
case "rpm":
|
|
if !slices.Contains(strings.Split(out, "\n"), "rpm") {
|
|
out := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "add", "rpm", pkg.CloneURL)
|
|
if len(strings.TrimSpace(out)) > 1 {
|
|
log.Println(out)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, pkgName := range oldPackageNames {
|
|
log.Println("fetching git:", pkgName)
|
|
remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "remote", "show", "-n"), "\n")
|
|
params := []string{"fetch", "--multiple"}
|
|
params = append(params, remotes...)
|
|
out := git.GitExecWithOutputOrPanic(pkgName, params...)
|
|
if len(strings.TrimSpace(out)) > 1 {
|
|
log.Println(out)
|
|
}
|
|
|
|
// check that nothing is broken with the update
|
|
if slices.Contains(remotes, "factory") {
|
|
head_branch := "factory"
|
|
out, err = git.GitExecWithOutput(pkgName, "rev-list", head_branch)
|
|
if err != nil {
|
|
head_branch = "HEAD"
|
|
out = git.GitExecWithOutputOrPanic(pkgName, "rev-list", head_branch)
|
|
}
|
|
old_revs := strings.Split(out, "\n")
|
|
added_revs := []string{}
|
|
out = git.GitExecWithOutputOrPanic(pkgName, "rev-list", head_branch, "^factory/factory")
|
|
added_revs = strings.Split(out, "\n")
|
|
added_rpm_revs := old_revs
|
|
if slices.Contains(remotes, "rpm") {
|
|
out = git.GitExecWithOutputOrPanic(pkgName, "rev-list", head_branch, "^rpm/factory")
|
|
added_rpm_revs = strings.Split(out, "\n")
|
|
}
|
|
|
|
if len(added_revs) == len(old_revs) && len(added_rpm_revs) == len(old_revs) {
|
|
log.Printf("Something is wrong with rev-ist for (len %d): %s\n", len(added_revs), pkgName)
|
|
|
|
// check if we have broken history in OBS and that Tree objects are still matching
|
|
newCommits, err := git.GitParseCommits(pkgName, old_revs)
|
|
if err != nil {
|
|
log.Panicln("failed to parse commitids", err)
|
|
}
|
|
factoryCommitIDs := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "rev-list", "factory/factory"), "\n")
|
|
factoryCommits, err := git.GitParseCommits(pkgName, factoryCommitIDs)
|
|
if err != nil {
|
|
log.Panicln("failed to parse factory commits")
|
|
}
|
|
|
|
// find first common trees
|
|
found := false
|
|
foundMatch:
|
|
for i := range newCommits {
|
|
for j := range factoryCommits {
|
|
if newCommits[i].Tree == factoryCommits[i].Tree {
|
|
log.Println("found match", i, "vs", j)
|
|
git.GitExecOrPanic(pkgName, "checkout", "-B", "factory", added_revs[j])
|
|
if i != 0 {
|
|
// chrry pick on top
|
|
for commitIndex := i - 1; commitIndex >= 0; i-- {
|
|
git.GitExecOrPanic(pkgName, "cherry-pick", factoryCommitIDs[commitIndex])
|
|
}
|
|
}
|
|
found = true
|
|
break foundMatch
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
reposOK = false
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
args := make([]string, 4, len(develProjectPackages)+4)
|
|
args[0] = "-p"
|
|
args[1] = prj
|
|
args[2] = "-r"
|
|
args[3] = git.GitPath
|
|
args = append(args, develProjectPackages...)
|
|
cmd = exec.Command("./git-importer", args...)
|
|
out, err = cmd.CombinedOutput()
|
|
log.Println(string(out))
|
|
if err != nil {
|
|
log.Panicln("Error returned by importer.", err)
|
|
}
|
|
|
|
if !reposOK {
|
|
log.Panicln("aborting import due to broken repos above...")
|
|
}
|
|
|
|
slices.SortFunc(factoryRepos, func(a, b *models.Repository) int {
|
|
if a.Name == b.Name {
|
|
orgOrderNo := func(org string) int {
|
|
switch org {
|
|
case "pool":
|
|
return 1
|
|
case "rpm":
|
|
return 2
|
|
}
|
|
return 0 // current devel to clone
|
|
}
|
|
|
|
return orgOrderNo(a.Owner.UserName) - orgOrderNo(b.Owner.UserName)
|
|
}
|
|
return strings.Compare(a.Name, b.Name)
|
|
})
|
|
factoryRepos = slices.CompactFunc(factoryRepos, func(a, b *models.Repository) bool {
|
|
return a.Name == b.Name
|
|
})
|
|
|
|
for _, pkg := range factoryRepos {
|
|
// update package
|
|
fork, err := client.Repository.CreateFork(repository.NewCreateForkParams().
|
|
WithOwner(pkg.Owner.UserName).
|
|
WithRepo(pkg.Name).
|
|
WithBody(&models.CreateForkOption{
|
|
Organization: org,
|
|
}), r.DefaultAuthentication)
|
|
if err != nil {
|
|
log.Panicln("Error while trying to create fork from pool", pkg.Name, err)
|
|
}
|
|
|
|
repo := fork.Payload
|
|
branchName := repo.DefaultBranch
|
|
|
|
if pkg.Owner.UserName == "pool" || pkg.Owner.UserName == "rpm" {
|
|
// forked a git-based devel project, so use the default branch name now
|
|
repoList, err := client.Repository.RepoListBranches(
|
|
repository.NewRepoListBranchesParams().WithOwner(org).WithRepo(pkg.Name),
|
|
r.DefaultAuthentication,
|
|
)
|
|
if err != nil {
|
|
log.Panicln("Cannot get list of branches for forked repo:", org, "/", pkg.Name)
|
|
}
|
|
priorityBranches := []string{
|
|
"devel",
|
|
"factory",
|
|
}
|
|
idx := len(priorityBranches)
|
|
for _, branch := range repoList.Payload {
|
|
i := slices.Index(priorityBranches, branch.Name)
|
|
if i > -1 && i < idx {
|
|
idx = i
|
|
}
|
|
}
|
|
branchName = priorityBranches[idx]
|
|
}
|
|
|
|
remotes := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "show")
|
|
if !slices.Contains(strings.Split(remotes, "\n"), "devel") {
|
|
git.GitExecOrPanic(pkg.Name, "remote", "add", "devel", repo.SSHURL)
|
|
git.GitExecOrPanic(pkg.Name, "fetch", "devel")
|
|
}
|
|
git.GitExecOrPanic(pkg.Name, "branch", "main", "-f", "devel/"+branchName)
|
|
git.GitExecOrPanic(pkg.Name, "push", "devel", "main")
|
|
|
|
_, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(&models.EditRepoOption{
|
|
DefaultBranch: "main",
|
|
DefaultMergeStyle: "fast-forward-only",
|
|
}), r.DefaultAuthentication)
|
|
|
|
if err != nil {
|
|
log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err)
|
|
}
|
|
}
|
|
|
|
for _, pkg := range develProjectPackages {
|
|
ret, err := client.Organization.CreateOrgRepo(organization.NewCreateOrgRepoParams().WithOrg(org).WithBody(
|
|
&models.CreateRepoOption{
|
|
ObjectFormatName: "sha256",
|
|
AutoInit: false,
|
|
Name: &pkg,
|
|
DefaultBranch: "main",
|
|
}),
|
|
r.DefaultAuthentication,
|
|
)
|
|
|
|
if err != nil {
|
|
log.Panicln("Error creating new package repository:", pkg, err)
|
|
}
|
|
|
|
repo := ret.Payload
|
|
remotes := git.GitExecWithOutputOrPanic(pkg, "remote", "show")
|
|
if !slices.Contains(strings.Split(remotes, "\n"), "devel") {
|
|
git.GitExecOrPanic(pkg, "remote", "add", "devel", repo.SSHURL)
|
|
}
|
|
git.GitExecOrPanic(pkg, "branch", "main", "-f", "factory")
|
|
git.GitExecOrPanic(pkg, "push", "devel", "main")
|
|
|
|
_, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(pkg).WithBody(&models.EditRepoOption{
|
|
DefaultBranch: "main",
|
|
DefaultMergeStyle: "fast-forward-only",
|
|
}), r.DefaultAuthentication)
|
|
|
|
if err != nil {
|
|
log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err)
|
|
}
|
|
}
|
|
}
|