autogits/devel-importer/main.go

package main

/*
 * This file is part of Autogits.
 *
 * Copyright © 2024 SUSE LLC
 *
 * Autogits is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License as published by the Free Software
 * Foundation, either version 2 of the License, or (at your option) any later
 * version.
 *
 * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 * PARTICULAR PURPOSE. See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * Foobar. If not, see <https://www.gnu.org/licenses/>.
 */

import (
	"errors"
	"flag"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"slices"
	"strings"
	"time"

	transport "github.com/go-openapi/runtime/client"
	"src.opensuse.org/autogits/common"
	apiclient "src.opensuse.org/autogits/common/gitea-generated/client"
	"src.opensuse.org/autogits/common/gitea-generated/client/organization"
	"src.opensuse.org/autogits/common/gitea-generated/client/repository"
	"src.opensuse.org/autogits/common/gitea-generated/models"
)

const commandLineHelp = `
  SYNTAX
    devel-importer <obs-project> <gitea-org>`

func printHelp() {
	log.Println(commandLineHelp)
}

func runObsCommand(args ...string) ([]byte, error) {
	cmd := exec.Command("osc", args...)
	return cmd.Output()
}

var DebugMode bool

func main() {
	if err := common.RequireGiteaSecretToken(); err != nil {
		log.Panicln("Missing GITEA_TOKEN")
	}

	if err := common.RequireObsSecretToken(); err != nil {
		log.Panicln("Missing OBS_PASSWORD and/or OBS_USER")
	}

	//workflowConfig := flag.String("config", "", "Repository and workflow definition file")
	giteaHost := flag.String("gitea", "src.opensuse.org", "Gitea instance")
	//rabbitUrl := flag.String("url", "amqps://rabbit.opensuse.org", "URL for RabbitMQ instance")
	flag.BoolVar(&DebugMode, "debug", false, "Extra debugging information")
	// revNew := flag.Int("nrevs", 20, "Number of new revisions in factory branch. Indicator of broken history import")
	purgeOnly := flag.Bool("purge", false, "Purges package repositories. Use with caution")
	debugGitPath := flag.String("git-path", "", "Path for temporary git directory. Only used if DebugMode")
	flag.Parse()

	git, err := common.CreateGitHandler("Autogits - Devel Importer", "not.exist", "devel-importer")
	if err != nil {
		log.Panicln("Failed to allocate git handler. Err:", err)
	}
	if DebugMode {
		log.Println(" - working directory:" + git.GitPath)
		if len(*debugGitPath) > 0 {
			git.Close()
			git.GitPath = *debugGitPath
		}
	} else {
		defer git.Close()
	}

	if flag.NArg() != 2 {
		printHelp()
		os.Exit(1)
	}

	prj := flag.Arg(0)
	org := flag.Arg(1)
	packageList, err := runObsCommand("ls", prj)

	if err != nil {
		log.Printf("Cannot list packages for project '%s'. Err: %v\n", prj, err)
	}
	packages := strings.Split(strings.TrimSpace(string(packageList)), "\n")
	slices.Sort(packages)
	for i := range packages {
		packages[i] = strings.Split(packages[i], ":")[0]
	}
	packages = slices.Compact(packages)

	log.Printf("%d packages: %s\n\n", len(packages), strings.Join(packages, " "))

	r := transport.New(*giteaHost, apiclient.DefaultBasePath, [](string){"https"})
	r.DefaultAuthentication = transport.BearerToken(common.GetGiteaToken())
	// r.SetDebug(true)
	client := apiclient.New(r, nil)

	if *purgeOnly {
		log.Println("Purging repositories...")
		for _, pkg := range packages {
			client.Repository.RepoDelete(repository.NewRepoDeleteParams().WithOwner(org).WithRepo(pkg), r.DefaultAuthentication)
		}
		os.Exit(10)
	}

	factoryRepos := make([]*models.Repository, 0, len(packages)*2)
	develProjectPackages := make([]string, 0, len(packages))
	for _, pkg := range packages {
		src_pkg_name := strings.Split(pkg, ":")
		repo, err := client.Repository.RepoGet(
			repository.NewRepoGetParams().
				WithDefaults().WithOwner("pool").WithRepo(src_pkg_name[0]),
			r.DefaultAuthentication)

		rpm, rpmErr := client.Repository.RepoGet(
			repository.NewRepoGetParams().WithDefaults().WithOwner("rpm").WithRepo(src_pkg_name[0]),
			r.DefaultAuthentication)

		if rpmErr == nil {
			factoryRepos = append(factoryRepos, rpm.Payload)
		} else {
			if !errors.Is(rpmErr, &repository.RepoGetNotFound{}) {
				log.Panicln(rpmErr)
			}

			log.Println("No RPM package?", src_pkg_name)
		}

		if err != nil {
			if !errors.Is(err, &repository.RepoGetNotFound{}) {
				log.Panicln(err)
			}

			log.Println("Cannot find src package:", src_pkg_name)
			develProjectPackages = append(develProjectPackages, src_pkg_name[0])
		} else {
			factoryRepos = append(factoryRepos, repo.Payload)
		}

	}
	log.Println("Num repos found:", len(factoryRepos))

	oldPackageNames := make([]string, 0, len(factoryRepos))
	for _, repo := range factoryRepos {
		oldPackageNames = append(oldPackageNames, repo.Name)
	}
	slices.Sort(oldPackageNames)
	oldPackageNames = slices.Compact(oldPackageNames)
	copy(oldPackageNames[2:], oldPackageNames)
	log.Println("Num of old packages:", len(oldPackageNames))

	// fork packags from pool
	cmd := exec.Command("./git-importer", append([]string{"-r", git.GitPath}, oldPackageNames...)...)
	out, err := cmd.CombinedOutput()
	log.Println(string(out))
	if err != nil {
		log.Println("Error returned by importer.", err)
	}

	reposOK := true
	log.Println("adding remotes...")
	for i := range factoryRepos {
		pkg := factoryRepos[i]

		// verify that package was created by `git-importer`, or it's scmsync package and clone it
		fi, err := os.Stat(filepath.Join(git.GitPath, pkg.Name))
		if os.IsNotExist(err) {
			// scmsync?
			devel_project, err := runObsCommand("develproject", "openSUSE:Factory", pkg.Name)
			if err != nil {
				log.Panicln(err)
			}
			d := strings.Split(strings.TrimSpace(string(devel_project)), "/")
			if len(d) != 2 {
				log.Panicln("expected devel project/package. got:", d)
			}
			obs, _ := common.NewObsClient("api.opensuse.org")
			meta, _ := obs.GetPackageMeta(d[0], d[1])
			if len(meta.ScmSync) > 0 {
				if err2 := git.CloneDevel("", pkg.Name, meta.ScmSync); err != nil {
					log.Panicln(err2)
				}
			}

			// try again, should now exist
			if fi, err = os.Stat(filepath.Join(git.GitPath, pkg.Name)); err != nil {
				log.Panicln(err)
			}
		} else if err != nil {
			log.Panicln(err)
		}

		if !fi.IsDir() {
			log.Panicln("Expected package file should be a directory. It's not.", fi)
		}

		// add remote repos
		out := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "show", "-n")
		switch pkg.Owner.UserName {
		case "pool":
			if !slices.Contains(strings.Split(out, "\n"), "factory") {
				out := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "add", "factory", pkg.CloneURL)
				if len(strings.TrimSpace(out)) > 1 {
					log.Println(out)
				}
			}
		case "rpm":
			if !slices.Contains(strings.Split(out, "\n"), "rpm") {
				out := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "add", "rpm", pkg.CloneURL)
				if len(strings.TrimSpace(out)) > 1 {
					log.Println(out)
				}
			}
		}
	}

	for _, pkgName := range oldPackageNames {
		log.Println("fetching git:", pkgName)
		remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "remote", "show", "-n"), "\n")
		params := []string{"fetch", "--multiple"}
		params = append(params, remotes...)
		out := git.GitExecWithOutputOrPanic(pkgName, params...)
		if len(strings.TrimSpace(out)) > 1 {
			log.Println(out)
		}

		// check that nothing is broken with the update
		if slices.Contains(remotes, "factory") {
			head_branch := "factory"
			out, err = git.GitExecWithOutput(pkgName, "rev-list", head_branch)
			if err != nil {
				head_branch = "HEAD"
				out = git.GitExecWithOutputOrPanic(pkgName, "rev-list", "HEAD")
			}
			old_revs := strings.Split(out, "\n")
			added_revs := []string{}
			out = git.GitExecWithOutputOrPanic(pkgName, "rev-list", head_branch, "^factory/factory")
			added_revs = strings.Split(out, "\n")
			added_rpm_revs := []string{}
			if slices.Contains(remotes, "rpm") {
				out = git.GitExecWithOutputOrPanic(pkgName, "rev-list", head_branch, "^rpm/factory")
				added_rpm_revs = strings.Split(out, "\n")
			}
			if len(added_revs) == len(old_revs) && len(added_rpm_revs) == len(old_revs) {
				log.Printf("Something is wrong with rev-ist for (len %d): %s\n", len(added_revs), pkgName)
				reposOK = false
			}
		}
	}

	args := make([]string, 4, len(develProjectPackages)+4)
	args[0] = "-p"
	args[1] = prj
	args[2] = "-r"
	args[3] = git.GitPath
	args = append(args, develProjectPackages...)
	cmd = exec.Command("./git-importer", args...)
	out, err = cmd.CombinedOutput()
	log.Println(out)
	if err != nil {
		log.Panicln("Error returned by importer.", err)
	}

	if !reposOK {
		log.Println("aborting import due to broken repos above...")
	}

	for _, pkg := range factoryRepos {
		// update package
		fork, err := client.Repository.CreateFork(repository.NewCreateForkParams().
			WithOwner("pool").
			WithRepo(pkg.Name).
			WithBody(&models.CreateForkOption{
				Organization: org,
			}), r.DefaultAuthentication)
		if err != nil {
			log.Panicln("Error while trying to create fork from pool/", pkg.Name, err)
		}

		repo := fork.Payload
		repoList, err := client.Repository.RepoListBranches(
			repository.NewRepoListBranchesParams().WithOwner(org).WithRepo(pkg.Name),
			r.DefaultAuthentication,
		)
		if err != nil {
			log.Panicln("Cannot get list of branches for forked repo:", org, "/", pkg.Name)
		}
		priorityBranches := []string{
			"devel",
			"factory",
		}
		idx := len(priorityBranches)
		for _, branch := range repoList.Payload {
			i := slices.Index(priorityBranches, branch.Name)
			if i > -1 && i < idx {
				idx = i
			}
		}

		branchName := priorityBranches[idx]
		remotes := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "show")
		if !slices.Contains(strings.Split(remotes, "\n"), "devel") {
			git.GitExecOrPanic(pkg.Name, "remote", "add", "devel", repo.SSHURL)
		}
		git.GitExecOrPanic(pkg.Name, "branch", "main", "-f", branchName)
		time.Sleep(2 * time.Second)
		git.GitExecOrPanic(pkg.Name, "push", "devel", "main")
		time.Sleep(2 * time.Second)

		_, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(repo.Name).WithBody(&models.EditRepoOption{
			DefaultBranch:     "main",
			DefaultMergeStyle: "fast-forward-only",
		}), r.DefaultAuthentication)

		if err != nil {
			log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err)
		}
	}

	for _, pkg := range develProjectPackages {
		ret, err := client.Organization.CreateOrgRepo(organization.NewCreateOrgRepoParams().WithOrg(org).WithBody(
			&models.CreateRepoOption{
				ObjectFormatName: "sha256",
				AutoInit:         false,
				Name:             &pkg,
				DefaultBranch:    "main",
			}),
			r.DefaultAuthentication,
		)

		if err != nil {
			log.Panicln("Error creating new package repository:", pkg, err)
		}

		repo := ret.Payload
		remotes := git.GitExecWithOutputOrPanic(pkg, "remote", "show")
		if !slices.Contains(strings.Split(remotes, "\n"), "devel") {
			git.GitExecOrPanic(pkg, "remote", "add", "devel", repo.SSHURL)
		}
		git.GitExecOrPanic(pkg, "branch", "main", "-f", "factory")
		time.Sleep(2 * time.Second)
		git.GitExecOrPanic(pkg, "push", "devel", "main")
		time.Sleep(2 * time.Second)

		_, err = client.Repository.RepoEdit(repository.NewRepoEditParams().WithOwner(org).WithRepo(pkg).WithBody(&models.EditRepoOption{
			DefaultBranch:     "main",
			DefaultMergeStyle: "fast-forward-only",
		}), r.DefaultAuthentication)

		if err != nil {
			log.Panicln("Failed to set default branch for package fork:", repo.Owner.UserName, "/", repo.Name, err)
		}
	}
}