devel-importer: adapt for scmsync packages

bots-common/git_utils.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -92,6 +93,27 @@ func (refs *GitReferences) addReference(id, branch string) {
 	refs.refs = append(refs.refs, GitReference{Branch: branch, Id: id})
 }
 
+func (e *GitHandler) CloneDevel(gitDir, outName, urlString string) error {
+	url, err := url.Parse(urlString)
+	branch := url.Fragment
+	url.Fragment = ""
+
+	params := []string{"clone", "-o", "devel"}
+	if len(branch) > 0 {
+		params = append(params, "-b", branch)
+	}
+	params = append(params, url.String(), outName)
+
+	if err != nil {
+		return fmt.Errorf("error parsing SSH URL. %w", err);
+	}
+	out, err := e.GitExecWithOutput(gitDir, params...)
+	if err != nil {
+		return fmt.Errorf("error cloning %s.\n%s\nerr: %w", urlString, out, err)
+	}
+	return nil
+}
+
 func (e *GitHandler) GitBranchHead(gitDir, branchName string) (string, error) {
 	id, err := e.GitExecWithOutput(gitDir, "rev-list", "-1", branchName)
 	if err != nil {

bots-common/obs_utils.go

@@ -87,6 +87,13 @@ type ProjectMeta struct {
 	UseForBuild Flags `xml:"useforbuild"`
 }
 
+type PackageMeta struct {
+	XMLName xml.Name `xml:"package"`
+	Name    string   `xml:"name,attr"`
+	Project string   `xml:"project,attr"`
+	ScmSync string   `xml:"scmsync"`
+}
+
 func parseProjectMeta(data []byte) (*ProjectMeta, error) {
 	var meta ProjectMeta
 	err := xml.Unmarshal(data, &meta)
@@ -128,6 +135,43 @@ func (c *ObsClient) GetProjectMeta(project string) (*ProjectMeta, error) {
 	return parseProjectMeta(data)
 }
 
+func (c *ObsClient) GetPackageMeta(project, pkg string) (*PackageMeta, error) {
+	req, err := http.NewRequest("GET", c.baseUrl.JoinPath("source", project, pkg, "_meta").String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.SetBasicAuth(c.user, c.password)
+	log.Printf("request: %#v", *req.URL)
+	log.Printf("headers: %#v", req.Header)
+	res, err := c.client.Do(req)
+
+	if err != nil {
+		return nil, err
+	}
+
+	switch res.StatusCode {
+	case 200:
+		break
+	case 404:
+		return nil, nil
+	default:
+		return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode)
+	}
+
+	data, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var meta PackageMeta
+	err = xml.Unmarshal(data, &meta)
+	if err != nil {
+		return nil, err
+	}
+
+	return &meta, nil
+}
+
 func ObsSafeProjectName(prjname string) string {
 	if len(prjname) < 1 {
 		return prjname

devel-importer/main.go

@@ -24,6 +24,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"slices"
 	"strings"
 	"time"
@@ -176,6 +177,38 @@ func main() {
 	for i := range factoryRepos {
 		pkg := factoryRepos[i]
 
+		// verify that package was created by `git-importer`, or it's scmsync package and clone it
+		fi, err := os.Stat(filepath.Join(git.GitPath, pkg.Name))
+		if os.IsNotExist(err) {
+			// scmsync?
+			devel_project, err := runObsCommand("develproject", "openSUSE:Factory", pkg.Name)
+			if err != nil {
+				log.Panicln(err)
+			}
+			d := strings.Split(strings.TrimSpace(string(devel_project)), "/")
+			if len(d) != 2 {
+				log.Panicln("expected devel project/package. got:", d)
+			}
+			obs, _ := common.NewObsClient("api.opensuse.org")
+			meta, _ := obs.GetPackageMeta(d[0], d[1])
+			if len(meta.ScmSync) > 0 {
+				if err2 := git.CloneDevel("", pkg.Name, meta.ScmSync); err != nil {
+					log.Panicln(err2)
+				}
+			}
+
+			// try again, should now exist
+			if fi, err = os.Stat(filepath.Join(git.GitPath, pkg.Name)); err != nil {
+				log.Panicln(err)
+			}
+		} else if err != nil {
+			log.Panicln(err)
+		}
+
+		if !fi.IsDir() {
+			log.Panicln("Expected package file should be a directory. It's not.", fi)
+		}
+
 		// add remote repos
 		out := git.GitExecWithOutputOrPanic(pkg.Name, "remote", "show", "-n")
 		switch pkg.Owner.UserName {
@@ -198,7 +231,10 @@ func main() {
 
 	for _, pkgName := range oldPackageNames {
 		log.Println("fetching git:", pkgName)
-		out := git.GitExecWithOutputOrPanic(pkgName, "fetch", "--multiple", "factory", "rpm")
+		remotes := common.SplitStringNoEmpty(git.GitExecWithOutputOrPanic(pkgName, "remote", "show", "-n"), "\n")
+		params := []string{"fetch", "--multiple"}
+		params = append(params, remotes...)
+		out := git.GitExecWithOutputOrPanic(pkgName, params...)
 		if len(strings.TrimSpace(out)) > 1 {
 			log.Println(out)
 		}
@@ -208,8 +244,11 @@ func main() {
 		old_revs := strings.Split(out, "\n")
 		out = git.GitExecWithOutputOrPanic(pkgName, "rev-list", "factory", "^factory/factory")
 		added_revs := strings.Split(out, "\n")
+		added_rpm_revs := []string{}
+		if slices.Contains(remotes, "rpm") {
 			out = git.GitExecWithOutputOrPanic(pkgName, "rev-list", "factory", "^rpm/factory")
-		added_rpm_revs := strings.Split(out, "\n")
+			added_rpm_revs = strings.Split(out, "\n")
+		}
 		if len(added_revs) == len(old_revs) && len(added_rpm_revs) == len(old_revs) {
 			log.Printf("Something is wrong with rev-ist for (len %d): %s\n", len(added_revs), pkgName)
 			reposOK = false