wip

bots-common/consts.go

@@ -24,6 +24,7 @@ const (
 	ObsPasswordEnv = "OBS_PASSWORD"
 
 	DefaultGitPrj      = "_ObsPrj"
+	PrjLinksFile       = "links.json"
 	GiteaRequestHeader = "X-Gitea-Event-Type"
 
 	Bot_BuildReview = "autogits_obs_staging_bot"

bots-common/gitea_utils.go

@@ -19,6 +19,7 @@ package common
  */
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io"
 	"os"
@@ -63,8 +64,8 @@ const (
 )
 
 type GiteaMaintainershipReader interface {
-	FetchMaintainershipFile(org, prjGit, branch string) ([]byte, error)
+	FetchMaintainershipFile(org, prjGit, branch string) ([]byte, string, error)
-	FetchMaintainershipDirFile(org, prjGit, branch, pkg string) ([]byte, error)
+	FetchMaintainershipDirFile(org, prjGit, branch, pkg string) ([]byte, string, error)
 }
 
 type GiteaPRFetcher interface {
@@ -94,7 +95,12 @@ type GiteaReviewer interface {
 	AddReviewComment(pr *models.PullRequest, state models.ReviewStateType, comment string) (*models.PullReview, error)
 }
 
+type GiteaRepoFetcher interface {
+	GetRepository(org, repo string) (*models.Repository, error)
+}
+
 type Gitea interface {
+	GiteaRepoFetcher
 	GiteaReviewRequester
 	GiteaReviewer
 	GiteaPRFetcher
@@ -108,8 +114,8 @@ type Gitea interface {
 	CreateRepositoryIfNotExist(git Git, org, repoName string) (*models.Repository, error)
 	CreatePullRequestIfNotExist(repo *models.Repository, srcId, targetId, title, body string) (*models.PullRequest, error)
 	GetAssociatedPrjGitPR(prjGitOrg, prjGitRepo, refOrg, refRepo string, Index int64) (*models.PullRequest, error)
-	GetRepositoryFileContent(org, repo, hash, path string) ([]byte, error)
+	GetRepositoryFileContent(org, repo, hash, path string) ([]byte, string, error)
-	GetPullRequestFileContent(pr *models.PullRequest, path string) ([]byte, error)
+	GetPullRequestFileContent(pr *models.PullRequest, path string) ([]byte, string, error)
 	GetRecentPullRequests(org, repo string) ([]*models.PullRequest, error)
 	GetRecentCommits(org, repo, branch string, commitNo int64) ([]*models.Commit, error)
 
@@ -132,11 +138,11 @@ func AllocateGiteaTransport(host string) Gitea {
 	return &r
 }
 
-func (gitea *GiteaTransport) FetchMaintainershipFile(org, repo, branch string) ([]byte, error) {
+func (gitea *GiteaTransport) FetchMaintainershipFile(org, repo, branch string) ([]byte, string, error) {
 	return gitea.GetRepositoryFileContent(org, repo, branch, MaintainershipFile)
 }
 
-func (gitea *GiteaTransport) FetchMaintainershipDirFile(org, repo, branch, pkg string) ([]byte, error) {
+func (gitea *GiteaTransport) FetchMaintainershipDirFile(org, repo, branch, pkg string) ([]byte, string, error) {
 	return gitea.GetRepositoryFileContent(org, repo, branch, path.Join(MaintainershipDir, pkg))
 }
 
@@ -153,6 +159,15 @@ func (gitea *GiteaTransport) GetPullRequest(org, project string, num int64) (*mo
 	return pr.Payload, err
 }
 
+func (gitea *GiteaTransport) GetRepository(org, pkg string) (*models.Repository, error) {
+	repo, err := gitea.client.Repository.RepoGet(repository.NewRepoGetParams().WithDefaults().WithOwner(org).WithRepo(pkg), gitea.transport.DefaultAuthentication)
+	if err != nil {
+		return nil, err
+	}
+
+	return repo.Payload, nil
+}
+
 func (gitea *GiteaTransport) GetPullRequestReviews(org, project string, PRnum int64) ([]*models.PullReview, error) {
 	limit := int64(20)
 	var page int64
@@ -463,35 +478,36 @@ func (gitea *GiteaTransport) AddReviewComment(pr *models.PullRequest, state mode
 	return c.Payload, nil
 }
 
-func (gitea *GiteaTransport) GetRepositoryFileContent(org, repo, hash, path string) ([]byte, error) {
+func (gitea *GiteaTransport) GetRepositoryFileContent(org, repo, hash, path string) ([]byte, string, error) {
-	var retData []byte
+	content, err := gitea.client.Repository.RepoGetContents(
-
+		repository.NewRepoGetContentsParams().WithOwner(org).WithRepo(repo).WithFilepath(path).WithRef(&hash),
-	dataOut := writeFunc(func(data []byte) (int, error) {
-		if len(data) == 0 {
-			return 0, nil
-		}
-		retData = data
-		return len(data), nil
-	})
-	_, err := gitea.client.Repository.RepoGetRawFile(
-		repository.NewRepoGetRawFileParams().
-			WithOwner(org).
-			WithRepo(repo).
-			WithFilepath(path).
-			WithRef(&hash),
 		gitea.transport.DefaultAuthentication,
-		dataOut,
-		repository.WithContentTypeApplicationOctetStream,
 	)
 
 	if err != nil {
-		return nil, err
+		return nil, "", err
+	}
+	if content.Payload.Encoding != "base64" {
+		return nil, "", fmt.Errorf("Unhandled content encoding: %s", content.Payload.Encoding)
 	}
 
-	return retData, nil
+	if content.Payload.Size > 10000000 {
+		return nil, "", fmt.Errorf("Content length is too large for %s/%s/%s#%s - %d bytes", org, repo, path, hash, content.Payload.Size)
+	}
+
+	data := make([]byte, content.Payload.Size)
+	n, err := base64.RawStdEncoding.Decode(data, []byte(content.Payload.Content))
+	if err != nil {
+		return nil, "", fmt.Errorf("Error decoding file %s/%s/%s#%s : %w", org, repo, path, hash, err)
+	}
+	if n != int(content.Payload.Size) {
+		return nil, "", fmt.Errorf("Decoded length doesn't match expected for %s/%s/%s#%s - %d vs %d bytes", org, repo, path, hash, content.Payload.Size, n)
+	}
+
+	return data, content.Payload.SHA, nil
 }
 
-func (gitea *GiteaTransport) GetPullRequestFileContent(pr *models.PullRequest, path string) ([]byte, error) {
+func (gitea *GiteaTransport) GetPullRequestFileContent(pr *models.PullRequest, path string) ([]byte, string, error) {
 	return gitea.GetRepositoryFileContent(pr.Head.Repo.Owner.UserName, pr.Head.Repo.Name, pr.Head.Sha, path)
 }
 

bots-common/maintainership.go

@@ -40,14 +40,14 @@ func parseMaintainershipData(data []byte) (*MaintainershipMap, error) {
 }
 
 func FetchProjectMaintainershipData(gitea GiteaMaintainershipReader, org, prjGit, branch string) (*MaintainershipMap, error) {
-	data, err := gitea.FetchMaintainershipDirFile(org, prjGit, branch, ProjectFileKey)
+	data, _, err := gitea.FetchMaintainershipDirFile(org, prjGit, branch, ProjectFileKey)
 	dir := true
 	if err != nil || data == nil {
 		dir = false
 		if _, notFound := err.(*repository.RepoGetRawFileNotFound); !notFound {
 			return nil, err
 		}
-		data, err = gitea.FetchMaintainershipFile(org, prjGit, branch)
+		data, _, err = gitea.FetchMaintainershipFile(org, prjGit, branch)
 		if err != nil || data == nil {
 			if _, notFound := err.(*repository.RepoGetRawFileNotFound); !notFound {
 				return nil, err
@@ -62,7 +62,8 @@ func FetchProjectMaintainershipData(gitea GiteaMaintainershipReader, org, prjGit
 	if m != nil {
 		m.IsDir = dir
 		m.FetchPackage = func(pkg string) ([]byte, error) {
-			return gitea.FetchMaintainershipDirFile(org, prjGit, branch, pkg)
+			data , _, err := gitea.FetchMaintainershipDirFile(org, prjGit, branch, pkg)
+			return data, err
 		}
 	}
 	return m, err

bots-common/maintainership_test.go

@@ -159,9 +159,9 @@ func TestMaintainership(t *testing.T) {
 
 			// tests with maintainership file
 			mi.EXPECT().FetchMaintainershipFile("foo", common.DefaultGitPrj, "bar").
-				Return(test.maintainersFile, test.maintainersFileErr)
+				Return(test.maintainersFile, "", test.maintainersFileErr)
 			mi.EXPECT().FetchMaintainershipDirFile("foo", common.DefaultGitPrj, "bar", common.ProjectFileKey).
-				Return(nil, notFoundError)
+				Return(nil, "", notFoundError)
 
 			runTests(t, mi)
 		})
@@ -172,13 +172,13 @@ func TestMaintainership(t *testing.T) {
 
 			// run same tests with directory maintainership data
 			for filename, data := range test.maintainersDir {
-				mi.EXPECT().FetchMaintainershipDirFile("foo", common.DefaultGitPrj, "bar", filename).Return(data, test.maintainersFileErr).AnyTimes()
+				mi.EXPECT().FetchMaintainershipDirFile("foo", common.DefaultGitPrj, "bar", filename).Return(data, "", test.maintainersFileErr).AnyTimes()
 			}
 			if _, found := test.maintainersDir[common.ProjectFileKey]; !found {
-				mi.EXPECT().FetchMaintainershipDirFile("foo", common.DefaultGitPrj, "bar", common.ProjectFileKey).Return(nil, test.maintainersFileErr).AnyTimes()
+				mi.EXPECT().FetchMaintainershipDirFile("foo", common.DefaultGitPrj, "bar", common.ProjectFileKey).Return(nil, "", test.maintainersFileErr).AnyTimes()
-				mi.EXPECT().FetchMaintainershipFile("foo", common.DefaultGitPrj, "bar").Return(nil, test.maintainersFileErr).AnyTimes()
+				mi.EXPECT().FetchMaintainershipFile("foo", common.DefaultGitPrj, "bar").Return(nil, "", test.maintainersFileErr).AnyTimes()
 			}
-			mi.EXPECT().FetchMaintainershipDirFile("foo", common.DefaultGitPrj, "bar", gomock.Any()).Return(nil, notFoundError).AnyTimes()
+			mi.EXPECT().FetchMaintainershipDirFile("foo", common.DefaultGitPrj, "bar", gomock.Any()).Return(nil, "", notFoundError).AnyTimes()
 
 			runTests(t, mi)
 		})

workflow-direct/links.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"fmt"
 	"slices"
 	"strings"
 
@@ -10,17 +9,18 @@ import (
 )
 
 type OrgLinks struct {
-	Source string // org/pkg#branch
+	Pkg          string // fork to track
-	Target string // pkg#branch
+	SourceBranch string // branch to follow, empty for default
+	TargetBranch string // branch in target repo, empty for default
 
-	SOid, TOid string // source and target oids
+	parentOrg, parentRepo string
 }
 
-func FetchProjectLinksFile(org, prj string) ([]byte, error) {
+func fetchProjectLinksFile(org, prj string) ([]byte, error) {
 	return nil, nil
 }
 
-func ParseProjectLinks(data []byte) ([]*OrgLinks, error) {
+func parseProjectLinks(data []byte) ([]*OrgLinks, error) {
 	values := make([]*OrgLinks, 0, 100)
 
 	if len(data) == 0 {
@@ -32,30 +32,47 @@ func ParseProjectLinks(data []byte) ([]*OrgLinks, error) {
 	}
 
 	slices.SortFunc(values, func(a, b *OrgLinks) int {
-		return strings.Compare(a.Target, b.Target)
+		return strings.Compare(a.Pkg, b.Pkg)
 	})
 
-	for _, link := range values {
-		if len(link.Source) < 5 {
-			return nil, fmt.Errorf("Invalid Source: %s", link.Source)
-		}
-		source_link := strings.Split(link.Source, "/")
-		if len(source_link) != 2 || len(source_link[0]) < 1 || len(source_link[1]) < 3 {
-			return nil, fmt.Errorf("Invalid Source: %s", link.Source)
-		}
-		branch := strings.Split(source_link[1], "#")
-		if len(branch) != 2 || len(branch[0]) < 1 || len(branch[1]) < 1 {
-			return nil, fmt.Errorf("Invalid Source: %s", link.Source)
-		}
-		target_link := strings.Split(link.Target, "#")
-		if len(target_link) != 2 || len(target_link[0]) < 1 || len(target_link[1]) < 1 {
-			return nil, fmt.Errorf("Missing Target for Source link: %s", link.Source)
-		}
-	}
-
 	return values, nil
 }
 
-func (link *OrgLinks) StateUpdater(git common.Git, gitea common.Gitea) {
+func ProcessProjectLinks(gitea common.Gitea, org, prjGit, branch string) ([]*OrgLinks, string, error) {
+	data, hash, err := gitea.GetRepositoryFileContent(org, prjGit, branch, common.PrjLinksFile)
+	if err != nil {
+		return nil, "", err
+	}
+	links, err := parseProjectLinks(data)
+	return links, hash, err
 }
 
+func ResolveLinks(org string, links []*OrgLinks, gitea common.GiteaRepoFetcher) {
+	for _, link := range links {
+		if repo, err := gitea.GetRepository(org, link.Pkg); err == nil {
+			link.parentOrg = repo.Parent.Owner.UserName
+			link.parentRepo = repo.Parent.Name
+			if len(link.SourceBranch) == 0 {
+				link.SourceBranch = repo.DefaultBranch
+			}
+			if len(link.TargetBranch) == 0 {
+				link.TargetBranch = repo.Parent.DefaultBranch
+			}
+		}
+	}
+}
+
+func ListenOrgsForUpdates(links []*OrgLinks) []string {
+	orgs := make([]string, 0, len(links))
+	for _, link := range links {
+		if !slices.Contains(orgs, link.parentOrg) {
+			orgs = append(orgs, link.parentOrg)
+		}
+	}
+
+	return orgs
+}
+
+func (link *OrgLinks) StateUpdater(git common.Git, gitea common.Gitea) {
+
+}

workflow-direct/links_test.go

@@ -6,6 +6,7 @@ import (
 
 	"go.uber.org/mock/gomock"
 
+	"src.opensuse.org/autogits/common"
 	mock_common "src.opensuse.org/autogits/common/mock"
 )
 
@@ -25,29 +26,28 @@ func TestLinkParsing(t *testing.T) {
 		},
 		{
 			name: "Single package linked",
-			data: []byte(`[{"Source": "pool/foo#main", "Target": "pkg#main"}]`),
+			data: []byte(`[{"Pkg": "foo"}]`),
 			links: []*OrgLinks{
 				&OrgLinks{
-					Source: "pool/foo#main",
+					Pkg: "foo",
-					Target: "pkg#main",
 				},
 			},
 		},
 		{
 			name: "Multiple packages linked, resorted",
-			data: []byte(`[{"Source": "pool/foo#main", "Target": "pkg#main"}, {"Source": "pool/abc#main", "Target": "abc#m"}, {"Source": "abc/aaa#q", "Target": "zzz#w"}]`),
+			data: []byte(`[{"Pkg": "foo"}, {"Pkg": "abc", "TargetBranch": "m"}, {"Pkg": "aaa", "SourceBranch": "q", TargetBranch": "w"}]`),
 			links: []*OrgLinks{
 				&OrgLinks{
-					Source: "pool/abc#main",
+					Pkg: "aaa",
-					Target: "abc#m",
+					SourceBranch: "q",
+					TargetBranch: "w",
 				},
 				&OrgLinks{
-					Source: "pool/foo#main",
+					Pkg: "abc",
-					Target: "pkg#main",
+					TargetBranch: "m",
 				},
 				&OrgLinks{
-					Source: "abc/aaa#q",
+					Pkg: "foo",
-					Target: "zzz#w",
 				},
 			},
 		},
@@ -80,7 +80,11 @@ func TestLinkParsing(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			res, err := ParseProjectLinks(test.data)
+			ctl := gomock.NewController(t)
+			gitea_mock := mock_common.NewMockGitea(ctl)
+			gitea_mock.EXPECT().GetRepositoryFileContent("org", "repo", "branch", common.PrjLinksFile).Return(test.data, "", nil)
+
+			res, _, err := ProcessProjectLinks(gitea_mock, "org", "repo", "branch")
 			if len(test.expected_err) > 0 && err == nil {
 				t.Error("Expected an error:", test.expected_err, "; but got nothing")
 			} else if err != nil && len(test.expected_err) == 0 {
@@ -112,13 +116,10 @@ func TestLinkUpdater(t *testing.T) {
 		{
 			name: "no-op update",
 			mock_setup: func(git *mock_common.MockGit, gitea *mock_common.MockGitea) {
+			
 				git.EXPECT().GitExecWithOutputOrPanic(gomock.Any(), "rev-list", "upstream/main", "^pkg_hash").Return("")
 			},
 			link: &OrgLinks{
-				Source: "foo/bar#branch1",
-				Target: "pkg#branch2",
-				SOid:   "bar_hash",
-				TOid:   "pkg_hash",
 			},
 		},
 	}