From 2b16810fa6460982eaf76700028da319ad662dc61348b3bccd1804103f7c9c82 Mon Sep 17 00:00:00 2001 From: Antonello Tartamo Date: Tue, 11 Nov 2025 15:40:49 +0100 Subject: [PATCH 1/8] draft: installcheck --- common/diff_parser.go | 33 ++++++ common/gitea_utils.go | 17 +++ common/obs_utils.go | 77 +++++++++++++ obs-installcheck-bot/main.go | 206 +++++++++++++++++++++++++++++++++++ obs-staging-bot/main.go | 18 +-- 5 files changed, 342 insertions(+), 9 deletions(-) create mode 100644 common/diff_parser.go create mode 100644 obs-installcheck-bot/main.go diff --git a/common/diff_parser.go b/common/diff_parser.go new file mode 100644 index 0000000..2e69030 --- /dev/null +++ b/common/diff_parser.go @@ -0,0 +1,33 @@ +package common + +import ( + "path" + "regexp" + "strings" +) + +// ParseSubprojectChangesFromDiff parses a diff string and returns a slice of package names that have changed. +// It identifies subproject changes by looking for "Subproject commit" lines within diff chunks. +func ParseSubprojectChangesFromDiff(diff string) []string { + var changedPackages []string + + // This regex finds diff chunks for subprojects. + // It looks for a `diff --git` line, followed by lines indicating a subproject commit change. + re := regexp.MustCompile(`diff --git a\/(.+) b\/(.+)\n`) + + matches := re.FindAllStringSubmatch(diff, -1) + + for _, match := range matches { + if len(match) > 1 { + // The package path is in the first capturing group. + // We use path.Base to get just the package name from the path (e.g., "rpms/iptraf-ng" -> "iptraf-ng"). + pkgPath := strings.TrimSpace(match[1]) + basePath := path.Base(pkgPath) + if basePath != ".gitmodules" && basePath != "" { + changedPackages = append(changedPackages, basePath) + } + } + } + + return changedPackages +} diff --git a/common/gitea_utils.go b/common/gitea_utils.go index c3358c9..c97fd2a 100644 --- a/common/gitea_utils.go +++ b/common/gitea_utils.go @@ -194,6 +194,7 @@ type Gitea interface { GetRecentPullRequests(org, repo, branch string) ([]*models.PullRequest, error) GetRecentCommits(org, repo, branch string, commitNo int64) ([]*models.Commit, error) GetPullRequests(org, project string) ([]*models.PullRequest, error) + GetPullRequestDiff(owner, repo string, index int64) (string, error) GetCurrentUser() (*models.User, error) } @@ -929,3 +930,19 @@ func (gitea *GiteaTransport) GetCurrentUser() (*models.User, error) { return user.GetPayload(), nil } + +func (gitea *GiteaTransport) GetPullRequestDiff(owner, repo string, index int64) (string, error) { + params := repository.NewRepoDownloadPullDiffOrPatchParams(). + WithDefaults(). + WithOwner(owner). + WithRepo(repo). + WithIndex(index). + WithDiffType("diff") + + result, err := gitea.client.Repository.RepoDownloadPullDiffOrPatch(params, gitea.transport.DefaultAuthentication) + if err != nil { + return "", err + } + + return result.Payload, nil +} diff --git a/common/obs_utils.go b/common/obs_utils.go index 817367e..319d536 100644 --- a/common/obs_utils.go +++ b/common/obs_utils.go @@ -630,6 +630,11 @@ type Binary struct { Mtime uint64 `xml:"mtime,attr"` } +type PackageBinariesList struct { + XMLName xml.Name `xml:"binarylist"` + Binary []Binary `xml:"binary"` +} + type BinaryList struct { Package string `xml:"package,attr"` Binary []Binary `xml:"binary"` @@ -932,3 +937,75 @@ func (c *ObsClient) BuildStatusWithState(project string, opts *BuildResultOption } return ret, err } + +func parsePackageBinaries(data []byte) (*PackageBinariesList, error) { + result := PackageBinariesList{} + err := xml.Unmarshal(data, &result) + if err != nil { + return nil, err + } + + return &result, nil +} + +func (c *ObsClient) GetBinariesFromPackage(project, repo, arch, pkg string) (*PackageBinariesList, error) { + req := []string{"build", project, repo, arch, pkg} + res, err := c.ObsRequest("GET", req, nil) + + if err != nil { + return nil, err + } + + switch res.StatusCode { + case 200: + break + case 404: + return nil, nil + default: + return nil, fmt.Errorf("Unexpected return code: %d %s %w", res.StatusCode, req, err) + } + + data, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + defer res.Body.Close() + + result, err := parsePackageBinaries(data) + if err != nil { + return nil, err + } + + if result != nil { + result.Binary = slices.DeleteFunc(result.Binary, func(b Binary) bool { + return !strings.HasSuffix(b.Filename, ".rpm") + }) + } + return result, nil +} + +func (c *ObsClient) GetBinary(project, repo, arch, pkg, filename string) ([]byte, error) { + req := []string{"build", project, repo, arch, pkg, filename} + res, err := c.ObsRequest("GET", req, nil) + + if err != nil { + return nil, err + } + + switch res.StatusCode { + case 200: + break + case 404: + return nil, nil + default: + return nil, fmt.Errorf("Unexpected return code: %d %s %w", res.StatusCode, req, err) + } + + data, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + defer res.Body.Close() + + return data, nil +} diff --git a/obs-installcheck-bot/main.go b/obs-installcheck-bot/main.go new file mode 100644 index 0000000..1ccbef1 --- /dev/null +++ b/obs-installcheck-bot/main.go @@ -0,0 +1,206 @@ +package main + +import ( + "flag" + "fmt" + "os/exec" + "path/filepath" + + "os" + "regexp" + "strconv" + + "github.com/opentracing/opentracing-go/log" + "src.opensuse.org/autogits/common" +) + +var GiteaUrl string +var ObsClient *common.ObsClient +var ProcessPROnly string + +const ObsProject = "Amazon:spal:2023" +const ObsRepository = "standard" + +var PathSpalPrimary string +var PathAl2023Primary string + +var ObsArch = []string{"x86_64", "aarch64"} + +const SkipSrcRpm = true + +func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, error) { + common.LogInfo("Processing PR", org+"/"+repo, "#"+strconv.FormatInt(id, 10)) + + common.LogDebug("Fetching PR:", org, repo, id) + pr, err := gitea.GetPullRequest(org, repo, id) + if err != nil { + common.LogError("No PR associated with review:", org, "/", repo, "#", id, "Error:", err) + return true, err + } + + dir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("%s-%s-%d-", org, repo, id)) + common.PanicOnError(err) + + diff, err := gitea.GetPullRequestDiff(org, repo, id) + common.LogDebug("PR Diff:\n", diff) + if err != nil { + common.LogError("Failed to get PR diff:", err) + return false, err + } + + changedPackages := common.ParseSubprojectChangesFromDiff(diff) + if len(changedPackages) > 0 { + common.LogInfo("Changed packages found in PR!") + + for _, arch := range ObsArch { + archDir := filepath.Join(dir, arch) + err := os.MkdirAll(archDir, 0755) + common.PanicOnError(err) + common.LogInfo("Created temporary directory for", arch, "at", archDir) + } + + for _, pkg := range changedPackages { + common.LogInfo("*** " + pkg) + + for _, arch := range ObsArch { + archDir := filepath.Join(dir, arch) + + var binaries *common.PackageBinariesList + binaries, err = ObsClient.GetBinariesFromPackage(ObsProject, ObsRepository, arch, pkg) + if err != nil { + common.LogError("Failed to get binaries:", err) + return false, err + } + + var filtered_binaries []string + + if binaries != nil { + common.LogInfo("Binaries for package", pkg, "and arch", arch+":") + for _, bin := range binaries.Binary { + common.LogInfo(" -", bin.Filename, "(size:", bin.Size, "mtime:", bin.Mtime, ")") + if SkipSrcRpm && regexp.MustCompile(`\.src\.rpm$`).MatchString(bin.Filename) { + common.LogInfo(" Skipping source RPM as requested.") + continue + } + filtered_binaries = append(filtered_binaries, bin.Filename) + } + } + + if len(filtered_binaries) > 0 { + for _, bin := range filtered_binaries { + common.LogInfo("Downloading binary:", bin) + binaryData, err := ObsClient.GetBinary(ObsProject, ObsRepository, arch, pkg, bin) + if err != nil { + common.LogError("Failed to download binary '"+bin+"':", err) + continue + } + if binaryData == nil { + common.LogInfo("Binary not found or is empty:", bin) + continue + } + filePath := filepath.Join(archDir, bin) + err = os.WriteFile(filePath, binaryData, 0644) + common.PanicOnError(err) + } + } + } + } + + for _, arch := range ObsArch { + archDir := filepath.Join(dir, arch) + // Call createrepo_c for the architecture directory + common.LogInfo("Creating repository for architecture:", arch, "in", archDir) + cmd := exec.Command("createrepo_c", archDir) + output, err := cmd.CombinedOutput() + if err != nil { + common.LogError("Failed to create repository for '"+arch+"':", err, "Output:", string(output)) + } + + common.LogInfo("Running installcheck for architecture:", arch) + installcheckCommand := fmt.Sprintf("installcheck %s %s %s --nocheck %s", arch, PathSpalPrimary, + filepath.Join(archDir, "repodata", "*primary.xml.zst"), + filepath.Join(PathAl2023Primary, arch+".primary.xml.gz")) + common.LogInfo("Running installcheck for architecture:", arch, "Command:", installcheckCommand) + installcheckCmd := exec.Command("bash", "-c", installcheckCommand) + installcheckOutput, installcheckErr := installcheckCmd.CombinedOutput() + var review_msg string + if installcheckErr != nil { + common.LogError("Failed to run installcheck for '"+arch+"':", installcheckErr, "Output:\n", string(installcheckOutput)) + review_msg = "Failed to run installcheck for '" + arch + "'\n```" + string(installcheckOutput) + "```" + + } else { + common.LogInfo("Installcheck for '"+arch+"' completed successfully. Output:", string(installcheckOutput)) + review_msg = "Installcheck for architecture '" + arch + "' completed successfully.\n" + } + + _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, review_msg) + if err != nil { + common.LogError("Failed to add review comment:", err) + } else { + common.LogInfo("Review comment added successfully") + } + } + + } else { + common.LogInfo("No subproject changes found in PR.") + } + + return true, nil +} + +func main() { + flag.StringVar(&GiteaUrl, "gitea-url", "https://src.opensuse.org", "Gitea instance") + ProcessPROnly := flag.String("pr", "", "Process only specific PR and ignore the rest. Use for debugging") + obsApiHost := flag.String("obs", "https://api.opensuse.org", "API for OBS instance") + debug := flag.Bool("debug", false, "Turns on debug logging") + flag.StringVar(&PathSpalPrimary, "path-spal-primary", "", "Path to the SPAL primary.xml.zst file (e.g., /bs/spal/next/repodata/*primary.xml.zst)") + flag.StringVar(&PathAl2023Primary, "path-al2023-primary", "", "Path to the AL2023 primary.xml.zst file (e.g., /bs/spal/main.db/x86_64.primary.xml.gz)") + flag.Parse() + + if *debug { + common.SetLoggingLevel(common.LogLevelDebug) + } else { + common.SetLoggingLevel(common.LogLevelInfo) + } + + common.LogDebug("OBS Gitea Host:", GiteaUrl) + common.LogDebug("OBS API Host:", *obsApiHost) + + if PathSpalPrimary == "" { + common.LogError("Error: --path-spal-primary is required.") + return + } + if PathAl2023Primary == "" { + common.LogError("Error: --path-al2023-primary is required.") + return + } + common.PanicOnErrorWithMsg(common.RequireGiteaSecretToken(), "Cannot find GITEA_TOKEN") + common.PanicOnErrorWithMsg(common.RequireObsSecretToken(), "Cannot find OBS_USER and OBS_PASSWORD") + + var err error + if ObsClient, err = common.NewObsClient(*obsApiHost); err != nil { + log.Error(err) + return + } + + if len(*ProcessPROnly) > 0 { + rx := regexp.MustCompile("^([^/#]+)/([^/#]+)#([0-9]+)$") + m := rx.FindStringSubmatch(*ProcessPROnly) + if m == nil { + common.LogError("Cannot find any PR matches in", *ProcessPROnly) + return + } + + gitea := common.AllocateGiteaTransport(GiteaUrl) + id, _ := strconv.ParseInt(m[3], 10, 64) + + ProcessPullRequest(gitea, m[1], m[2], id) + return + } + + /*for { + PollWorkNotifications(GiteaUrl) + common.LogInfo("Poll cycle finished") + time.Sleep(5 * time.Minute) + }*/ +} diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index fd14c37..6177f42 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -388,10 +388,10 @@ func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, git templateMeta.Name = stagingProject + ":" + subProjectName // freeze tag for now if len(templateMeta.ScmSync) > 0 { - repository, err := url.Parse(templateMeta.ScmSync) - if err != nil { - panic(err) - } + repository, err := url.Parse(templateMeta.ScmSync) + if err != nil { + panic(err) + } common.LogDebug("getting data for ", repository.EscapedPath()) split := strings.Split(repository.EscapedPath(), "/") @@ -399,15 +399,15 @@ func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, git common.LogDebug("getting commit for ", org, " repo ", repo, " fragment ", repository.Fragment) branch, err := gitea.GetCommit(org, repo, repository.Fragment) - if err != nil { - panic(err) - } + if err != nil { + panic(err) + } // set expanded commit url - repository.Fragment = branch.SHA + repository.Fragment = branch.SHA templateMeta.ScmSync = repository.String() common.LogDebug("Setting scmsync url to ", templateMeta.ScmSync) - } + } // Cleanup ReleaseTarget and modify affected path entries for idx, r := range templateMeta.Repositories { templateMeta.Repositories[idx].ReleaseTargets = nil -- 2.51.1 From 33f4838ac99179e191842832af20da57113907a1144ad039b956a0b0f6c42f44 Mon Sep 17 00:00:00 2001 From: Antonello Tartamo Date: Wed, 12 Nov 2025 15:30:19 +0100 Subject: [PATCH 2/8] enabled notifications and keep info from staging.config --- common/config.go | 12 ++ obs-installcheck-bot/main.go | 299 +++++++++++++++++++++++++---------- 2 files changed, 224 insertions(+), 87 deletions(-) diff --git a/common/config.go b/common/config.go index d4a6331..ee13188 100644 --- a/common/config.go +++ b/common/config.go @@ -273,6 +273,15 @@ func (config *AutogitConfig) GetRemoteBranch() string { return "origin_" + config.Branch } +type RepositoryConfig struct { + Architectures []string +} + +type InstallcheckConfig struct { + Repos map[string][]string `json:"repos"` + NoCheckRepos map[string][]string `json:"nocheck_repos"` +} + type StagingConfig struct { ObsProject string RebuildAll bool @@ -281,6 +290,9 @@ type StagingConfig struct { // if set, then only use pull request numbers as unique identifiers StagingProject string QA []QAConfig + + Repositories map[string]RepositoryConfig `json:",omitempty"` + Installcheck InstallcheckConfig `json:",omitempty"` } func ParseStagingConfig(data []byte) (*StagingConfig, error) { diff --git a/obs-installcheck-bot/main.go b/obs-installcheck-bot/main.go index 1ccbef1..92b07af 100644 --- a/obs-installcheck-bot/main.go +++ b/obs-installcheck-bot/main.go @@ -5,6 +5,9 @@ import ( "fmt" "os/exec" "path/filepath" + "runtime/debug" + "strings" + "time" "os" "regexp" @@ -12,30 +15,136 @@ import ( "github.com/opentracing/opentracing-go/log" "src.opensuse.org/autogits/common" + "src.opensuse.org/autogits/common/gitea-generated/models" ) var GiteaUrl string var ObsClient *common.ObsClient var ProcessPROnly string - -const ObsProject = "Amazon:spal:2023" -const ObsRepository = "standard" - -var PathSpalPrimary string -var PathAl2023Primary string - -var ObsArch = []string{"x86_64", "aarch64"} +var IsDryRun bool +var ListPullNotificationsOnly bool const SkipSrcRpm = true +func ParseNotificationToPR(thread *models.NotificationThread) (org string, repo string, num int64, err error) { + rx := regexp.MustCompile(`^https://src\.(?:open)?suse\.(?:org|de)/api/v\d+/repos/(?[-_a-zA-Z0-9]+)/(?[-_a-zA-Z0-9]+)/issues/(?[0-9]+)$`) + notification := thread.Subject + match := rx.FindStringSubmatch(notification.URL) + if match == nil { + err = fmt.Errorf("Unexpected notification format: %s", notification.URL) + return + } + + org = match[1] + repo = match[2] + num, err = strconv.ParseInt(match[3], 10, 64) + return +} + +func ProcessPullNotification(gitea common.Gitea, thread *models.NotificationThread) { + defer func() { + err := recover() + if err != nil { + common.LogError(err) + common.LogError(string(debug.Stack())) + } + }() + + org, repo, num, err := ParseNotificationToPR(thread) + if err != nil { + common.LogError(err.Error()) + return + } + common.LogInfo("processing PR:", org, "/", repo, "#", num) + + done, err := ProcessPullRequest(gitea, org, repo, num) + if !IsDryRun && err == nil && done { + gitea.SetNotificationRead(thread.ID) + } else if err != nil { + common.LogError(err) + } +} + +func PollWorkNotifications(giteaUrl string) { + gitea := common.AllocateGiteaTransport(giteaUrl) + data, err := gitea.GetNotifications(common.GiteaNotificationType_Pull, nil) + + if err != nil { + common.LogError(err) + return + } + + if data != nil { + common.LogDebug("Processing", len(data), "notifications.") + for _, notification := range data { + common.LogInfo("notification", notification.ID, "--", notification.Subject.HTMLURL) + + if !ListPullNotificationsOnly { + switch notification.Subject.Type { + case "Pull": + ProcessPullNotification(gitea, notification) + default: + if !IsDryRun { + gitea.SetNotificationRead(notification.ID) + } + } + } + } + } +} + func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, error) { common.LogInfo("Processing PR", org+"/"+repo, "#"+strconv.FormatInt(id, 10)) - common.LogDebug("Fetching PR:", org, repo, id) pr, err := gitea.GetPullRequest(org, repo, id) if err != nil { common.LogError("No PR associated with review:", org, "/", repo, "#", id, "Error:", err) - return true, err + return false, err + } + + data, _, err := gitea.GetRepositoryFileContent(org, repo, pr.Head.Sha, common.StagingConfigFile) + if err != nil { + common.LogError("Cannot find 'staging.config' in PR#", pr.Index, ":", err) + _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.ProjectConfigFile) + if err != nil { + common.LogError("Failed to add review comment:", err) + } + } + + var config *common.StagingConfig + + config, err = common.ParseStagingConfig(data) + if err != nil { + common.LogDebug("Failed to parse staging config. Using defaults. Error:", err) + _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Failed to parse staging config.") + if err != nil { + common.LogError("Failed to add review comment:", err) + } + return false, err + } + + if config.StagingProject == "" || + config.Repositories == nil || len(config.Repositories) == 0 || + config.Installcheck.Repos == nil || len(config.Installcheck.Repos) == 0 { + common.LogError("Wrong staging config format in PR#", pr.Index) + _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Wrong staging config format.") + if err != nil { + common.LogError("Failed to add review comment:", err) + } + return false, err + } + + config.StagingProject = config.StagingProject + ":" + strconv.FormatInt(pr.Index, 10) + + common.LogInfo("Using staging project:", config.StagingProject) + for repoName, repoConfig := range config.Repositories { + common.LogInfo("Using OBS repository:", repoName, "architectures: ", repoConfig.Architectures) + } + for arch, repos := range config.Installcheck.Repos { + common.LogInfo("Using installcheck repos for architecture:", arch, ":", repos) + } + for arch, repos := range config.Installcheck.NoCheckRepos { + common.LogInfo("Using installcheck nocheck repos for architecture:", arch, ":", repos) } dir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("%s-%s-%d-", org, repo, id)) @@ -52,88 +161,112 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e if len(changedPackages) > 0 { common.LogInfo("Changed packages found in PR!") - for _, arch := range ObsArch { - archDir := filepath.Join(dir, arch) - err := os.MkdirAll(archDir, 0755) - common.PanicOnError(err) - common.LogInfo("Created temporary directory for", arch, "at", archDir) + for repoName, repoConfig := range config.Repositories { + for _, arch := range repoConfig.Architectures { + archDir := filepath.Join(dir, repoName, arch) + err := os.MkdirAll(archDir, 0755) + common.PanicOnError(err) + common.LogInfo("Created temporary directory for", repo, "arch", arch, "at", archDir) + } } for _, pkg := range changedPackages { common.LogInfo("*** " + pkg) - for _, arch := range ObsArch { - archDir := filepath.Join(dir, arch) + for repoName, repoConfig := range config.Repositories { + for _, arch := range repoConfig.Architectures { + archDir := filepath.Join(dir, repoName, arch) - var binaries *common.PackageBinariesList - binaries, err = ObsClient.GetBinariesFromPackage(ObsProject, ObsRepository, arch, pkg) - if err != nil { - common.LogError("Failed to get binaries:", err) - return false, err - } - - var filtered_binaries []string - - if binaries != nil { - common.LogInfo("Binaries for package", pkg, "and arch", arch+":") - for _, bin := range binaries.Binary { - common.LogInfo(" -", bin.Filename, "(size:", bin.Size, "mtime:", bin.Mtime, ")") - if SkipSrcRpm && regexp.MustCompile(`\.src\.rpm$`).MatchString(bin.Filename) { - common.LogInfo(" Skipping source RPM as requested.") - continue - } - filtered_binaries = append(filtered_binaries, bin.Filename) + var binaries *common.PackageBinariesList + binaries, err = ObsClient.GetBinariesFromPackage(config.StagingProject, repoName, arch, pkg) + if err != nil || binaries == nil { + common.LogError("Failed to get list of binaries:", err) + return false, err } - } - if len(filtered_binaries) > 0 { - for _, bin := range filtered_binaries { - common.LogInfo("Downloading binary:", bin) - binaryData, err := ObsClient.GetBinary(ObsProject, ObsRepository, arch, pkg, bin) - if err != nil { - common.LogError("Failed to download binary '"+bin+"':", err) - continue + var filtered_binaries []string + + if binaries != nil { + common.LogInfo("Binaries for package", pkg, "and arch", arch+":") + for _, bin := range binaries.Binary { + common.LogInfo(" -", bin.Filename, "(size:", bin.Size, "mtime:", bin.Mtime, ")") + if SkipSrcRpm && regexp.MustCompile(`\.src\.rpm$`).MatchString(bin.Filename) { + common.LogInfo(" Skipping source RPM as requested.") + continue + } + filtered_binaries = append(filtered_binaries, bin.Filename) } - if binaryData == nil { - common.LogInfo("Binary not found or is empty:", bin) - continue + } + + if len(filtered_binaries) > 0 { + for _, bin := range filtered_binaries { + common.LogInfo("Downloading binary:", bin) + binaryData, err := ObsClient.GetBinary(config.StagingProject, repoName, arch, pkg, bin) + if err != nil { + common.LogError("Failed to download binary '"+bin+"':", err) + return false, err + } + if binaryData == nil { + common.LogInfo("Binary not found or is empty:", bin) + return false, err + } + filePath := filepath.Join(archDir, bin) + err = os.WriteFile(filePath, binaryData, 0644) + common.PanicOnError(err) } - filePath := filepath.Join(archDir, bin) - err = os.WriteFile(filePath, binaryData, 0644) - common.PanicOnError(err) } } } } - for _, arch := range ObsArch { - archDir := filepath.Join(dir, arch) - // Call createrepo_c for the architecture directory - common.LogInfo("Creating repository for architecture:", arch, "in", archDir) - cmd := exec.Command("createrepo_c", archDir) - output, err := cmd.CombinedOutput() - if err != nil { - common.LogError("Failed to create repository for '"+arch+"':", err, "Output:", string(output)) + var review_msg string = "" + var anyInstallcheckFailed bool = false + + for repoName, repoConfig := range config.Repositories { + for _, arch := range repoConfig.Architectures { + archDir := filepath.Join(dir, repoName, arch) + // Call createrepo_c for the architecture directory + common.LogInfo("Creating repository for architecture:", arch, "in", archDir) + cmd := exec.Command("createrepo_c", archDir) + output, err := cmd.CombinedOutput() + if err != nil { + common.LogError("Failed to create repository for '"+arch+"' in repo '"+repoName+"':", err, "Output:", string(output)) + } + + common.LogInfo("Running installcheck for architecture:", arch) + + var installcheckArgs []string + installcheckArgs = append(installcheckArgs, arch) + if reposToInclude, ok := config.Installcheck.Repos[arch]; ok { + installcheckArgs = append(installcheckArgs, reposToInclude...) + } + installcheckArgs = append(installcheckArgs, filepath.Join(archDir, "repodata", "*primary.xml.zst")) + if reposToExclude, ok := config.Installcheck.NoCheckRepos[arch]; ok { + installcheckArgs = append(installcheckArgs, "--nocheck") + installcheckArgs = append(installcheckArgs, reposToExclude...) + } + installcheckCommand := "installcheck " + strings.Join(installcheckArgs, " ") + common.LogInfo("Running installcheck for repository "+repoName+" and architecture:", arch, "Command:", installcheckCommand) + installcheckCmd := exec.Command("bash", "-c", installcheckCommand) + installcheckOutput, installcheckErr := installcheckCmd.CombinedOutput() + if installcheckErr != nil { + common.LogError("Failed to run installcheck for repository '"+repoName+"' and architecture '"+arch+"':", installcheckErr, "Output:\n", string(installcheckOutput)) + anyInstallcheckFailed = true + review_msg += "Failed to run installcheck for repository '" + repoName + "' and architecture '" + arch + "'\n```" + string(installcheckOutput) + "```\n\n" + + } else { + common.LogInfo("Installcheck for repository '"+repoName+"' and architecture '"+arch+"' completed successfully. Output:", string(installcheckOutput)) + review_msg += "Installcheck for repository '" + repoName + "' and architecture '" + arch + "' completed successfully.\n\n" + } } + } - common.LogInfo("Running installcheck for architecture:", arch) - installcheckCommand := fmt.Sprintf("installcheck %s %s %s --nocheck %s", arch, PathSpalPrimary, - filepath.Join(archDir, "repodata", "*primary.xml.zst"), - filepath.Join(PathAl2023Primary, arch+".primary.xml.gz")) - common.LogInfo("Running installcheck for architecture:", arch, "Command:", installcheckCommand) - installcheckCmd := exec.Command("bash", "-c", installcheckCommand) - installcheckOutput, installcheckErr := installcheckCmd.CombinedOutput() - var review_msg string - if installcheckErr != nil { - common.LogError("Failed to run installcheck for '"+arch+"':", installcheckErr, "Output:\n", string(installcheckOutput)) - review_msg = "Failed to run installcheck for '" + arch + "'\n```" + string(installcheckOutput) + "```" - + if !IsDryRun { + if anyInstallcheckFailed { + _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, review_msg) } else { - common.LogInfo("Installcheck for '"+arch+"' completed successfully. Output:", string(installcheckOutput)) - review_msg = "Installcheck for architecture '" + arch + "' completed successfully.\n" + _, err = gitea.AddReviewComment(pr, common.ReviewStateApproved, review_msg) } - - _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, review_msg) if err != nil { common.LogError("Failed to add review comment:", err) } else { @@ -149,12 +282,12 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e } func main() { + flag.BoolVar(&ListPullNotificationsOnly, "list-notifications-only", false, "Only lists notifications without acting on them") flag.StringVar(&GiteaUrl, "gitea-url", "https://src.opensuse.org", "Gitea instance") ProcessPROnly := flag.String("pr", "", "Process only specific PR and ignore the rest. Use for debugging") - obsApiHost := flag.String("obs", "https://api.opensuse.org", "API for OBS instance") - debug := flag.Bool("debug", false, "Turns on debug logging") - flag.StringVar(&PathSpalPrimary, "path-spal-primary", "", "Path to the SPAL primary.xml.zst file (e.g., /bs/spal/next/repodata/*primary.xml.zst)") - flag.StringVar(&PathAl2023Primary, "path-al2023-primary", "", "Path to the AL2023 primary.xml.zst file (e.g., /bs/spal/main.db/x86_64.primary.xml.gz)") + obsApiHost := flag.String("obs", "https://api.opensuse.org", "API for OBS instance") // Keep this as it's for the OBS client itself + debug := flag.Bool("debug", false, "Turns on debug logging") // Keep this for logging control + flag.BoolVar(&IsDryRun, "dry", false, "Dry-run, don't actually post reviews") flag.Parse() if *debug { @@ -166,14 +299,6 @@ func main() { common.LogDebug("OBS Gitea Host:", GiteaUrl) common.LogDebug("OBS API Host:", *obsApiHost) - if PathSpalPrimary == "" { - common.LogError("Error: --path-spal-primary is required.") - return - } - if PathAl2023Primary == "" { - common.LogError("Error: --path-al2023-primary is required.") - return - } common.PanicOnErrorWithMsg(common.RequireGiteaSecretToken(), "Cannot find GITEA_TOKEN") common.PanicOnErrorWithMsg(common.RequireObsSecretToken(), "Cannot find OBS_USER and OBS_PASSWORD") @@ -198,9 +323,9 @@ func main() { return } - /*for { + for { PollWorkNotifications(GiteaUrl) common.LogInfo("Poll cycle finished") time.Sleep(5 * time.Minute) - }*/ + } } -- 2.51.1 From 5d27c8d1e7e1191cc7d42d8993bdf3fd4e11e461439f248481bc912149168158 Mon Sep 17 00:00:00 2001 From: Antonello Tartamo Date: Wed, 12 Nov 2025 15:32:48 +0100 Subject: [PATCH 3/8] reverted unwanted changes on staging bot --- obs-staging-bot/main.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/obs-staging-bot/main.go b/obs-staging-bot/main.go index 6177f42..fd14c37 100644 --- a/obs-staging-bot/main.go +++ b/obs-staging-bot/main.go @@ -388,10 +388,10 @@ func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, git templateMeta.Name = stagingProject + ":" + subProjectName // freeze tag for now if len(templateMeta.ScmSync) > 0 { - repository, err := url.Parse(templateMeta.ScmSync) - if err != nil { - panic(err) - } + repository, err := url.Parse(templateMeta.ScmSync) + if err != nil { + panic(err) + } common.LogDebug("getting data for ", repository.EscapedPath()) split := strings.Split(repository.EscapedPath(), "/") @@ -399,15 +399,15 @@ func CreateQASubProject(stagingConfig *common.StagingConfig, git common.Git, git common.LogDebug("getting commit for ", org, " repo ", repo, " fragment ", repository.Fragment) branch, err := gitea.GetCommit(org, repo, repository.Fragment) - if err != nil { - panic(err) - } + if err != nil { + panic(err) + } // set expanded commit url - repository.Fragment = branch.SHA + repository.Fragment = branch.SHA templateMeta.ScmSync = repository.String() common.LogDebug("Setting scmsync url to ", templateMeta.ScmSync) - } + } // Cleanup ReleaseTarget and modify affected path entries for idx, r := range templateMeta.Repositories { templateMeta.Repositories[idx].ReleaseTargets = nil -- 2.51.1 From 71d409b7222d8c102bb4cfc77b88755d8fd126c2255f4bfbfd376db82d916a91 Mon Sep 17 00:00:00 2001 From: Antonello Tartamo Date: Wed, 12 Nov 2025 17:13:57 +0100 Subject: [PATCH 4/8] avoid spamming comments and defer remove directory --- obs-installcheck-bot/main.go | 39 ++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/obs-installcheck-bot/main.go b/obs-installcheck-bot/main.go index 92b07af..8b56be3 100644 --- a/obs-installcheck-bot/main.go +++ b/obs-installcheck-bot/main.go @@ -93,6 +93,28 @@ func PollWorkNotifications(giteaUrl string) { } } +func addUniqueReviewComment(gitea common.Gitea, pr *models.PullRequest, state models.ReviewStateType, comment string) error { + reviews, err := gitea.GetPullRequestReviews(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Index) + if err != nil { + return fmt.Errorf("failed to get pull request reviews: %w", err) + } + + botUser, err := gitea.GetCurrentUser() + if err != nil { + return fmt.Errorf("failed to get current user: %w", err) + } + + for _, review := range reviews { + if review.User != nil && review.User.UserName == botUser.UserName && review.Body == comment { + common.LogInfo("Review comment already exists, skipping.") + return nil + } + } + + _, err = gitea.AddReviewComment(pr, state, comment) + return err +} + func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, error) { common.LogInfo("Processing PR", org+"/"+repo, "#"+strconv.FormatInt(id, 10)) @@ -105,7 +127,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e data, _, err := gitea.GetRepositoryFileContent(org, repo, pr.Head.Sha, common.StagingConfigFile) if err != nil { common.LogError("Cannot find 'staging.config' in PR#", pr.Index, ":", err) - _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.ProjectConfigFile) + err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.StagingConfigFile) if err != nil { common.LogError("Failed to add review comment:", err) } @@ -116,7 +138,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e config, err = common.ParseStagingConfig(data) if err != nil { common.LogDebug("Failed to parse staging config. Using defaults. Error:", err) - _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Failed to parse staging config.") + err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Failed to parse staging config.") if err != nil { common.LogError("Failed to add review comment:", err) } @@ -127,7 +149,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e config.Repositories == nil || len(config.Repositories) == 0 || config.Installcheck.Repos == nil || len(config.Installcheck.Repos) == 0 { common.LogError("Wrong staging config format in PR#", pr.Index) - _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, "Wrong staging config format.") + err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Wrong staging config format.") if err != nil { common.LogError("Failed to add review comment:", err) } @@ -149,6 +171,11 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e dir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("%s-%s-%d-", org, repo, id)) common.PanicOnError(err) + if IsDryRun { + common.LogInfo("will keep temp directory:", dir) + } else { + defer os.RemoveAll(dir) + } diff, err := gitea.GetPullRequestDiff(org, repo, id) common.LogDebug("PR Diff:\n", diff) @@ -240,7 +267,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e if reposToInclude, ok := config.Installcheck.Repos[arch]; ok { installcheckArgs = append(installcheckArgs, reposToInclude...) } - installcheckArgs = append(installcheckArgs, filepath.Join(archDir, "repodata", "*primary.xml.zst")) + installcheckArgs = append(installcheckArgs, filepath.Join(archDir, "repodata", "*primary.xml.*")) if reposToExclude, ok := config.Installcheck.NoCheckRepos[arch]; ok { installcheckArgs = append(installcheckArgs, "--nocheck") installcheckArgs = append(installcheckArgs, reposToExclude...) @@ -263,9 +290,9 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e if !IsDryRun { if anyInstallcheckFailed { - _, err = gitea.AddReviewComment(pr, common.ReviewStateRequestChanges, review_msg) + err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, review_msg) } else { - _, err = gitea.AddReviewComment(pr, common.ReviewStateApproved, review_msg) + err = addUniqueReviewComment(gitea, pr, common.ReviewStateApproved, review_msg) } if err != nil { common.LogError("Failed to add review comment:", err) -- 2.51.1 From 3450505cc51ce161be52eb49091ef47999ec22528cd167533c81a761b9ba4f79 Mon Sep 17 00:00:00 2001 From: Antonello Tartamo Date: Thu, 13 Nov 2025 10:29:45 +0100 Subject: [PATCH 5/8] added waitforapprover feature --- obs-installcheck-bot/main.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/obs-installcheck-bot/main.go b/obs-installcheck-bot/main.go index 8b56be3..dc0e99e 100644 --- a/obs-installcheck-bot/main.go +++ b/obs-installcheck-bot/main.go @@ -22,6 +22,7 @@ var GiteaUrl string var ObsClient *common.ObsClient var ProcessPROnly string var IsDryRun bool +var WaitForApprover string var ListPullNotificationsOnly bool const SkipSrcRpm = true @@ -124,6 +125,27 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e return false, err } + if WaitForApprover != "" { + common.LogInfo("Waiting for approver:", WaitForApprover) + reviews, err := gitea.GetPullRequestReviews(org, repo, id) + if err != nil { + return false, fmt.Errorf("failed to get pull request reviews: %w", err) + } + + approved := false + for _, review := range reviews { + if review.User != nil && review.User.UserName == WaitForApprover && review.State == common.ReviewStateApproved { + approved = true + break + } + } + + if !approved { + common.LogInfo("PR not yet approved by", WaitForApprover, ". Skipping for now.") + return false, nil // Not an error, just not ready to process. + } + } + data, _, err := gitea.GetRepositoryFileContent(org, repo, pr.Head.Sha, common.StagingConfigFile) if err != nil { common.LogError("Cannot find 'staging.config' in PR#", pr.Index, ":", err) @@ -315,6 +337,7 @@ func main() { obsApiHost := flag.String("obs", "https://api.opensuse.org", "API for OBS instance") // Keep this as it's for the OBS client itself debug := flag.Bool("debug", false, "Turns on debug logging") // Keep this for logging control flag.BoolVar(&IsDryRun, "dry", false, "Dry-run, don't actually post reviews") + flag.StringVar(&WaitForApprover, "wait-for-approver", "", "Username of the user whose approval is required before processing the PR") flag.Parse() if *debug { -- 2.51.1 From c7da5bc39b9a33aa8235db551032cbe2479f4949e666c1db552d53cd65160064 Mon Sep 17 00:00:00 2001 From: Antonello Tartamo Date: Thu, 13 Nov 2025 18:27:24 +0100 Subject: [PATCH 6/8] optimized version: just downloading the rpm headers --- common/cpio_utils.go | 147 +++++++++++++++++++++++++++++++++++ common/obs_utils.go | 80 +++++++++++++++++++ obs-installcheck-bot/main.go | 68 ++++++++-------- 3 files changed, 259 insertions(+), 36 deletions(-) create mode 100644 common/cpio_utils.go diff --git a/common/cpio_utils.go b/common/cpio_utils.go new file mode 100644 index 0000000..4433bf5 --- /dev/null +++ b/common/cpio_utils.go @@ -0,0 +1,147 @@ +package common + +/* + * This file is part of Autogits. + * + * Copyright © 2024 SUSE LLC + * + * Autogits is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 2 of the License, or (at your option) any later + * version. + * + * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * Foobar. If not, see . + */ + +import ( + "encoding/binary" + "fmt" + "io" + "regexp" + "strconv" + "strings" +) + +// CpioHeader represents the CPIO "newc" format header. +// The fields are stored as 8-char hex strings. +type CpioHeader struct { + Magic [6]byte + Ino [8]byte + Mode [8]byte + UID [8]byte + GID [8]byte + NLink [8]byte + MTime [8]byte + FileSize [8]byte + DevMajor [8]byte + DevMinor [8]byte + RDevMajor [8]byte + RDevMinor [8]byte + NameSize [8]byte + Check [8]byte +} + +// cpioMagic is the magic number for the "newc" format. +var cpioMagic = []byte("070701") + +// cpioNameRe is a regex to parse filenames inside the CPIO archive. +var cpioNameRe = regexp.MustCompile(`^([^/]+)-([0-9a-f]{32})$`) + +// parseHex parses an 8-char hex string from a byte slice into an int64. +func parseHex(hexBytes []byte) (int64, error) { + return strconv.ParseInt(string(hexBytes), 16, 64) +} + +// ExtractCPIOStream reads a CPIO stream and extracts files to destDir. +func ExtractCPIOStream(stream io.Reader) ([]byte, error) { + for { + var fileContent []byte + var hdr CpioHeader + if err := binary.Read(stream, binary.LittleEndian, &hdr); err != nil { + if err == io.EOF { + return nil, fmt.Errorf("unexpected EOF while reading CPIO header") + } + return nil, fmt.Errorf("failed to read CPIO header: %w", err) + } + + if string(hdr.Magic[:]) != string(cpioMagic) { + return nil, fmt.Errorf("CPIO format magic %s not implemented", string(hdr.Magic[:])) + } + + nameSize, err := parseHex(hdr.NameSize[:]) + if err != nil { + return nil, fmt.Errorf("failed to parse name size: %w", err) + } + + fileSize, err := parseHex(hdr.FileSize[:]) + if err != nil { + return nil, fmt.Errorf("failed to parse file size: %w", err) + } + + // Read filename, which includes a null terminator. + nameBuf := make([]byte, nameSize) + if _, err := io.ReadFull(stream, nameBuf); err != nil { // The filename is null-terminated. + return nil, fmt.Errorf("failed to read filename: %w", err) + } + // The filename is null-terminated. + filename := strings.TrimRight(string(nameBuf), "\x00") + + // The new-ascii format has padding for 4-byte alignment after the header and filename. + // The total size of header + filename is len(hdr) + nameSize. + headerAndNameSize := binary.Size(hdr) + int(nameSize) + if padding := (4 - (headerAndNameSize % 4)) % 4; padding > 0 { + if _, err := io.CopyN(io.Discard, stream, int64(padding)); err != nil { // End of the CPIO archive. + return nil, fmt.Errorf("failed to read header padding: %w", err) + } + } + + if filename == "TRAILER!!!" { + // End of the CPIO archive. + break + } + + if filename == ".errors" { + content := make([]byte, fileSize) + if _, err := io.ReadFull(stream, content); err != nil { // Create a temporary file to download into. + return nil, fmt.Errorf("failed to read .errors content: %w", err) + } + return nil, fmt.Errorf("download has errors: %s", string(content)) + } + + if match := cpioNameRe.FindStringSubmatch(filename); match != nil { + fileContent = make([]byte, fileSize) + if _, err := io.ReadFull(stream, fileContent); err != nil { + return nil, fmt.Errorf("failed to read file content: %w", err) + } + + } else { + // For any other file, just discard its content. + LogInfo("Unhandled file %s in archive, discarding.", filename) + if _, err := io.CopyN(io.Discard, stream, fileSize); err != nil { + return nil, fmt.Errorf("failed to discard unhandled file content: %w", err) + } + } + + // The file data is also padded to a 4-byte boundary. + if padding := (4 - (fileSize % 4)) % 4; padding > 0 { + if _, err := io.CopyN(io.Discard, stream, int64(padding)); err != nil { + return nil, fmt.Errorf("failed to read file data padding: %w", err) + } + } + return fileContent, nil + } + + // Check for any trailing data. + if n, err := io.Copy(io.Discard, stream); err != nil && err != io.EOF { + return nil, fmt.Errorf("error reading trailing data: %w", err) + } else if n > 0 { + LogInfo("Warning: %d bytes of unexpected trailing data in CPIO stream", n) + } + + return nil, nil +} diff --git a/common/obs_utils.go b/common/obs_utils.go index 319d536..3351ed4 100644 --- a/common/obs_utils.go +++ b/common/obs_utils.go @@ -635,6 +635,19 @@ type PackageBinariesList struct { Binary []Binary `xml:"binary"` } +// BinaryVersion represents a binary in a binaryversionlist response. +type BinaryVersion struct { + Name string `xml:"name,attr"` + SizeK string `xml:"sizek,attr"` + HdrMD5 string `xml:"hdrmd5,attr"` +} + +// BinaryVersionList represents a list of binaries from a binaryversionlist response. +type BinaryVersionList struct { + XMLName xml.Name `xml:"binaryversionlist"` + Binary []BinaryVersion `xml:"binary"` +} + type BinaryList struct { Package string `xml:"package,attr"` Binary []Binary `xml:"binary"` @@ -1009,3 +1022,70 @@ func (c *ObsClient) GetBinary(project, repo, arch, pkg, filename string) ([]byte return data, nil } + +func parsePackageBinarieVersions(data []byte) (*BinaryVersionList, error) { + result := BinaryVersionList{} + err := xml.Unmarshal(data, &result) + if err != nil { + return nil, err + } + + return &result, nil +} + +func (c *ObsClient) GetBinariesFromRepo(project, repo, arch string) (*BinaryVersionList, error) { + u := c.baseUrl.JoinPath("build", project, repo, arch, "_repository") + query := u.Query() + query.Add("view", "binaryversions") + query.Add("nometa", "1") + u.RawQuery = query.Encode() + res, err := c.ObsRequestRaw("GET", u.String(), nil) + if err != nil { + return nil, err + } + + switch res.StatusCode { + case 200: + break + case 404: + return nil, ObsProjectNotFound{project} + default: + return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode) + } + + data, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + defer res.Body.Close() + + result, err := parsePackageBinarieVersions(data) + if err != nil { + return nil, err + } + + return result, nil +} + +func (c *ObsClient) GetBinaryHeader(project, repo, arch, filename string) ([]byte, error) { + u := c.baseUrl.JoinPath("build", project, repo, arch, "_repository") + query := u.Query() + query.Add("view", "cpioheaders") + query.Add("binary", strings.TrimSuffix(filename, ".rpm")) + u.RawQuery = query.Encode() + res, err := c.ObsRequestRaw("GET", u.String(), nil) + if err != nil { + return nil, err + } + + switch res.StatusCode { + case 200: + break + case 404: + return nil, ObsProjectNotFound{project} + default: + return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode) + } + + return ExtractCPIOStream(res.Body) +} diff --git a/obs-installcheck-bot/main.go b/obs-installcheck-bot/main.go index dc0e99e..ecbe5d9 100644 --- a/obs-installcheck-bot/main.go +++ b/obs-installcheck-bot/main.go @@ -219,50 +219,46 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e } } - for _, pkg := range changedPackages { - common.LogInfo("*** " + pkg) + var nameIgnoreRe = regexp.MustCompile(`-debug(info|source|info-32bit)\.rpm$`) - for repoName, repoConfig := range config.Repositories { - for _, arch := range repoConfig.Architectures { - archDir := filepath.Join(dir, repoName, arch) + for repoName, repoConfig := range config.Repositories { + for _, arch := range repoConfig.Architectures { + archDir := filepath.Join(dir, repoName, arch) - var binaries *common.PackageBinariesList - binaries, err = ObsClient.GetBinariesFromPackage(config.StagingProject, repoName, arch, pkg) - if err != nil || binaries == nil { - common.LogError("Failed to get list of binaries:", err) - return false, err - } + var binaries *common.BinaryVersionList + binaries, err = ObsClient.GetBinariesFromRepo(config.StagingProject, repoName, arch) + if err != nil || binaries == nil { + common.LogError("Failed to get list of binaries:", err) + return false, err + } - var filtered_binaries []string + var filtered_binaries []string - if binaries != nil { - common.LogInfo("Binaries for package", pkg, "and arch", arch+":") - for _, bin := range binaries.Binary { - common.LogInfo(" -", bin.Filename, "(size:", bin.Size, "mtime:", bin.Mtime, ")") - if SkipSrcRpm && regexp.MustCompile(`\.src\.rpm$`).MatchString(bin.Filename) { - common.LogInfo(" Skipping source RPM as requested.") - continue - } - filtered_binaries = append(filtered_binaries, bin.Filename) + if binaries != nil { + common.LogInfo("Binaries for repo ", repoName, " and arch", arch+":") + for _, bin := range binaries.Binary { + common.LogInfo(" -", bin.Name, "(md5:", bin.HdrMD5, "size:", bin.SizeK, ")") + if strings.HasSuffix(bin.Name, ".rpm") && !nameIgnoreRe.MatchString(bin.Name) { + filtered_binaries = append(filtered_binaries, bin.Name) } } + } - if len(filtered_binaries) > 0 { - for _, bin := range filtered_binaries { - common.LogInfo("Downloading binary:", bin) - binaryData, err := ObsClient.GetBinary(config.StagingProject, repoName, arch, pkg, bin) - if err != nil { - common.LogError("Failed to download binary '"+bin+"':", err) - return false, err - } - if binaryData == nil { - common.LogInfo("Binary not found or is empty:", bin) - return false, err - } - filePath := filepath.Join(archDir, bin) - err = os.WriteFile(filePath, binaryData, 0644) - common.PanicOnError(err) + if len(filtered_binaries) > 0 { + for _, bin := range filtered_binaries { + common.LogInfo("Downloading binary header:", bin) + binaryData, err := ObsClient.GetBinaryHeader(config.StagingProject, repoName, arch, bin) + if err != nil { + common.LogError("Failed to download binary '"+bin+"':", err) + return false, err } + if binaryData == nil { + common.LogInfo("Binary not found or is empty:", bin) + return false, err + } + filePath := filepath.Join(archDir, bin) + err = os.WriteFile(filePath, binaryData, 0644) + common.PanicOnError(err) } } } -- 2.51.1 From 79a2067a9bc123756c9a4623cdc7fbf290f0f5e357c9d626a72a8589b2806535 Mon Sep 17 00:00:00 2001 From: Antonello Tartamo Date: Wed, 19 Nov 2025 13:50:22 +0100 Subject: [PATCH 7/8] avoid bot keep re-running in case of errors --- obs-installcheck-bot/main.go | 98 +++++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 17 deletions(-) diff --git a/obs-installcheck-bot/main.go b/obs-installcheck-bot/main.go index ecbe5d9..3620f98 100644 --- a/obs-installcheck-bot/main.go +++ b/obs-installcheck-bot/main.go @@ -56,11 +56,15 @@ func ProcessPullNotification(gitea common.Gitea, thread *models.NotificationThre common.LogError(err.Error()) return } - common.LogInfo("processing PR:", org, "/", repo, "#", num) done, err := ProcessPullRequest(gitea, org, repo, num) - if !IsDryRun && err == nil && done { - gitea.SetNotificationRead(thread.ID) + if !IsDryRun && done { + err = gitea.SetNotificationRead(thread.ID) + if err != nil { + common.LogError(err) + } else { + common.LogInfo("Notification marked as read") + } } else if err != nil { common.LogError(err) } @@ -117,19 +121,29 @@ func addUniqueReviewComment(gitea common.Gitea, pr *models.PullRequest, state mo } func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, error) { - common.LogInfo("Processing PR", org+"/"+repo, "#"+strconv.FormatInt(id, 10)) + var done bool = false + + common.LogInfo("Processing PR", org+"/"+repo+"#"+strconv.FormatInt(id, 10)) pr, err := gitea.GetPullRequest(org, repo, id) if err != nil { common.LogError("No PR associated with review:", org, "/", repo, "#", id, "Error:", err) - return false, err + done = false + return done, err + } + + if pr.State != "open" { + common.LogInfo("PR is not open, skipping.") + done = true + return done, nil } if WaitForApprover != "" { common.LogInfo("Waiting for approver:", WaitForApprover) reviews, err := gitea.GetPullRequestReviews(org, repo, id) if err != nil { - return false, fmt.Errorf("failed to get pull request reviews: %w", err) + done = false + return done, fmt.Errorf("failed to get pull request reviews: %w", err) } approved := false @@ -140,9 +154,12 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e } } - if !approved { - common.LogInfo("PR not yet approved by", WaitForApprover, ". Skipping for now.") - return false, nil // Not an error, just not ready to process. + if approved { + common.LogInfo("PR approved by", WaitForApprover) + } else { + common.LogInfo("PR not yet approved by", WaitForApprover+". Skipping for now.") + done = false + return done, nil // Not an error, just not ready to process. } } @@ -153,18 +170,21 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e if err != nil { common.LogError("Failed to add review comment:", err) } + done = true + return done, err } var config *common.StagingConfig config, err = common.ParseStagingConfig(data) if err != nil { - common.LogDebug("Failed to parse staging config. Using defaults. Error:", err) + common.LogDebug("Failed to parse staging config. Error:", err) err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Failed to parse staging config.") if err != nil { common.LogError("Failed to add review comment:", err) } - return false, err + done = true + return done, fmt.Errorf("failed to parse staging config: %w", err) } if config.StagingProject == "" || @@ -175,7 +195,8 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e if err != nil { common.LogError("Failed to add review comment:", err) } - return false, err + done = true + return done, fmt.Errorf("wrong staging config format in PR: %w", err) } config.StagingProject = config.StagingProject + ":" + strconv.FormatInt(pr.Index, 10) @@ -203,7 +224,8 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e common.LogDebug("PR Diff:\n", diff) if err != nil { common.LogError("Failed to get PR diff:", err) - return false, err + done = false + return done, err } changedPackages := common.ParseSubprojectChangesFromDiff(diff) @@ -229,7 +251,12 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e binaries, err = ObsClient.GetBinariesFromRepo(config.StagingProject, repoName, arch) if err != nil || binaries == nil { common.LogError("Failed to get list of binaries:", err) - return false, err + err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Failed to get list of binaries for repo '"+repoName+"' arch '"+arch+"': "+err.Error()) + if err != nil { + common.LogError("Failed to add review comment:", err) + } + done = true + return done, err } var filtered_binaries []string @@ -245,16 +272,46 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e } if len(filtered_binaries) > 0 { + // Add check if all changed packages are present in the binaries + for _, changedPkg := range changedPackages { + found := false + for _, bin := range filtered_binaries { + if strings.HasPrefix(bin, changedPkg+"-") { + common.LogDebug("Changed package", changedPkg, "found in binaries for repo", repoName, "arch", arch) + found = true + break + } + } + if !found { + common.LogError("Changed package", changedPkg, "NOT found in binaries for repo", repoName, "arch", arch) + // staging bot can approve the request even if some packages are missing + done = false + return done, fmt.Errorf("changed package %s not found in binaries for repo %s arch %s", changedPkg, repoName, arch) + } + } + + // Download the filtered binaries + common.LogInfo("Downloading", len(filtered_binaries), "binaries for repo", repoName, "arch", arch) for _, bin := range filtered_binaries { common.LogInfo("Downloading binary header:", bin) binaryData, err := ObsClient.GetBinaryHeader(config.StagingProject, repoName, arch, bin) if err != nil { common.LogError("Failed to download binary '"+bin+"':", err) - return false, err + err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Failed to download binary '"+bin+"': "+err.Error()) + if err != nil { + common.LogError("Failed to add review comment:", err) + } + done = true + return done, err } if binaryData == nil { common.LogInfo("Binary not found or is empty:", bin) - return false, err + err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Binary not found or is empty: '"+bin+"'") + if err != nil { + common.LogError("Failed to add review comment:", err) + } + done = true + return done, err } filePath := filepath.Join(archDir, bin) err = os.WriteFile(filePath, binaryData, 0644) @@ -317,13 +374,20 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e } else { common.LogInfo("Review comment added successfully") } + done = true } } else { + done = true common.LogInfo("No subproject changes found in PR.") + + err = addUniqueReviewComment(gitea, pr, common.ReviewStateApproved, "No packages changed in the PR, nothing to check.") + if err != nil { + common.LogError("Failed to add review comment:", err) + } } - return true, nil + return done, nil } func main() { -- 2.51.1 From 2053454ab950fe0507a5d51765013edb99f3dc1383ed2ff6b0d5198aa5c33fe5 Mon Sep 17 00:00:00 2001 From: Antonello Tartamo Date: Wed, 26 Nov 2025 14:12:44 +0100 Subject: [PATCH 8/8] added some fixes --- common/diff_parser.go | 3 ++- obs-installcheck-bot/main.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/common/diff_parser.go b/common/diff_parser.go index 2e69030..8795cc1 100644 --- a/common/diff_parser.go +++ b/common/diff_parser.go @@ -23,7 +23,8 @@ func ParseSubprojectChangesFromDiff(diff string) []string { // We use path.Base to get just the package name from the path (e.g., "rpms/iptraf-ng" -> "iptraf-ng"). pkgPath := strings.TrimSpace(match[1]) basePath := path.Base(pkgPath) - if basePath != ".gitmodules" && basePath != "" { + // TODO: parse _manifest files to get real package names + if basePath != ".gitmodules" && basePath != "_config" && basePath != "" { changedPackages = append(changedPackages, basePath) } } diff --git a/obs-installcheck-bot/main.go b/obs-installcheck-bot/main.go index 3620f98..909053b 100644 --- a/obs-installcheck-bot/main.go +++ b/obs-installcheck-bot/main.go @@ -110,7 +110,7 @@ func addUniqueReviewComment(gitea common.Gitea, pr *models.PullRequest, state mo } for _, review := range reviews { - if review.User != nil && review.User.UserName == botUser.UserName && review.Body == comment { + if review.User != nil && review.User.UserName == botUser.UserName && review.Body == comment && review.State == common.ReviewStatePending { common.LogInfo("Review comment already exists, skipping.") return nil } @@ -276,7 +276,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e for _, changedPkg := range changedPackages { found := false for _, bin := range filtered_binaries { - if strings.HasPrefix(bin, changedPkg+"-") { + if strings.HasPrefix(bin, changedPkg+"-") || bin == changedPkg+".rpm" { common.LogDebug("Changed package", changedPkg, "found in binaries for repo", repoName, "arch", arch) found = true break @@ -286,7 +286,7 @@ func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, e common.LogError("Changed package", changedPkg, "NOT found in binaries for repo", repoName, "arch", arch) // staging bot can approve the request even if some packages are missing done = false - return done, fmt.Errorf("changed package %s not found in binaries for repo %s arch %s", changedPkg, repoName, arch) + return done, nil } } -- 2.51.1