WIP: feature/installcheck #99
@@ -273,6 +273,15 @@ func (config *AutogitConfig) GetRemoteBranch() string {
|
||||
return "origin_" + config.Branch
|
||||
}
|
||||
|
||||
type RepositoryConfig struct {
|
||||
Architectures []string
|
||||
}
|
||||
|
||||
type InstallcheckConfig struct {
|
||||
Repos map[string][]string `json:"repos"`
|
||||
|
|
||||
NoCheckRepos map[string][]string `json:"nocheck_repos"`
|
||||
}
|
||||
|
||||
type StagingConfig struct {
|
||||
ObsProject string
|
||||
RebuildAll bool
|
||||
@@ -281,6 +290,9 @@ type StagingConfig struct {
|
||||
// if set, then only use pull request numbers as unique identifiers
|
||||
StagingProject string
|
||||
QA []QAConfig
|
||||
|
||||
Repositories map[string]RepositoryConfig `json:",omitempty"`
|
||||
Installcheck InstallcheckConfig `json:",omitempty"`
|
||||
|
adamm
commented
pointer here, *InstallcheckConfig, would be omitted if empty. map[string]*RepositoryConfig pointer here, *InstallcheckConfig, would be omitted if empty.
map[string]*RepositoryConfig
|
||||
}
|
||||
|
||||
func ParseStagingConfig(data []byte) (*StagingConfig, error) {
|
||||
|
||||
147
common/cpio_utils.go
Normal file
@@ -0,0 +1,147 @@
|
||||
package common
|
||||
|
||||
/*
|
||||
* This file is part of Autogits.
|
||||
*
|
||||
* Copyright © 2024 SUSE LLC
|
||||
*
|
||||
* Autogits is free software: you can redistribute it and/or modify it under
|
||||
* the terms of the GNU General Public License as published by the Free Software
|
||||
* Foundation, either version 2 of the License, or (at your option) any later
|
||||
* version.
|
||||
*
|
||||
* Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
|
||||
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* Foobar. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
adamm
commented
Foobar? :-) Foobar? :-)
|
||||
*/
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CpioHeader represents the CPIO "newc" format header.
|
||||
// The fields are stored as 8-char hex strings.
|
||||
type CpioHeader struct {
|
||||
Magic [6]byte
|
||||
Ino [8]byte
|
||||
Mode [8]byte
|
||||
UID [8]byte
|
||||
GID [8]byte
|
||||
NLink [8]byte
|
||||
MTime [8]byte
|
||||
FileSize [8]byte
|
||||
DevMajor [8]byte
|
||||
DevMinor [8]byte
|
||||
RDevMajor [8]byte
|
||||
RDevMinor [8]byte
|
||||
NameSize [8]byte
|
||||
Check [8]byte
|
||||
}
|
||||
|
||||
// cpioMagic is the magic number for the "newc" format.
|
||||
var cpioMagic = []byte("070701")
|
||||
|
||||
// cpioNameRe is a regex to parse filenames inside the CPIO archive.
|
||||
var cpioNameRe = regexp.MustCompile(`^([^/]+)-([0-9a-f]{32})$`)
|
||||
|
||||
// parseHex parses an 8-char hex string from a byte slice into an int64.
|
||||
func parseHex(hexBytes []byte) (int64, error) {
|
||||
return strconv.ParseInt(string(hexBytes), 16, 64)
|
||||
}
|
||||
|
||||
// ExtractCPIOStream reads a CPIO stream and extracts files to destDir.
|
||||
func ExtractCPIOStream(stream io.Reader) ([]byte, error) {
|
||||
for {
|
||||
var fileContent []byte
|
||||
var hdr CpioHeader
|
||||
if err := binary.Read(stream, binary.LittleEndian, &hdr); err != nil {
|
||||
if err == io.EOF {
|
||||
return nil, fmt.Errorf("unexpected EOF while reading CPIO header")
|
||||
}
|
||||
return nil, fmt.Errorf("failed to read CPIO header: %w", err)
|
||||
}
|
||||
|
||||
if string(hdr.Magic[:]) != string(cpioMagic) {
|
||||
return nil, fmt.Errorf("CPIO format magic %s not implemented", string(hdr.Magic[:]))
|
||||
}
|
||||
|
||||
nameSize, err := parseHex(hdr.NameSize[:])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse name size: %w", err)
|
||||
}
|
||||
|
||||
fileSize, err := parseHex(hdr.FileSize[:])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse file size: %w", err)
|
||||
}
|
||||
|
||||
// Read filename, which includes a null terminator.
|
||||
nameBuf := make([]byte, nameSize)
|
||||
if _, err := io.ReadFull(stream, nameBuf); err != nil { // The filename is null-terminated.
|
||||
return nil, fmt.Errorf("failed to read filename: %w", err)
|
||||
}
|
||||
// The filename is null-terminated.
|
||||
filename := strings.TrimRight(string(nameBuf), "\x00")
|
||||
|
||||
// The new-ascii format has padding for 4-byte alignment after the header and filename.
|
||||
// The total size of header + filename is len(hdr) + nameSize.
|
||||
headerAndNameSize := binary.Size(hdr) + int(nameSize)
|
||||
if padding := (4 - (headerAndNameSize % 4)) % 4; padding > 0 {
|
||||
if _, err := io.CopyN(io.Discard, stream, int64(padding)); err != nil { // End of the CPIO archive.
|
||||
return nil, fmt.Errorf("failed to read header padding: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if filename == "TRAILER!!!" {
|
||||
// End of the CPIO archive.
|
||||
break
|
||||
}
|
||||
|
||||
if filename == ".errors" {
|
||||
content := make([]byte, fileSize)
|
||||
if _, err := io.ReadFull(stream, content); err != nil { // Create a temporary file to download into.
|
||||
return nil, fmt.Errorf("failed to read .errors content: %w", err)
|
||||
}
|
||||
return nil, fmt.Errorf("download has errors: %s", string(content))
|
||||
}
|
||||
|
||||
if match := cpioNameRe.FindStringSubmatch(filename); match != nil {
|
||||
fileContent = make([]byte, fileSize)
|
||||
if _, err := io.ReadFull(stream, fileContent); err != nil {
|
||||
return nil, fmt.Errorf("failed to read file content: %w", err)
|
||||
}
|
||||
|
||||
} else {
|
||||
// For any other file, just discard its content.
|
||||
LogInfo("Unhandled file %s in archive, discarding.", filename)
|
||||
if _, err := io.CopyN(io.Discard, stream, fileSize); err != nil {
|
||||
return nil, fmt.Errorf("failed to discard unhandled file content: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The file data is also padded to a 4-byte boundary.
|
||||
if padding := (4 - (fileSize % 4)) % 4; padding > 0 {
|
||||
if _, err := io.CopyN(io.Discard, stream, int64(padding)); err != nil {
|
||||
return nil, fmt.Errorf("failed to read file data padding: %w", err)
|
||||
}
|
||||
}
|
||||
return fileContent, nil
|
||||
}
|
||||
|
||||
// Check for any trailing data.
|
||||
if n, err := io.Copy(io.Discard, stream); err != nil && err != io.EOF {
|
||||
return nil, fmt.Errorf("error reading trailing data: %w", err)
|
||||
} else if n > 0 {
|
||||
LogInfo("Warning: %d bytes of unexpected trailing data in CPIO stream", n)
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
34
common/diff_parser.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ParseSubprojectChangesFromDiff parses a diff string and returns a slice of package names that have changed.
|
||||
|
adamm
commented
should be
not packages, repositories. For packages we need to parser the .gitmoduels and substitute the submodule mount point to repo name. fortunately only few exceptions here, like the packages with > slice of package names that have changed
should be
> slice of repository names that have changed
not packages, repositories. For packages we need to parser the .gitmoduels and substitute the submodule mount point to repo name.
fortunately only few exceptions here, like the packages with `+`
|
||||
// It identifies subproject changes by looking for "Subproject commit" lines within diff chunks.
|
||||
func ParseSubprojectChangesFromDiff(diff string) []string {
|
||||
var changedPackages []string
|
||||
|
||||
// This regex finds diff chunks for subprojects.
|
||||
// It looks for a `diff --git` line, followed by lines indicating a subproject commit change.
|
||||
re := regexp.MustCompile(`diff --git a\/(.+) b\/(.+)\n`)
|
||||
|
adamm
commented
This probably needs This probably needs `^` and `$` to mark the front end end of the line.
|
||||
|
||||
matches := re.FindAllStringSubmatch(diff, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) > 1 {
|
||||
// The package path is in the first capturing group.
|
||||
// We use path.Base to get just the package name from the path (e.g., "rpms/iptraf-ng" -> "iptraf-ng").
|
||||
pkgPath := strings.TrimSpace(match[1])
|
||||
basePath := path.Base(pkgPath)
|
||||
// TODO: parse _manifest files to get real package names
|
||||
if basePath != ".gitmodules" && basePath != "_config" && basePath != "" {
|
||||
changedPackages = append(changedPackages, basePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return changedPackages
|
||||
}
|
||||
@@ -194,6 +194,7 @@ type Gitea interface {
|
||||
GetRecentPullRequests(org, repo, branch string) ([]*models.PullRequest, error)
|
||||
GetRecentCommits(org, repo, branch string, commitNo int64) ([]*models.Commit, error)
|
||||
GetPullRequests(org, project string) ([]*models.PullRequest, error)
|
||||
GetPullRequestDiff(owner, repo string, index int64) (string, error)
|
||||
|
||||
GetCurrentUser() (*models.User, error)
|
||||
}
|
||||
@@ -929,3 +930,19 @@ func (gitea *GiteaTransport) GetCurrentUser() (*models.User, error) {
|
||||
|
||||
return user.GetPayload(), nil
|
||||
}
|
||||
|
||||
func (gitea *GiteaTransport) GetPullRequestDiff(owner, repo string, index int64) (string, error) {
|
||||
params := repository.NewRepoDownloadPullDiffOrPatchParams().
|
||||
WithDefaults().
|
||||
WithOwner(owner).
|
||||
WithRepo(repo).
|
||||
WithIndex(index).
|
||||
WithDiffType("diff")
|
||||
|
||||
result, err := gitea.client.Repository.RepoDownloadPullDiffOrPatch(params, gitea.transport.DefaultAuthentication)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return result.Payload, nil
|
||||
}
|
||||
|
||||
@@ -630,6 +630,24 @@ type Binary struct {
|
||||
Mtime uint64 `xml:"mtime,attr"`
|
||||
}
|
||||
|
||||
type PackageBinariesList struct {
|
||||
XMLName xml.Name `xml:"binarylist"`
|
||||
Binary []Binary `xml:"binary"`
|
||||
}
|
||||
|
||||
// BinaryVersion represents a binary in a binaryversionlist response.
|
||||
type BinaryVersion struct {
|
||||
Name string `xml:"name,attr"`
|
||||
SizeK string `xml:"sizek,attr"`
|
||||
HdrMD5 string `xml:"hdrmd5,attr"`
|
||||
}
|
||||
|
||||
// BinaryVersionList represents a list of binaries from a binaryversionlist response.
|
||||
type BinaryVersionList struct {
|
||||
XMLName xml.Name `xml:"binaryversionlist"`
|
||||
Binary []BinaryVersion `xml:"binary"`
|
||||
}
|
||||
|
||||
type BinaryList struct {
|
||||
Package string `xml:"package,attr"`
|
||||
Binary []Binary `xml:"binary"`
|
||||
@@ -932,3 +950,142 @@ func (c *ObsClient) BuildStatusWithState(project string, opts *BuildResultOption
|
||||
}
|
||||
return ret, err
|
||||
}
|
||||
|
||||
func parsePackageBinaries(data []byte) (*PackageBinariesList, error) {
|
||||
result := PackageBinariesList{}
|
||||
err := xml.Unmarshal(data, &result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *ObsClient) GetBinariesFromPackage(project, repo, arch, pkg string) (*PackageBinariesList, error) {
|
||||
req := []string{"build", project, repo, arch, pkg}
|
||||
res, err := c.ObsRequest("GET", req, nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch res.StatusCode {
|
||||
case 200:
|
||||
break
|
||||
case 404:
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("Unexpected return code: %d %s %w", res.StatusCode, req, err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
result, err := parsePackageBinaries(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if result != nil {
|
||||
result.Binary = slices.DeleteFunc(result.Binary, func(b Binary) bool {
|
||||
return !strings.HasSuffix(b.Filename, ".rpm")
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *ObsClient) GetBinary(project, repo, arch, pkg, filename string) ([]byte, error) {
|
||||
req := []string{"build", project, repo, arch, pkg, filename}
|
||||
res, err := c.ObsRequest("GET", req, nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch res.StatusCode {
|
||||
case 200:
|
||||
break
|
||||
case 404:
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("Unexpected return code: %d %s %w", res.StatusCode, req, err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func parsePackageBinarieVersions(data []byte) (*BinaryVersionList, error) {
|
||||
result := BinaryVersionList{}
|
||||
err := xml.Unmarshal(data, &result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *ObsClient) GetBinariesFromRepo(project, repo, arch string) (*BinaryVersionList, error) {
|
||||
u := c.baseUrl.JoinPath("build", project, repo, arch, "_repository")
|
||||
query := u.Query()
|
||||
query.Add("view", "binaryversions")
|
||||
query.Add("nometa", "1")
|
||||
u.RawQuery = query.Encode()
|
||||
res, err := c.ObsRequestRaw("GET", u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch res.StatusCode {
|
||||
case 200:
|
||||
break
|
||||
case 404:
|
||||
return nil, ObsProjectNotFound{project}
|
||||
default:
|
||||
return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
result, err := parsePackageBinarieVersions(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *ObsClient) GetBinaryHeader(project, repo, arch, filename string) ([]byte, error) {
|
||||
u := c.baseUrl.JoinPath("build", project, repo, arch, "_repository")
|
||||
query := u.Query()
|
||||
query.Add("view", "cpioheaders")
|
||||
query.Add("binary", strings.TrimSuffix(filename, ".rpm"))
|
||||
u.RawQuery = query.Encode()
|
||||
res, err := c.ObsRequestRaw("GET", u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch res.StatusCode {
|
||||
case 200:
|
||||
break
|
||||
case 404:
|
||||
return nil, ObsProjectNotFound{project}
|
||||
default:
|
||||
return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
return ExtractCPIOStream(res.Body)
|
||||
}
|
||||
|
||||
441
obs-installcheck-bot/main.go
Normal file
@@ -0,0 +1,441 @@
|
||||||
package main
|
||||||
|
||||||
import (
|
||||||
"flag"
|
||||||
"fmt"
|
||||||
"os/exec"
|
||||||
"path/filepath"
|
||||||
"runtime/debug"
|
||||||
"strings"
|
||||||
"time"
|
||||||
|
||||||
"os"
|
||||||
"regexp"
|
||||||
"strconv"
|
||||||
|
||||||
"github.com/opentracing/opentracing-go/log"
|
||||||
"src.opensuse.org/autogits/common"
|
||||||
"src.opensuse.org/autogits/common/gitea-generated/models"
|
||||||
)
|
||||||
|
||||||
var GiteaUrl string
|
||||||
var ObsClient *common.ObsClient
|
||||||
var ProcessPROnly string
|
||||||
var IsDryRun bool
|
||||||
var WaitForApprover string
|
||||||
var ListPullNotificationsOnly bool
|
||||||
|
||||||
const SkipSrcRpm = true
|
||||||
|
||||||
func ParseNotificationToPR(thread *models.NotificationThread) (org string, repo string, num int64, err error) {
|
||||||
rx := regexp.MustCompile(`^https://src\.(?:open)?suse\.(?:org|de)/api/v\d+/repos/(?<org>[-_a-zA-Z0-9]+)/(?<project>[-_a-zA-Z0-9]+)/issues/(?<num>[0-9]+)$`)
|
||||||
|
adamm
commented
Need to add
maybe better, this duplication from the staging bot can be put as a const in TODO: this could be a URL parser and just fetch parts of the path as org, project and issue number. Need to add `\.` to the org and project names, in case those have a . in them.
`^https://src\.(?:open)?suse\.(?:org|de)/api/v\d+/repos/(?<org>[-\._a-zA-Z0-9]+)/(?<project>[-\._a-zA-Z0-9]+)/issues/(?<num>[0-9]+)$`
maybe better, this duplication from the staging bot can be put as a const in `common/consts.go` and references from both places.
TODO: this could be a URL parser and just fetch parts of the path as org, project and issue number.
|
||||||
notification := thread.Subject
|
||||||
match := rx.FindStringSubmatch(notification.URL)
|
||||||
if match == nil {
|
||||||
err = fmt.Errorf("Unexpected notification format: %s", notification.URL)
|
||||||
return
|
||||||
}
|
||||||
|
||||||
org = match[1]
|
||||||
repo = match[2]
|
||||||
num, err = strconv.ParseInt(match[3], 10, 64)
|
||||||
return
|
||||||
}
|
||||||
|
||||||
func ProcessPullNotification(gitea common.Gitea, thread *models.NotificationThread) {
|
||||||
defer func() {
|
||||||
err := recover()
|
||||||
if err != nil {
|
||||||
common.LogError(err)
|
||||||
common.LogError(string(debug.Stack()))
|
||||||
}
|
||||||
}()
|
||||||
|
||||||
org, repo, num, err := ParseNotificationToPR(thread)
|
||||||
if err != nil {
|
||||||
common.LogError(err.Error())
|
||||||
return
|
||||||
}
|
||||||
|
||||||
done, err := ProcessPullRequest(gitea, org, repo, num)
|
||||||
if !IsDryRun && done {
|
||||||
err = gitea.SetNotificationRead(thread.ID)
|
||||||
if err != nil {
|
||||||
common.LogError(err)
|
||||||
} else {
|
||||||
common.LogInfo("Notification marked as read")
|
||||||
}
|
||||||
} else if err != nil {
|
||||||
common.LogError(err)
|
||||||
}
|
||||||
}
|
||||||
|
||||||
func PollWorkNotifications(giteaUrl string) {
|
||||||
gitea := common.AllocateGiteaTransport(giteaUrl)
|
||||||
data, err := gitea.GetNotifications(common.GiteaNotificationType_Pull, nil)
|
||||||
|
||||||
if err != nil {
|
||||||
common.LogError(err)
|
||||||
return
|
||||||
}
|
||||||
|
||||||
if data != nil {
|
||||||
common.LogDebug("Processing", len(data), "notifications.")
|
||||||
for _, notification := range data {
|
||||||
common.LogInfo("notification", notification.ID, "--", notification.Subject.HTMLURL)
|
||||||
|
||||||
if !ListPullNotificationsOnly {
|
||||||
switch notification.Subject.Type {
|
||||||
case "Pull":
|
||||||
ProcessPullNotification(gitea, notification)
|
||||||
default:
|
||||||
if !IsDryRun {
|
||||||
gitea.SetNotificationRead(notification.ID)
|
||||||
}
|
||||||
}
|
||||||
}
|
||||||
}
|
||||||
}
|
||||||
}
|
||||||
|
||||||
func addUniqueReviewComment(gitea common.Gitea, pr *models.PullRequest, state models.ReviewStateType, comment string) error {
|
||||||
reviews, err := gitea.GetPullRequestReviews(pr.Base.Repo.Owner.UserName, pr.Base.Repo.Name, pr.Index)
|
||||||
if err != nil {
|
||||||
return fmt.Errorf("failed to get pull request reviews: %w", err)
|
||||||
}
|
||||||
|
||||||
botUser, err := gitea.GetCurrentUser()
|
||||||
if err != nil {
|
||||||
return fmt.Errorf("failed to get current user: %w", err)
|
||||||
}
|
||||||
|
||||||
for _, review := range reviews {
|
||||||
if review.User != nil && review.User.UserName == botUser.UserName && review.Body == comment && review.State == common.ReviewStatePending {
|
||||||
common.LogInfo("Review comment already exists, skipping.")
|
||||||
return nil
|
||||||
}
|
||||||
}
|
||||||
|
||||||
_, err = gitea.AddReviewComment(pr, state, comment)
|
||||||
return err
|
||||||
}
|
||||||
|
||||||
func ProcessPullRequest(gitea common.Gitea, org, repo string, id int64) (bool, error) {
|
||||||
var done bool = false
|
||||||
|
||||||
common.LogInfo("Processing PR", org+"/"+repo+"#"+strconv.FormatInt(id, 10))
|
||||||
|
||||||
pr, err := gitea.GetPullRequest(org, repo, id)
|
||||||
if err != nil {
|
||||||
common.LogError("No PR associated with review:", org, "/", repo, "#", id, "Error:", err)
|
||||||
done = false
|
||||||
return done, err
|
||||||
}
|
||||||
|
||||||
if pr.State != "open" {
|
||||||
common.LogInfo("PR is not open, skipping.")
|
||||||
done = true
|
||||||
return done, nil
|
||||||
}
|
||||||
|
||||||
if WaitForApprover != "" {
|
||||||
common.LogInfo("Waiting for approver:", WaitForApprover)
|
||||||
reviews, err := gitea.GetPullRequestReviews(org, repo, id)
|
||||||
if err != nil {
|
||||||
done = false
|
||||||
return done, fmt.Errorf("failed to get pull request reviews: %w", err)
|
||||||
}
|
||||||
|
||||||
approved := false
|
||||||
for _, review := range reviews {
|
||||||
if review.User != nil && review.User.UserName == WaitForApprover && review.State == common.ReviewStateApproved {
|
||||||
approved = true
|
||||||
break
|
||||||
}
|
||||||
}
|
||||||
|
||||||
if approved {
|
||||||
common.LogInfo("PR approved by", WaitForApprover)
|
||||||
} else {
|
||||||
common.LogInfo("PR not yet approved by", WaitForApprover+". Skipping for now.")
|
||||||
done = false
|
||||||
return done, nil // Not an error, just not ready to process.
|
||||||
}
|
||||||
}
|
||||||
|
||||||
data, _, err := gitea.GetRepositoryFileContent(org, repo, pr.Head.Sha, common.StagingConfigFile)
|
||||||
if err != nil {
|
||||||
common.LogError("Cannot find 'staging.config' in PR#", pr.Index, ":", err)
|
||||||
err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Cannot find project config in PR: "+common.StagingConfigFile)
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to add review comment:", err)
|
||||||
}
|
||||||
done = true
|
||||||
return done, err
|
||||||
}
|
||||||
|
||||||
var config *common.StagingConfig
|
||||||
|
||||||
config, err = common.ParseStagingConfig(data)
|
||||||
if err != nil {
|
||||||
common.LogDebug("Failed to parse staging config. Error:", err)
|
||||||
err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Failed to parse staging config.")
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to add review comment:", err)
|
||||||
}
|
||||||
done = true
|
||||||
return done, fmt.Errorf("failed to parse staging config: %w", err)
|
||||||
}
|
||||||
|
||||||
if config.StagingProject == "" ||
|
||||||
config.Repositories == nil || len(config.Repositories) == 0 ||
|
||||||
config.Installcheck.Repos == nil || len(config.Installcheck.Repos) == 0 {
|
||||||
common.LogError("Wrong staging config format in PR#", pr.Index)
|
||||||
err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Wrong staging config format.")
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to add review comment:", err)
|
||||||
}
|
||||||
done = true
|
||||||
return done, fmt.Errorf("wrong staging config format in PR: %w", err)
|
||||||
}
|
||||||
|
||||||
config.StagingProject = config.StagingProject + ":" + strconv.FormatInt(pr.Index, 10)
|
||||||
|
||||||
common.LogInfo("Using staging project:", config.StagingProject)
|
||||||
for repoName, repoConfig := range config.Repositories {
|
||||||
common.LogInfo("Using OBS repository:", repoName, "architectures: ", repoConfig.Architectures)
|
||||||
}
|
||||||
for arch, repos := range config.Installcheck.Repos {
|
||||||
common.LogInfo("Using installcheck repos for architecture:", arch, ":", repos)
|
||||||
}
|
||||||
for arch, repos := range config.Installcheck.NoCheckRepos {
|
||||||
common.LogInfo("Using installcheck nocheck repos for architecture:", arch, ":", repos)
|
||||||
}
|
||||||
|
||||||
dir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("%s-%s-%d-", org, repo, id))
|
||||||
common.PanicOnError(err)
|
||||||
if IsDryRun {
|
||||||
common.LogInfo("will keep temp directory:", dir)
|
||||||
} else {
|
||||||
defer os.RemoveAll(dir)
|
||||||
}
|
||||||
|
||||||
diff, err := gitea.GetPullRequestDiff(org, repo, id)
|
||||||
common.LogDebug("PR Diff:\n", diff)
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to get PR diff:", err)
|
||||||
done = false
|
||||||
return done, err
|
||||||
}
|
||||||
|
||||||
changedPackages := common.ParseSubprojectChangesFromDiff(diff)
|
||||||
if len(changedPackages) > 0 {
|
||||||
common.LogInfo("Changed packages found in PR!")
|
||||||
|
adamm
commented
Here we already have this functionality not by parsing a diff, but at looking at changes to the submodules. But maybe this is better if subdirectories need to be taken into account.
common/git_utils.go
Line 865 in cc675c1b24
(so nothing to change here, just a comment) Here we already have this functionality not by parsing a diff, but at looking at changes to the submodules. But maybe this is better if subdirectories need to be taken into account.
https://src.opensuse.org/git-workflow/autogits/src/commit/cc675c1b249a4a86eaf85f70ffa8caaa432b9e14f046247eb83c1d780499628f/common/git_utils.go#L865
(so nothing to change here, just a comment)
|
||||||
|
||||||
for repoName, repoConfig := range config.Repositories {
|
||||||
for _, arch := range repoConfig.Architectures {
|
||||||
archDir := filepath.Join(dir, repoName, arch)
|
||||||
err := os.MkdirAll(archDir, 0755)
|
||||||
common.PanicOnError(err)
|
||||||
common.LogInfo("Created temporary directory for", repo, "arch", arch, "at", archDir)
|
||||||
}
|
||||||
}
|
||||||
|
||||||
var nameIgnoreRe = regexp.MustCompile(`-debug(info|source|info-32bit)\.rpm$`)
|
||||||
|
||||||
for repoName, repoConfig := range config.Repositories {
|
||||||
for _, arch := range repoConfig.Architectures {
|
||||||
archDir := filepath.Join(dir, repoName, arch)
|
||||||
|
||||||
var binaries *common.BinaryVersionList
|
||||||
binaries, err = ObsClient.GetBinariesFromRepo(config.StagingProject, repoName, arch)
|
||||||
if err != nil || binaries == nil {
|
||||||
common.LogError("Failed to get list of binaries:", err)
|
||||||
err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Failed to get list of binaries for repo '"+repoName+"' arch '"+arch+"': "+err.Error())
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to add review comment:", err)
|
||||||
}
|
||||||
done = true
|
||||||
return done, err
|
||||||
}
|
||||||
|
||||||
var filtered_binaries []string
|
||||||
|
||||||
if binaries != nil {
|
||||||
common.LogInfo("Binaries for repo ", repoName, " and arch", arch+":")
|
||||||
for _, bin := range binaries.Binary {
|
||||||
common.LogInfo(" -", bin.Name, "(md5:", bin.HdrMD5, "size:", bin.SizeK, ")")
|
||||||
if strings.HasSuffix(bin.Name, ".rpm") && !nameIgnoreRe.MatchString(bin.Name) {
|
||||||
filtered_binaries = append(filtered_binaries, bin.Name)
|
||||||
}
|
||||||
}
|
||||||
}
|
||||||
|
||||||
if len(filtered_binaries) > 0 {
|
||||||
// Add check if all changed packages are present in the binaries
|
||||||
for _, changedPkg := range changedPackages {
|
||||||
found := false
|
||||||
for _, bin := range filtered_binaries {
|
||||||
if strings.HasPrefix(bin, changedPkg+"-") || bin == changedPkg+".rpm" {
|
||||||
common.LogDebug("Changed package", changedPkg, "found in binaries for repo", repoName, "arch", arch)
|
||||||
found = true
|
||||||
break
|
||||||
}
|
||||||
}
|
||||||
if !found {
|
||||||
common.LogError("Changed package", changedPkg, "NOT found in binaries for repo", repoName, "arch", arch)
|
||||||
// staging bot can approve the request even if some packages are missing
|
||||||
done = false
|
||||||
return done, nil
|
||||||
}
|
||||||
}
|
||||||
|
||||||
// Download the filtered binaries
|
||||||
common.LogInfo("Downloading", len(filtered_binaries), "binaries for repo", repoName, "arch", arch)
|
||||||
for _, bin := range filtered_binaries {
|
||||||
common.LogInfo("Downloading binary header:", bin)
|
||||||
binaryData, err := ObsClient.GetBinaryHeader(config.StagingProject, repoName, arch, bin)
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to download binary '"+bin+"':", err)
|
||||||
err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Failed to download binary '"+bin+"': "+err.Error())
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to add review comment:", err)
|
||||||
}
|
||||||
done = true
|
||||||
return done, err
|
||||||
}
|
||||||
if binaryData == nil {
|
||||||
common.LogInfo("Binary not found or is empty:", bin)
|
||||||
err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, "Binary not found or is empty: '"+bin+"'")
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to add review comment:", err)
|
||||||
}
|
||||||
done = true
|
||||||
return done, err
|
||||||
}
|
||||||
filePath := filepath.Join(archDir, bin)
|
||||||
err = os.WriteFile(filePath, binaryData, 0644)
|
||||||
common.PanicOnError(err)
|
||||||
}
|
||||||
}
|
||||||
}
|
||||||
}
|
||||||
|
||||||
var review_msg string = ""
|
||||||
var anyInstallcheckFailed bool = false
|
||||||
|
||||||
for repoName, repoConfig := range config.Repositories {
|
||||||
for _, arch := range repoConfig.Architectures {
|
||||||
archDir := filepath.Join(dir, repoName, arch)
|
||||||
// Call createrepo_c for the architecture directory
|
||||||
common.LogInfo("Creating repository for architecture:", arch, "in", archDir)
|
||||||
cmd := exec.Command("createrepo_c", archDir)
|
||||||
output, err := cmd.CombinedOutput()
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to create repository for '"+arch+"' in repo '"+repoName+"':", err, "Output:", string(output))
|
||||||
}
|
||||||
|
||||||
common.LogInfo("Running installcheck for architecture:", arch)
|
||||||
|
||||||
var installcheckArgs []string
|
||||||
installcheckArgs = append(installcheckArgs, arch)
|
||||||
if reposToInclude, ok := config.Installcheck.Repos[arch]; ok {
|
||||||
installcheckArgs = append(installcheckArgs, reposToInclude...)
|
||||||
}
|
||||||
installcheckArgs = append(installcheckArgs, filepath.Join(archDir, "repodata", "*primary.xml.*"))
|
||||||
if reposToExclude, ok := config.Installcheck.NoCheckRepos[arch]; ok {
|
||||||
installcheckArgs = append(installcheckArgs, "--nocheck")
|
||||||
installcheckArgs = append(installcheckArgs, reposToExclude...)
|
||||||
}
|
||||||
installcheckCommand := "installcheck " + strings.Join(installcheckArgs, " ")
|
||||||
common.LogInfo("Running installcheck for repository "+repoName+" and architecture:", arch, "Command:", installcheckCommand)
|
||||||
installcheckCmd := exec.Command("bash", "-c", installcheckCommand)
|
||||||
installcheckOutput, installcheckErr := installcheckCmd.CombinedOutput()
|
||||||
if installcheckErr != nil {
|
||||||
common.LogError("Failed to run installcheck for repository '"+repoName+"' and architecture '"+arch+"':", installcheckErr, "Output:\n", string(installcheckOutput))
|
||||||
anyInstallcheckFailed = true
|
||||||
review_msg += "Failed to run installcheck for repository '" + repoName + "' and architecture '" + arch + "'\n```" + string(installcheckOutput) + "```\n\n"
|
||||||
|
||||||
} else {
|
||||||
common.LogInfo("Installcheck for repository '"+repoName+"' and architecture '"+arch+"' completed successfully. Output:", string(installcheckOutput))
|
||||||
review_msg += "Installcheck for repository '" + repoName + "' and architecture '" + arch + "' completed successfully.\n\n"
|
||||||
}
|
||||||
}
|
||||||
}
|
||||||
|
||||||
if !IsDryRun {
|
||||||
if anyInstallcheckFailed {
|
||||||
err = addUniqueReviewComment(gitea, pr, common.ReviewStateRequestChanges, review_msg)
|
||||||
} else {
|
||||||
err = addUniqueReviewComment(gitea, pr, common.ReviewStateApproved, review_msg)
|
||||||
}
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to add review comment:", err)
|
||||||
} else {
|
||||||
common.LogInfo("Review comment added successfully")
|
||||||
}
|
||||||
done = true
|
||||||
}
|
||||||
|
||||||
} else {
|
||||||
done = true
|
||||||
common.LogInfo("No subproject changes found in PR.")
|
||||||
|
||||||
err = addUniqueReviewComment(gitea, pr, common.ReviewStateApproved, "No packages changed in the PR, nothing to check.")
|
||||||
if err != nil {
|
||||||
common.LogError("Failed to add review comment:", err)
|
||||||
}
|
||||||
}
|
||||||
|
||||||
return done, nil
|
||||||
}
|
||||||
|
||||||
func main() {
|
||||||
flag.BoolVar(&ListPullNotificationsOnly, "list-notifications-only", false, "Only lists notifications without acting on them")
|
||||||
flag.StringVar(&GiteaUrl, "gitea-url", "https://src.opensuse.org", "Gitea instance")
|
||||||
ProcessPROnly := flag.String("pr", "", "Process only specific PR and ignore the rest. Use for debugging")
|
||||||
obsApiHost := flag.String("obs", "https://api.opensuse.org", "API for OBS instance") // Keep this as it's for the OBS client itself
|
||||||
debug := flag.Bool("debug", false, "Turns on debug logging") // Keep this for logging control
|
||||||
flag.BoolVar(&IsDryRun, "dry", false, "Dry-run, don't actually post reviews")
|
||||||
flag.StringVar(&WaitForApprover, "wait-for-approver", "", "Username of the user whose approval is required before processing the PR")
|
||||||
flag.Parse()
|
||||||
|
||||||
if *debug {
|
||||||
common.SetLoggingLevel(common.LogLevelDebug)
|
||||||
} else {
|
||||||
common.SetLoggingLevel(common.LogLevelInfo)
|
||||||
}
|
||||||
|
||||||
common.LogDebug("OBS Gitea Host:", GiteaUrl)
|
||||||
common.LogDebug("OBS API Host:", *obsApiHost)
|
||||||
|
||||||
common.PanicOnErrorWithMsg(common.RequireGiteaSecretToken(), "Cannot find GITEA_TOKEN")
|
||||||
common.PanicOnErrorWithMsg(common.RequireObsSecretToken(), "Cannot find OBS_USER and OBS_PASSWORD")
|
||||||
|
||||||
var err error
|
||||||
if ObsClient, err = common.NewObsClient(*obsApiHost); err != nil {
|
||||||
log.Error(err)
|
||||||
return
|
||||||
}
|
||||||
|
||||||
if len(*ProcessPROnly) > 0 {
|
||||||
rx := regexp.MustCompile("^([^/#]+)/([^/#]+)#([0-9]+)$")
|
||||||
m := rx.FindStringSubmatch(*ProcessPROnly)
|
||||||
if m == nil {
|
||||||
common.LogError("Cannot find any PR matches in", *ProcessPROnly)
|
||||||
return
|
||||||
}
|
||||||
|
||||||
gitea := common.AllocateGiteaTransport(GiteaUrl)
|
||||||
id, _ := strconv.ParseInt(m[3], 10, 64)
|
||||||
|
||||||
ProcessPullRequest(gitea, m[1], m[2], id)
|
||||||
return
|
||||||
}
|
||||||
|
||||||
for {
|
||||||
PollWorkNotifications(GiteaUrl)
|
||||||
common.LogInfo("Poll cycle finished")
|
||||||
time.Sleep(5 * time.Minute)
|
||||||
}
|
||||||
}
|
||||||
This needs documentation. This probably needs to be extended
https://src.opensuse.org/git-workflow/autogits/src/branch/main/obs-staging-bot#configuration-file
Need example of how these things are defined. map of what to what.
stringis not very descriptive here.