Files
autogits/common/obs_utils.go
Adam Majer 3d24dce5c0 common: rabbit refactor
Generalize interface to allow processing of any events, not just
Gitea events.
2025-07-26 13:54:51 +02:00

908 lines
23 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package common
/*
* This file is part of Autogits.
*
* Copyright © 2024 SUSE LLC
*
* Autogits is free software: you can redistribute it and/or modify it under
* the terms of the GNU General Public License as published by the Free Software
* Foundation, either version 2 of the License, or (at your option) any later
* version.
*
* Autogits is distributed in the hope that it will be useful, but WITHOUT ANY
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* Foobar. If not, see <https://www.gnu.org/licenses/>.
*/
import (
"bytes"
"encoding/base64"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os/exec"
"regexp"
"slices"
"strings"
"time"
)
//go:generate mockgen -source=obs_utils.go -destination=mock/obs_utils.go -typed
type BuildResultOptions struct {
BinaryList bool
OldState string
LastBuild bool
}
type ObsStatusFetcherWithState interface {
BuildStatusWithState(project string, opts *BuildResultOptions, packages ...string) (*BuildResultList, error)
}
type ObsClient struct {
baseUrl *url.URL
client *http.Client
user, password string
cookie string
sshkey string
sshkeyfile string
HomeProject string
}
func NewObsClient(host string) (*ObsClient, error) {
baseUrl, err := url.Parse(host)
if err != nil {
return nil, err
}
return &ObsClient{
baseUrl: baseUrl,
client: &http.Client{},
user: obsUser,
password: obsPassword,
sshkey: obsSshkey,
sshkeyfile: obsSshkeyFile,
HomeProject: fmt.Sprintf("home:%s", obsUser),
}, nil
}
type ReleaseTargetMeta struct {
Project string `xml:"project,attr"`
Repository string `xml:"repository,attr"`
Trigger string `xml:"trigger,attr"`
}
type RepositoryPathMeta struct {
Project string `xml:"project,attr"`
Repository string `xml:"repository,attr"`
}
type RepositoryMeta struct {
Name string `xml:"name,attr"`
BuildTrigger string `xml:"rebuild,attr,omitempty"`
BlockMode string `xml:"block,attr,omitempty"`
LinkedBuild string `xml:"linkedbuild,attr,omitempty"`
ReleaseTargets []ReleaseTargetMeta `xml:"releasetarget"`
Paths []RepositoryPathMeta `xml:"path"`
Archs []string `xml:"arch"`
}
type PersonRepoMeta struct {
XMLName xml.Name `xml:"person"`
UserID string `xml:"userid,attr"`
Role string `xml:"role,attr,omitempty"`
}
type PersonGroup struct {
XMLName xml.Name `xml:"person"`
Persons []PersonRepoMeta `xml:"person"`
}
type GroupRepoMeta struct {
GroupID string `xml:"groupid,attr"`
Role string `xml:"role,attr"`
}
type Flags struct {
Contents string `xml:",innerxml"`
}
type ProjectMeta struct {
XMLName xml.Name `xml:"project"`
Name string `xml:"name,attr"`
Title string `xml:"title"`
Description string `xml:"description"`
Url string `xml:"url,omitempty"`
ScmSync string `xml:"scmsync"`
Persons []PersonRepoMeta `xml:"person"`
Groups []GroupRepoMeta `xml:"group"`
Repositories []RepositoryMeta `xml:"repository"`
BuildFlags Flags `xml:"build"`
PublicFlags Flags `xml:"publish"`
DebugFlags Flags `xml:"debuginfo"`
UseForBuild Flags `xml:"useforbuild"`
}
type PackageMeta struct {
XMLName xml.Name `xml:"package"`
Name string `xml:"name,attr"`
Project string `xml:"project,attr"`
ScmSync string `xml:"scmsync"`
Persons []PersonRepoMeta `xml:"person"`
Groups []GroupRepoMeta `xml:"group"`
}
type UserMeta struct {
XMLName xml.Name `xml:"person"`
Login string `xml:"login"`
Email string `xml:"email"`
Name string `xml:"realname"`
State string `xml:"state"`
}
type GroupMeta struct {
XMLName xml.Name `xml:"group"`
Title string `xml:"title"`
Persons PersonGroup `xml:"person"`
}
type RequestStateMeta struct {
XMLName xml.Name `xml:"state"`
State string `xml:"name,attr"`
}
type RequestActionTarget struct {
XMLName xml.Name
Project string `xml:"project,attr"`
Package string `xml:"package,attr"`
Revision *string `xml:"rev,attr,optional"`
}
type RequestActionMeta struct {
XMLName xml.Name `xml:"action"`
Type string `xml:"type,attr"`
Source *RequestActionTarget `xml:"source,optional"`
Target *RequestActionTarget `xml:"target,optional"`
}
type RequestMeta struct {
XMLName xml.Name `xml:"request"`
Id int `xml:"id,attr"`
Creator string `xml:"creator,attr"`
Action *RequestActionMeta `xml:"action"`
State RequestStateMeta `xml:"state"`
}
func parseProjectMeta(data []byte) (*ProjectMeta, error) {
var meta ProjectMeta
err := xml.Unmarshal(data, &meta)
if err != nil {
return nil, err
}
return &meta, nil
}
const (
RequestStatus_Unknown = "unknown"
RequestStatus_Accepted = "accepted"
RequestStatus_Superseded = "superseded"
RequestStatus_Declined = "declined"
RequestStatus_Revoked = "revoked"
RequestStatus_New = "new"
RequestStatus_Review = "review"
)
func (status *RequestStateMeta) IsFinal() bool {
switch status.State {
case RequestStatus_Declined, RequestStatus_Revoked, RequestStatus_Accepted, RequestStatus_Superseded:
return true
}
return false
}
func parseRequestXml(data []byte) (*RequestMeta, error) {
ret := RequestMeta{}
LogDebug("parsing: ", string(data))
if err := xml.Unmarshal(data, &ret); err != nil {
return nil, err
}
return &ret, nil
}
func (c *ObsClient) CreateSubmitRequest(sourcePrj, sourcePkg, targetPrj string) (*RequestMeta, error) {
url := c.baseUrl.JoinPath("request")
query := url.Query()
query.Add("cmd", "create")
url.RawQuery = query.Encode()
request := `<request>
<action type="submit">
<source project="` + sourcePrj + `" package="` + sourcePkg + `">
</source>
<target project="` + targetPrj + `" package="` + sourcePkg + `">
</target>
</action>
</request>`
res, err := c.ObsRequestRaw("POST", url.String(), strings.NewReader(request))
if err != nil {
return nil, err
} else if res.StatusCode != 200 {
return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode)
}
data, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
return parseRequestXml(data)
}
func (c *ObsClient) RequestStatus(requestID int) (*RequestMeta, error) {
res, err := c.ObsRequest("GET", []string{"request", fmt.Sprint(requestID)}, nil)
if err != nil {
return nil, err
}
if res.StatusCode != 200 {
return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode)
}
data, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
return parseRequestXml(data)
}
func (c *ObsClient) GetGroupMeta(gid string) (*GroupMeta, error) {
res, err := c.ObsRequest("GET", []string{"group", gid}, nil)
if err != nil {
return nil, err
}
switch res.StatusCode {
case 200:
break
case 404:
return nil, nil
default:
return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode)
}
data, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
defer res.Body.Close()
var meta GroupMeta
err = xml.Unmarshal(data, &meta)
if err != nil {
return nil, err
}
return &meta, nil
}
func (c *ObsClient) GetUserMeta(uid string) (*UserMeta, error) {
res, err := c.ObsRequest("GET", []string{"person", uid}, nil)
if err != nil {
return nil, err
}
switch res.StatusCode {
case 200:
break
case 404:
return nil, nil
default:
return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode)
}
data, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
defer res.Body.Close()
var meta UserMeta
err = xml.Unmarshal(data, &meta)
if err != nil {
return nil, err
}
return &meta, nil
}
func (c *ObsClient) ObsRequest(method string, url_path []string, body io.Reader) (*http.Response, error) {
return c.ObsRequestRaw(method, c.baseUrl.JoinPath(url_path...).String(), body)
}
func (c *ObsClient) ObsRequestRaw(method string, url string, body io.Reader) (*http.Response, error) {
req, err := http.NewRequest(method, url, body)
if err != nil {
return nil, err
}
if body != nil {
req.Body = io.NopCloser(body)
}
if c.cookie != "" {
req.Header.Add("cookie", c.cookie)
}
res, err := c.client.Do(req)
if err != nil && res == nil {
LogDebug("No res headers:", err)
return res, err
}
if err == nil && res.StatusCode == 200 {
auth_cookie := res.Header.Get("set-cookie")
if auth_cookie != "" {
c.cookie = auth_cookie
}
return res, nil
}
if res.StatusCode == 401 {
if c.sshkey == "" {
LogDebug("setting basic auth")
req.SetBasicAuth(c.user, c.password)
} else {
www := res.Header.Get("www-authenticate")
// log.Printf("www-authenticate %s", www)
re := regexp.MustCompile(`Signature realm="(.*?)",headers="\(created\)`)
match := re.FindStringSubmatch(www)
if len(match) < 1 {
return nil, errors.New("No realm found")
}
realm := string(match[1])
// SSH Sign
cmd := exec.Command("ssh-keygen", "-Y", "sign", "-f", c.sshkeyfile, "-n", realm, "-q")
now := time.Now().Unix()
sigdata := fmt.Sprintf("(created): %d", now)
cmd.Stdin = strings.NewReader(sigdata)
stdout, err := cmd.Output()
if err != nil {
exitCode := -1337
if cmd.ProcessState != nil {
exitCode = cmd.ProcessState.ExitCode()
}
return nil, errors.New(fmt.Sprintf("ssh-keygen signature creation failed: %d", exitCode))
}
reg := regexp.MustCompile("(?s)-----BEGIN SSH SIGNATURE-----\n(.*?)\n-----END SSH SIGNATURE-----")
match = reg.FindStringSubmatch(string(stdout))
if len(match) < 2 {
return nil, errors.New("could not extract ssh signature")
}
signature, err := base64.StdEncoding.DecodeString(match[1])
if err != nil {
return nil, err
}
signatureBase64 := base64.StdEncoding.EncodeToString(signature)
authorization := fmt.Sprintf(`keyId="%s",algorithm="ssh",headers="(created)",created=%d,signature="%s"`,
c.user, now, signatureBase64)
// log.Printf("Add Authorization Signature ", authorization)
req.Header.Add("Authorization", "Signature "+authorization)
}
// Another time with authentification header
LogDebug("Trying again with authorization for", req.URL.String())
res, err = c.client.Do(req)
if err != nil {
if res != nil {
LogError("Authentification failed:", res.StatusCode)
}
return nil, err
}
}
if err == nil {
// Store the cookie for next call
auth_cookie := res.Header.Get("set-cookie")
if auth_cookie != "" {
c.cookie = auth_cookie
}
}
return res, err
}
func (c *ObsClient) GetProjectMeta(project string) (*ProjectMeta, error) {
req := []string{"source", project, "_meta"}
res, err := c.ObsRequest("GET", req, nil)
if err != nil {
return nil, err
}
switch res.StatusCode {
case 200:
break
case 404:
return nil, nil
default:
return nil, fmt.Errorf("Unexpected return code: %d %s %w", res.StatusCode, req, err)
}
data, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
defer res.Body.Close()
return parseProjectMeta(data)
}
func (c *ObsClient) GetPackageMeta(project, pkg string) (*PackageMeta, error) {
res, err := c.ObsRequest("GET", []string{"source", project, pkg, "_meta"}, nil)
if err != nil {
return nil, err
}
switch res.StatusCode {
case 200:
break
case 404:
return nil, nil
default:
return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode)
}
data, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
defer res.Body.Close()
var meta PackageMeta
err = xml.Unmarshal(data, &meta)
if err != nil {
return nil, err
}
return &meta, nil
}
func ObsSafeProjectName(prjname string) string {
if len(prjname) < 1 {
return prjname
} else if len(prjname) > 200 {
prjname = prjname[:199]
}
switch prjname[0] {
case '_', '.', ':':
prjname = "X" + prjname[1:]
// no UTF-8 in OBS :(
// prjname = "_" + prjname[1:]
// case ':':
// prjname = "" + prjname[1:]
// case '.':
// prjname = "" + prjname[1:]
}
return prjname
}
var ValidBlockModes []string = []string{"all", "local", "never"}
var ValidPrjLinkModes []string = []string{"off", "localdep", "alldirect", "all"}
var ValidTriggerModes []string = []string{"transitive", "direct", "local"}
func (c *ObsClient) SetProjectMeta(meta *ProjectMeta) error {
for _, repo := range meta.Repositories {
if len(repo.BlockMode) > 0 && !slices.Contains(ValidBlockModes, repo.BlockMode) {
return fmt.Errorf("Invalid repository block mode: '%s'", repo.BlockMode)
}
if len(repo.BuildTrigger) > 0 && !slices.Contains(ValidTriggerModes, repo.BuildTrigger) {
return fmt.Errorf("Invalid repository trigger mode: '%s'", repo.BuildTrigger)
}
if len(repo.LinkedBuild) > 0 && !slices.Contains(ValidPrjLinkModes, repo.LinkedBuild) {
return fmt.Errorf("Invalid linked project rebuild mode: '%s'", repo.LinkedBuild)
}
}
xml, err := xml.Marshal(meta)
if err != nil {
return err
}
res, err := c.ObsRequest("PUT", []string{"source", meta.Name, "_meta"}, io.NopCloser(bytes.NewReader(xml)))
if err != nil {
return err
}
defer res.Body.Close()
switch res.StatusCode {
case 200:
break
default:
return fmt.Errorf("Unexpected return code: %d", res.StatusCode)
}
return nil
}
func (c *ObsClient) DeleteProject(project string) error {
url := c.baseUrl.JoinPath("source", project)
query := url.Query()
query.Add("force", "1")
url.RawQuery = query.Encode()
res, err := c.ObsRequestRaw("DELETE", url.String(), nil)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("Unexpected return code: %d", res.StatusCode)
}
return nil
}
func (c *ObsClient) BuildLog(prj, pkg, repo, arch string) (io.ReadCloser, error) {
url := c.baseUrl.JoinPath("build", prj, repo, arch, pkg, "_log")
query := url.Query()
query.Add("nostream", "1")
query.Add("start", "0")
url.RawQuery = query.Encode()
res, err := c.ObsRequestRaw("GET", url.String(), nil)
if err != nil {
return nil, err
}
return res.Body, nil
}
type PackageBuildStatus struct {
Package string `xml:"package,attr"`
Code string `xml:"code,attr"`
Details string `xml:"details"`
}
type BuildResult struct {
Project string `xml:"project,attr"`
Repository string `xml:"repository,attr"`
Arch string `xml:"arch,attr"`
Code string `xml:"code,attr"`
Dirty bool `xml:"dirty,attr"`
ScmSync string `xml:"scmsync"`
ScmInfo string `xml:"scminfo"`
Status []PackageBuildStatus `xml:"status"`
Binaries []BinaryList `xml:"binarylist"`
}
type Binary struct {
Size uint64 `xml:"size,attr"`
Filename string `xml:"filename,attr"`
Mtime uint64 `xml:"mtime,attr"`
}
type BinaryList struct {
Package string `xml:"package,attr"`
Binary []Binary `xml:"binary"`
}
type BuildResultList struct {
XMLName xml.Name `xml:"resultlist"`
State string `xml:"state,attr"`
Result []BuildResult `xml:"result"`
isLastBuild bool
}
func (r *BuildResultList) GetPackageList() []string {
pkgList := make([]string, 0, 3*len(r.Result[0].Status)/2)
for ridx, res := range r.Result {
for _, status := range res.Status {
if ridx == 0 {
pkgList = append(pkgList, status.Package)
} else if idx, found := slices.BinarySearch(pkgList, status.Package); !found {
pkgList = slices.Insert(pkgList, idx, status.Package)
}
}
if ridx == 0 {
slices.Sort(pkgList)
}
}
return pkgList
}
func (r *BuildResultList) BuildResultSummary() (success, finished bool) {
if r == nil {
return false, false
}
finished = len(r.Result) > 0 && len(r.Result[0].Status) > 0
success = finished
for _, resultSet := range r.Result {
repoDetail, ok := ObsRepoStatusDetails[resultSet.Code]
if !ok {
panic("Unknown repo result code: " + resultSet.Code)
}
finished = r.isLastBuild || repoDetail.Finished
if !finished || resultSet.Dirty {
return
}
for _, result := range resultSet.Status {
detail, ok := ObsBuildStatusDetails[result.Code]
if !ok {
panic("Unknown result code: " + result.Code)
}
if r.isLastBuild && result.Code == "unknown" {
// it means the package has never build yet,
// but we don't know the reason
detail.Finished = true
}
finished = finished && detail.Finished
success = success && detail.Success
if !finished {
return
}
}
}
return
}
type ObsBuildStatusDetail struct {
Code string
Description string
Finished bool
Success bool
}
var ObsBuildStatusDetails map[string]ObsBuildStatusDetail = map[string]ObsBuildStatusDetail{
"succeeded": ObsBuildStatusDetail{
Code: "succeeded",
Description: "Package has built successfully and can be used to build further packages.",
Finished: true,
Success: true,
},
"failed": ObsBuildStatusDetail{
Code: "failed",
Description: "The package does not build successfully. No packages have been created. Packages that depend on this package will be built using any previously created packages, if they exist.",
Finished: true,
Success: false,
},
"unresolvable": ObsBuildStatusDetail{
Code: "unresolvable",
Description: "The build can not begin, because required packages are either missing or not explicitly defined.",
Finished: true,
Success: false,
},
"broken": ObsBuildStatusDetail{
Code: "broken",
Description: "The sources either contain no build description (e.g. specfile), automatic source processing failed or a merge conflict does exist.",
Finished: true,
Success: false,
},
"blocked": ObsBuildStatusDetail{
Code: "blocked",
Description: "This package waits for other packages to be built. These can be in the same or other projects.",
Finished: false,
},
"scheduled": ObsBuildStatusDetail{
Code: "scheduled",
Description: "A package has been marked for building, but the build has not started yet.",
Finished: false,
},
"dispatching": ObsBuildStatusDetail{
Code: "dispatching",
Description: "A package is being copied to a build host. This is an intermediate state before building.",
Finished: false,
},
"building": ObsBuildStatusDetail{
Code: "building",
Description: "The package is currently being built.",
Finished: false,
},
"signing": ObsBuildStatusDetail{
Code: "signing",
Description: "The package has been built successfully and is assigned to get signed.",
Finished: false,
},
"finished": ObsBuildStatusDetail{
Code: "finished",
Description: "The package has been built and signed, but has not yet been picked up by the scheduler. This is an intermediate state prior to 'succeeded' or 'failed'.",
Finished: false,
},
"disabled": ObsBuildStatusDetail{
Code: "disabled",
Description: "The package has been disabled from building in project or package metadata. Packages that depend on this package will be built using any previously created packages, if they still exist.",
Finished: true,
Success: true,
},
"excluded": ObsBuildStatusDetail{
Code: "excluded",
Description: "The package build has been disabled in package build description (for example in the .spec file) or does not provide a matching build description for the target.",
Finished: true,
Success: true,
},
"locked": ObsBuildStatusDetail{
Code: "locked",
Description: "The package is frozen",
Finished: true,
Success: true,
},
"unknown": ObsBuildStatusDetail{
Code: "unknown",
Description: "The scheduler has not yet evaluated this package. Should be a short intermediate state for new packages. When used for lastbuild state it means it was never possible to attempt a build",
Finished: false,
},
"error": ObsBuildStatusDetail{
Code: "Error",
Description: "Unknown status code",
},
}
var ObsRepoStatusDetails map[string]ObsBuildStatusDetail = map[string]ObsBuildStatusDetail{
// repo status
"published": ObsBuildStatusDetail{
Code: "published",
Description: "Repository has been published",
Finished: true,
},
"publishing": ObsBuildStatusDetail{
Code: "publishing",
Description: "Repository is being created right now",
Finished: true,
},
"unpublished": ObsBuildStatusDetail{
Code: "unpublished",
Description: "Build finished, but repository publishing is disabled",
Finished: true,
},
"building": ObsBuildStatusDetail{
Code: "building",
Description: "Build jobs exist for the repository",
Finished: false,
},
"finished": ObsBuildStatusDetail{
Code: "finished",
Description: "Build jobs have been processed, new repository is not yet created",
Finished: true,
},
"blocked": ObsBuildStatusDetail{
Code: "blocked",
Description: "No build possible at the moment, waiting for jobs in other repositories",
Finished: false,
},
"broken": ObsBuildStatusDetail{
Code: "broken",
Description: "The repository setup is broken, build or publish not possible",
Finished: true,
},
"scheduling": ObsBuildStatusDetail{
Code: "scheduling",
Description: "The repository state is being calculated right now",
Finished: false,
},
"unknown": ObsBuildStatusDetail{
Code: "unknown",
Description: "The repository state has not been seen by the scheduler yet",
Finished: false,
},
}
func parseBuildResults(data []byte) (*BuildResultList, error) {
result := BuildResultList{}
err := xml.Unmarshal(data, &result)
if err != nil {
return nil, err
}
return &result, nil
}
type ObsProjectNotFound struct {
Project string
}
func (obs ObsProjectNotFound) Error() string {
return fmt.Sprintf("OBS project is not found: %s", obs.Project)
}
func (c *ObsClient) ProjectConfig(project string) (string, error) {
res, err := c.ObsRequest("GET", []string{"source", project, "_config"}, nil)
if err != nil {
return "", err
}
if data, err := io.ReadAll(res.Body); err == nil {
defer res.Body.Close()
return string(data), nil
} else {
return "", err
}
}
func (c *ObsClient) BuildStatus(project string, packages ...string) (*BuildResultList, error) {
return c.BuildStatusWithState(project, &BuildResultOptions{}, packages...)
}
func (c *ObsClient) LastBuildResults(project string, packages ...string) (*BuildResultList, error) {
return c.BuildStatusWithState(project, &BuildResultOptions{LastBuild: true}, packages...)
}
func (c *ObsClient) BuildStatusWithState(project string, opts *BuildResultOptions, packages ...string) (*BuildResultList, error) {
u := c.baseUrl.JoinPath("build", project, "_result")
query := u.Query()
query.Add("view", "status")
if opts.BinaryList {
query.Add("view", "binarylist")
}
query.Add("multibuild", "1")
if len(opts.OldState) > 0 {
query.Add("oldstate", opts.OldState)
}
if opts.LastBuild {
query.Add("lastbuild", "1")
}
if len(packages) > 0 {
for _, pkg := range packages {
query.Add("package", pkg)
}
}
u.RawQuery = query.Encode()
res, err := c.ObsRequestRaw("GET", u.String(), nil)
if err != nil {
return nil, err
}
switch res.StatusCode {
case 200:
break
case 404:
return nil, ObsProjectNotFound{project}
default:
return nil, fmt.Errorf("Unexpected return code: %d", res.StatusCode)
}
data, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
defer res.Body.Close()
ret, err := parseBuildResults(data)
if ret != nil {
ret.isLastBuild = opts.LastBuild
}
return ret, err
}