diff --git a/common/config.go b/common/config.go index cc93820..5d014ad 100644 --- a/common/config.go +++ b/common/config.go @@ -59,6 +59,7 @@ type AutogitConfig struct { Reviewers []string // only used by `pr` workflow ReviewGroups []ReviewGroup Committers []string // group in addition to Reviewers and Maintainers that can order the bot around, mostly as helper for factory-maintainers + Subdirs []string // list of directories to sort submodules into. Needed b/c _manifest cannot list non-existent directories ManualMergeOnly bool // only merge with "Merge OK" comment by Project Maintainers and/or Package Maintainers and/or reviewers ManualMergeProject bool // require merge of ProjectGit PRs with "Merge OK" by ProjectMaintainers and/or reviewers diff --git a/common/obs_utils.go b/common/obs_utils.go index c611ec8..696541c 100644 --- a/common/obs_utils.go +++ b/common/obs_utils.go @@ -562,7 +562,21 @@ func (c *ObsClient) DeleteProject(project string) error { } return nil +} +func (c *ObsClient) BuildLog(prj, pkg, repo, arch string) (io.ReadCloser, error) { + url := c.baseUrl.JoinPath("build", prj, repo, arch, pkg, "_log") + query := url.Query() + query.Add("nostream", "1") + query.Add("start", "0") + url.RawQuery = query.Encode() + res, err := c.ObsRequestRaw("GET", url.String(), nil) + + if err != nil { + return nil, err + } + + return res.Body, nil } type PackageBuildStatus struct { diff --git a/common/listen.go b/common/rabbitmq.go similarity index 55% rename from common/listen.go rename to common/rabbitmq.go index 88b8540..4384717 100644 --- a/common/listen.go +++ b/common/rabbitmq.go @@ -22,55 +22,32 @@ import ( "crypto/tls" "fmt" "net/url" - "runtime/debug" - "slices" "strings" "time" rabbitmq "github.com/rabbitmq/amqp091-go" ) -const RequestType_CreateBrachTag = "create" -const RequestType_DeleteBranchTag = "delete" -const RequestType_Fork = "fork" -const RequestType_Issue = "issues" -const RequestType_IssueAssign = "issue_assign" -const RequestType_IssueComment = "issue_comment" -const RequestType_IssueLabel = "issue_label" -const RequestType_IssueMilestone = "issue_milestone" -const RequestType_Push = "push" -const RequestType_Repository = "repository" -const RequestType_Release = "release" -const RequestType_PR = "pull_request" -const RequestType_PRAssign = "pull_request_assign" -const RequestType_PRLabel = "pull_request_label" -const RequestType_PRComment = "pull_request_comment" -const RequestType_PRMilestone = "pull_request_milestone" -const RequestType_PRSync = "pull_request_sync" -const RequestType_PRReviewAccepted = "pull_request_review_approved" -const RequestType_PRReviewRejected = "pull_request_review_rejected" -const RequestType_PRReviewRequest = "pull_request_review_request" -const RequestType_PRReviewComment = "pull_request_review_comment" -const RequestType_Wiki = "wiki" - -type RequestProcessor interface { - ProcessFunc(*Request) error -} - -type ListenDefinitions struct { +type RabbitConnection struct { RabbitURL *url.URL // amqps://user:password@host/queue - GitAuthor string - Handlers map[string]RequestProcessor - Orgs []string + queueName string + ch *rabbitmq.Channel topics []string topicSubChanges chan string // +topic = subscribe, -topic = unsubscribe } +type RabbitProcessor interface { + GenerateTopics() []string + + Connection() *RabbitConnection + ProcessRabbitMessage(msg RabbitMessage) error +} + type RabbitMessage rabbitmq.Delivery -func (l *ListenDefinitions) processTopicChanges(ch *rabbitmq.Channel, queueName string) { +func (l *RabbitConnection) ProcessTopicChanges() { for { topic, ok := <-l.topicSubChanges if !ok { @@ -80,11 +57,11 @@ func (l *ListenDefinitions) processTopicChanges(ch *rabbitmq.Channel, queueName LogDebug(" topic change:", topic) switch topic[0] { case '+': - if err := ch.QueueBind(queueName, topic[1:], "pubsub", false, nil); err != nil { + if err := l.ch.QueueBind(l.queueName, topic[1:], "pubsub", false, nil); err != nil { LogError(err) } case '-': - if err := ch.QueueUnbind(queueName, topic[1:], "pubsub", nil); err != nil { + if err := l.ch.QueueUnbind(l.queueName, topic[1:], "pubsub", nil); err != nil { LogError(err) } default: @@ -93,7 +70,7 @@ func (l *ListenDefinitions) processTopicChanges(ch *rabbitmq.Channel, queueName } } -func (l *ListenDefinitions) processRabbitMQ(msgCh chan<- RabbitMessage) error { +func (l *RabbitConnection) ProcessRabbitMQ(msgCh chan<- RabbitMessage) error { queueName := l.RabbitURL.Path l.RabbitURL.Path = "" @@ -152,7 +129,7 @@ func (l *ListenDefinitions) processRabbitMQ(msgCh chan<- RabbitMessage) error { LogDebug(" -- listening to topics:") l.topicSubChanges = make(chan string) defer close(l.topicSubChanges) - go l.processTopicChanges(ch, q.Name) + go l.ProcessTopicChanges() for _, topic := range l.topics { l.topicSubChanges <- "+" + topic @@ -174,18 +151,18 @@ func (l *ListenDefinitions) processRabbitMQ(msgCh chan<- RabbitMessage) error { } } -func (l *ListenDefinitions) connectAndProcessRabbitMQ(ch chan<- RabbitMessage) { +func (l *RabbitConnection) ConnectAndProcessRabbitMQ(ch chan<- RabbitMessage) { defer func() { if r := recover(); r != nil { LogError(r) LogError("'crash' RabbitMQ worker. Recovering... reconnecting...") time.Sleep(5 * time.Second) - go l.connectAndProcessRabbitMQ(ch) + go l.ConnectAndProcessRabbitMQ(ch) } }() for { - err := l.processRabbitMQ(ch) + err := l.ProcessRabbitMQ(ch) if err != nil { LogError("Error in RabbitMQ connection. %#v", err) LogInfo("Reconnecting in 2 seconds...") @@ -194,49 +171,20 @@ func (l *ListenDefinitions) connectAndProcessRabbitMQ(ch chan<- RabbitMessage) { } } -func (l *ListenDefinitions) connectToRabbitMQ() chan RabbitMessage { +func (l *RabbitConnection) ConnectToRabbitMQ(processor RabbitProcessor) <-chan RabbitMessage { + LogInfo("RabbitMQ connection:", l.RabbitURL.String()) + + l.RabbitURL.User = url.UserPassword(rabbitUser, rabbitPassword) + l.topics = processor.GenerateTopics() + ch := make(chan RabbitMessage, 100) - go l.connectAndProcessRabbitMQ(ch) + go l.ConnectAndProcessRabbitMQ(ch) return ch } -func ProcessEvent(f RequestProcessor, request *Request) { - defer func() { - if r := recover(); r != nil { - LogError("panic caught") - if err, ok := r.(error); !ok { - LogError(err) - } - LogError(string(debug.Stack())) - } - }() - - if err := f.ProcessFunc(request); err != nil { - LogError(err) - } - -} - -func (l *ListenDefinitions) generateTopics() []string { - topics := make([]string, 0, len(l.Handlers)*len(l.Orgs)) - scope := "suse" - if l.RabbitURL.Hostname() == "rabbit.opensuse.org" { - scope = "opensuse" - } - - for _, org := range l.Orgs { - for requestType, _ := range l.Handlers { - topics = append(topics, fmt.Sprintf("%s.src.%s.%s.#", scope, org, requestType)) - } - } - - slices.Sort(topics) - return slices.Compact(topics) -} - -func (l *ListenDefinitions) UpdateTopics() { - newTopics := l.generateTopics() +func (l *RabbitConnection) UpdateTopics(processor RabbitProcessor) { + newTopics := processor.GenerateTopics() j := 0 next_new_topic: @@ -273,14 +221,8 @@ next_new_topic: l.topics = newTopics } -func (l *ListenDefinitions) ProcessRabbitMQEvents() error { - LogInfo("RabbitMQ connection:", l.RabbitURL.String()) - LogDebug("# Handlers:", len(l.Handlers)) - LogDebug("# Orgs:", len(l.Orgs)) - - l.RabbitURL.User = url.UserPassword(rabbitUser, rabbitPassword) - l.topics = l.generateTopics() - ch := l.connectToRabbitMQ() +func ProcessRabbitMQEvents(processor RabbitProcessor) error { + ch := processor.Connection().ConnectToRabbitMQ(processor) for { msg, ok := <-ch @@ -289,36 +231,8 @@ func (l *ListenDefinitions) ProcessRabbitMQEvents() error { } LogDebug("event:", msg.RoutingKey) - - route := strings.Split(msg.RoutingKey, ".") - if len(route) > 3 { - reqType := route[3] - org := route[2] - - if !slices.Contains(l.Orgs, org) { - LogInfo("Got event for unhandeled org:", org) - continue - } - - LogDebug("org:", org, "type:", reqType) - if handler, found := l.Handlers[reqType]; found { - /* h, err := CreateRequestHandler() - if err != nil { - log.Println("Cannot create request handler", err) - continue - } - */ - req, err := ParseRequestJSON(reqType, msg.Body) - if err != nil { - LogError("Error parsing request JSON:", err) - continue - } else { - LogDebug("processing req", req.Type) - // h.Request = req - ProcessEvent(handler, req) - - } - } + if err := processor.ProcessRabbitMessage(msg); err != nil { + LogError("Error processing", msg.RoutingKey, err) } } } diff --git a/common/rabbitmq_gitea.go b/common/rabbitmq_gitea.go new file mode 100644 index 0000000..4df1d0f --- /dev/null +++ b/common/rabbitmq_gitea.go @@ -0,0 +1,128 @@ +package common + +/* + * This file is part of Autogits. + * + * Copyright © 2024 SUSE LLC + * + * Autogits is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 2 of the License, or (at your option) any later + * version. + * + * Autogits is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * Foobar. If not, see . + */ + +import ( + "fmt" + "runtime/debug" + "slices" + "strings" +) + +const RequestType_CreateBrachTag = "create" +const RequestType_DeleteBranchTag = "delete" +const RequestType_Fork = "fork" +const RequestType_Issue = "issues" +const RequestType_IssueAssign = "issue_assign" +const RequestType_IssueComment = "issue_comment" +const RequestType_IssueLabel = "issue_label" +const RequestType_IssueMilestone = "issue_milestone" +const RequestType_Push = "push" +const RequestType_Repository = "repository" +const RequestType_Release = "release" +const RequestType_PR = "pull_request" +const RequestType_PRAssign = "pull_request_assign" +const RequestType_PRLabel = "pull_request_label" +const RequestType_PRComment = "pull_request_comment" +const RequestType_PRMilestone = "pull_request_milestone" +const RequestType_PRSync = "pull_request_sync" +const RequestType_PRReviewAccepted = "pull_request_review_approved" +const RequestType_PRReviewRejected = "pull_request_review_rejected" +const RequestType_PRReviewRequest = "pull_request_review_request" +const RequestType_PRReviewComment = "pull_request_review_comment" +const RequestType_Wiki = "wiki" + +type RequestProcessor interface { + ProcessFunc(*Request) error +} + +type RabbitMQGiteaEventsProcessor struct { + Handlers map[string]RequestProcessor + Orgs []string + + c *RabbitConnection +} + +func (gitea *RabbitMQGiteaEventsProcessor) Connection() *RabbitConnection { + if gitea.c == nil { + gitea.c = &RabbitConnection{} + } + return gitea.c +} + +func (gitea *RabbitMQGiteaEventsProcessor) GenerateTopics() []string { + topics := make([]string, 0, len(gitea.Handlers)*len(gitea.Orgs)) + scope := "suse" + if gitea.c.RabbitURL.Hostname() == "rabbit.opensuse.org" { + scope = "opensuse" + } + + for _, org := range gitea.Orgs { + for requestType, _ := range gitea.Handlers { + topics = append(topics, fmt.Sprintf("%s.src.%s.%s.#", scope, org, requestType)) + } + } + + slices.Sort(topics) + return slices.Compact(topics) +} + +func (gitea *RabbitMQGiteaEventsProcessor) ProcessRabbitMessage(msg RabbitMessage) error { + route := strings.Split(msg.RoutingKey, ".") + if len(route) > 3 { + reqType := route[3] + org := route[2] + + if !slices.Contains(gitea.Orgs, org) { + LogInfo("Got event for unhandeled org:", org) + return nil + } + + LogDebug("org:", org, "type:", reqType) + if handler, found := gitea.Handlers[reqType]; found { + req, err := ParseRequestJSON(reqType, msg.Body) + if err != nil { + LogError("Error parsing request JSON:", err) + return nil + } else { + LogDebug("processing req", req.Type) + // h.Request = req + ProcessEvent(handler, req) + } + } + } + + return fmt.Errorf("Invalid routing key: %s", route) +} + +func ProcessEvent(f RequestProcessor, request *Request) { + defer func() { + if r := recover(); r != nil { + LogError("panic caught") + if err, ok := r.(error); !ok { + LogError(err) + } + LogError(string(debug.Stack())) + } + }() + + if err := f.ProcessFunc(request); err != nil { + LogError(err) + } +} diff --git a/common/rabbitmq_obs.go b/common/rabbitmq_obs.go new file mode 100644 index 0000000..69b5825 --- /dev/null +++ b/common/rabbitmq_obs.go @@ -0,0 +1,20 @@ +package common + +type RabbitMQObsBuildStatusProcessor struct { + c *RabbitConnection +} + +func (o *RabbitMQObsBuildStatusProcessor) GenerateTopics() []string { +} + +func (o *RabbitMQObsBuildStatusProcessor) Connection() *RabbitConnection { + if o.c == nil { + o.c = &RabbitConnection{} + } + + return o.c +} + +func (o *RabbitMQObsBuildStatusProcessor) ProcessRabbitMessage(msg RabbitMessage) error { +} + diff --git a/common/listen_test.go b/common/rabbitmq_test.go similarity index 84% rename from common/listen_test.go rename to common/rabbitmq_test.go index ed0dc78..48b7412 100644 --- a/common/listen_test.go +++ b/common/rabbitmq_test.go @@ -50,11 +50,13 @@ func TestListenDefinitionsTopicUpdate(t *testing.T) { u, _ := url.Parse("amqps://rabbit.example.com") for _, test := range tests { t.Run(test.name, func(t *testing.T) { - l := ListenDefinitions{ - Orgs: test.orgs1, - Handlers: make(map[string]RequestProcessor), - topicSubChanges: make(chan string, len(test.topicDelta)*10), - RabbitURL: u, + l := &RabbitMQGiteaEventsProcessor{ + Orgs: test.orgs1, + Handlers: make(map[string]RequestProcessor), + c: &RabbitConnection{ + RabbitURL: u, + topicSubChanges: make(chan string, len(test.topicDelta)*10), + }, } slices.Sort(test.topicDelta) @@ -64,11 +66,11 @@ func TestListenDefinitionsTopicUpdate(t *testing.T) { } changes := []string{} - l.UpdateTopics() + l.c.UpdateTopics(l) a: for { select { - case c := <-l.topicSubChanges: + case c := <-l.c.topicSubChanges: changes = append(changes, c) default: changes = []string{} @@ -78,13 +80,13 @@ func TestListenDefinitionsTopicUpdate(t *testing.T) { l.Orgs = test.orgs2 - l.UpdateTopics() + l.c.UpdateTopics(l) changes = []string{} b: for { select { - case c := <-l.topicSubChanges: + case c := <-l.c.topicSubChanges: changes = append(changes, c) default: slices.Sort(changes) diff --git a/common/submodules.go b/common/submodules.go index 06bf910..680368c 100644 --- a/common/submodules.go +++ b/common/submodules.go @@ -113,6 +113,10 @@ func (s *Submodule) parseKeyValue(line string) error { return nil } +func (s *Submodule) ManifestSubmodulePath(manifest *Manifest) string { + return manifest.SubdirForPackage(s.Path) +} + func ParseSubmodulesFile(reader io.Reader) ([]Submodule, error) { data, err := io.ReadAll(reader) if err != nil { diff --git a/obs-status-service/main.go b/obs-status-service/main.go index 8ec1d35..b0b65e0 100644 --- a/obs-status-service/main.go +++ b/obs-status-service/main.go @@ -22,6 +22,7 @@ import ( "bytes" "flag" "fmt" + "io" "log" "net/http" @@ -78,7 +79,7 @@ func ProjectStatusSummarySvg(project string) []byte { return ret.Bytes() } -func PackageStatusSummarySvg(status common.PackageBuildStatus) []byte { +func PackageStatusSummarySvg(status *common.PackageBuildStatus) []byte { buildStatus, ok := common.ObsBuildStatusDetails[status.Code] if !ok { buildStatus = common.ObsBuildStatusDetails["error"] @@ -108,10 +109,13 @@ func main() { key := flag.String("key-file", "", "Private key for the TLS certificate") listen := flag.String("listen", "[::1]:8080", "Listening string") disableTls := flag.Bool("no-tls", false, "Disable TLS") - obsHost := flag.String("obs-host", "api.opensuse.org", "OBS API endpoint for package status information") + obsHost := flag.String("obs-host", "https://api.opensuse.org", "OBS API endpoint for package status information") flag.BoolVar(&debug, "debug", false, "Enable debug logging") + RabbitMQHost := flag.String("rabbit-mq", "amqps://rabbit.opensuse.org", "RabbitMQ message bus server") + Topic := flag.String("topic", "opensuse.obs", "RabbitMQ topic prefix") flag.Parse() + common.PanicOnError(common.RequireObsSecretToken()) var err error @@ -143,21 +147,25 @@ func main() { res.Header().Add("content-type", "image/svg+xml") - prjStatus := GetCurrentStatus(prj) - if prjStatus == nil { + status := GetDetailedBuildStatus(prj, pkg, repo, arch) + res.Write(PackageStatusSummarySvg(status)) + }) + http.HandleFunc("GET /{Project}/{Package}/{Repository}/{Arch}/buildlog", func(res http.ResponseWriter, req *http.Request) { + prj := req.PathValue("Project") + pkg := req.PathValue("Package") + repo := req.PathValue("Repository") + arch := req.PathValue("Arch") + +// status := GetDetailedBuildStatus(prj, pkg, repo, arch) + data, err := obs.BuildLog(prj, pkg, repo, arch) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + common.LogError("Failed to fetch build log for:", prj, pkg, repo, arch, err) return } - - for _, r := range prjStatus.Result { - if r.Arch == arch && r.Repository == repo { - for _, status := range r.Status { - if status.Package == pkg { - res.Write(PackageStatusSummarySvg(status)) - return - } - } - } - } + defer data.Close() + + io.Copy(res, data) }) go ProcessUpdates() diff --git a/obs-status-service/status.go b/obs-status-service/status.go index bf83770..5d097e9 100644 --- a/obs-status-service/status.go +++ b/obs-status-service/status.go @@ -34,6 +34,25 @@ func GetCurrentStatus(project string) *common.BuildResultList { } } +func GetDetailedBuildStatus(prj, pkg, repo, arch string) *common.PackageBuildStatus { + prjStatus := GetCurrentStatus(prj) + if prjStatus == nil { + return nil + } + + for _, r := range prjStatus.Result { + if r.Arch == arch && r.Repository == repo { + for _, status := range r.Status { + if status.Package == pkg { + return &status + } + } + } + } + + return nil +} + func ProcessUpdates() { for { msg := <-StatusUpdateCh diff --git a/workflow-direct/main.go b/workflow-direct/main.go index 0e89933..4fe532c 100644 --- a/workflow-direct/main.go +++ b/workflow-direct/main.go @@ -526,7 +526,7 @@ func main() { log.Fatal(err) } - var defs common.ListenDefinitions + defs := &common.RabbitMQGiteaEventsProcessor{} var err error if len(*basePath) == 0 { @@ -557,7 +557,7 @@ func main() { } log.Println("*** Reconfiguring ***") updateConfiguration(*configFilename, &defs.Orgs) - defs.UpdateTopics() + defs.Connection().UpdateTopics(defs) } }() signal.Notify(signalChannel, syscall.SIGHUP) @@ -573,18 +573,17 @@ func main() { updateConfiguration(*configFilename, &defs.Orgs) - defs.GitAuthor = GitAuthor - defs.RabbitURL, err = url.Parse(*rabbitUrl) + defs.Connection().RabbitURL, err = url.Parse(*rabbitUrl) if err != nil { log.Panicf("cannot parse server URL. Err: %#v\n", err) } go consistencyCheckProcess() - log.Println("defs:", defs) + log.Println("defs:", *defs) defs.Handlers = make(map[string]common.RequestProcessor) defs.Handlers[common.RequestType_Push] = &PushActionProcessor{} defs.Handlers[common.RequestType_Repository] = &RepositoryActionProcessor{} - log.Fatal(defs.ProcessRabbitMQEvents()) + log.Fatal(common.ProcessRabbitMQEvents(defs)) } diff --git a/workflow-pr/main.go b/workflow-pr/main.go index 219ee1f..38a4168 100644 --- a/workflow-pr/main.go +++ b/workflow-pr/main.go @@ -162,9 +162,9 @@ func main() { checker := CreateDefaultStateChecker(*checkOnStart, req, Gitea, time.Duration(*checkIntervalHours)*time.Hour) go checker.ConsistencyCheckProcess() - listenDefs := common.ListenDefinitions{ + listenDefs := &common.RabbitMQGiteaEventsProcessor{ Orgs: orgs, - GitAuthor: GitAuthor, +// GitAuthor: GitAuthor, Handlers: map[string]common.RequestProcessor{ common.RequestType_PR: req, common.RequestType_PRSync: req, @@ -172,7 +172,7 @@ func main() { common.RequestType_PRReviewRejected: req, }, } - listenDefs.RabbitURL, _ = url.Parse(*rabbitUrl) + listenDefs.Connection().RabbitURL, _ = url.Parse(*rabbitUrl) - common.PanicOnError(listenDefs.ProcessRabbitMQEvents()) + common.PanicOnError(common.ProcessRabbitMQEvents(listenDefs)) }