From 36f3820a4c0f9ec4e615c3481aa335f9ca3f9fbc836797590cab03c65616a3f3 Mon Sep 17 00:00:00 2001 From: Adam Majer Date: Tue, 13 Aug 2024 16:42:20 +0200 Subject: [PATCH] . --- bots-common/listen.go | 2 +- gitea-events-rabbitmq-publisher/README.md | 17 ++++ gitea-events-rabbitmq-publisher/go.mod | 39 +++++++++ gitea-events-rabbitmq-publisher/go.sum | 81 +++++++++++++++++++ gitea-events-rabbitmq-publisher/main.go | 45 +++++++++++ gitea-events-rabbitmq-publisher/rabbitmq.go | 90 +++++++++++++++++++++ 6 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 gitea-events-rabbitmq-publisher/README.md create mode 100644 gitea-events-rabbitmq-publisher/go.mod create mode 100644 gitea-events-rabbitmq-publisher/go.sum create mode 100644 gitea-events-rabbitmq-publisher/main.go create mode 100644 gitea-events-rabbitmq-publisher/rabbitmq.go diff --git a/bots-common/listen.go b/bots-common/listen.go index 846229c..e58ef0e 100644 --- a/bots-common/listen.go +++ b/bots-common/listen.go @@ -29,7 +29,7 @@ func StartServerWithAddress(listenDefs ListenDefinitions, addr string) { http.HandleFunc("/"+listenDefs.Url, func(res http.ResponseWriter, req *http.Request) { h := CreateRequestHandler(listenDefs.GitAuthor, listenDefs.Url) - // defer h.Close() + defer h.Close() if len(req.Header.Get("Content-Type")) == 0 || req.Header["Content-Type"][0] != "application/json" || diff --git a/gitea-events-rabbitmq-publisher/README.md b/gitea-events-rabbitmq-publisher/README.md new file mode 100644 index 0000000..a03948c --- /dev/null +++ b/gitea-events-rabbitmq-publisher/README.md @@ -0,0 +1,17 @@ +gitea-events-rabbitmq-publisher +============= + +Publishes all events that go to a webhook to RabbitMQ instance. + +Areas of Responsibility +----------------------- + +* Listen for webhook events from Gitea +* Forward them to RabbitMQ + + queue events locally if rabbit is temporarily not working + + +Target Usage +------------ + +Any projects that requires reliable messanging diff --git a/gitea-events-rabbitmq-publisher/go.mod b/gitea-events-rabbitmq-publisher/go.mod new file mode 100644 index 0000000..f58ddb8 --- /dev/null +++ b/gitea-events-rabbitmq-publisher/go.mod @@ -0,0 +1,39 @@ +module src.opensuse.org/autogits/obs-staging-bot + +go 1.22.3 + +replace src.opensuse.org/autogits/common => ../bots-common + +require ( + github.com/mattn/go-sqlite3 v1.14.22 + github.com/rabbitmq/amqp091-go v1.10.0 + src.opensuse.org/autogits/common v0.0.0-00010101000000-000000000000 +) + +require ( + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/runtime v0.28.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/validate v0.24.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + go.mongodb.org/mongo-driver v1.14.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/sync v0.7.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/gitea-events-rabbitmq-publisher/go.sum b/gitea-events-rabbitmq-publisher/go.sum new file mode 100644 index 0000000..86afe29 --- /dev/null +++ b/gitea-events-rabbitmq-publisher/go.sum @@ -0,0 +1,81 @@ +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= +github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ= +github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= +go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/gitea-events-rabbitmq-publisher/main.go b/gitea-events-rabbitmq-publisher/main.go new file mode 100644 index 0000000..95af583 --- /dev/null +++ b/gitea-events-rabbitmq-publisher/main.go @@ -0,0 +1,45 @@ +package main + +import ( + "log" + "net/http" + + "src.opensuse.org/autogits/common" +) + +const ( + ListenAddr = "[::1]:8001" + RabbitForwarderPath = "rabbitmq-forwarder" +) + +func main() { + common.RequireGiteaSecretToken() + + http.HandleFunc("/"+RabbitForwarderPath, func(res http.ResponseWriter, req *http.Request) { + if len(req.Header.Get("Content-Type")) == 0 || + req.Header["Content-Type"][0] != "application/json" || + req.Method != "POST" { + + res.WriteHeader(http.StatusInternalServerError) + return + } + + hdr := req.Header[common.GiteaRequestHeader] + if len(hdr) != 1 { + // h.LogError("Unsupported number of %s headers: %d: %#v\n", GiteaRequestHeader, len(hdr), hdr) + // h.WriteError() + res.WriteHeader(http.StatusInternalServerError) + return + } + reqType := hdr[0] + err := PublicMessage(reqType, req.Body) + if err != nil { + log.Printf("hook (%s) processing error: %v\n", reqType, err) + res.WriteHeader(http.StatusBadRequest) + } else { + res.WriteHeader(http.StatusOK) + } + }) + + log.Fatal(http.ListenAndServe(ListenAddr, nil)) +} diff --git a/gitea-events-rabbitmq-publisher/rabbitmq.go b/gitea-events-rabbitmq-publisher/rabbitmq.go new file mode 100644 index 0000000..62e595a --- /dev/null +++ b/gitea-events-rabbitmq-publisher/rabbitmq.go @@ -0,0 +1,90 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io" + "log" + "time" + + _ "github.com/mattn/go-sqlite3" + rabbitmq "github.com/rabbitmq/amqp091-go" +) + +func failOnError(err error, msg string) { + if err != nil { + log.Panicf("%s: %s", err, msg) + } +} + +type Message struct { + Topic string + Body string +} + +var messageQueue chan Messag + +func PublicMessage(giteaWebhookType string, msgBody io.Reader) error { + data, err := io.ReadAll(msgBody) + if !json.Valid(data) { + return fmt.Errorf("Invalid JSON in request") + } + + messageQueue <- Message { + Topic: "opensuse.gitea." + giteaWebhookType + "." + + } +} + +func ConnectToExchangeForPublish(host, username, password string) { + defer func() { + if r := recover(); r != nil { + log.Print("recovering... reconnecting...\n") + time.Sleep(5 * time.Second) + go ConnectToExchangeForPublish(host, username, password) + } + }() + if messageQueue == nil { + messageQueue = make(chan Message, 10240) + } + + auth := "" + if len(username) > 0 && len(password) > 0 { + auth = username + ":" + password + "@" + } + + connection, err := rabbitmq.DialTLS("amqps://"+auth+host, &tls.Config{ + ServerName: host, + }) + failOnError(err, "Cannot connect to rabbit.opensuse.org") + defer connection.Close() + + ch, err := connection.Channel() + failOnError(err, "Cannot create a channel") + defer ch.Close() + + err = ch.ExchangeDeclarePassive("pubsub", "topic", true, false, false, false, nil) + failOnError(err, "Cannot find pubsub exchange") + + for { + msg, ok := <-messageQueue + if !ok { + log.Printf("channel/connection closed?\n") + + if connection.IsClosed() { + // reconnect + log.Printf("reconnecting...") + time.Sleep(5 * time.Second) + go ConnectToExchangeForPublish(host, username, password) + } + return + } + + ch.Publish("pubsub", msg.Topic, false, false, rabbitmq.Publishing { + ContentType: "application/json", + AppId: RabbitForwarderPath, + Body: []byte(msg.Body), + }) + } +} +