From 907e7be5450b10a9ebb9270adb7ac9ba5d6a9909 Mon Sep 17 00:00:00 2001 From: olegburov Date: Mon, 31 Aug 2020 17:26:57 -0700 Subject: [PATCH] Bump Logstash hook for logrus to `v1.0.0`. Signed-off-by: olegburov --- go.mod | 2 +- go.sum | 4 +- registry/registry.go | 3 +- .../logrus-logstash-hook/.travis.yml | 13 +- .../logrus-logstash-hook/CHANGELOG.md | 6 + .../logrus-logstash-hook/README.md | 76 ++-------- .../logrus-logstash-hook/hook.go | 126 +++++++++++++++++ .../logrus-logstash-hook/logstash.go | 133 ------------------ .../logstash_formatter.go | 81 ----------- vendor/modules.txt | 2 +- 10 files changed, 157 insertions(+), 289 deletions(-) create mode 100644 vendor/github.com/bshuster-repo/logrus-logstash-hook/hook.go delete mode 100644 vendor/github.com/bshuster-repo/logrus-logstash-hook/logstash.go delete mode 100644 vendor/github.com/bshuster-repo/logrus-logstash-hook/logstash_formatter.go diff --git a/go.mod b/go.mod index 61d7ec16..1cc54a45 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/aws/aws-sdk-go v1.34.9 github.com/bitly/go-simplejson v0.5.0 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect - github.com/bshuster-repo/logrus-logstash-hook v0.4.1 + github.com/bshuster-repo/logrus-logstash-hook v1.0.0 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect diff --git a/go.sum b/go.sum index 284c0cf8..4e9ff1d1 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkN github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/bshuster-repo/logrus-logstash-hook v0.4.1 h1:pgAtgj+A31JBVtEHu2uHuEx0n+2ukqUJnS2vVe5pQNA= -github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= +github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ= diff --git a/registry/registry.go b/registry/registry.go index 95dc3caa..8f891b17 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -18,6 +18,7 @@ import ( "github.com/bugsnag/bugsnag-go" "github.com/docker/go-metrics" gorhandlers "github.com/gorilla/handlers" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/yvasiyarov/gorelic" @@ -285,7 +286,7 @@ func configureLogging(ctx context.Context, config *configuration.Configuration) }) case "logstash": log.SetFormatter(&logstash.LogstashFormatter{ - TimestampFormat: time.RFC3339Nano, + Formatter: &logrus.JSONFormatter{TimestampFormat: time.RFC3339Nano}, }) default: // just let the library use default on empty string. diff --git a/vendor/github.com/bshuster-repo/logrus-logstash-hook/.travis.yml b/vendor/github.com/bshuster-repo/logrus-logstash-hook/.travis.yml index 60c00ef6..0df15979 100644 --- a/vendor/github.com/bshuster-repo/logrus-logstash-hook/.travis.yml +++ b/vendor/github.com/bshuster-repo/logrus-logstash-hook/.travis.yml @@ -1,13 +1,10 @@ language: go sudo: false -matrix: - include: - - go: 1.3 - - go: 1.4 - - go: 1.5 - - go: 1.6 - - go: tip +go: + - "1.11.x" + - "1.12.x" + - "tip" install: - # Skip @@ -15,5 +12,5 @@ install: script: - go get -t -v ./... - diff -u <(echo -n) <(gofmt -d .) - - go tool vet . + - go vet . - go test -v -race ./... diff --git a/vendor/github.com/bshuster-repo/logrus-logstash-hook/CHANGELOG.md b/vendor/github.com/bshuster-repo/logrus-logstash-hook/CHANGELOG.md index 31c8b5f3..293d799a 100644 --- a/vendor/github.com/bshuster-repo/logrus-logstash-hook/CHANGELOG.md +++ b/vendor/github.com/bshuster-repo/logrus-logstash-hook/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 1.0 + + * Remove the old API: `NewConnWith`, `WithPrefix` and etc and move to a simple `New` function. + * Prefix is no longer supported in this package. + * Change the Hook structure to have only two members: `logrus.Formatter` and `io.Writer`. + ## 0.4 * Update the name of the package from `logrus_logstash` to `logrustash` diff --git a/vendor/github.com/bshuster-repo/logrus-logstash-hook/README.md b/vendor/github.com/bshuster-repo/logrus-logstash-hook/README.md index 9cc4378c..f5cf1638 100644 --- a/vendor/github.com/bshuster-repo/logrus-logstash-hook/README.md +++ b/vendor/github.com/bshuster-repo/logrus-logstash-hook/README.md @@ -1,30 +1,35 @@ -# Logstash hook for logrus :walrus: [![Build Status](https://travis-ci.org/bshuster-repo/logrus-logstash-hook.svg?branch=master)](https://travis-ci.org/bshuster-repo/logrus-logstash-hook) -Use this hook to send the logs to [Logstash](https://www.elastic.co/products/logstash) over both UDP and TCP. +# Logstash hook for logrus :walrus: +[![Build Status](https://travis-ci.org/bshuster-repo/logrus-logstash-hook.svg?branch=master)](https://travis-ci.org/bshuster-repo/logrus-logstash-hook) +[![Go Report Status](https://goreportcard.com/badge/github.com/bshuster-repo/logrus-logstash-hook)](https://goreportcard.com/report/github.com/bshuster-repo/logrus-logstash-hook) -## Usage +Use this hook to send the logs to [Logstash](https://www.elastic.co/products/logstash). + +# Usage ```go package main import ( - "github.com/sirupsen/logrus" "github.com/bshuster-repo/logrus-logstash-hook" + "github.com/sirupsen/logrus" + "net" ) func main() { log := logrus.New() - hook, err := logrustash.NewHook("tcp", "172.17.0.2:9999", "myappName") - + conn, err := net.Dial("tcp", "logstash.mycompany.net:8911") if err != nil { log.Fatal(err) } + hook := logrustash.New(conn, logrustash.DefaultFormatter(logrus.Fields{"type": "myappName"})) + log.Hooks.Add(hook) ctx := log.WithFields(logrus.Fields{ - "method": "main", + "method": "main", }) - ... ctx.Info("Hello World!") } + ``` This is how it will look like: @@ -41,61 +46,8 @@ This is how it will look like: "type" => "myappName" } ``` -## Hook Fields -Fields can be added to the hook, which will always be in the log context. -This can be done when creating the hook: -```go - -hook, err := logrustash.NewHookWithFields("tcp", "172.17.0.2:9999", "myappName", logrus.Fields{ - "hostname": os.Hostname(), - "serviceName": "myServiceName", -}) -``` - -Or afterwards: - -```go - -hook.WithFields(logrus.Fields{ - "hostname": os.Hostname(), - "serviceName": "myServiceName", -}) -``` -This allows you to set up the hook so logging is available immediately, and add important fields as they become available. - -Single fields can be added/updated using 'WithField': - -```go - -hook.WithField("status", "running") -``` - - - -## Field prefix - -The hook allows you to send logging to logstash and also retain the default std output in text format. -However to keep this console output readable some fields might need to be omitted from the default non-hooked log output. -Each hook can be configured with a prefix used to identify fields which are only to be logged to the logstash connection. -For example if you don't want to see the hostname and serviceName on each log line in the console output you can add a prefix: - -```go - - -hook, err := logrustash.NewHookWithFields("tcp", "172.17.0.2:9999", "myappName", logrus.Fields{ - "_hostname": os.Hostname(), - "_serviceName": "myServiceName", -}) -... -hook.WithPrefix("_") -``` - -There are also constructors available which allow you to specify the prefix from the start. -The std-out will not have the '\_hostname' and '\_servicename' fields, and the logstash output will, but the prefix will be dropped from the name. - - -# Authors +# Maintainers Name | Github | Twitter | ------------ | --------- | ---------- | diff --git a/vendor/github.com/bshuster-repo/logrus-logstash-hook/hook.go b/vendor/github.com/bshuster-repo/logrus-logstash-hook/hook.go new file mode 100644 index 00000000..cd229a75 --- /dev/null +++ b/vendor/github.com/bshuster-repo/logrus-logstash-hook/hook.go @@ -0,0 +1,126 @@ +package logrustash + +import ( + "io" + "sync" + + "github.com/sirupsen/logrus" +) + +// Hook represents a Logstash hook. +// It has two fields: writer to write the entry to Logstash and +// formatter to format the entry to a Logstash format before sending. +// +// To initialize it use the `New` function. +// +type Hook struct { + writer io.Writer + formatter logrus.Formatter +} + +// New returns a new logrus.Hook for Logstash. +// +// To create a new hook that sends logs to `tcp://logstash.corp.io:9999`: +// +// conn, _ := net.Dial("tcp", "logstash.corp.io:9999") +// hook := logrustash.New(conn, logrustash.DefaultFormatter()) +func New(w io.Writer, f logrus.Formatter) logrus.Hook { + return Hook{ + writer: w, + formatter: f, + } +} + +// Fire takes, formats and sends the entry to Logstash. +// Hook's formatter is used to format the entry into Logstash format +// and Hook's writer is used to write the formatted entry to the Logstash instance. +func (h Hook) Fire(e *logrus.Entry) error { + dataBytes, err := h.formatter.Format(e) + if err != nil { + return err + } + _, err = h.writer.Write(dataBytes) + return err +} + +// Levels returns all logrus levels. +func (h Hook) Levels() []logrus.Level { + return logrus.AllLevels +} + +// Using a pool to re-use of old entries when formatting Logstash messages. +// It is used in the Fire function. +var entryPool = sync.Pool{ + New: func() interface{} { + return &logrus.Entry{} + }, +} + +// copyEntry copies the entry `e` to a new entry and then adds all the fields in `fields` that are missing in the new entry data. +// It uses `entryPool` to re-use allocated entries. +func copyEntry(e *logrus.Entry, fields logrus.Fields) *logrus.Entry { + ne := entryPool.Get().(*logrus.Entry) + ne.Message = e.Message + ne.Level = e.Level + ne.Time = e.Time + ne.Data = logrus.Fields{} + for k, v := range fields { + ne.Data[k] = v + } + for k, v := range e.Data { + ne.Data[k] = v + } + return ne +} + +// releaseEntry puts the given entry back to `entryPool`. It must be called if copyEntry is called. +func releaseEntry(e *logrus.Entry) { + entryPool.Put(e) +} + +// LogstashFormatter represents a Logstash format. +// It has logrus.Formatter which formats the entry and logrus.Fields which +// are added to the JSON message if not given in the entry data. +// +// Note: use the `DefaultFormatter` function to set a default Logstash formatter. +type LogstashFormatter struct { + logrus.Formatter + logrus.Fields +} + +var ( + logstashFields = logrus.Fields{"@version": "1", "type": "log"} + logstashFieldMap = logrus.FieldMap{ + logrus.FieldKeyTime: "@timestamp", + logrus.FieldKeyMsg: "message", + } +) + +// DefaultFormatter returns a default Logstash formatter: +// A JSON format with "@version" set to "1" (unless set differently in `fields`, +// "type" to "log" (unless set differently in `fields`), +// "@timestamp" to the log time and "message" to the log message. +// +// Note: to set a different configuration use the `LogstashFormatter` structure. +func DefaultFormatter(fields logrus.Fields) logrus.Formatter { + for k, v := range logstashFields { + if _, ok := fields[k]; !ok { + fields[k] = v + } + } + + return LogstashFormatter{ + Formatter: &logrus.JSONFormatter{FieldMap: logstashFieldMap}, + Fields: fields, + } +} + +// Format formats an entry to a Logstash format according to the given Formatter and Fields. +// +// Note: the given entry is copied and not changed during the formatting process. +func (f LogstashFormatter) Format(e *logrus.Entry) ([]byte, error) { + ne := copyEntry(e, f.Fields) + dataBytes, err := f.Formatter.Format(ne) + releaseEntry(ne) + return dataBytes, err +} diff --git a/vendor/github.com/bshuster-repo/logrus-logstash-hook/logstash.go b/vendor/github.com/bshuster-repo/logrus-logstash-hook/logstash.go deleted file mode 100644 index 1f2e5a0c..00000000 --- a/vendor/github.com/bshuster-repo/logrus-logstash-hook/logstash.go +++ /dev/null @@ -1,133 +0,0 @@ -package logrustash - -import ( - "net" - "strings" - - "github.com/sirupsen/logrus" -) - -// Hook represents a connection to a Logstash instance -type Hook struct { - conn net.Conn - appName string - alwaysSentFields logrus.Fields - hookOnlyPrefix string - TimeFormat string -} - -// NewHook creates a new hook to a Logstash instance, which listens on -// `protocol`://`address`. -func NewHook(protocol, address, appName string) (*Hook, error) { - return NewHookWithFields(protocol, address, appName, make(logrus.Fields)) -} - -// NewHookWithConn creates a new hook to a Logstash instance, using the supplied connection -func NewHookWithConn(conn net.Conn, appName string) (*Hook, error) { - return NewHookWithFieldsAndConn(conn, appName, make(logrus.Fields)) -} - -// NewHookWithFields creates a new hook to a Logstash instance, which listens on -// `protocol`://`address`. alwaysSentFields will be sent with every log entry. -func NewHookWithFields(protocol, address, appName string, alwaysSentFields logrus.Fields) (*Hook, error) { - return NewHookWithFieldsAndPrefix(protocol, address, appName, alwaysSentFields, "") -} - -// NewHookWithFieldsAndPrefix creates a new hook to a Logstash instance, which listens on -// `protocol`://`address`. alwaysSentFields will be sent with every log entry. prefix is used to select fields to filter -func NewHookWithFieldsAndPrefix(protocol, address, appName string, alwaysSentFields logrus.Fields, prefix string) (*Hook, error) { - conn, err := net.Dial(protocol, address) - if err != nil { - return nil, err - } - return NewHookWithFieldsAndConnAndPrefix(conn, appName, alwaysSentFields, prefix) -} - -// NewHookWithFieldsAndConn creates a new hook to a Logstash instance using the supplied connection -func NewHookWithFieldsAndConn(conn net.Conn, appName string, alwaysSentFields logrus.Fields) (*Hook, error) { - return NewHookWithFieldsAndConnAndPrefix(conn, appName, alwaysSentFields, "") -} - -//NewHookWithFieldsAndConnAndPrefix creates a new hook to a Logstash instance using the suppolied connection and prefix -func NewHookWithFieldsAndConnAndPrefix(conn net.Conn, appName string, alwaysSentFields logrus.Fields, prefix string) (*Hook, error) { - return &Hook{conn: conn, appName: appName, alwaysSentFields: alwaysSentFields, hookOnlyPrefix: prefix}, nil -} - -//NewFilterHook makes a new hook which does not forward to logstash, but simply enforces the prefix rules -func NewFilterHook() *Hook { - return NewFilterHookWithPrefix("") -} - -//NewFilterHookWithPrefix make a new hook which does not forward to logstash, but simply enforces the specified prefix -func NewFilterHookWithPrefix(prefix string) *Hook { - return &Hook{conn: nil, appName: "", alwaysSentFields: make(logrus.Fields), hookOnlyPrefix: prefix} -} - -func (h *Hook) filterHookOnly(entry *logrus.Entry) { - if h.hookOnlyPrefix != "" { - for key := range entry.Data { - if strings.HasPrefix(key, h.hookOnlyPrefix) { - delete(entry.Data, key) - } - } - } - -} - -//WithPrefix sets a prefix filter to use in all subsequent logging -func (h *Hook) WithPrefix(prefix string) { - h.hookOnlyPrefix = prefix -} - -func (h *Hook) WithField(key string, value interface{}) { - h.alwaysSentFields[key] = value -} - -func (h *Hook) WithFields(fields logrus.Fields) { - //Add all the new fields to the 'alwaysSentFields', possibly overwriting exising fields - for key, value := range fields { - h.alwaysSentFields[key] = value - } -} - -func (h *Hook) Fire(entry *logrus.Entry) error { - //make sure we always clear the hookonly fields from the entry - defer h.filterHookOnly(entry) - - // Add in the alwaysSentFields. We don't override fields that are already set. - for k, v := range h.alwaysSentFields { - if _, inMap := entry.Data[k]; !inMap { - entry.Data[k] = v - } - } - - //For a filteringHook, stop here - if h.conn == nil { - return nil - } - - formatter := LogstashFormatter{Type: h.appName} - if h.TimeFormat != "" { - formatter.TimestampFormat = h.TimeFormat - } - - dataBytes, err := formatter.FormatWithPrefix(entry, h.hookOnlyPrefix) - if err != nil { - return err - } - if _, err = h.conn.Write(dataBytes); err != nil { - return err - } - return nil -} - -func (h *Hook) Levels() []logrus.Level { - return []logrus.Level{ - logrus.PanicLevel, - logrus.FatalLevel, - logrus.ErrorLevel, - logrus.WarnLevel, - logrus.InfoLevel, - logrus.DebugLevel, - } -} diff --git a/vendor/github.com/bshuster-repo/logrus-logstash-hook/logstash_formatter.go b/vendor/github.com/bshuster-repo/logrus-logstash-hook/logstash_formatter.go deleted file mode 100644 index 64bc5c38..00000000 --- a/vendor/github.com/bshuster-repo/logrus-logstash-hook/logstash_formatter.go +++ /dev/null @@ -1,81 +0,0 @@ -package logrustash - -import ( - "encoding/json" - "fmt" - "strings" - "time" - - "github.com/sirupsen/logrus" -) - -// Formatter generates json in logstash format. -// Logstash site: http://logstash.net/ -type LogstashFormatter struct { - Type string // if not empty use for logstash type field. - - // TimestampFormat sets the format used for timestamps. - TimestampFormat string -} - -func (f *LogstashFormatter) Format(entry *logrus.Entry) ([]byte, error) { - return f.FormatWithPrefix(entry, "") -} - -func (f *LogstashFormatter) FormatWithPrefix(entry *logrus.Entry, prefix string) ([]byte, error) { - fields := make(logrus.Fields) - for k, v := range entry.Data { - //remvove the prefix when sending the fields to logstash - if prefix != "" && strings.HasPrefix(k, prefix) { - k = strings.TrimPrefix(k, prefix) - } - - switch v := v.(type) { - case error: - // Otherwise errors are ignored by `encoding/json` - // https://github.com/Sirupsen/logrus/issues/377 - fields[k] = v.Error() - default: - fields[k] = v - } - } - - fields["@version"] = "1" - - timeStampFormat := f.TimestampFormat - - if timeStampFormat == "" { - timeStampFormat = time.RFC3339 - } - - fields["@timestamp"] = entry.Time.Format(timeStampFormat) - - // set message field - v, ok := entry.Data["message"] - if ok { - fields["fields.message"] = v - } - fields["message"] = entry.Message - - // set level field - v, ok = entry.Data["level"] - if ok { - fields["fields.level"] = v - } - fields["level"] = entry.Level.String() - - // set type field - if f.Type != "" { - v, ok = entry.Data["type"] - if ok { - fields["fields.type"] = v - } - fields["type"] = f.Type - } - - serialized, err := json.Marshal(fields) - if err != nil { - return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) - } - return append(serialized, '\n'), nil -} diff --git a/vendor/modules.txt b/vendor/modules.txt index fd1138fd..220a0f3b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -57,7 +57,7 @@ github.com/aws/aws-sdk-go/service/sts github.com/aws/aws-sdk-go/service/sts/stsiface # github.com/beorn7/perks v1.0.1 github.com/beorn7/perks/quantile -# github.com/bshuster-repo/logrus-logstash-hook v0.4.1 +# github.com/bshuster-repo/logrus-logstash-hook v1.0.0 github.com/bshuster-repo/logrus-logstash-hook # github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd github.com/bugsnag/bugsnag-go