rpm/golang-github-prometheus-prometheus
SHA256
1
0
Fork 0
forked from pool/golang-github-prometheus-prometheus
Code Pull Requests Activity

Accepting request 750102 from server:monitoring

Browse Source 
OBS-URL: https://build.opensuse.org/request/show/750102
OBS-URL: https://build.opensuse.org/package/show/openSUSE:Factory/golang-github-prometheus-prometheus?expand=0&rev=6
This commit is contained in:
Dominique Leuenberger 2019-11-25 10:22:31 +00:00 committed by Git OBS Bridge
commit 85200fb256
3 changed files with 218 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

314
0003-Add-Uyuni-service-discovery.patch
View File

@ -1,43 +1,3 @@
From 757642fdefcc3a6f08d36d5db9ff5e9b46104193 Mon Sep 17 00:00:00 2001
From: Joao Cavalheiro <jcavalheiro@suse.de>
Date: Wed, 22 May 2019 16:39:25 +0100
Subject: [PATCH] Add Uyuni service discovery
---
discovery/config/config.go | 3 +
discovery/manager.go | 6 +
discovery/uyuni/uyuni.go | 219 ++++++++++
vendor/github.com/kolo/xmlrpc/LICENSE | 19 +
vendor/github.com/kolo/xmlrpc/README.md | 89 ++++
vendor/github.com/kolo/xmlrpc/client.go | 170 ++++++++
vendor/github.com/kolo/xmlrpc/client_test.go | 141 +++++++
vendor/github.com/kolo/xmlrpc/decoder.go | 473 ++++++++++++++++++++++
vendor/github.com/kolo/xmlrpc/decoder_test.go | 234 +++++++++++
vendor/github.com/kolo/xmlrpc/encoder.go | 171 ++++++++
vendor/github.com/kolo/xmlrpc/encoder_test.go | 58 +++
vendor/github.com/kolo/xmlrpc/fixtures/cp1251.xml | 6 +
vendor/github.com/kolo/xmlrpc/request.go | 57 +++
vendor/github.com/kolo/xmlrpc/response.go | 52 +++
vendor/github.com/kolo/xmlrpc/response_test.go | 84 ++++
vendor/github.com/kolo/xmlrpc/test_server.rb | 25 ++
vendor/github.com/kolo/xmlrpc/xmlrpc.go | 19 +
17 files changed, 1826 insertions(+)
create mode 100644 discovery/uyuni/uyuni.go
create mode 100644 vendor/github.com/kolo/xmlrpc/LICENSE
create mode 100644 vendor/github.com/kolo/xmlrpc/README.md
create mode 100644 vendor/github.com/kolo/xmlrpc/client.go
create mode 100644 vendor/github.com/kolo/xmlrpc/client_test.go
create mode 100644 vendor/github.com/kolo/xmlrpc/decoder.go
create mode 100644 vendor/github.com/kolo/xmlrpc/decoder_test.go
create mode 100644 vendor/github.com/kolo/xmlrpc/encoder.go
create mode 100644 vendor/github.com/kolo/xmlrpc/encoder_test.go
create mode 100644 vendor/github.com/kolo/xmlrpc/fixtures/cp1251.xml
create mode 100644 vendor/github.com/kolo/xmlrpc/request.go
create mode 100644 vendor/github.com/kolo/xmlrpc/response.go
create mode 100644 vendor/github.com/kolo/xmlrpc/response_test.go
create mode 100644 vendor/github.com/kolo/xmlrpc/test_server.rb
create mode 100644 vendor/github.com/kolo/xmlrpc/xmlrpc.go
diff --git a/discovery/config/config.go b/discovery/config/config.go diff --git a/discovery/config/config.go b/discovery/config/config.go
index 820de1f7..27d8c0cc 100644 index 820de1f7..27d8c0cc 100644
--- a/discovery/config/config.go --- a/discovery/config/config.go
@ -85,11 +45,11 @@ index 1dbdecc8..ac621f3e 100644
return &StaticProvider{TargetGroups: cfg.StaticConfigs}, nil return &StaticProvider{TargetGroups: cfg.StaticConfigs}, nil
diff --git a/discovery/uyuni/uyuni.go b/discovery/uyuni/uyuni.go diff --git a/discovery/uyuni/uyuni.go b/discovery/uyuni/uyuni.go
new file mode 100644 new file mode 100644
index 00000000..60f741a5 index 00000000..fcaaad0f
--- /dev/null --- /dev/null
+++ b/discovery/uyuni/uyuni.go +++ b/discovery/uyuni/uyuni.go
@@ -0,0 +1,219 @@ @@ -0,0 +1,340 @@
+// Copyright 2017 The Prometheus Authors +// Copyright 2019 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License"); +// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. +// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at +// You may obtain a copy of the License at
@ -106,8 +66,13 @@ index 00000000..60f741a5
+ +
+import ( +import (
+ "context" + "context"
+ "encoding/json"
+ "fmt" + "fmt"
+ "net/http" + "net/http"
+ "net/url"
+ "regexp"
+ "strings"
+ "sync"
+ "time" + "time"
+ +
+ "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log"
@ -130,6 +95,9 @@ index 00000000..60f741a5
+ RefreshInterval: model.Duration(1 * time.Minute), + RefreshInterval: model.Duration(1 * time.Minute),
+} +}
+ +
+// Regular expression to extract port from formula data
+var monFormulaRegex = regexp.MustCompile(`--(?:telemetry\.address|web\.listen-address)=\":([0-9]*)\"`)
+
+// SDConfig is the configuration for Uyuni based service discovery. +// SDConfig is the configuration for Uyuni based service discovery.
+type SDConfig struct { +type SDConfig struct {
+ Host string `yaml:"host"` + Host string `yaml:"host"`
@ -140,23 +108,38 @@ index 00000000..60f741a5
+ +
+// Uyuni API Response structures +// Uyuni API Response structures
+type clientRef struct { +type clientRef struct {
+ Id int `xmlrpc:"id"` + ID int `xmlrpc:"id"`
+ Name string `xmlrpc:"name"` + Name string `xmlrpc:"name"`
+} +}
+ +
+type clientDetail struct { +type systemDetail struct {
+ Id int `xmlrpc:"id"` + ID int `xmlrpc:"id"`
+ Hostname string `xmlrpc:"hostname"` + Hostname string `xmlrpc:"hostname"`
+ Entitlements []string `xmlrpc:"addon_entitlements"` + Entitlements []string `xmlrpc:"addon_entitlements"`
+} +}
+ +
+type exporterConfig struct { +type groupDetail struct {
+ Enabled bool `xmlrpc:"enabled"` + ID int `xmlrpc:"id"`
+ Subscribed int `xmlrpc:"subscribed"`
+ SystemGroupName string `xmlrpc:"system_group_name"`
+} +}
+ +
+type formulaData struct { +type networkInfo struct {
+ NodeExporter exporterConfig `xmlrpc:"node_exporter"` + IP string `xmlrpc:"ip"`
+ PostgresExporter exporterConfig `xmlrpc:"postgres_exporter"` +}
+
+type exporterConfig struct {
+ Args string `xmlrpc:"args"`
+ Enabled bool `xmlrpc:"enabled"`
+}
+
+// Discovery periodically performs Uyuni API requests. It implements the Discoverer interface.
+type Discovery struct {
+ *refresh.Discovery
+ client *http.Client
+ interval time.Duration
+ sdConfig *SDConfig
+ logger log.Logger
+} +}
+ +
+// UnmarshalYAML implements the yaml.Unmarshaler interface. +// UnmarshalYAML implements the yaml.Unmarshaler interface.
@ -183,55 +166,79 @@ index 00000000..60f741a5
+ return nil + return nil
+} +}
+ +
+// Attempt to login in SUSE Manager Server and get an auth token +// Attempt to login in Uyuni Server and get an auth token
+func Login(rpcclient *xmlrpc.Client, user string, pass string) (string, error) { +func login(rpcclient *xmlrpc.Client, user string, pass string) (string, error) {
+ var result string + var result string
+ err := rpcclient.Call("auth.login", []interface{}{user, pass}, &result) + err := rpcclient.Call("auth.login", []interface{}{user, pass}, &result)
+ return result, err + return result, err
+} +}
+ +
+// Logout from SUSE Manager API +// Logout from Uyuni API
+func Logout(rpcclient *xmlrpc.Client, token string) error { +func logout(rpcclient *xmlrpc.Client, token string) error {
+ err := rpcclient.Call("auth.logout", token, nil) + err := rpcclient.Call("auth.logout", token, nil)
+ return err + return err
+} +}
+ +
+// Get client list +// Get system list
+func ListSystems(rpcclient *xmlrpc.Client, token string) ([]clientRef, error) { +func listSystems(rpcclient *xmlrpc.Client, token string) ([]clientRef, error) {
+ var result []clientRef + var result []clientRef
+ err := rpcclient.Call("system.listSystems", token, &result) + err := rpcclient.Call("system.listSystems", token, &result)
+ return result, err + return result, err
+} +}
+ +
+// Get client details +// Get system details
+func GetSystemDetails(rpcclient *xmlrpc.Client, token string, systemId int) (clientDetail, error) { +func getSystemDetails(rpcclient *xmlrpc.Client, token string, systemID int) (systemDetail, error) {
+ var result clientDetail + var result systemDetail
+ err := rpcclient.Call("system.getDetails", []interface{}{token, systemId}, &result) + err := rpcclient.Call("system.getDetails", []interface{}{token, systemID}, &result)
+ return result, err + return result, err
+} +}
+ +
+// List client FQDNs +// Get list of groups a system belongs to
+func ListSystemFQDNs(rpcclient *xmlrpc.Client, token string, systemId int) ([]string, error) { +func listSystemGroups(rpcclient *xmlrpc.Client, token string, systemID int) ([]groupDetail, error) {
+ var result []string + var result []groupDetail
+ err := rpcclient.Call("system.listFqdns", []interface{}{token, systemId}, &result) + err := rpcclient.Call("system.listGroups", []interface{}{token, systemID}, &result)
+ return result, err
+}
+
+// GetSystemNetworkInfo lists client FQDNs
+func getSystemNetworkInfo(rpcclient *xmlrpc.Client, token string, systemID int) (networkInfo, error) {
+ var result networkInfo
+ err := rpcclient.Call("system.getNetwork", []interface{}{token, systemID}, &result)
+ return result, err + return result, err
+} +}
+ +
+// Get formula data for a given system +// Get formula data for a given system
+func getSystemFormulaData(rpcclient *xmlrpc.Client, token string, systemId int, formulaName string) (formulaData, error) { +func getSystemFormulaData(rpcclient *xmlrpc.Client, token string, systemID int, formulaName string) (map[string]exporterConfig, error) {
+ var result formulaData + var result map[string]exporterConfig
+ err := rpcclient.Call("formula.getSystemFormulaData", []interface{}{token, systemId, formulaName}, &result) + err := rpcclient.Call("formula.getSystemFormulaData", []interface{}{token, systemID, formulaName}, &result)
+ return result, err + return result, err
+} +}
+ +
+// Discovery periodically performs Uyuni API requests. It implements +// Get formula data for a given group
+// the Discoverer interface. +func getGroupFormulaData(rpcclient *xmlrpc.Client, token string, groupID int, formulaName string) (map[string]exporterConfig, error) {
+type Discovery struct { + var result map[string]exporterConfig
+ *refresh.Discovery + err := rpcclient.Call("formula.getGroupFormulaData", []interface{}{token, groupID, formulaName}, &result)
+ client *http.Client + return result, err
+ interval time.Duration +}
+ sdConfig *SDConfig +
+ logger log.Logger +// Get exporter port configuration from Formula
+func extractPortFromFormulaData(args string) (string, error) {
+ tokens := monFormulaRegex.FindStringSubmatch(args)
+ if len(tokens) < 1 {
+ return "", errors.New("Unable to find port in args: " + args)
+ }
+ return tokens[1], nil
+}
+
+// Take a current formula structure and override values if the new config is set
+// Used for calculating final formula values when using groups
+func getCombinedFormula(combined map[string]exporterConfig, new map[string]exporterConfig) map[string]exporterConfig {
+ for k, v := range new {
+ if v.Enabled {
+ combined[k] = v
+ }
+ }
+ return combined
+} +}
+ +
+// NewDiscovery returns a new file discovery for the given paths. +// NewDiscovery returns a new file discovery for the given paths.
@ -239,7 +246,7 @@ index 00000000..60f741a5
+ d := &Discovery{ + d := &Discovery{
+ interval: time.Duration(conf.RefreshInterval), + interval: time.Duration(conf.RefreshInterval),
+ sdConfig: conf, + sdConfig: conf,
+ logger: logger, + logger: logger,
+ } + }
+ d.Discovery = refresh.NewDiscovery( + d.Discovery = refresh.NewDiscovery(
+ logger, + logger,
@ -253,59 +260,133 @@ index 00000000..60f741a5
+func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { +func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
+ +
+ config := d.sdConfig + config := d.sdConfig
+ apiUrl := config.Host + "/rpc/api" + apiURL := config.Host + "/rpc/api"
+ +
+ rpcclient, _ := xmlrpc.NewClient(apiUrl, nil) + // Check if the URL is valid and create rpc client
+ + _, err := url.ParseRequestURI(apiURL)
+ token, err := Login(rpcclient, config.User, config.Pass)
+ if err != nil { + if err != nil {
+ return nil, errors.Wrap(err, "Unable to login to SUSE Manager API") + return nil, errors.Wrap(err, "Uyuni Server URL is not valid")
+ } + }
+ rpc, _ := xmlrpc.NewClient(apiURL, nil)
+ tg := &targetgroup.Group{Source: config.Host}
+ +
+ clientList, err := ListSystems(rpcclient, token) + // Login into Uyuni API and get auth token
+ token, err := login(rpc, config.User, config.Pass)
+ if err != nil {
+ return nil, errors.Wrap(err, "Unable to login to Uyuni API")
+ }
+ // Get list of managed clients from Uyuni API
+ clientList, err := listSystems(rpc, token)
+ if err != nil { + if err != nil {
+ return nil, errors.Wrap(err, "Unable to get list of systems") + return nil, errors.Wrap(err, "Unable to get list of systems")
+ } + }
+ +
+ tg := &targetgroup.Group{ + // Iterate list of clients
+ Source: config.Host,
+ }
+
+ if len(clientList) == 0 { + if len(clientList) == 0 {
+ fmt.Printf("\tFound 0 systems.\n") + fmt.Printf("\tFound 0 systems.\n")
+ } else { + } else {
+ for _, client := range clientList { + startTime := time.Now()
+ fqdns := []string{} + var wg sync.WaitGroup
+ formulas := formulaData{} + wg.Add(len(clientList))
+ details, err := GetSystemDetails(rpcclient, token, client.Id) +
+ if err != nil { + for _, cl := range clientList {
+ level.Error(d.logger).Log("msg", "Unable to get system details","clientId", client.Id, "err", err) +
+ continue; + go func(client clientRef) {
+ } + defer wg.Done()
+ // Check if system is to be monitored + rpcclient, _ := xmlrpc.NewClient(apiURL, nil)
+ for _, v := range details.Entitlements { + netInfo := networkInfo{}
+ if v == "monitoring_entitled" { + formulas := map[string]exporterConfig{}
+ fqdns, err = ListSystemFQDNs(rpcclient, token, client.Id) + groups := []groupDetail{}
+ formulas, err = getSystemFormulaData(rpcclient, token, client.Id, "prometheus-exporters") +
+ if (formulas.NodeExporter.Enabled) { + // Get the system details
+ labels := model.LabelSet{} + details, err := getSystemDetails(rpcclient, token, client.ID)
+ addr := fmt.Sprintf("%s:%d", fqdns[len(fqdns)-1], 9100) +
+ labels[model.AddressLabel] = model.LabelValue(addr) + if err != nil {
+ tg.Targets = append(tg.Targets, labels) + level.Error(d.logger).Log("msg", "Unable to get system details", "clientId", client.ID, "err", err)
+ } + return
+ if (formulas.PostgresExporter.Enabled) { + }
+ labels := model.LabelSet{} + jsonDetails, _ := json.Marshal(details)
+ addr := fmt.Sprintf("%s:%d", fqdns[len(fqdns)-1], 9187) + level.Debug(d.logger).Log("msg", "System details", "details", jsonDetails)
+ labels[model.AddressLabel] = model.LabelValue(addr) +
+ tg.Targets = append(tg.Targets, labels) + // Check if system is monitoring entitled
+ for _, v := range details.Entitlements {
+ if v == "monitoring_entitled" { // golang has no native method to check if an element is part of a slice
+
+ // Get network details
+ netInfo, err = getSystemNetworkInfo(rpcclient, token, client.ID)
+ if err != nil {
+ level.Error(d.logger).Log("msg", "getSystemNetworkInfo failed", "clientId", client.ID, "err", err)
+ return
+ }
+
+ // Get list of groups this system is assigned to
+ candidateGroups, err := listSystemGroups(rpcclient, token, client.ID)
+ if err != nil {
+ level.Error(d.logger).Log("msg", "listSystemGroups failed", "clientId", client.ID, "err", err)
+ return
+ }
+ groups := []string{}
+ for _, g := range candidateGroups {
+ // get list of group formulas
+ // TODO: Put the resulting data on a map so that we do not have to repeat the call below for every system
+ if g.Subscribed == 1 {
+ groupFormulas, err := getGroupFormulaData(rpcclient, token, g.ID, "prometheus-exporters")
+ if err != nil {
+ level.Error(d.logger).Log("msg", "getGroupFormulaData failed", "groupId", client.ID, "err", err)
+ return
+ }
+ formulas = getCombinedFormula(formulas, groupFormulas)
+ // replace spaces with dashes on all group names
+ groups = append(groups, strings.ToLower(strings.ReplaceAll(g.SystemGroupName, " ", "-")))
+ }
+ }
+
+ // Get system formula list
+ systemFormulas, err := getSystemFormulaData(rpcclient, token, client.ID, "prometheus-exporters")
+ if err != nil {
+ level.Error(d.logger).Log("msg", "getSystemFormulaData failed", "clientId", client.ID, "err", err)
+ return
+ }
+ formulas = getCombinedFormula(formulas, systemFormulas)
+
+ // Iterate list of formulas and check for enabled exporters
+ for k, v := range formulas {
+ if v.Enabled {
+ port, err := extractPortFromFormulaData(v.Args)
+ if err != nil {
+ level.Error(d.logger).Log("msg", "Invalid exporter port", "clientId", client.ID, "err", err)
+ return
+ }
+ targets := model.LabelSet{}
+ addr := fmt.Sprintf("%s:%s", netInfo.IP, port)
+ targets[model.AddressLabel] = model.LabelValue(addr)
+ targets["exporter"] = model.LabelValue(k)
+ targets["hostname"] = model.LabelValue(details.Hostname)
+ targets["groups"] = model.LabelValue(strings.Join(groups, ","))
+ for _, g := range groups {
+ gname := fmt.Sprintf("grp:%s", g)
+ targets[model.LabelName(gname)] = model.LabelValue("active")
+ }
+ tg.Targets = append(tg.Targets, targets)
+ }
+ }
+ } + }
+ } + }
+ } + // Log debug information
+ + if netInfo.IP != "" {
+ level.Debug(d.logger).Log("msg", "Found system", "host", details.Hostname, "entitlements", fmt.Sprintf("%+v", details.Entitlements), "FQDN", fmt.Sprintf("%+v", fqdns), "formulas", fmt.Sprintf("%+v", formulas)) + level.Info(d.logger).Log("msg", "Found monitored system", "Host", details.Hostname,
+ "Entitlements", fmt.Sprintf("%+v", details.Entitlements),
+ "Network", fmt.Sprintf("%+v", netInfo), "Groups",
+ fmt.Sprintf("%+v", groups), "Formulas", fmt.Sprintf("%+v", formulas))
+ }
+ rpcclient.Close()
+ }(cl)
+ } + }
+ wg.Wait()
+ level.Info(d.logger).Log("msg", "Total discovery time", "time", time.Since(startTime))
+ } + }
+ Logout(rpcclient, token) + logout(rpc, token)
+ rpc.Close()
+ return []*targetgroup.Group{tg}, nil + return []*targetgroup.Group{tg}, nil
+} +}
diff --git a/vendor/github.com/kolo/xmlrpc/LICENSE b/vendor/github.com/kolo/xmlrpc/LICENSE diff --git a/vendor/github.com/kolo/xmlrpc/LICENSE b/vendor/github.com/kolo/xmlrpc/LICENSE
@ -1991,6 +2072,3 @@ index 00000000..8766403a
+ +
+// Base64 represents value in base64 encoding +// Base64 represents value in base64 encoding
+type Base64 string +type Base64 string
--
2.16.4

22
golang-github-prometheus-prometheus.changes
View File

@ -1,3 +1,25 @@
-------------------------------------------------------------------
Wed Nov 20 22:33:20 UTC 2019 - Michał Rostecki <mrostecki@opensuse.org>
- Remove firewalld files. They are installed in the main firewalld
package.
-------------------------------------------------------------------
Wed Nov 20 15:32:20 UTC 2019 - Joao Cavalheiro <jcavalheiro@suse.com>
- Update Uyuni/SUSE Manager service discovery patch
+ Modified 0003-Add-Uyuni-service-discovery.patch
+ Fixes crashes when systems have no FQDN
+ Adds Parallel calls to Uyuni API, meaningful performance increase
+ Adds Support for system group labels
-------------------------------------------------------------------
Mon Sep 23 10:19:03 UTC 2019 - Michał Rostecki <mrostecki@opensuse.org>
- Do not install the firewalld config file on Tumbleweed (on
versions newer than Leap 15.1). It's installed in the main
firewalld package.
------------------------------------------------------------------- -------------------------------------------------------------------
Fri Aug 16 06:46:24 UTC 2019 - Jan Fajerski <jan.fajerski@suse.com> Fri Aug 16 06:46:24 UTC 2019 - Jan Fajerski <jan.fajerski@suse.com>

6
golang-github-prometheus-prometheus.spec
View File

@ -86,9 +86,6 @@ cp -fr console_libraries/ consoles/ %{buildroot}%{_datarootdir}/prometheus
install -m 0755 -d %{buildroot}%{_fillupdir} install -m 0755 -d %{buildroot}%{_fillupdir}
install -m 0644 %{SOURCE3} %{buildroot}%{_fillupdir}/sysconfig.prometheus install -m 0644 %{SOURCE3} %{buildroot}%{_fillupdir}/sysconfig.prometheus
install -m 0755 -d %{buildroot}%{_libdir}/firewalld/services/
install -m 0644 %{SOURCE4} %{buildroot}%{_libdir}/firewalld/services/prometheus.xml
install -Dd -m 0750 %{buildroot}%{_localstatedir}/lib/prometheus install -Dd -m 0750 %{buildroot}%{_localstatedir}/lib/prometheus
install -Dd -m 0750 %{buildroot}%{_localstatedir}/lib/prometheus/data install -Dd -m 0750 %{buildroot}%{_localstatedir}/lib/prometheus/data
install -Dd -m 0750 %{buildroot}%{_localstatedir}/lib/prometheus/metrics install -Dd -m 0750 %{buildroot}%{_localstatedir}/lib/prometheus/metrics
@ -128,8 +125,5 @@ getent passwd %{prometheus_user} >/dev/null || %{_sbindir}/useradd -r -g %{prome
%dir %attr(0700,%{prometheus_user},%{prometheus_group}) %{_sharedstatedir}/prometheus/metrics %dir %attr(0700,%{prometheus_user},%{prometheus_group}) %{_sharedstatedir}/prometheus/metrics
%dir %{_sysconfdir}/prometheus %dir %{_sysconfdir}/prometheus
%config(noreplace) %{_sysconfdir}/prometheus/prometheus.yml %config(noreplace) %{_sysconfdir}/prometheus/prometheus.yml
%dir %{_libdir}/firewalld
%dir %{_libdir}/firewalld/services
%{_libdir}/firewalld/services/prometheus.xml
%changelog %changelog