golang-github-prometheus-prometheus/0003-Add-Uyuni-service-discovery.patch

From 239bb9d32cb40409c8f2085700e85033f8642670 Mon Sep 17 00:00:00 2001
From: Joao Cavalheiro <jcavalheiro@suse.de>
Date: Mon, 27 Jul 2020 17:42:33 +0200
Subject: [PATCH 3/3] Add Uyuni service discovery

---
 discovery/install/install.go  |   1 +
 discovery/uyuni/uyuni.go      | 411 ++++++++++++++++++++++++++++++++++
 discovery/uyuni/uyuni_test.go |  46 ++++
 go.mod                        |   3 +-
 go.sum                        |   2 +
 5 files changed, 462 insertions(+), 1 deletion(-)
 create mode 100644 discovery/uyuni/uyuni.go
 create mode 100644 discovery/uyuni/uyuni_test.go

diff --git a/discovery/install/install.go b/discovery/install/install.go
index 3e6f0f388..484d48db8 100644
--- a/discovery/install/install.go
+++ b/discovery/install/install.go
@@ -31,5 +31,6 @@ import (
 	_ "github.com/prometheus/prometheus/discovery/openstack"    // register openstack
 	_ "github.com/prometheus/prometheus/discovery/scaleway"     // register scaleway
 	_ "github.com/prometheus/prometheus/discovery/triton"       // register triton
+	_ "github.com/prometheus/prometheus/discovery/uyuni"        // register uyuni
 	_ "github.com/prometheus/prometheus/discovery/zookeeper"    // register zookeeper
 )
diff --git a/discovery/uyuni/uyuni.go b/discovery/uyuni/uyuni.go
new file mode 100644
index 000000000..8163a3bf0
--- /dev/null
+++ b/discovery/uyuni/uyuni.go
@@ -0,0 +1,411 @@
+// Copyright 2019 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package uyuni
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/kolo/xmlrpc"
+	"github.com/pkg/errors"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/prometheus/discovery"
+	"github.com/prometheus/prometheus/discovery/refresh"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+)
+
+const (
+	monitoringEntitlementLabel    = "monitoring_entitled"
+	prometheusExporterFormulaName = "prometheus-exporters"
+	uyuniXMLRPCAPIPath            = "/rpc/api"
+)
+
+// DefaultSDConfig is the default Uyuni SD configuration.
+var DefaultSDConfig = SDConfig{
+	RefreshInterval: model.Duration(1 * time.Minute),
+}
+
+// Regular expression to extract port from formula data
+var monFormulaRegex = regexp.MustCompile(`--(?:telemetry\.address|web\.listen-address)=\":([0-9]*)\"`)
+
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
+// SDConfig is the configuration for Uyuni based service discovery.
+type SDConfig struct {
+	Host            string         `yaml:"host"`
+	User            string         `yaml:"username"`
+	Pass            string         `yaml:"password"`
+	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
+}
+
+// Uyuni API Response structures
+type systemGroupID struct {
+	GroupID   int    `xmlrpc:"id"`
+	GroupName string `xmlrpc:"name"`
+}
+
+type networkInfo struct {
+	SystemID    int    `xmlrpc:"system_id"`
+	Hostname    string `xmlrpc:"hostname"`
+	PrimaryFQDN string `xmlrpc:"primary_fqdn"`
+	IP          string `xmlrpc:"ip"`
+}
+
+type tlsConfig struct {
+	Enabled bool `xmlrpc:"enabled"`
+}
+
+type exporterConfig struct {
+	Address string `xmlrpc:"address"`
+	Args    string `xmlrpc:"args"`
+	Enabled bool   `xmlrpc:"enabled"`
+}
+
+type proxiedExporterConfig struct {
+	ProxyIsEnabled  bool                      `xmlrpc:"proxy_enabled"`
+	ProxyPort       float32                   `xmlrpc:"proxy_port"`
+	TLSConfig       tlsConfig                 `xmlrpc:"tls"`
+	ExporterConfigs map[string]exporterConfig `xmlrpc:"exporters"`
+}
+
+// Discovery periodically performs Uyuni API requests. It implements the Discoverer interface.
+type Discovery struct {
+	*refresh.Discovery
+	interval time.Duration
+	sdConfig *SDConfig
+	logger   log.Logger
+}
+
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "uyuni" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(c, opts.Logger), nil
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	*c = DefaultSDConfig
+	type plain SDConfig
+	err := unmarshal((*plain)(c))
+
+	if err != nil {
+		return err
+	}
+	if c.Host == "" {
+		return errors.New("Uyuni SD configuration requires a Host")
+	}
+	if c.User == "" {
+		return errors.New("Uyuni SD configuration requires a Username")
+	}
+	if c.Pass == "" {
+		return errors.New("Uyuni SD configuration requires a Password")
+	}
+	if c.RefreshInterval <= 0 {
+		return errors.New("Uyuni SD configuration requires RefreshInterval to be a positive integer")
+	}
+	return nil
+}
+
+// Attempt to login in Uyuni Server and get an auth token
+func login(rpcclient *xmlrpc.Client, user string, pass string) (string, error) {
+	var result string
+	err := rpcclient.Call("auth.login", []interface{}{user, pass}, &result)
+	return result, err
+}
+
+// Logout from Uyuni API
+func logout(rpcclient *xmlrpc.Client, token string) error {
+	err := rpcclient.Call("auth.logout", token, nil)
+	return err
+}
+
+// Get the system groups information of monitored clients
+func getSystemGroupsInfoOfMonitoredClients(rpcclient *xmlrpc.Client, token string) (map[int][]systemGroupID, error) {
+	var systemGroupsInfos []struct {
+		SystemID     int             `xmlrpc:"id"`
+		SystemGroups []systemGroupID `xmlrpc:"system_groups"`
+	}
+	err := rpcclient.Call("system.listSystemGroupsForSystemsWithEntitlement", []interface{}{token, monitoringEntitlementLabel}, &systemGroupsInfos)
+	if err != nil {
+		return nil, err
+	}
+	result := make(map[int][]systemGroupID)
+	for _, systemGroupsInfo := range systemGroupsInfos {
+		result[systemGroupsInfo.SystemID] = systemGroupsInfo.SystemGroups
+	}
+	return result, nil
+}
+
+// GetSystemNetworkInfo lists client FQDNs
+func getNetworkInformationForSystems(rpcclient *xmlrpc.Client, token string, systemIDs []int) (map[int]networkInfo, error) {
+	var networkInfos []networkInfo
+	err := rpcclient.Call("system.getNetworkForSystems", []interface{}{token, systemIDs}, &networkInfos)
+	if err != nil {
+		return nil, err
+	}
+	result := make(map[int]networkInfo)
+	for _, networkInfo := range networkInfos {
+		result[networkInfo.SystemID] = networkInfo
+	}
+	return result, nil
+}
+
+// Get formula data for a given system
+func getExporterDataForSystems(
+	rpcclient *xmlrpc.Client,
+	token string,
+	systemIDs []int,
+) (map[int]proxiedExporterConfig, error) {
+	var combinedFormulaData []struct {
+		SystemID        int                   `xmlrpc:"system_id"`
+		ExporterConfigs proxiedExporterConfig `xmlrpc:"formula_values"`
+	}
+	err := rpcclient.Call(
+		"formula.getCombinedFormulaDataByServerIds",
+		[]interface{}{token, prometheusExporterFormulaName, systemIDs},
+		&combinedFormulaData)
+	if err != nil {
+		return nil, err
+	}
+	result := make(map[int]proxiedExporterConfig)
+	for _, combinedFormulaData := range combinedFormulaData {
+		result[combinedFormulaData.SystemID] = combinedFormulaData.ExporterConfigs
+	}
+	return result, nil
+}
+
+// extractPortFromFormulaData gets exporter port configuration from the formula.
+// args takes precedence over address.
+func extractPortFromFormulaData(args string, address string) (string, error) {
+	// first try args
+	var port string
+	tokens := monFormulaRegex.FindStringSubmatch(args)
+	if len(tokens) < 1 {
+		err := "Unable to find port in args: " + args
+		// now try address
+		_, addrPort, addrErr := net.SplitHostPort(address)
+		if addrErr != nil || len(addrPort) == 0 {
+			if addrErr != nil {
+				err = strings.Join([]string{addrErr.Error(), err}, " ")
+			}
+			return "", errors.New(err)
+		}
+		port = addrPort
+	} else {
+		port = tokens[1]
+	}
+
+	return port, nil
+}
+
+// NewDiscovery returns a new file discovery for the given paths.
+func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery {
+	d := &Discovery{
+		interval: time.Duration(conf.RefreshInterval),
+		sdConfig: conf,
+		logger:   logger,
+	}
+	d.Discovery = refresh.NewDiscovery(
+		logger,
+		"uyuni",
+		time.Duration(conf.RefreshInterval),
+		d.refresh,
+	)
+	return d
+}
+
+func initializeExporterTargets(
+	targets *[]model.LabelSet,
+	exporterName string, config exporterConfig,
+	proxyPort string,
+	errors *[]error,
+) {
+	if !(config.Enabled) {
+		return
+	}
+	var port string
+	if len(proxyPort) == 0 {
+		exporterPort, err := extractPortFromFormulaData(config.Args, config.Address)
+		if err != nil {
+			*errors = append(*errors, err)
+			return
+		}
+		port = exporterPort
+	} else {
+		port = proxyPort
+	}
+
+	labels := model.LabelSet{}
+	labels["exporter"] = model.LabelValue(exporterName)
+	// for now set only port number here
+	labels[model.AddressLabel] = model.LabelValue(port)
+	if len(proxyPort) > 0 {
+		labels[model.ParamLabelPrefix+"module"] = model.LabelValue(exporterName)
+	}
+	*targets = append(*targets, labels)
+}
+
+func (d *Discovery) getTargetsForSystem(
+	systemID int,
+	systemGroupsIDs []systemGroupID,
+	networkInfo networkInfo,
+	combinedFormulaData proxiedExporterConfig,
+) []model.LabelSet {
+
+	var labelSets []model.LabelSet
+	var errors []error
+	var proxyPortNumber string
+	var hostname string
+	if combinedFormulaData.ProxyIsEnabled {
+		proxyPortNumber = fmt.Sprintf("%d", int(combinedFormulaData.ProxyPort))
+	}
+	if len(networkInfo.PrimaryFQDN) > 0 {
+		hostname = networkInfo.PrimaryFQDN
+	} else {
+		hostname = networkInfo.Hostname
+	}
+	for exporterName, formulaValues := range combinedFormulaData.ExporterConfigs {
+		initializeExporterTargets(&labelSets, exporterName, formulaValues, proxyPortNumber, &errors)
+	}
+	managedGroupNames := getSystemGroupNames(systemGroupsIDs)
+	for _, labels := range labelSets {
+		// add hostname to the address label
+		addr := fmt.Sprintf("%s:%s", hostname, labels[model.AddressLabel])
+		labels[model.AddressLabel] = model.LabelValue(addr)
+		labels["hostname"] = model.LabelValue(hostname)
+		labels["groups"] = model.LabelValue(strings.Join(managedGroupNames, ","))
+		if combinedFormulaData.ProxyIsEnabled {
+			labels[model.MetricsPathLabel] = "/proxy"
+		}
+		if combinedFormulaData.TLSConfig.Enabled {
+			labels[model.SchemeLabel] = "https"
+		}
+		_ = level.Debug(d.logger).Log("msg", "Configured target", "Labels", fmt.Sprintf("%+v", labels))
+	}
+	for _, err := range errors {
+		level.Error(d.logger).Log("msg", "Invalid exporter port", "clientId", systemID, "err", err)
+	}
+
+	return labelSets
+}
+
+func getSystemGroupNames(systemGroupsIDs []systemGroupID) []string {
+	managedGroupNames := make([]string, 0, len(systemGroupsIDs))
+	for _, systemGroupInfo := range systemGroupsIDs {
+		managedGroupNames = append(managedGroupNames, systemGroupInfo.GroupName)
+	}
+
+	if len(managedGroupNames) == 0 {
+		managedGroupNames = []string{"No group"}
+	}
+	return managedGroupNames
+}
+
+func (d *Discovery) getTargetsForSystems(
+	rpcClient *xmlrpc.Client,
+	token string,
+	systemGroupIDsBySystemID map[int][]systemGroupID,
+) ([]model.LabelSet, error) {
+
+	result := make([]model.LabelSet, 0)
+
+	systemIDs := make([]int, 0, len(systemGroupIDsBySystemID))
+	for systemID := range systemGroupIDsBySystemID {
+		systemIDs = append(systemIDs, systemID)
+	}
+
+	combinedFormulaDataBySystemID, err := getExporterDataForSystems(rpcClient, token, systemIDs)
+	if err != nil {
+		return nil, errors.Wrap(err, "Unable to get systems combined formula data")
+	}
+	networkInfoBySystemID, err := getNetworkInformationForSystems(rpcClient, token, systemIDs)
+	if err != nil {
+		return nil, errors.Wrap(err, "Unable to get the systems network information")
+	}
+
+	for _, systemID := range systemIDs {
+		targets := d.getTargetsForSystem(
+			systemID,
+			systemGroupIDsBySystemID[systemID],
+			networkInfoBySystemID[systemID],
+			combinedFormulaDataBySystemID[systemID])
+		result = append(result, targets...)
+
+		// Log debug information
+		if networkInfoBySystemID[systemID].IP != "" {
+			level.Debug(d.logger).Log("msg", "Found monitored system",
+				"PrimaryFQDN", networkInfoBySystemID[systemID].PrimaryFQDN,
+				"Hostname", networkInfoBySystemID[systemID].Hostname,
+				"Network", fmt.Sprintf("%+v", networkInfoBySystemID[systemID]),
+				"Groups", fmt.Sprintf("%+v", systemGroupIDsBySystemID[systemID]),
+				"Formulas", fmt.Sprintf("%+v", combinedFormulaDataBySystemID[systemID]))
+		}
+	}
+	return result, nil
+}
+
+func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
+	config := d.sdConfig
+	apiURL := config.Host + uyuniXMLRPCAPIPath
+
+	startTime := time.Now()
+
+	// Check if the URL is valid and create rpc client
+	_, err := url.ParseRequestURI(apiURL)
+	if err != nil {
+		return nil, errors.Wrap(err, "Uyuni Server URL is not valid")
+	}
+
+	rpcClient, _ := xmlrpc.NewClient(apiURL, nil)
+
+	token, err := login(rpcClient, config.User, config.Pass)
+	if err != nil {
+		return nil, errors.Wrap(err, "Unable to login to Uyuni API")
+	}
+	systemGroupIDsBySystemID, err := getSystemGroupsInfoOfMonitoredClients(rpcClient, token)
+	if err != nil {
+		return nil, errors.Wrap(err, "Unable to get the managed system groups information of monitored clients")
+	}
+
+	targets := make([]model.LabelSet, 0)
+	if len(systemGroupIDsBySystemID) > 0 {
+		targetsForSystems, err := d.getTargetsForSystems(rpcClient, token, systemGroupIDsBySystemID)
+		if err != nil {
+			return nil, err
+		}
+		targets = append(targets, targetsForSystems...)
+		level.Info(d.logger).Log("msg", "Total discovery time", "time", time.Since(startTime))
+	} else {
+		fmt.Printf("\tFound 0 systems.\n")
+	}
+
+	err = logout(rpcClient, token)
+	if err != nil {
+		level.Warn(d.logger).Log("msg", "Failed to log out from Uyuni API", "err", err)
+	}
+	rpcClient.Close()
+	return []*targetgroup.Group{&targetgroup.Group{Targets: targets, Source: config.Host}}, nil
+}
diff --git a/discovery/uyuni/uyuni_test.go b/discovery/uyuni/uyuni_test.go
new file mode 100644
index 000000000..c5fa8cc9e
--- /dev/null
+++ b/discovery/uyuni/uyuni_test.go
@@ -0,0 +1,46 @@
+// Copyright 2019 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package uyuni
+
+import "testing"
+
+func TestExtractPortFromFormulaData(t *testing.T) {
+	type args struct {
+		args    string
+		address string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{name: `TestArgs`, args: args{args: `--web.listen-address=":9100"`}, want: `9100`},
+		{name: `TestAddress`, args: args{address: `:9100`}, want: `9100`},
+		{name: `TestArgsAndAddress`, args: args{args: `--web.listen-address=":9100"`, address: `9999`}, want: `9100`},
+		{name: `TestMissingPort`, args: args{args: `localhost`}, wantErr: true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := extractPortFromFormulaData(tt.args.args, tt.args.address)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("extractPortFromFormulaData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("extractPortFromFormulaData() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/go.mod b/go.mod
index dd22c9be7..b283af28a 100644
--- a/go.mod
+++ b/go.mod
@@ -34,6 +34,7 @@ require (
 	github.com/hetznercloud/hcloud-go v1.25.0
 	github.com/influxdata/influxdb v1.8.5
 	github.com/json-iterator/go v1.1.11
+	github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b
 	github.com/miekg/dns v1.1.41
 	github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
@@ -110,4 +111,4 @@ exclude (
 	k8s.io/client-go v8.0.0+incompatible
 	k8s.io/client-go v9.0.0+incompatible
 	k8s.io/client-go v9.0.0-invalid+incompatible
-)
+)
\ No newline at end of file
diff --git a/go.sum b/go.sum
index 03a0abd29..e2fff275e 100644
--- a/go.sum
+++ b/go.sum
@@ -556,6 +556,8 @@ github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
 github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
 github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b h1:iNjcivnc6lhbvJA3LD622NPrUponluJrBWPIwGG/3Bg=
+github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
-- 
2.31.1