Adding upstream version 1.34.4.

Signed-off-by: Daniel Baumann <daniel@debian.org>

plugins/inputs/vault/README.md

@@ -0,0 +1,50 @@
+# Hashicorp Vault Input Plugin
+
+The Vault plugin could grab metrics from every Vault agent of the
+cluster. Telegraf may be present in every node and connect to the agent
+locally. In this case should be something like `http://127.0.0.1:8200`.
+
+> Tested on vault 1.8.5
+
+## Global configuration options <!-- @/docs/includes/plugin_config.md -->
+
+In addition to the plugin-specific configuration settings, plugins support
+additional global and plugin configuration settings. These settings are used to
+modify metrics, tags, and field or create aliases and configure ordering, etc.
+See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
+
+[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
+
+## Configuration
+
+```toml @sample.conf
+# Read metrics from the Vault API
+[[inputs.vault]]
+  ## URL for the Vault agent
+  # url = "http://127.0.0.1:8200"
+
+  ## Use Vault token for authorization.
+  ## Vault token configuration is mandatory.
+  ## If both are empty or both are set, an error is thrown.
+  # token_file = "/path/to/auth/token"
+  ## OR
+  token = "s.CDDrgg5zPv5ssI0Z2P4qxJj2"
+
+  ## Set response_timeout (default 5 seconds)
+  # response_timeout = "5s"
+
+  ## Optional TLS Config
+  # tls_ca = /path/to/cafile
+  # tls_cert = /path/to/certfile
+  # tls_key = /path/to/keyfile
+```
+
+## Metrics
+
+For a more deep understanding of Vault monitoring, please have a look at the
+following Vault documentation:
+
+- [https://www.vaultproject.io/docs/internals/telemetry](https://www.vaultproject.io/docs/internals/telemetry)
+- [https://learn.hashicorp.com/tutorials/vault/monitor-telemetry-audit-splunk?in=vault/monitoring](https://learn.hashicorp.com/tutorials/vault/monitor-telemetry-audit-splunk?in=vault/monitoring)
+
+## Example Output

plugins/inputs/vault/sample.conf

@@ -0,0 +1,19 @@
+# Read metrics from the Vault API
+[[inputs.vault]]
+  ## URL for the Vault agent
+  # url = "http://127.0.0.1:8200"
+
+  ## Use Vault token for authorization.
+  ## Vault token configuration is mandatory.
+  ## If both are empty or both are set, an error is thrown.
+  # token_file = "/path/to/auth/token"
+  ## OR
+  token = "s.CDDrgg5zPv5ssI0Z2P4qxJj2"
+
+  ## Set response_timeout (default 5 seconds)
+  # response_timeout = "5s"
+
+  ## Optional TLS Config
+  # tls_ca = /path/to/cafile
+  # tls_cert = /path/to/certfile
+  # tls_key = /path/to/keyfile

plugins/inputs/vault/testdata/response_key_metrics.json

@@ -0,0 +1,40 @@
+{
+	"Gauges": [
+		{
+			"Name": "vault.core.unsealed",
+			"Value": 1,
+			"Labels": {
+				"cluster": "vault-cluster-23b671c7"
+			}
+		}
+	],
+	"Counters": [
+		{
+			"Name": "vault.raft.replication.appendEntries.logs",
+			"Count": 130,
+			"Rate": 0.2,
+			"Sum": 2,
+			"Min": 0,
+			"Max": 1,
+			"Mean": 0.015384615384615385,
+			"Stddev": 0.12355304447984486,
+			"Labels": {
+				"peer_id": "clustnode-02"
+			}
+		}
+	],
+	"Samples": [
+		{
+			"Name": "vault.token.lookup",
+			"Count": 5135,
+			"Rate": 87.21228296905755,
+			"Sum": 872.1228296905756,
+			"Min": 0.06690400093793869,
+			"Max": 16.22449493408203,
+			"Mean": 0.1698389152269865,
+			"Stddev": 0.24637634000854705,
+			"Labels": {}
+		}
+	],
+	"Timestamp": "2021-11-30 15:49:00 +0000 UTC"
+}

plugins/inputs/vault/vault.go

@@ -0,0 +1,198 @@
+//go:generate ../../../tools/readme_config_includer/generator
+package vault
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/internal"
+	common_http "github.com/influxdata/telegraf/plugins/common/http"
+	"github.com/influxdata/telegraf/plugins/inputs"
+)
+
+//go:embed sample.conf
+var sampleConfig string
+
+const timeLayout = "2006-01-02 15:04:05 -0700 MST"
+
+type Vault struct {
+	URL       string          `toml:"url"`
+	TokenFile string          `toml:"token_file"`
+	Token     string          `toml:"token"`
+	Log       telegraf.Logger `toml:"-"`
+	common_http.HTTPClientConfig
+
+	client *http.Client
+}
+
+func (*Vault) SampleConfig() string {
+	return sampleConfig
+}
+
+func (n *Vault) Init() error {
+	if n.URL == "" {
+		n.URL = "http://127.0.0.1:8200"
+	}
+
+	if n.TokenFile == "" && n.Token == "" {
+		return errors.New("token missing")
+	}
+
+	if n.TokenFile != "" && n.Token != "" {
+		return errors.New("both token_file and token are set")
+	}
+
+	if n.TokenFile != "" {
+		token, err := os.ReadFile(n.TokenFile)
+		if err != nil {
+			return fmt.Errorf("reading file failed: %w", err)
+		}
+		n.Token = strings.TrimSpace(string(token))
+	}
+
+	ctx := context.Background()
+	client, err := n.HTTPClientConfig.CreateClient(ctx, n.Log)
+	if err != nil {
+		return fmt.Errorf("creating client failed: %w", err)
+	}
+	n.client = client
+
+	return nil
+}
+
+func (*Vault) Start(telegraf.Accumulator) error {
+	return nil
+}
+
+func (n *Vault) Gather(acc telegraf.Accumulator) error {
+	sysMetrics, err := n.loadJSON(n.URL + "/v1/sys/metrics")
+	if err != nil {
+		return err
+	}
+
+	return buildVaultMetrics(acc, sysMetrics)
+}
+
+func (n *Vault) Stop() {
+	if n.client != nil {
+		n.client.CloseIdleConnections()
+	}
+}
+
+func (n *Vault) loadJSON(url string) (*sysMetrics, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("X-Vault-Token", n.Token)
+	req.Header.Add("Accept", "application/json")
+
+	resp, err := n.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error making HTTP request to %q: %w", url, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%s returned HTTP status %s", url, resp.Status)
+	}
+
+	var metrics sysMetrics
+	err = json.NewDecoder(resp.Body).Decode(&metrics)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing json response: %w", err)
+	}
+
+	return &metrics, nil
+}
+
+// buildVaultMetrics, it builds all the metrics and adds them to the accumulator
+func buildVaultMetrics(acc telegraf.Accumulator, sysMetrics *sysMetrics) error {
+	t, err := internal.ParseTimestamp(timeLayout, sysMetrics.Timestamp, nil)
+	if err != nil {
+		return fmt.Errorf("error parsing time: %w", err)
+	}
+
+	for _, counters := range sysMetrics.Counters {
+		tags := make(map[string]string)
+		for key, val := range counters.baseInfo.Labels {
+			convertedVal, err := internal.ToString(val)
+			if err != nil {
+				return fmt.Errorf("converting counter %s=%v failed: %w", key, val, err)
+			}
+			tags[key] = convertedVal
+		}
+
+		fields := map[string]interface{}{
+			"count":  counters.Count,
+			"rate":   counters.Rate,
+			"sum":    counters.Sum,
+			"min":    counters.Min,
+			"max":    counters.Max,
+			"mean":   counters.Mean,
+			"stddev": counters.Stddev,
+		}
+		acc.AddCounter(counters.baseInfo.Name, fields, tags, t)
+	}
+
+	for _, gauges := range sysMetrics.Gauges {
+		tags := make(map[string]string)
+		for key, val := range gauges.baseInfo.Labels {
+			convertedVal, err := internal.ToString(val)
+			if err != nil {
+				return fmt.Errorf("converting gauges %s=%v failed: %w", key, val, err)
+			}
+			tags[key] = convertedVal
+		}
+
+		fields := map[string]interface{}{
+			"value": gauges.Value,
+		}
+
+		acc.AddGauge(gauges.Name, fields, tags, t)
+	}
+
+	for _, summary := range sysMetrics.Summaries {
+		tags := make(map[string]string)
+		for key, val := range summary.baseInfo.Labels {
+			convertedVal, err := internal.ToString(val)
+			if err != nil {
+				return fmt.Errorf("converting summary %s=%v failed: %w", key, val, err)
+			}
+			tags[key] = convertedVal
+		}
+
+		fields := map[string]interface{}{
+			"count":  summary.Count,
+			"rate":   summary.Rate,
+			"sum":    summary.Sum,
+			"stddev": summary.Stddev,
+			"min":    summary.Min,
+			"max":    summary.Max,
+			"mean":   summary.Mean,
+		}
+		acc.AddCounter(summary.Name, fields, tags, t)
+	}
+
+	return nil
+}
+
+func init() {
+	inputs.Add("vault", func() telegraf.Input {
+		return &Vault{
+			HTTPClientConfig: common_http.HTTPClientConfig{
+				ResponseHeaderTimeout: config.Duration(5 * time.Second),
+			},
+		}
+	})
+}

plugins/inputs/vault/vault_metrics.go

@@ -0,0 +1,40 @@
+package vault
+
+type sysMetrics struct {
+	Timestamp string    `json:"timestamp"`
+	Gauges    []gauge   `json:"Gauges"`
+	Counters  []counter `json:"Counters"`
+	Summaries []summary `json:"Samples"`
+}
+
+type baseInfo struct {
+	Name   string                 `json:"Name"`
+	Labels map[string]interface{} `json:"Labels"`
+}
+
+type gauge struct {
+	baseInfo
+	Value int `json:"Value"`
+}
+
+type counter struct {
+	baseInfo
+	Count  int     `json:"Count"`
+	Rate   float64 `json:"Rate"`
+	Sum    int     `json:"Sum"`
+	Min    int     `json:"Min"`
+	Max    int     `json:"Max"`
+	Mean   float64 `json:"Mean"`
+	Stddev float64 `json:"Stddev"`
+}
+
+type summary struct {
+	baseInfo
+	Count  int     `json:"Count"`
+	Rate   float64 `json:"Rate"`
+	Sum    float64 `json:"Sum"`
+	Min    float64 `json:"Min"`
+	Max    float64 `json:"Max"`
+	Mean   float64 `json:"Mean"`
+	Stddev float64 `json:"Stddev"`
+}

plugins/inputs/vault/vault_test.go

@@ -0,0 +1,228 @@
+package vault
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/go-connections/nat"
+	"github.com/stretchr/testify/require"
+	"github.com/testcontainers/testcontainers-go/wait"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/testutil"
+)
+
+func TestVaultStats(t *testing.T) {
+	var applyTests = []struct {
+		name     string
+		expected []telegraf.Metric
+	}{
+		{
+			name: "Metrics",
+			expected: []telegraf.Metric{
+				testutil.MustMetric(
+					"vault.raft.replication.appendEntries.logs",
+					map[string]string{
+						"peer_id": "clustnode-02",
+					},
+					map[string]interface{}{
+						"count":  int(130),
+						"rate":   float64(0.2),
+						"sum":    int(2),
+						"min":    int(0),
+						"max":    int(1),
+						"mean":   float64(0.015384615384615385),
+						"stddev": float64(0.12355304447984486),
+					},
+					time.Unix(1638287340, 0),
+					1,
+				),
+				testutil.MustMetric(
+					"vault.core.unsealed",
+					map[string]string{
+						"cluster": "vault-cluster-23b671c7",
+					},
+					map[string]interface{}{
+						"value": int(1),
+					},
+					time.Unix(1638287340, 0),
+					2,
+				),
+				testutil.MustMetric(
+					"vault.token.lookup",
+					map[string]string{},
+					map[string]interface{}{
+						"count":  int(5135),
+						"max":    float64(16.22449493408203),
+						"mean":   float64(0.1698389152269865),
+						"min":    float64(0.06690400093793869),
+						"rate":   float64(87.21228296905755),
+						"stddev": float64(0.24637634000854705),
+						"sum":    float64(872.1228296905756),
+					},
+					time.Unix(1638287340, 0),
+					1,
+				),
+			},
+		},
+	}
+
+	for _, tt := range applyTests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.RequestURI == "/v1/sys/metrics" {
+					responseKeyMetrics, err := os.ReadFile("testdata/response_key_metrics.json")
+					if err != nil {
+						w.WriteHeader(http.StatusInternalServerError)
+						t.Error(err)
+						return
+					}
+
+					if _, err = fmt.Fprintln(w, string(responseKeyMetrics)); err != nil {
+						w.WriteHeader(http.StatusInternalServerError)
+						t.Error(err)
+						return
+					}
+					w.WriteHeader(http.StatusOK)
+				}
+			}))
+			defer ts.Close()
+
+			plugin := &Vault{
+				URL:   ts.URL,
+				Token: "s.CDDrgg5zPv5ssI0Z2P4qxJj2",
+			}
+			err := plugin.Init()
+			require.NoError(t, err)
+
+			acc := testutil.Accumulator{}
+			err = plugin.Gather(&acc)
+			require.NoError(t, err)
+
+			testutil.RequireMetricsEqual(t, tt.expected, acc.GetTelegrafMetrics())
+		})
+	}
+}
+
+func TestRedirect(t *testing.T) {
+	expected := []telegraf.Metric{
+		testutil.MustMetric(
+			"vault.raft.replication.appendEntries.logs",
+			map[string]string{
+				"peer_id": "clustnode-02",
+			},
+			map[string]interface{}{
+				"count":  int(130),
+				"rate":   float64(0.2),
+				"sum":    int(2),
+				"min":    int(0),
+				"max":    int(1),
+				"mean":   float64(0.015384615384615385),
+				"stddev": float64(0.12355304447984486),
+			},
+			time.Unix(1638287340, 0),
+			1,
+		),
+		testutil.MustMetric(
+			"vault.core.unsealed",
+			map[string]string{
+				"cluster": "vault-cluster-23b671c7",
+			},
+			map[string]interface{}{
+				"value": int(1),
+			},
+			time.Unix(1638287340, 0),
+			2,
+		),
+		testutil.MustMetric(
+			"vault.token.lookup",
+			map[string]string{},
+			map[string]interface{}{
+				"count":  int(5135),
+				"max":    float64(16.22449493408203),
+				"mean":   float64(0.1698389152269865),
+				"min":    float64(0.06690400093793869),
+				"rate":   float64(87.21228296905755),
+				"stddev": float64(0.24637634000854705),
+				"sum":    float64(872.1228296905756),
+			},
+			time.Unix(1638287340, 0),
+			1,
+		),
+	}
+
+	response, err := os.ReadFile("testdata/response_key_metrics.json")
+	require.NoError(t, err)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.RequestURI {
+		case "/v1/sys/metrics":
+			redirectURL := "http://" + r.Host + "/custom/metrics"
+			http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect)
+		case "/custom/metrics":
+			if _, err := w.Write(response); err != nil {
+				w.WriteHeader(http.StatusInternalServerError)
+				t.Error(err)
+				return
+			}
+			w.WriteHeader(http.StatusOK)
+		}
+	}))
+	defer server.Close()
+
+	// Setup the plugin
+	plugin := &Vault{
+		URL:   server.URL,
+		Token: "s.CDDrgg5zPv5ssI0Z2P4qxJj2",
+	}
+	require.NoError(t, plugin.Init())
+
+	var acc testutil.Accumulator
+	require.NoError(t, plugin.Gather(&acc))
+	actual := acc.GetTelegrafMetrics()
+	testutil.RequireMetricsEqual(t, expected, actual)
+}
+
+func TestIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	// Start the docker container
+	cntnr := testutil.Container{
+		Image:        "vault:1.13.3",
+		ExposedPorts: []string{"8200"},
+		Env: map[string]string{
+			"VAULT_DEV_ROOT_TOKEN_ID": "root",
+		},
+		HostConfigModifier: func(hc *container.HostConfig) {
+			hc.CapAdd = []string{"IPC_LOCK"}
+		},
+		WaitingFor: wait.ForAll(
+			wait.ForLog("Root Token: root"),
+			wait.ForListeningPort(nat.Port("8200")),
+		),
+	}
+	require.NoError(t, cntnr.Start(), "failed to start container")
+	defer cntnr.Terminate()
+
+	// Setup the plugin
+	port := cntnr.Ports["8200"]
+	plugin := &Vault{
+		URL:   "http://" + cntnr.Address + ":" + port,
+		Token: "root",
+	}
+	require.NoError(t, plugin.Init())
+
+	// Collect the metrics and compare
+	var acc testutil.Accumulator
+	require.Eventually(t, func() bool {
+		require.NoError(t, plugin.Gather(&acc))
+		return len(acc.GetTelegrafMetrics()) > 50
+	}, 5*time.Second, 100*time.Millisecond)
+}