Adding upstream version 1.34.4.

Signed-off-by: Daniel Baumann <daniel@debian.org>

plugins/outputs/cratedb/README.md

@@ -0,0 +1,79 @@
+# CrateDB Output Plugin
+
+This plugin writes metrics to [CrateDB][cratedb] via its
+[PostgreSQL protocol][psql_protocol].
+
+⭐ Telegraf v1.5.0
+🏷️ cloud, datastore
+💻 all
+
+[cratedb]: https://crate.io/
+[psql_protocol]: https://crate.io/docs/crate/reference/protocols/postgres.html
+
+## Table Schema
+
+The plugin requires a table with the following schema.
+
+```sql
+CREATE TABLE IF NOT EXISTS my_metrics (
+  "hash_id" LONG INDEX OFF,
+  "timestamp" TIMESTAMP,
+  "name" STRING,
+  "tags" OBJECT(DYNAMIC),
+  "fields" OBJECT(DYNAMIC),
+  "day" TIMESTAMP GENERATED ALWAYS AS date_trunc('day', "timestamp"),
+  PRIMARY KEY ("timestamp", "hash_id","day")
+) PARTITIONED BY("day");
+```
+
+The plugin can create this table for you automatically via the `table_create`
+config option, see below.
+
+## Global configuration options <!-- @/docs/includes/plugin_config.md -->
+
+In addition to the plugin-specific configuration settings, plugins support
+additional global and plugin configuration settings. These settings are used to
+modify metrics, tags, and field or create aliases and configure ordering, etc.
+See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
+
+[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
+
+## Startup error behavior options <!-- @/docs/includes/startup_error_behavior.md -->
+
+In addition to the plugin-specific and global configuration settings the plugin
+supports options for specifying the behavior when experiencing startup errors
+using the `startup_error_behavior` setting. Available values are:
+
+- `error`:  Telegraf with stop and exit in case of startup errors. This is the
+            default behavior.
+- `ignore`: Telegraf will ignore startup errors for this plugin and disables it
+            but continues processing for all other plugins.
+- `retry`:  Telegraf will try to startup the plugin in every gather or write
+            cycle in case of startup errors. The plugin is disabled until
+            the startup succeeds.
+- `probe`:  Telegraf will probe the plugin's function (if possible) and disables the plugin
+            in case probing fails. If the plugin does not support probing, Telegraf will
+            behave as if `ignore` was set instead.
+
+## Configuration
+
+```toml @sample.conf
+# Configuration for CrateDB to send metrics to.
+[[outputs.cratedb]]
+  ## Connection parameters for accessing the database see
+  ##   https://pkg.go.dev/github.com/jackc/pgx/v4#ParseConfig
+  ## for available options
+  url = "postgres://user:password@localhost/schema?sslmode=disable"
+
+  ## Timeout for all CrateDB queries.
+  # timeout = "5s"
+
+  ## Name of the table to store metrics in.
+  # table = "metrics"
+
+  ## If true, and the metrics table does not exist, create it automatically.
+  # table_create = false
+
+  ## The character(s) to replace any '.' in an object key with
+  # key_separator = "_"
+```

plugins/outputs/cratedb/cratedb.go

@@ -0,0 +1,262 @@
+//go:generate ../../../tools/readme_config_includer/generator
+package cratedb
+
+import (
+	"context"
+	"crypto/sha512"
+	"database/sql"
+	_ "embed"
+	"encoding/binary"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	_ "github.com/jackc/pgx/v4/stdlib" // to register stdlib from PostgreSQL Driver and Toolkit
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/outputs"
+)
+
+//go:embed sample.conf
+var sampleConfig string
+
+const MaxInt64 = int64(^uint64(0) >> 1)
+
+const tableCreationQuery = `
+CREATE TABLE IF NOT EXISTS %s (
+	"hash_id" LONG INDEX OFF,
+	"timestamp" TIMESTAMP,
+	"name" STRING,
+	"tags" OBJECT(DYNAMIC),
+	"fields" OBJECT(DYNAMIC),
+	"day" TIMESTAMP GENERATED ALWAYS AS date_trunc('day', "timestamp"),
+	PRIMARY KEY ("timestamp", "hash_id","day")
+) PARTITIONED BY("day");
+`
+
+type CrateDB struct {
+	URL          string          `toml:"url"`
+	Timeout      config.Duration `toml:"timeout"`
+	Table        string          `toml:"table"`
+	TableCreate  bool            `toml:"table_create"`
+	KeySeparator string          `toml:"key_separator"`
+
+	db *sql.DB
+}
+
+func (*CrateDB) SampleConfig() string {
+	return sampleConfig
+}
+
+func (c *CrateDB) Init() error {
+	// Set defaults
+	if c.KeySeparator == "" {
+		c.KeySeparator = "_"
+	}
+	if c.Table == "" {
+		c.Table = "metrics"
+	}
+
+	return nil
+}
+
+func (c *CrateDB) Connect() error {
+	if c.db == nil {
+		db, err := sql.Open("pgx", c.URL)
+		if err != nil {
+			return err
+		}
+		c.db = db
+	}
+
+	if c.TableCreate {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(c.Timeout))
+		defer cancel()
+
+		query := fmt.Sprintf(tableCreationQuery, c.Table)
+		if _, err := c.db.ExecContext(ctx, query); err != nil {
+			return &internal.StartupError{Err: err, Retry: true}
+		}
+	}
+
+	return nil
+}
+
+func (c *CrateDB) Write(metrics []telegraf.Metric) error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(c.Timeout))
+	defer cancel()
+
+	generatedSQL, err := insertSQL(c.Table, c.KeySeparator, metrics)
+	if err != nil {
+		return err
+	}
+
+	_, err = c.db.ExecContext(ctx, generatedSQL)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func insertSQL(table, keyReplacement string, metrics []telegraf.Metric) (string, error) {
+	rows := make([]string, 0, len(metrics))
+	for _, m := range metrics {
+		cols := []interface{}{
+			hashID(m),
+			m.Time().UTC(),
+			m.Name(),
+			m.Tags(),
+			m.Fields(),
+		}
+
+		escapedCols := make([]string, 0, len(cols))
+		for _, col := range cols {
+			escaped, err := escapeValue(col, keyReplacement)
+			if err != nil {
+				return "", err
+			}
+			escapedCols = append(escapedCols, escaped)
+		}
+		rows = append(rows, `(`+strings.Join(escapedCols, ", ")+`)`)
+	}
+	query := `INSERT INTO ` + table + ` ("hash_id", "timestamp", "name", "tags", "fields")
+VALUES
+` + strings.Join(rows, " ,\n") + `;`
+	return query, nil
+}
+
+// escapeValue returns a string version of val that is suitable for being used
+// inside of a VALUES expression or similar. Unsupported types return an error.
+//
+// Warning: This is not ideal from a security perspective, but unfortunately
+// CrateDB does not support enough of the PostgreSQL wire protocol to allow
+// using pgx with $1, $2 placeholders [1]. Security conscious users of this
+// plugin should probably refrain from using it in combination with untrusted
+// inputs.
+//
+// [1] https://github.com/influxdata/telegraf/pull/3210#issuecomment-339273371
+func escapeValue(val interface{}, keyReplacement string) (string, error) {
+	switch t := val.(type) {
+	case string:
+		return escapeString(t, `'`), nil
+	case int64, float64:
+		return fmt.Sprint(t), nil
+	case uint64:
+		// The long type is the largest integer type in CrateDB and is the
+		// size of a signed int64.  If our value is too large send the largest
+		// possible value.
+		if t <= uint64(MaxInt64) {
+			return strconv.FormatInt(int64(t), 10), nil
+		}
+		return strconv.FormatInt(MaxInt64, 10), nil
+	case bool:
+		return strconv.FormatBool(t), nil
+	case time.Time:
+		// see https://crate.io/docs/crate/reference/sql/data_types.html#timestamp
+		return escapeValue(t.Format("2006-01-02T15:04:05.999-0700"), keyReplacement)
+	case map[string]string:
+		return escapeObject(convertMap(t), keyReplacement)
+	case map[string]interface{}:
+		return escapeObject(t, keyReplacement)
+	default:
+		// This might be panic worthy under normal circumstances, but it's probably
+		// better to not shut down the entire telegraf process because of one
+		// misbehaving plugin.
+		return "", fmt.Errorf("unexpected type: %T: %#v", t, t)
+	}
+}
+
+// convertMap converts m from map[string]string to map[string]interface{} by
+// copying it. Generics, oh generics where art thou?
+func convertMap(m map[string]string) map[string]interface{} {
+	c := make(map[string]interface{}, len(m))
+	for k, v := range m {
+		c[k] = v
+	}
+	return c
+}
+
+func escapeObject(m map[string]interface{}, keyReplacement string) (string, error) {
+	// There is a decent chance that the implementation below doesn't catch all
+	// edge cases, but it's hard to tell since the format seems to be a bit
+	// underspecified.
+	// See https://crate.io/docs/crate/reference/sql/data_types.html#object
+
+	// We find all keys and sort them first because iterating a map in go is
+	// randomized and we need consistent output for our unit tests.
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	// Now we build our key = val pairs
+	pairs := make([]string, 0, len(m))
+	for _, k := range keys {
+		key := escapeString(strings.ReplaceAll(k, ".", keyReplacement), `"`)
+
+		// escape the value of the value at k (potentially recursive)
+		val, err := escapeValue(m[k], keyReplacement)
+		if err != nil {
+			return "", err
+		}
+
+		pairs = append(pairs, key+" = "+val)
+	}
+	return `{` + strings.Join(pairs, ", ") + `}`, nil
+}
+
+// escapeString wraps s in the given quote string and replaces all occurrences
+// of it inside of s with a double quote.
+func escapeString(s, quote string) string {
+	return quote + strings.ReplaceAll(s, quote, quote+quote) + quote
+}
+
+// hashID returns a cryptographic hash int64 hash that includes the metric name
+// and tags. It's used instead of m.HashID() because it's not considered stable
+// and because a cryptographic hash makes more sense for the use case of
+// deduplication.
+// [1] https://github.com/influxdata/telegraf/pull/3210#discussion_r148411201
+func hashID(m telegraf.Metric) int64 {
+	h := sha512.New()
+	h.Write([]byte(m.Name()))
+	tags := m.Tags()
+	tmp := make([]string, 0, len(tags))
+	for k, v := range tags {
+		tmp = append(tmp, k+v)
+	}
+	sort.Strings(tmp)
+
+	for _, s := range tmp {
+		h.Write([]byte(s))
+	}
+	sum := h.Sum(nil)
+
+	// Note: We have to convert from uint64 to int64 below because CrateDB only
+	// supports a signed 64 bit LONG type:
+	//
+	// CREATE TABLE my_long (val LONG);
+	// INSERT INTO my_long(val) VALUES (14305102049502225714);
+	// -> ERROR:  SQLParseException: For input string: "14305102049502225714"
+	return int64(binary.LittleEndian.Uint64(sum))
+}
+
+func (c *CrateDB) Close() error {
+	if c.db == nil {
+		return nil
+	}
+	return c.db.Close()
+}
+
+func init() {
+	outputs.Add("cratedb", func() telegraf.Output {
+		return &CrateDB{
+			Timeout: config.Duration(time.Second * 5),
+		}
+	})
+}

plugins/outputs/cratedb/cratedb_test.go

@@ -0,0 +1,312 @@
+package cratedb
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"github.com/testcontainers/testcontainers-go/wait"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/metric"
+	"github.com/influxdata/telegraf/models"
+	"github.com/influxdata/telegraf/testutil"
+)
+
+const servicePort = "5432"
+
+func createTestContainer(t *testing.T) *testutil.Container {
+	container := testutil.Container{
+		Image:        "crate",
+		ExposedPorts: []string{servicePort},
+		Entrypoint: []string{
+			"/docker-entrypoint.sh",
+			"-Cdiscovery.type=single-node",
+		},
+		WaitingFor: wait.ForAll(
+			wait.ForListeningPort(servicePort),
+			wait.ForLog("recovered [0] indices into cluster_state"),
+		),
+	}
+	err := container.Start()
+	require.NoError(t, err, "failed to start container")
+
+	return &container
+}
+
+func TestConnectAndWriteIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	container := createTestContainer(t)
+	defer container.Terminate()
+	url := fmt.Sprintf("postgres://crate@%s:%s/test", container.Address, container.Ports[servicePort])
+
+	db, err := sql.Open("pgx", url)
+	require.NoError(t, err)
+	defer db.Close()
+
+	c := &CrateDB{
+		URL:         url,
+		Table:       "testing",
+		Timeout:     config.Duration(time.Second * 5),
+		TableCreate: true,
+	}
+
+	metrics := testutil.MockMetrics()
+	require.NoError(t, c.Connect())
+	require.NoError(t, c.Write(metrics))
+
+	// The code below verifies that the metrics were written. We have to select
+	// the rows using their primary keys in order to take advantage of
+	// read-after-write consistency in CrateDB.
+	for _, m := range metrics {
+		var id int64
+		row := db.QueryRow("SELECT hash_id FROM testing WHERE hash_id = ? AND timestamp = ?", hashID(m), m.Time())
+		require.NoError(t, row.Scan(&id))
+		// We could check the whole row, but this is meant to be more of a smoke
+		// test, so just checking the HashID seems fine.
+		require.Equal(t, id, hashID(m))
+	}
+
+	require.NoError(t, c.Close())
+}
+
+func TestConnectionIssueAtStartup(t *testing.T) {
+	// Test case for https://github.com/influxdata/telegraf/issues/13278
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	container := testutil.Container{
+		Image:        "crate",
+		ExposedPorts: []string{servicePort},
+		Entrypoint: []string{
+			"/docker-entrypoint.sh",
+			"-Cdiscovery.type=single-node",
+		},
+		WaitingFor: wait.ForAll(
+			wait.ForListeningPort(servicePort),
+			wait.ForLog("recovered [0] indices into cluster_state"),
+		),
+	}
+	require.NoError(t, container.Start(), "failed to start container")
+	defer container.Terminate()
+	url := fmt.Sprintf("postgres://crate@%s:%s/test", container.Address, container.Ports[servicePort])
+
+	// Pause the container for connectivity issues
+	require.NoError(t, container.Pause())
+
+	// Create a model to be able to use the startup retry strategy
+	plugin := &CrateDB{
+		URL:         url,
+		Table:       "testing",
+		Timeout:     config.Duration(time.Second * 5),
+		TableCreate: true,
+	}
+	model := models.NewRunningOutput(
+		plugin,
+		&models.OutputConfig{
+			Name:                 "cratedb",
+			StartupErrorBehavior: "retry",
+		},
+		1000, 1000,
+	)
+	require.NoError(t, model.Init())
+
+	// The connect call should succeed even though the table creation was not
+	// successful due to the "retry" strategy
+	require.NoError(t, model.Connect())
+
+	// Writing the metrics in this state should fail because we are not fully
+	// started up
+	metrics := testutil.MockMetrics()
+	for _, m := range metrics {
+		model.AddMetric(m)
+	}
+	require.ErrorIs(t, model.WriteBatch(), internal.ErrNotConnected)
+
+	// Unpause the container, now writes should succeed
+	require.NoError(t, container.Resume())
+	require.NoError(t, model.WriteBatch())
+	defer model.Close()
+
+	// Verify that the metrics were actually written
+	for _, m := range metrics {
+		mid := hashID(m)
+		row := plugin.db.QueryRow("SELECT hash_id FROM testing WHERE hash_id = ? AND timestamp = ?", mid, m.Time())
+
+		var id int64
+		require.NoError(t, row.Scan(&id))
+		require.Equal(t, id, mid)
+	}
+}
+
+func TestInsertSQL(t *testing.T) {
+	tests := []struct {
+		Metrics []telegraf.Metric
+		Want    string
+	}{
+		{
+			Metrics: testutil.MockMetrics(),
+			Want: strings.TrimSpace(`
+INSERT INTO my_table ("hash_id", "timestamp", "name", "tags", "fields")
+VALUES
+(-4023501406646044814, '2009-11-10T23:00:00+0000', 'test1', {"tag1" = 'value1'}, {"value" = 1});
+`),
+		},
+	}
+
+	for _, test := range tests {
+		if got, err := insertSQL("my_table", "_", test.Metrics); err != nil {
+			t.Error(err)
+		} else if got != test.Want {
+			t.Errorf("got:\n%s\n\nwant:\n%s", got, test.Want)
+		}
+	}
+}
+
+type escapeValueTest struct {
+	Value interface{}
+	Want  string
+}
+
+func escapeValueTests() []escapeValueTest {
+	return []escapeValueTest{
+		// string
+		{`foo`, `'foo'`},
+		{`foo'bar 'yeah`, `'foo''bar ''yeah'`},
+		// int types
+		{int64(123), `123`},
+		{uint64(123), `123`},
+		{uint64(MaxInt64) + 1, `9223372036854775807`},
+		{true, `true`},
+		{false, `false`},
+		// float types
+		{float64(123.456), `123.456`},
+		// time.Time
+		{time.Date(2017, 8, 7, 16, 44, 52, 123*1000*1000, time.FixedZone("Dreamland", 5400)), `'2017-08-07T16:44:52.123+0130'`},
+		// map[string]string
+		{map[string]string{}, `{}`},
+		{map[string]string(nil), `{}`},
+		{map[string]string{"foo": "bar"}, `{"foo" = 'bar'}`},
+		{map[string]string{"foo": "bar", "one": "more"}, `{"foo" = 'bar', "one" = 'more'}`},
+		{map[string]string{"f.oo": "bar", "o.n.e": "more"}, `{"f_oo" = 'bar', "o_n_e" = 'more'}`},
+		// map[string]interface{}
+		{map[string]interface{}{}, `{}`},
+		{map[string]interface{}(nil), `{}`},
+		{map[string]interface{}{"foo": "bar"}, `{"foo" = 'bar'}`},
+		{map[string]interface{}{"foo": "bar", "one": "more"}, `{"foo" = 'bar', "one" = 'more'}`},
+		{map[string]interface{}{"foo": map[string]interface{}{"one": "more"}}, `{"foo" = {"one" = 'more'}}`},
+		{map[string]interface{}{`fo"o`: `b'ar`, `ab'c`: `xy"z`, `on"""e`: `mo'''re`}, `{"ab'c" = 'xy"z', "fo""o" = 'b''ar', "on""""""e" = 'mo''''''re'}`},
+	}
+}
+
+func TestEscapeValueIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	container := createTestContainer(t)
+	defer container.Terminate()
+	url := fmt.Sprintf("postgres://crate@%s:%s/test", container.Address, container.Ports[servicePort])
+
+	db, err := sql.Open("pgx", url)
+	require.NoError(t, err)
+	defer db.Close()
+
+	tests := escapeValueTests()
+	for _, test := range tests {
+		got, err := escapeValue(test.Value, "_")
+		require.NoError(t, err, "value: %#v", test.Value)
+
+		// This is a smoke test that will blow up if our escaping causing a SQL
+		// syntax error, which may allow for an attack.=
+		var reply interface{}
+		row := db.QueryRow("SELECT ?", got)
+		require.NoError(t, row.Scan(&reply))
+	}
+}
+
+func TestEscapeValue(t *testing.T) {
+	tests := escapeValueTests()
+	for _, test := range tests {
+		got, err := escapeValue(test.Value, "_")
+		require.NoError(t, err, "value: %#v", test.Value)
+		require.Equal(t, test.Want, got)
+	}
+}
+
+func TestCircumventingStringEscape(t *testing.T) {
+	value, err := escapeObject(map[string]interface{}{"a.b": "c"}, `_"`)
+	require.NoError(t, err)
+	require.Equal(t, `{"a_""b" = 'c'}`, value)
+}
+
+func Test_hashID(t *testing.T) {
+	tests := []struct {
+		Name   string
+		Tags   map[string]string
+		Fields map[string]interface{}
+		Want   int64
+	}{
+		{
+			Name:   "metric1",
+			Tags:   map[string]string{"tag1": "val1", "tag2": "val2"},
+			Fields: map[string]interface{}{"field1": "val1", "field2": "val2"},
+			Want:   8973971082006474188,
+		},
+
+		// This metric has a different tag order (in a perhaps non-ideal attempt to
+		// trigger different pseudo-random map iteration)) and fields (none)
+		// compared to the previous metric, but should still get the same hash.
+		{
+			Name:   "metric1",
+			Tags:   map[string]string{"tag2": "val2", "tag1": "val1"},
+			Fields: map[string]interface{}{"field3": "val3"},
+			Want:   8973971082006474188,
+		},
+
+		// Different metric name -> different hash
+		{
+			Name:   "metric2",
+			Tags:   map[string]string{"tag1": "val1", "tag2": "val2"},
+			Fields: map[string]interface{}{"field1": "val1", "field2": "val2"},
+			Want:   306487682448261783,
+		},
+
+		// Different tag val -> different hash
+		{
+			Name:   "metric1",
+			Tags:   map[string]string{"tag1": "new-val", "tag2": "val2"},
+			Fields: map[string]interface{}{"field1": "val1", "field2": "val2"},
+			Want:   1938713695181062970,
+		},
+
+		// Different tag key -> different hash
+		{
+			Name:   "metric1",
+			Tags:   map[string]string{"new-key": "val1", "tag2": "val2"},
+			Fields: map[string]interface{}{"field1": "val1", "field2": "val2"},
+			Want:   7678889081527706328,
+		},
+	}
+
+	for i, test := range tests {
+		m := metric.New(
+			test.Name,
+			test.Tags,
+			test.Fields,
+			time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
+		)
+		if got := hashID(m); got != test.Want {
+			t.Errorf("test #%d: got=%d want=%d", i, got, test.Want)
+		}
+	}
+}

plugins/outputs/cratedb/sample.conf

@@ -0,0 +1,18 @@
+# Configuration for CrateDB to send metrics to.
+[[outputs.cratedb]]
+  ## Connection parameters for accessing the database see
+  ##   https://pkg.go.dev/github.com/jackc/pgx/v4#ParseConfig
+  ## for available options
+  url = "postgres://user:password@localhost/schema?sslmode=disable"
+
+  ## Timeout for all CrateDB queries.
+  # timeout = "5s"
+
+  ## Name of the table to store metrics in.
+  # table = "metrics"
+
+  ## If true, and the metrics table does not exist, create it automatically.
+  # table_create = false
+
+  ## The character(s) to replace any '.' in an object key with
+  # key_separator = "_"