Adding upstream version 1.34.4.

Signed-off-by: Daniel Baumann <daniel@debian.org>

plugins/outputs/opentelemetry/README.md

@@ -0,0 +1,116 @@
+# OpenTelemetry Output Plugin
+
+This plugin writes metrics to [OpenTelemetry][opentelemetry] servers and agents
+via gRPC.
+
+⭐ Telegraf v1.20.0
+🏷️ logging, messaging
+💻 all
+
+[opentelemetry]: https://opentelemetry.io
+
+## Global configuration options <!-- @/docs/includes/plugin_config.md -->
+
+In addition to the plugin-specific configuration settings, plugins support
+additional global and plugin configuration settings. These settings are used to
+modify metrics, tags, and field or create aliases and configure ordering, etc.
+See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
+
+[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
+
+## Configuration
+
+```toml @sample.conf
+# Send OpenTelemetry metrics over gRPC
+[[outputs.opentelemetry]]
+  ## Override the default (localhost:4317) OpenTelemetry gRPC service
+  ## address:port
+  # service_address = "localhost:4317"
+
+  ## Override the default (5s) request timeout
+  # timeout = "5s"
+
+  ## Optional TLS Config.
+  ##
+  ## Root certificates for verifying server certificates encoded in PEM format.
+  # tls_ca = "/etc/telegraf/ca.pem"
+  ## The public and private key pairs for the client encoded in PEM format.
+  ## May contain intermediate certificates.
+  # tls_cert = "/etc/telegraf/cert.pem"
+  # tls_key = "/etc/telegraf/key.pem"
+  ## Use TLS, but skip TLS chain and host verification.
+  # insecure_skip_verify = false
+  ## Send the specified TLS server name via SNI.
+  # tls_server_name = "foo.example.com"
+
+  ## Override the default (gzip) compression used to send data.
+  ## Supports: "gzip", "none"
+  # compression = "gzip"
+
+  ## NOTE: Due to the way TOML is parsed, tables must be at the END of the
+  ## plugin definition, otherwise additional config options are read as part of
+  ## the table
+
+  ## Configuration options for the Coralogix dialect
+  ## Enable the following section of you use this plugin with a Coralogix endpoint
+  # [outputs.opentelemetry.coralogix]
+  #   ## Your Coralogix private key (required).
+  #   ## Please note that this is sensitive data!
+  #   private_key = "your_coralogix_key"
+  #
+  #   ## Application and subsystem names for the metrics (required)
+  #   application = "$NAMESPACE"
+  #   subsystem = "$HOSTNAME"
+
+  ## Additional OpenTelemetry resource attributes
+  # [outputs.opentelemetry.attributes]
+  # "service.name" = "demo"
+
+  ## Additional gRPC request metadata
+  # [outputs.opentelemetry.headers]
+  # key1 = "value1"
+```
+
+## Supported dialects
+
+### Coralogix
+
+This plugins supports sending data to a [Coralogix](https://coralogix.com)
+server by enabling the corresponding dialect by uncommenting
+the `[output.opentelemetry.coralogix]` section.
+
+There, you can find the required setting to interact with the server.
+
+- The `private_key` is your Private Key, which you can find in
+  `Settings > Send Your Data`.
+- The `application`, is your application name, which will be added to your
+  `metric attributes`.
+- The `subsystem`, is your subsystem, which will be added to your metric
+  attributes.
+
+More information in the
+[Getting Started page](https://coralogix.com/docs/guide-first-steps-coralogix/).
+
+### Schema
+
+The InfluxDB->OpenTelemetry conversion [schema][] and [implementation][] are
+hosted on [GitHub][repo].
+
+For metrics, two input schemata exist.  Line protocol with measurement name
+`prometheus` is assumed to have a schema matching [Prometheus input
+plugin](../../inputs/prometheus/README.md) when `metric_version = 2`.  Line
+protocol with other measurement names is assumed to have schema matching
+[Prometheus input plugin](../../inputs/prometheus/README.md) when
+`metric_version = 1`.  If both schema assumptions fail, then the line protocol
+data is interpreted as:
+
+- Metric type = gauge (or counter, if indicated by the input plugin)
+- Metric name = `[measurement]_[field key]`
+- Metric value = line protocol field value, cast to float
+- Metric labels = line protocol tags
+
+Also see the [OpenTelemetry input plugin](../../inputs/opentelemetry/README.md).
+
+[schema]: https://github.com/influxdata/influxdb-observability/blob/main/docs/index.md
+[implementation]: https://github.com/influxdata/influxdb-observability/tree/main/influx2otel
+[repo]: https://github.com/influxdata/influxdb-observability

plugins/outputs/opentelemetry/logger.go

@@ -0,0 +1,16 @@
+package opentelemetry
+
+import (
+	"strings"
+
+	"github.com/influxdata/telegraf"
+)
+
+type otelLogger struct {
+	telegraf.Logger
+}
+
+func (l otelLogger) Debug(msg string, kv ...interface{}) {
+	format := msg + strings.Repeat(" %s=%q", len(kv)/2)
+	l.Logger.Debugf(format, kv...)
+}

plugins/outputs/opentelemetry/opentelemetry.go

@@ -0,0 +1,215 @@
+//go:generate ../../../tools/readme_config_includer/generator
+package opentelemetry
+
+import (
+	"context"
+	ntls "crypto/tls"
+	_ "embed"
+	"sort"
+	"time"
+
+	"github.com/influxdata/influxdb-observability/common"
+	"github.com/influxdata/influxdb-observability/influx2otel"
+	"go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	_ "google.golang.org/grpc/encoding/gzip" // Blank import to allow gzip encoding
+	"google.golang.org/grpc/metadata"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/common/tls"
+	"github.com/influxdata/telegraf/plugins/outputs"
+)
+
+var userAgent = internal.ProductToken()
+
+//go:embed sample.conf
+var sampleConfig string
+
+type OpenTelemetry struct {
+	ServiceAddress string `toml:"service_address"`
+
+	tls.ClientConfig
+	Timeout     config.Duration   `toml:"timeout"`
+	Compression string            `toml:"compression"`
+	Headers     map[string]string `toml:"headers"`
+	Attributes  map[string]string `toml:"attributes"`
+	Coralogix   *CoralogixConfig  `toml:"coralogix"`
+
+	Log telegraf.Logger `toml:"-"`
+
+	metricsConverter     *influx2otel.LineProtocolToOtelMetrics
+	grpcClientConn       *grpc.ClientConn
+	metricsServiceClient pmetricotlp.GRPCClient
+	callOptions          []grpc.CallOption
+}
+
+type CoralogixConfig struct {
+	AppName    string `toml:"application"`
+	SubSystem  string `toml:"subsystem"`
+	PrivateKey string `toml:"private_key"`
+}
+
+func (*OpenTelemetry) SampleConfig() string {
+	return sampleConfig
+}
+
+func (o *OpenTelemetry) Connect() error {
+	logger := &otelLogger{o.Log}
+
+	if o.ServiceAddress == "" {
+		o.ServiceAddress = defaultServiceAddress
+	}
+	if o.Timeout <= 0 {
+		o.Timeout = defaultTimeout
+	}
+	if o.Compression == "" {
+		o.Compression = defaultCompression
+	}
+	if o.Coralogix != nil {
+		if o.Headers == nil {
+			o.Headers = make(map[string]string)
+		}
+		o.Headers["ApplicationName"] = o.Coralogix.AppName
+		o.Headers["ApiName"] = o.Coralogix.SubSystem
+		o.Headers["Authorization"] = "Bearer " + o.Coralogix.PrivateKey
+	}
+
+	metricsConverter, err := influx2otel.NewLineProtocolToOtelMetrics(logger)
+	if err != nil {
+		return err
+	}
+
+	var grpcTLSDialOption grpc.DialOption
+	if tlsConfig, err := o.ClientConfig.TLSConfig(); err != nil {
+		return err
+	} else if tlsConfig != nil {
+		grpcTLSDialOption = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
+	} else if o.Coralogix != nil {
+		// For coralogix, we enforce GRPC connection with TLS
+		grpcTLSDialOption = grpc.WithTransportCredentials(credentials.NewTLS(&ntls.Config{}))
+	} else {
+		grpcTLSDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
+	}
+
+	grpcClientConn, err := grpc.NewClient(o.ServiceAddress, grpcTLSDialOption, grpc.WithUserAgent(userAgent))
+	if err != nil {
+		return err
+	}
+
+	metricsServiceClient := pmetricotlp.NewGRPCClient(grpcClientConn)
+
+	o.metricsConverter = metricsConverter
+	o.grpcClientConn = grpcClientConn
+	o.metricsServiceClient = metricsServiceClient
+
+	if o.Compression != "" && o.Compression != "none" {
+		o.callOptions = append(o.callOptions, grpc.UseCompressor(o.Compression))
+	}
+
+	return nil
+}
+
+func (o *OpenTelemetry) Close() error {
+	if o.grpcClientConn != nil {
+		err := o.grpcClientConn.Close()
+		o.grpcClientConn = nil
+		return err
+	}
+	return nil
+}
+
+// Split metrics up by timestamp and send to Google Cloud Stackdriver
+func (o *OpenTelemetry) Write(metrics []telegraf.Metric) error {
+	metricBatch := make(map[int64][]telegraf.Metric)
+	timestamps := make([]int64, 0, len(metrics))
+	for _, metric := range metrics {
+		timestamp := metric.Time().UnixNano()
+		if existingSlice, ok := metricBatch[timestamp]; ok {
+			metricBatch[timestamp] = append(existingSlice, metric)
+		} else {
+			metricBatch[timestamp] = []telegraf.Metric{metric}
+			timestamps = append(timestamps, timestamp)
+		}
+	}
+
+	// sort the timestamps we collected
+	sort.Slice(timestamps, func(i, j int) bool { return timestamps[i] < timestamps[j] })
+
+	o.Log.Debugf("Received %d metrics and split into %d groups by timestamp", len(metrics), len(metricBatch))
+	for _, timestamp := range timestamps {
+		if err := o.sendBatch(metricBatch[timestamp]); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (o *OpenTelemetry) sendBatch(metrics []telegraf.Metric) error {
+	batch := o.metricsConverter.NewBatch()
+	for _, metric := range metrics {
+		var vType common.InfluxMetricValueType
+		switch metric.Type() {
+		case telegraf.Gauge:
+			vType = common.InfluxMetricValueTypeGauge
+		case telegraf.Untyped:
+			vType = common.InfluxMetricValueTypeUntyped
+		case telegraf.Counter:
+			vType = common.InfluxMetricValueTypeSum
+		case telegraf.Histogram:
+			vType = common.InfluxMetricValueTypeHistogram
+		case telegraf.Summary:
+			vType = common.InfluxMetricValueTypeSummary
+		default:
+			o.Log.Warnf("Unrecognized metric type %v", metric.Type())
+			continue
+		}
+		err := batch.AddPoint(metric.Name(), metric.Tags(), metric.Fields(), metric.Time(), vType)
+		if err != nil {
+			o.Log.Warnf("Failed to add point: %v", err)
+			continue
+		}
+	}
+
+	md := pmetricotlp.NewExportRequestFromMetrics(batch.GetMetrics())
+	if md.Metrics().ResourceMetrics().Len() == 0 {
+		return nil
+	}
+
+	if len(o.Attributes) > 0 {
+		for i := 0; i < md.Metrics().ResourceMetrics().Len(); i++ {
+			for k, v := range o.Attributes {
+				md.Metrics().ResourceMetrics().At(i).Resource().Attributes().PutStr(k, v)
+			}
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(o.Timeout))
+
+	if len(o.Headers) > 0 {
+		ctx = metadata.NewOutgoingContext(ctx, metadata.New(o.Headers))
+	}
+	defer cancel()
+	_, err := o.metricsServiceClient.Export(ctx, md, o.callOptions...)
+	return err
+}
+
+const (
+	defaultServiceAddress = "localhost:4317"
+	defaultTimeout        = config.Duration(5 * time.Second)
+	defaultCompression    = "gzip"
+)
+
+func init() {
+	outputs.Add("opentelemetry", func() telegraf.Output {
+		return &OpenTelemetry{
+			ServiceAddress: defaultServiceAddress,
+			Timeout:        defaultTimeout,
+			Compression:    defaultCompression,
+		}
+	})
+}

plugins/outputs/opentelemetry/opentelemetry_test.go

@@ -0,0 +1,146 @@
+package opentelemetry
+
+import (
+	"context"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/influxdata/influxdb-observability/common"
+	"github.com/influxdata/influxdb-observability/influx2otel"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/pdata/pcommon"
+	"go.opentelemetry.io/collector/pdata/pmetric"
+	"go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/testutil"
+)
+
+func TestOpenTelemetry(t *testing.T) {
+	expect := pmetric.NewMetrics()
+	{
+		rm := expect.ResourceMetrics().AppendEmpty()
+		rm.Resource().Attributes().PutStr("host.name", "potato")
+		rm.Resource().Attributes().PutStr("attr-key", "attr-val")
+		ilm := rm.ScopeMetrics().AppendEmpty()
+		ilm.Scope().SetName("My Library Name")
+		m := ilm.Metrics().AppendEmpty()
+		m.SetName("cpu_temp")
+		m.SetEmptyGauge()
+		dp := m.Gauge().DataPoints().AppendEmpty()
+		dp.Attributes().PutStr("foo", "bar")
+		dp.SetTimestamp(pcommon.Timestamp(1622848686000000000))
+		dp.SetDoubleValue(87.332)
+	}
+	m := newMockOtelService(t)
+	t.Cleanup(m.Cleanup)
+
+	metricsConverter, err := influx2otel.NewLineProtocolToOtelMetrics(common.NoopLogger{})
+	require.NoError(t, err)
+	plugin := &OpenTelemetry{
+		ServiceAddress:       m.Address(),
+		Timeout:              config.Duration(time.Second),
+		Headers:              map[string]string{"test": "header1"},
+		Attributes:           map[string]string{"attr-key": "attr-val"},
+		metricsConverter:     metricsConverter,
+		grpcClientConn:       m.GrpcClient(),
+		metricsServiceClient: pmetricotlp.NewGRPCClient(m.GrpcClient()),
+		Log:                  testutil.Logger{},
+	}
+
+	input := testutil.MustMetric(
+		"cpu_temp",
+		map[string]string{
+			"foo":               "bar",
+			"otel.library.name": "My Library Name",
+			"host.name":         "potato",
+		},
+		map[string]interface{}{
+			"gauge": 87.332,
+		},
+		time.Unix(0, 1622848686000000000))
+
+	err = plugin.Write([]telegraf.Metric{input})
+	require.NoError(t, err)
+
+	got := m.GotMetrics()
+
+	marshaller := pmetric.JSONMarshaler{}
+	expectJSON, err := marshaller.MarshalMetrics(expect)
+	require.NoError(t, err)
+
+	gotJSON, err := marshaller.MarshalMetrics(got)
+	require.NoError(t, err)
+
+	require.JSONEq(t, string(expectJSON), string(gotJSON))
+}
+
+var _ pmetricotlp.GRPCServer = (*mockOtelService)(nil)
+
+type mockOtelService struct {
+	pmetricotlp.UnimplementedGRPCServer
+	t          *testing.T
+	listener   net.Listener
+	grpcServer *grpc.Server
+	grpcClient *grpc.ClientConn
+
+	metrics pmetric.Metrics
+}
+
+func newMockOtelService(t *testing.T) *mockOtelService {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	grpcServer := grpc.NewServer()
+
+	mockOtelService := &mockOtelService{
+		t:          t,
+		listener:   listener,
+		grpcServer: grpcServer,
+	}
+
+	pmetricotlp.RegisterGRPCServer(grpcServer, mockOtelService)
+	go func() {
+		if err := grpcServer.Serve(listener); err != nil {
+			t.Error(err)
+		}
+	}()
+
+	grpcClient, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.NoError(t, err)
+	require.True(t, grpcClient.WaitForStateChange(t.Context(), connectivity.Connecting))
+	mockOtelService.grpcClient = grpcClient
+
+	return mockOtelService
+}
+
+func (m *mockOtelService) Cleanup() {
+	require.NoError(m.t, m.grpcClient.Close())
+	m.grpcServer.Stop()
+}
+
+func (m *mockOtelService) GrpcClient() *grpc.ClientConn {
+	return m.grpcClient
+}
+
+func (m *mockOtelService) GotMetrics() pmetric.Metrics {
+	return m.metrics
+}
+
+func (m *mockOtelService) Address() string {
+	return m.listener.Addr().String()
+}
+
+func (m *mockOtelService) Export(ctx context.Context, request pmetricotlp.ExportRequest) (pmetricotlp.ExportResponse, error) {
+	m.metrics = pmetric.NewMetrics()
+	request.Metrics().CopyTo(m.metrics)
+	ctxMetadata, ok := metadata.FromIncomingContext(ctx)
+	require.Equal(m.t, []string{"header1"}, ctxMetadata.Get("test"))
+	require.True(m.t, ok)
+	return pmetricotlp.NewExportResponse(), nil
+}

plugins/outputs/opentelemetry/sample.conf

@@ -0,0 +1,48 @@
+# Send OpenTelemetry metrics over gRPC
+[[outputs.opentelemetry]]
+  ## Override the default (localhost:4317) OpenTelemetry gRPC service
+  ## address:port
+  # service_address = "localhost:4317"
+
+  ## Override the default (5s) request timeout
+  # timeout = "5s"
+
+  ## Optional TLS Config.
+  ##
+  ## Root certificates for verifying server certificates encoded in PEM format.
+  # tls_ca = "/etc/telegraf/ca.pem"
+  ## The public and private key pairs for the client encoded in PEM format.
+  ## May contain intermediate certificates.
+  # tls_cert = "/etc/telegraf/cert.pem"
+  # tls_key = "/etc/telegraf/key.pem"
+  ## Use TLS, but skip TLS chain and host verification.
+  # insecure_skip_verify = false
+  ## Send the specified TLS server name via SNI.
+  # tls_server_name = "foo.example.com"
+
+  ## Override the default (gzip) compression used to send data.
+  ## Supports: "gzip", "none"
+  # compression = "gzip"
+
+  ## NOTE: Due to the way TOML is parsed, tables must be at the END of the
+  ## plugin definition, otherwise additional config options are read as part of
+  ## the table
+
+  ## Configuration options for the Coralogix dialect
+  ## Enable the following section of you use this plugin with a Coralogix endpoint
+  # [outputs.opentelemetry.coralogix]
+  #   ## Your Coralogix private key (required).
+  #   ## Please note that this is sensitive data!
+  #   private_key = "your_coralogix_key"
+  #
+  #   ## Application and subsystem names for the metrics (required)
+  #   application = "$NAMESPACE"
+  #   subsystem = "$HOSTNAME"
+
+  ## Additional OpenTelemetry resource attributes
+  # [outputs.opentelemetry.attributes]
+  # "service.name" = "demo"
+
+  ## Additional gRPC request metadata
+  # [outputs.opentelemetry.headers]
+  # key1 = "value1"