Adding upstream version 1.34.4.

Signed-off-by: Daniel Baumann <daniel@debian.org>

plugins/processors/noise/README.md

@@ -0,0 +1,97 @@
+# Noise Processor Plugin
+
+The _Noise_ processor is used to add noise to numerical field values. For each
+field a noise is generated using a defined probability density function and
+added to the value. The function type can be configured as _Laplace_, _Gaussian_
+or _Uniform_.  Depending on the function, various parameters need to be
+configured:
+
+## Global configuration options <!-- @/docs/includes/plugin_config.md -->
+
+In addition to the plugin-specific configuration settings, plugins support
+additional global and plugin configuration settings. These settings are used to
+modify metrics, tags, and field or create aliases and configure ordering, etc.
+See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
+
+[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
+
+## Configuration
+
+```toml @sample.conf
+# Adds noise to numerical fields
+[[processors.noise]]
+  ## Specified the type of the random distribution.
+  ## Can be "laplacian", "gaussian" or "uniform".
+  # type = "laplacian
+
+  ## Center of the distribution.
+  ## Only used for Laplacian and Gaussian distributions.
+  # mu = 0.0
+
+  ## Scale parameter for the Laplacian or Gaussian distribution
+  # scale = 1.0
+
+  ## Upper and lower bound of the Uniform distribution
+  # min = -1.0
+  # max = 1.0
+
+  ## Apply the noise only to numeric fields matching the filter criteria below.
+  ## Excludes takes precedence over includes.
+  # include_fields = []
+  # exclude_fields = []
+```
+
+Depending on the choice of the distribution function, the respective parameters
+must be set. Default settings are `noise_type = "laplacian"` with `mu = 0.0` and
+`scale = 1.0`:
+
+Using the `include_fields` and `exclude_fields` options a filter can be
+configured to apply noise only to numeric fields matching it.  The following
+distribution functions are available.
+
+### Laplacian
+
+- `noise_type = laplacian`
+- `scale`: also referred to as _diversity_ parameter, regulates the width & height of the function, a bigger `scale` value means a higher probability of larger noise, default set to 1.0
+- `mu`: location of the curve, default set to 0.0
+
+### Gaussian
+
+- `noise_type = gaussian`
+- `mu`: mean value, default set to 0.0
+- `scale`: standard deviation, default set to 1.0
+
+### Uniform
+
+- `noise_type = uniform`
+- `min`: minimal interval value, default set to -1.0
+- `max`: maximal interval value, default set to 1.0
+
+## Example
+
+Add noise to each value the _inputs.cpu_ plugin generates, except for the
+_usage\_steal_, _usage\_user_, _uptime\_format_, _usage\_idle_ field and all
+fields of the metrics _swap_, _disk_ and _net_:
+
+```toml
+[[inputs.cpu]]
+  percpu = true
+  totalcpu = true
+  collect_cpu_time = false
+  report_active = false
+
+[[processors.noise]]
+  scale = 1.0
+  mu = 0.0
+  noise_type = "laplacian"
+  include_fields = []
+  exclude_fields = ["usage_steal", "usage_user", "uptime_format", "usage_idle" ]
+  namedrop = ["swap", "disk", "net"]
+```
+
+Result of noise added to the _cpu_ metric:
+
+```diff
+- cpu map[cpu:cpu11 host:98d5b8dbad1c] map[usage_guest:0 usage_guest_nice:0 usage_idle:94.3999999994412 usage_iowait:0 usage_irq:0.1999999999998181 usage_nice:0 usage_softirq:0.20000000000209184 usage_steal:0 usage_system:1.2000000000080036 usage_user:4.000000000014552]
++ cpu map[cpu:cpu11 host:98d5b8dbad1c] map[usage_guest:1.0078071583066057 usage_guest_nice:0.523063861602435 usage_idle:95.53920223476884 usage_iowait:0.5162661526251292 usage_irq:0.7138529816101375 usage_nice:0.6119678488887954 usage_softirq:0.5573585443688622 usage_steal:0.2006120911289802 usage_system:1.2954475820198437 usage_user:6.885664792615023]
+```

plugins/processors/noise/noise.go

@@ -0,0 +1,136 @@
+//go:generate ../../../tools/readme_config_includer/generator
+package noise
+
+import (
+	_ "embed"
+	"fmt"
+	"math"
+	"reflect"
+
+	"gonum.org/v1/gonum/stat/distuv"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/filter"
+	"github.com/influxdata/telegraf/plugins/processors"
+)
+
+//go:embed sample.conf
+var sampleConfig string
+
+const (
+	defaultScale     = 1.0
+	defaultMin       = -1.0
+	defaultMax       = 1.0
+	defaultMu        = 0.0
+	defaultNoiseType = "laplacian"
+)
+
+type Noise struct {
+	Scale         float64         `toml:"scale"`
+	Min           float64         `toml:"min"`
+	Max           float64         `toml:"max"`
+	Mu            float64         `toml:"mu"`
+	IncludeFields []string        `toml:"include_fields"`
+	ExcludeFields []string        `toml:"exclude_fields"`
+	NoiseType     string          `toml:"type"`
+	Log           telegraf.Logger `toml:"-"`
+	generator     distuv.Rander
+	fieldFilter   filter.Filter
+}
+
+// generates a random noise value depending on the defined probability density
+// function and adds that to the original value. If any integer overflows
+// happen during the calculation, the result is set to MaxInt or 0 (for uint)
+func (p *Noise) addNoise(value interface{}) interface{} {
+	n := p.generator.Rand()
+	switch v := value.(type) {
+	case int:
+	case int8:
+	case int16:
+	case int32:
+	case int64:
+		if v > 0 && (n > math.Nextafter(float64(math.MaxInt64), 0) || int64(n) > math.MaxInt64-v) {
+			p.Log.Debug("Int64 overflow, setting value to MaxInt64")
+			return int64(math.MaxInt64)
+		}
+		if v < 0 && (n < math.Nextafter(float64(math.MinInt64), 0) || int64(n) < math.MinInt64-v) {
+			p.Log.Debug("Int64 (negative) overflow, setting value to MinInt64")
+			return int64(math.MinInt64)
+		}
+		return v + int64(n)
+	case uint:
+	case uint8:
+	case uint16:
+	case uint32:
+	case uint64:
+		if n < 0 {
+			if uint64(-n) > v {
+				p.Log.Debug("Uint64 (negative) overflow, setting value to 0")
+				return uint64(0)
+			}
+			return v - uint64(-n)
+		}
+		if n > math.Nextafter(float64(math.MaxUint64), 0) || uint64(n) > math.MaxUint64-v {
+			p.Log.Debug("Uint64 overflow, setting value to MaxUint64")
+			return uint64(math.MaxUint64)
+		}
+		return v + uint64(n)
+	case float32:
+		return v + float32(n)
+	case float64:
+		return v + n
+	default:
+		p.Log.Debugf("Value (%v) type invalid: [%v] is not an int, uint or float", v, reflect.TypeOf(value))
+	}
+	return value
+}
+
+func (*Noise) SampleConfig() string {
+	return sampleConfig
+}
+
+// Creates a filter for Include and Exclude fields and sets the desired noise
+// distribution
+func (p *Noise) Init() error {
+	fieldFilter, err := filter.NewIncludeExcludeFilter(p.IncludeFields, p.ExcludeFields)
+	if err != nil {
+		return fmt.Errorf("creating fieldFilter failed: %w", err)
+	}
+	p.fieldFilter = fieldFilter
+
+	switch p.NoiseType {
+	case "", "laplacian":
+		p.generator = &distuv.Laplace{Mu: p.Mu, Scale: p.Scale}
+	case "uniform":
+		p.generator = &distuv.Uniform{Min: p.Min, Max: p.Max}
+	case "gaussian":
+		p.generator = &distuv.Normal{Mu: p.Mu, Sigma: p.Scale}
+	default:
+		return fmt.Errorf("unknown distribution type %q", p.NoiseType)
+	}
+	return nil
+}
+
+func (p *Noise) Apply(metrics ...telegraf.Metric) []telegraf.Metric {
+	for _, metric := range metrics {
+		for _, field := range metric.FieldList() {
+			if !p.fieldFilter.Match(field.Key) {
+				continue
+			}
+			field.Value = p.addNoise(field.Value)
+		}
+	}
+	return metrics
+}
+
+func init() {
+	processors.Add("noise", func() telegraf.Processor {
+		return &Noise{
+			NoiseType: defaultNoiseType,
+			Mu:        defaultMu,
+			Scale:     defaultScale,
+			Min:       defaultMin,
+			Max:       defaultMax,
+		}
+	})
+}

plugins/processors/noise/noise_test.go

@@ -0,0 +1,467 @@
+package noise
+
+import (
+	"math"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"gonum.org/v1/gonum/stat/distuv"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/metric"
+	"github.com/influxdata/telegraf/testutil"
+)
+
+type testDistribution struct {
+	value float64
+}
+
+func (t *testDistribution) Rand() float64 {
+	return t.value
+}
+
+// Verifies that field values are modified by the Laplace noise
+func TestAddNoiseToMetric(t *testing.T) {
+	generators := []string{"laplacian", "gaussian", "uniform"}
+	for _, generator := range generators {
+		p := Noise{
+			NoiseType: generator,
+			Scale:     1.0,
+			Mu:        0.0,
+			Min:       -1,
+			Max:       1,
+			Log:       testutil.Logger{},
+		}
+		require.NoError(t, p.Init())
+		for _, m := range testutil.MockMetrics() {
+			after := p.Apply(m.Copy())
+			require.Len(t, after, 1)
+			require.NotEqual(t, m, after[0])
+		}
+	}
+}
+
+// Verifies that a given noise is added correctly to values
+func TestAddNoise(t *testing.T) {
+	tests := []struct {
+		name         string
+		input        []telegraf.Metric
+		expected     []telegraf.Metric
+		distribution distuv.Rander
+	}{
+		{
+			name: "int64",
+			input: []telegraf.Metric{
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": int64(5)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": int64(-10)},
+					time.Unix(0, 0),
+				),
+			},
+			expected: []telegraf.Metric{
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": int64(4)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": int64(-11)},
+					time.Unix(0, 0),
+				),
+			},
+			distribution: &testDistribution{value: -1.5},
+		},
+		{
+			name: "uint64",
+			input: []telegraf.Metric{
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(25)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(0)},
+					time.Unix(0, 0),
+				),
+			},
+			expected: []telegraf.Metric{
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(26)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(1)},
+					time.Unix(0, 0),
+				),
+			},
+			distribution: &testDistribution{value: 1.5},
+		},
+		{
+			name: "float64",
+			input: []telegraf.Metric{
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": float64(0.0005)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": float64(1000.5)},
+					time.Unix(0, 0),
+				),
+			},
+			expected: []telegraf.Metric{
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": float64(5.0005)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": float64(1005.5)},
+					time.Unix(0, 0),
+				),
+			},
+			distribution: &testDistribution{value: 5.0},
+		},
+		{
+			name: "float64",
+			input: []telegraf.Metric{
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": float64(0.0005)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": float64(1000.5)},
+					time.Unix(0, 0),
+				),
+			},
+			expected: []telegraf.Metric{
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": float64(-0.4995)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{"value": float64(1000)},
+					time.Unix(0, 0),
+				),
+			},
+			distribution: &testDistribution{value: -0.5},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			plugin := Noise{
+				NoiseType: "laplacian",
+				Scale:     1.0,
+				Log:       testutil.Logger{},
+			}
+			require.NoError(t, plugin.Init())
+			plugin.generator = tt.distribution
+
+			actual := plugin.Apply(tt.input...)
+			testutil.RequireMetricsEqual(t, tt.expected, actual)
+		})
+	}
+}
+
+// Tests that int64 & uint64 overflow errors are caught
+func TestAddNoiseOverflowCheck(t *testing.T) {
+	tests := []struct {
+		name         string
+		input        []telegraf.Metric
+		expected     []telegraf.Metric
+		distribution distuv.Rander
+	}{
+		{
+			name: "underflow",
+			input: []telegraf.Metric{
+				testutil.MustMetric("underflow_int64",
+					map[string]string{},
+					map[string]interface{}{"value": int64(math.MinInt64)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("underflow_uint64_1",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(5)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("underflow_uint64_2",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(0)},
+					time.Unix(0, 0),
+				),
+			},
+			expected: []telegraf.Metric{
+				testutil.MustMetric("underflow_int64",
+					map[string]string{},
+					map[string]interface{}{"value": int64(math.MinInt64)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("underflow_uint64_1",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(4)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("underflow_uint64_2",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(0)},
+					time.Unix(0, 0),
+				),
+			},
+			distribution: &testDistribution{value: -1.0},
+		},
+		{
+			name: "overflow",
+			input: []telegraf.Metric{
+				testutil.MustMetric("overflow_int64",
+					map[string]string{},
+					map[string]interface{}{"value": int64(math.MaxInt64)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("overflow_uint",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(math.MaxUint)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("overflow_uint64",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(math.MaxUint64)},
+					time.Unix(0, 0),
+				),
+			},
+			expected: []telegraf.Metric{
+				testutil.MustMetric("overflow_int64",
+					map[string]string{},
+					map[string]interface{}{"value": int64(math.MaxInt64)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("overflow_uint",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(math.MaxUint)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("overflow_uint64",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(math.MaxUint64)},
+					time.Unix(0, 0),
+				),
+			},
+			distribution: &testDistribution{value: 0.0},
+		},
+		{
+			name: "non-numeric fields",
+			input: []telegraf.Metric{
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{
+						"a": "test",
+						"b": true,
+					},
+					time.Unix(0, 0),
+				),
+			},
+			expected: []telegraf.Metric{
+				testutil.MustMetric("cpu",
+					map[string]string{},
+					map[string]interface{}{
+						"a": "test",
+						"b": true,
+					},
+					time.Unix(0, 0),
+				),
+			},
+			distribution: &testDistribution{value: 1.0},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			plugin := Noise{
+				NoiseType: "laplacian",
+				Scale:     1.0,
+				Log:       testutil.Logger{},
+			}
+			require.NoError(t, plugin.Init())
+			plugin.generator = tt.distribution
+
+			actual := plugin.Apply(tt.input...)
+			testutil.RequireMetricsEqual(t, tt.expected, actual)
+		})
+	}
+}
+
+// Verifies that even addNoise() modifies 0 values as well
+func TestAddNoiseWithZeroValue(t *testing.T) {
+	tests := []struct {
+		name         string
+		input        []telegraf.Metric
+		expected     []telegraf.Metric
+		distribution distuv.Rander
+	}{
+		{
+			name: "zeros",
+			input: []telegraf.Metric{
+				testutil.MustMetric("zero_uint64",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(0)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("zero_int64",
+					map[string]string{},
+					map[string]interface{}{"value": int64(0)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("zero_float",
+					map[string]string{},
+					map[string]interface{}{"value": float64(0.0)},
+					time.Unix(0, 0),
+				),
+			},
+			expected: []telegraf.Metric{
+				testutil.MustMetric("zero_uint64",
+					map[string]string{},
+					map[string]interface{}{"value": uint64(13)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("zero_int64",
+					map[string]string{},
+					map[string]interface{}{"value": int64(13)},
+					time.Unix(0, 0),
+				),
+				testutil.MustMetric("zero_float",
+					map[string]string{},
+					map[string]interface{}{"value": float64(13.37)},
+					time.Unix(0, 0),
+				),
+			},
+			distribution: &testDistribution{value: 13.37},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			plugin := Noise{
+				NoiseType: "laplacian",
+				Scale:     1.0,
+				Log:       testutil.Logger{},
+			}
+			require.NoError(t, plugin.Init())
+			plugin.generator = tt.distribution
+
+			actual := plugin.Apply(tt.input...)
+			testutil.RequireMetricsEqual(t, tt.expected, actual)
+		})
+	}
+}
+
+// Verifies that any invalid generator setting (not "laplacian", "gaussian" or
+// "uniform") raises an error
+func TestInvalidDistributionFunction(t *testing.T) {
+	p := Noise{
+		NoiseType: "invalid",
+		Log:       testutil.Logger{},
+	}
+	err := p.Init()
+	require.EqualError(t, err, "unknown distribution type \"invalid\"")
+}
+
+func TestTracking(t *testing.T) {
+	// Setup raw input and expected output
+	inputRaw := []telegraf.Metric{
+		metric.New(
+			"zero_uint64",
+			map[string]string{},
+			map[string]interface{}{"value": uint64(0)},
+			time.Unix(0, 0),
+		),
+		metric.New(
+			"zero_int64",
+			map[string]string{},
+			map[string]interface{}{"value": int64(0)},
+			time.Unix(0, 0),
+		),
+		metric.New(
+			"zero_float",
+			map[string]string{},
+			map[string]interface{}{"value": float64(0.0)},
+			time.Unix(0, 0),
+		),
+	}
+
+	expected := []telegraf.Metric{
+		metric.New(
+			"zero_uint64",
+			map[string]string{},
+			map[string]interface{}{"value": uint64(13)},
+			time.Unix(0, 0),
+		),
+		metric.New(
+			"zero_int64",
+			map[string]string{},
+			map[string]interface{}{"value": int64(13)},
+			time.Unix(0, 0),
+		),
+		metric.New(
+			"zero_float",
+			map[string]string{},
+			map[string]interface{}{"value": float64(13.37)},
+			time.Unix(0, 0),
+		),
+	}
+
+	// Create fake notification for testing
+	var mu sync.Mutex
+	delivered := make([]telegraf.DeliveryInfo, 0, len(inputRaw))
+	notify := func(di telegraf.DeliveryInfo) {
+		mu.Lock()
+		defer mu.Unlock()
+		delivered = append(delivered, di)
+	}
+
+	// Convert raw input to tracking metric
+	input := make([]telegraf.Metric, 0, len(inputRaw))
+	for _, m := range inputRaw {
+		tm, _ := metric.WithTracking(m, notify)
+		input = append(input, tm)
+	}
+
+	// Prepare and start the plugin
+	plugin := &Noise{
+		NoiseType: "laplacian",
+		Scale:     1.0,
+		Log:       testutil.Logger{},
+	}
+	require.NoError(t, plugin.Init())
+	plugin.generator = &testDistribution{value: 13.37}
+
+	// Process expected metrics and compare with resulting metrics
+	actual := plugin.Apply(input...)
+	testutil.RequireMetricsEqual(t, expected, actual)
+
+	// Simulate output acknowledging delivery
+	for _, m := range actual {
+		m.Accept()
+	}
+
+	// Check delivery
+	require.Eventuallyf(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+		return len(input) == len(delivered)
+	}, time.Second, 100*time.Millisecond, "%d delivered but %d expected", len(delivered), len(expected))
+}

plugins/processors/noise/sample.conf

@@ -0,0 +1,21 @@
+# Adds noise to numerical fields
+[[processors.noise]]
+  ## Specified the type of the random distribution.
+  ## Can be "laplacian", "gaussian" or "uniform".
+  # type = "laplacian
+
+  ## Center of the distribution.
+  ## Only used for Laplacian and Gaussian distributions.
+  # mu = 0.0
+
+  ## Scale parameter for the Laplacian or Gaussian distribution
+  # scale = 1.0
+
+  ## Upper and lower bound of the Uniform distribution
+  # min = -1.0
+  # max = 1.0
+
+  ## Apply the noise only to numeric fields matching the filter criteria below.
+  ## Excludes takes precedence over includes.
+  # include_fields = []
+  # exclude_fields = []