Adding upstream version 1.34.4.

Signed-off-by: Daniel Baumann <daniel@debian.org>

agent/README.md

@@ -0,0 +1,6 @@
+# Agent
+
+For a complete list of configuration options and details about the agent, please
+see the [configuration][] document's agent section.
+
+[configuration]: ../docs/CONFIGURATION.md#agent

agent/accumulator.go

@@ -0,0 +1,160 @@
+package agent
+
+import (
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/metric"
+)
+
+type MetricMaker interface {
+	LogName() string
+	MakeMetric(m telegraf.Metric) telegraf.Metric
+	Log() telegraf.Logger
+}
+
+type accumulator struct {
+	maker     MetricMaker
+	metrics   chan<- telegraf.Metric
+	precision time.Duration
+}
+
+func NewAccumulator(
+	maker MetricMaker,
+	metrics chan<- telegraf.Metric,
+) telegraf.Accumulator {
+	acc := accumulator{
+		maker:     maker,
+		metrics:   metrics,
+		precision: time.Nanosecond,
+	}
+	return &acc
+}
+
+func (ac *accumulator) AddFields(
+	measurement string,
+	fields map[string]interface{},
+	tags map[string]string,
+	t ...time.Time,
+) {
+	ac.addMeasurement(measurement, tags, fields, telegraf.Untyped, t...)
+}
+
+func (ac *accumulator) AddGauge(
+	measurement string,
+	fields map[string]interface{},
+	tags map[string]string,
+	t ...time.Time,
+) {
+	ac.addMeasurement(measurement, tags, fields, telegraf.Gauge, t...)
+}
+
+func (ac *accumulator) AddCounter(
+	measurement string,
+	fields map[string]interface{},
+	tags map[string]string,
+	t ...time.Time,
+) {
+	ac.addMeasurement(measurement, tags, fields, telegraf.Counter, t...)
+}
+
+func (ac *accumulator) AddSummary(
+	measurement string,
+	fields map[string]interface{},
+	tags map[string]string,
+	t ...time.Time,
+) {
+	ac.addMeasurement(measurement, tags, fields, telegraf.Summary, t...)
+}
+
+func (ac *accumulator) AddHistogram(
+	measurement string,
+	fields map[string]interface{},
+	tags map[string]string,
+	t ...time.Time,
+) {
+	ac.addMeasurement(measurement, tags, fields, telegraf.Histogram, t...)
+}
+
+func (ac *accumulator) AddMetric(m telegraf.Metric) {
+	m.SetTime(m.Time().Round(ac.precision))
+	if m := ac.maker.MakeMetric(m); m != nil {
+		ac.metrics <- m
+	}
+}
+
+func (ac *accumulator) addMeasurement(
+	measurement string,
+	tags map[string]string,
+	fields map[string]interface{},
+	tp telegraf.ValueType,
+	t ...time.Time,
+) {
+	m := metric.New(measurement, tags, fields, ac.getTime(t), tp)
+	if m := ac.maker.MakeMetric(m); m != nil {
+		ac.metrics <- m
+	}
+}
+
+// AddError passes a runtime error to the accumulator.
+// The error will be tagged with the plugin name and written to the log.
+func (ac *accumulator) AddError(err error) {
+	if err == nil {
+		return
+	}
+	ac.maker.Log().Errorf("Error in plugin: %v", err)
+}
+
+func (ac *accumulator) SetPrecision(precision time.Duration) {
+	ac.precision = precision
+}
+
+func (ac *accumulator) getTime(t []time.Time) time.Time {
+	var timestamp time.Time
+	if len(t) > 0 {
+		timestamp = t[0]
+	} else {
+		timestamp = time.Now()
+	}
+	return timestamp.Round(ac.precision)
+}
+
+func (ac *accumulator) WithTracking(maxTracked int) telegraf.TrackingAccumulator {
+	return &trackingAccumulator{
+		Accumulator: ac,
+		delivered:   make(chan telegraf.DeliveryInfo, maxTracked),
+	}
+}
+
+type trackingAccumulator struct {
+	telegraf.Accumulator
+	delivered chan telegraf.DeliveryInfo
+}
+
+func (a *trackingAccumulator) AddTrackingMetric(m telegraf.Metric) telegraf.TrackingID {
+	dm, id := metric.WithTracking(m, a.onDelivery)
+	a.AddMetric(dm)
+	return id
+}
+
+func (a *trackingAccumulator) AddTrackingMetricGroup(group []telegraf.Metric) telegraf.TrackingID {
+	db, id := metric.WithGroupTracking(group, a.onDelivery)
+	for _, m := range db {
+		a.AddMetric(m)
+	}
+	return id
+}
+
+func (a *trackingAccumulator) Delivered() <-chan telegraf.DeliveryInfo {
+	return a.delivered
+}
+
+func (a *trackingAccumulator) onDelivery(info telegraf.DeliveryInfo) {
+	select {
+	case a.delivered <- info:
+	default:
+		// This is a programming error in the input.  More items were sent for
+		// tracking than space requested.
+		panic("channel is full")
+	}
+}

agent/accumulator_test.go

@@ -0,0 +1,160 @@
+package agent
+
+import (
+	"bytes"
+	"errors"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/logger"
+	"github.com/influxdata/telegraf/testutil"
+)
+
+func TestAddFields(t *testing.T) {
+	metrics := make(chan telegraf.Metric, 10)
+	defer close(metrics)
+	a := NewAccumulator(&TestMetricMaker{}, metrics)
+
+	tags := map[string]string{"foo": "bar"}
+	fields := map[string]interface{}{
+		"usage": float64(99),
+	}
+	now := time.Now()
+	a.AddCounter("acctest", fields, tags, now)
+
+	testm := <-metrics
+
+	require.Equal(t, "acctest", testm.Name())
+	actual, ok := testm.GetField("usage")
+
+	require.True(t, ok)
+	require.InDelta(t, float64(99), actual, testutil.DefaultDelta)
+
+	actual, ok = testm.GetTag("foo")
+	require.True(t, ok)
+	require.Equal(t, "bar", actual)
+
+	tm := testm.Time()
+	// okay if monotonic clock differs
+	require.True(t, now.Equal(tm))
+
+	tp := testm.Type()
+	require.Equal(t, telegraf.Counter, tp)
+}
+
+func TestAccAddError(t *testing.T) {
+	errBuf := bytes.NewBuffer(nil)
+	logger.RedirectLogging(errBuf)
+	defer logger.RedirectLogging(os.Stderr)
+
+	metrics := make(chan telegraf.Metric, 10)
+	defer close(metrics)
+	a := NewAccumulator(&TestMetricMaker{}, metrics)
+
+	a.AddError(errors.New("foo"))
+	a.AddError(errors.New("bar"))
+	a.AddError(errors.New("baz"))
+
+	errs := bytes.Split(errBuf.Bytes(), []byte{'\n'})
+	require.Len(t, errs, 4) // 4 because of trailing newline
+	require.Contains(t, string(errs[0]), "TestPlugin")
+	require.Contains(t, string(errs[0]), "foo")
+	require.Contains(t, string(errs[1]), "TestPlugin")
+	require.Contains(t, string(errs[1]), "bar")
+	require.Contains(t, string(errs[2]), "TestPlugin")
+	require.Contains(t, string(errs[2]), "baz")
+}
+
+func TestSetPrecision(t *testing.T) {
+	tests := []struct {
+		name      string
+		unset     bool
+		precision time.Duration
+		timestamp time.Time
+		expected  time.Time
+	}{
+		{
+			name:      "default precision is nanosecond",
+			unset:     true,
+			timestamp: time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC),
+			expected:  time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC),
+		},
+		{
+			name:      "second interval",
+			precision: time.Second,
+			timestamp: time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC),
+			expected:  time.Date(2006, time.February, 10, 12, 0, 0, 0, time.UTC),
+		},
+		{
+			name:      "microsecond interval",
+			precision: time.Microsecond,
+			timestamp: time.Date(2006, time.February, 10, 12, 0, 0, 82912748, time.UTC),
+			expected:  time.Date(2006, time.February, 10, 12, 0, 0, 82913000, time.UTC),
+		},
+		{
+			name:      "2 second precision",
+			precision: 2 * time.Second,
+			timestamp: time.Date(2006, time.February, 10, 12, 0, 2, 4, time.UTC),
+			expected:  time.Date(2006, time.February, 10, 12, 0, 2, 0, time.UTC),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			metrics := make(chan telegraf.Metric, 10)
+
+			a := NewAccumulator(&TestMetricMaker{}, metrics)
+			if !tt.unset {
+				a.SetPrecision(tt.precision)
+			}
+
+			a.AddFields("acctest",
+				map[string]interface{}{"value": float64(101)},
+				map[string]string{},
+				tt.timestamp,
+			)
+
+			testm := <-metrics
+			require.Equal(t, tt.expected, testm.Time())
+
+			close(metrics)
+		})
+	}
+}
+
+func TestAddTrackingMetricGroupEmpty(t *testing.T) {
+	ch := make(chan telegraf.Metric, 10)
+	metrics := make([]telegraf.Metric, 0)
+	acc := NewAccumulator(&TestMetricMaker{}, ch).WithTracking(1)
+
+	id := acc.AddTrackingMetricGroup(metrics)
+
+	select {
+	case tracking := <-acc.Delivered():
+		require.Equal(t, tracking.ID(), id)
+	default:
+		t.Fatal("empty group should be delivered immediately")
+	}
+}
+
+type TestMetricMaker struct {
+}
+
+func (*TestMetricMaker) Name() string {
+	return "TestPlugin"
+}
+
+func (tm *TestMetricMaker) LogName() string {
+	return tm.Name()
+}
+
+func (*TestMetricMaker) MakeMetric(metric telegraf.Metric) telegraf.Metric {
+	return metric
+}
+
+func (*TestMetricMaker) Log() telegraf.Logger {
+	return logger.New("TestPlugin", "test", "")
+}

agent/agent_posix.go

@@ -0,0 +1,19 @@
+//go:build !windows
+
+package agent
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+const flushSignal = syscall.SIGUSR1
+
+func watchForFlushSignal(flushRequested chan os.Signal) {
+	signal.Notify(flushRequested, flushSignal)
+}
+
+func stopListeningForFlushSignal(flushRequested chan os.Signal) {
+	signal.Stop(flushRequested)
+}

agent/agent_test.go

@@ -0,0 +1,259 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/require"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/models"
+	_ "github.com/influxdata/telegraf/plugins/aggregators/all"
+	_ "github.com/influxdata/telegraf/plugins/inputs/all"
+	_ "github.com/influxdata/telegraf/plugins/outputs/all"
+	"github.com/influxdata/telegraf/plugins/parsers/influx"
+	_ "github.com/influxdata/telegraf/plugins/processors/all"
+	"github.com/influxdata/telegraf/testutil"
+)
+
+func TestAgent_OmitHostname(t *testing.T) {
+	c := config.NewConfig()
+	c.Agent.OmitHostname = true
+	_ = NewAgent(c)
+	require.NotContains(t, c.Tags, "host")
+}
+
+func TestAgent_LoadPlugin(t *testing.T) {
+	c := config.NewConfig()
+	c.InputFilters = []string{"mysql"}
+	err := c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a := NewAgent(c)
+	require.Len(t, a.Config.Inputs, 1)
+
+	c = config.NewConfig()
+	c.InputFilters = []string{"foo"}
+	err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a = NewAgent(c)
+	require.Empty(t, a.Config.Inputs)
+
+	c = config.NewConfig()
+	c.InputFilters = []string{"mysql", "foo"}
+	err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a = NewAgent(c)
+	require.Len(t, a.Config.Inputs, 1)
+
+	c = config.NewConfig()
+	c.InputFilters = []string{"mysql", "redis"}
+	err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a = NewAgent(c)
+	require.Len(t, a.Config.Inputs, 2)
+
+	c = config.NewConfig()
+	c.InputFilters = []string{"mysql", "foo", "redis", "bar"}
+	err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a = NewAgent(c)
+	require.Len(t, a.Config.Inputs, 2)
+}
+
+func TestAgent_LoadOutput(t *testing.T) {
+	c := config.NewConfig()
+	c.OutputFilters = []string{"influxdb"}
+	err := c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a := NewAgent(c)
+	require.Len(t, a.Config.Outputs, 2)
+
+	c = config.NewConfig()
+	c.OutputFilters = []string{"kafka"}
+	err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a = NewAgent(c)
+	require.Len(t, a.Config.Outputs, 1)
+
+	c = config.NewConfig()
+	err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a = NewAgent(c)
+	require.Len(t, a.Config.Outputs, 3)
+
+	c = config.NewConfig()
+	c.OutputFilters = []string{"foo"}
+	err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a = NewAgent(c)
+	require.Empty(t, a.Config.Outputs)
+
+	c = config.NewConfig()
+	c.OutputFilters = []string{"influxdb", "foo"}
+	err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a = NewAgent(c)
+	require.Len(t, a.Config.Outputs, 2)
+
+	c = config.NewConfig()
+	c.OutputFilters = []string{"influxdb", "kafka"}
+	err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	require.Len(t, c.Outputs, 3)
+	a = NewAgent(c)
+	require.Len(t, a.Config.Outputs, 3)
+
+	c = config.NewConfig()
+	c.OutputFilters = []string{"influxdb", "foo", "kafka", "bar"}
+	err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
+	require.NoError(t, err)
+	a = NewAgent(c)
+	require.Len(t, a.Config.Outputs, 3)
+}
+
+func TestWindow(t *testing.T) {
+	parse := func(s string) time.Time {
+		tm, err := time.Parse(time.RFC3339, s)
+		if err != nil {
+			panic(err)
+		}
+		return tm
+	}
+
+	tests := []struct {
+		name          string
+		start         time.Time
+		roundInterval bool
+		period        time.Duration
+		since         time.Time
+		until         time.Time
+	}{
+		{
+			name:          "round with exact alignment",
+			start:         parse("2018-03-27T00:00:00Z"),
+			roundInterval: true,
+			period:        30 * time.Second,
+			since:         parse("2018-03-27T00:00:00Z"),
+			until:         parse("2018-03-27T00:00:30Z"),
+		},
+		{
+			name:          "round with alignment needed",
+			start:         parse("2018-03-27T00:00:05Z"),
+			roundInterval: true,
+			period:        30 * time.Second,
+			since:         parse("2018-03-27T00:00:00Z"),
+			until:         parse("2018-03-27T00:00:30Z"),
+		},
+		{
+			name:          "no round with exact alignment",
+			start:         parse("2018-03-27T00:00:00Z"),
+			roundInterval: false,
+			period:        30 * time.Second,
+			since:         parse("2018-03-27T00:00:00Z"),
+			until:         parse("2018-03-27T00:00:30Z"),
+		},
+		{
+			name:          "no found with alignment needed",
+			start:         parse("2018-03-27T00:00:05Z"),
+			roundInterval: false,
+			period:        30 * time.Second,
+			since:         parse("2018-03-27T00:00:05Z"),
+			until:         parse("2018-03-27T00:00:35Z"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			since, until := updateWindow(tt.start, tt.roundInterval, tt.period)
+			require.Equal(t, tt.since, since, "since")
+			require.Equal(t, tt.until, until, "until")
+		})
+	}
+}
+
+func TestCases(t *testing.T) {
+	// Get all directories in testcases
+	folders, err := os.ReadDir("testcases")
+	require.NoError(t, err)
+
+	// Make sure tests contains data
+	require.NotEmpty(t, folders)
+
+	for _, f := range folders {
+		// Only handle folders
+		if !f.IsDir() {
+			continue
+		}
+
+		fname := f.Name()
+		testdataPath := filepath.Join("testcases", fname)
+		configFilename := filepath.Join(testdataPath, "telegraf.conf")
+		expectedFilename := filepath.Join(testdataPath, "expected.out")
+
+		t.Run(fname, func(t *testing.T) {
+			// Get parser to parse input and expected output
+			parser := &influx.Parser{}
+			require.NoError(t, parser.Init())
+
+			expected, err := testutil.ParseMetricsFromFile(expectedFilename, parser)
+			require.NoError(t, err)
+			require.NotEmpty(t, expected)
+
+			// Load the config and inject the mock output to be able to verify
+			// the resulting metrics
+			cfg := config.NewConfig()
+			require.NoError(t, cfg.LoadAll(configFilename))
+			require.Empty(t, cfg.Outputs, "No output(s) allowed in the config!")
+
+			// Setup the agent and run the agent in "once" mode
+			agent := NewAgent(cfg)
+			ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+			defer cancel()
+			actual, err := collect(ctx, agent, 0)
+			require.NoError(t, err)
+
+			// Process expected metrics and compare with resulting metrics
+			options := []cmp.Option{
+				testutil.IgnoreTags("host"),
+				testutil.IgnoreTime(),
+			}
+			testutil.RequireMetricsEqual(t, expected, actual, options...)
+		})
+	}
+}
+
+// Implement a "test-mode" like call but collect the metrics
+func collect(ctx context.Context, a *Agent, wait time.Duration) ([]telegraf.Metric, error) {
+	var received []telegraf.Metric
+	var mu sync.Mutex
+
+	src := make(chan telegraf.Metric, 100)
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for m := range src {
+			mu.Lock()
+			received = append(received, m)
+			mu.Unlock()
+			m.Reject()
+		}
+	}()
+
+	if err := a.runTest(ctx, wait, src); err != nil {
+		return nil, err
+	}
+	wg.Wait()
+
+	if models.GlobalGatherErrors.Get() != 0 {
+		return received, fmt.Errorf("input plugins recorded %d errors", models.GlobalGatherErrors.Get())
+	}
+	return received, nil
+}

agent/agent_windows.go

@@ -0,0 +1,13 @@
+//go:build windows
+
+package agent
+
+import "os"
+
+func watchForFlushSignal(_ chan os.Signal) {
+	// not supported
+}
+
+func stopListeningForFlushSignal(_ chan os.Signal) {
+	// not supported
+}

agent/testcases/aggregators-rerun-processors/expected.out

@@ -0,0 +1,2 @@
+metric value=420
+metric value_min=4200,value_max=4200

agent/testcases/aggregators-rerun-processors/input.influx

@@ -0,0 +1 @@
+metric value=42.0

agent/testcases/aggregators-rerun-processors/telegraf.conf

@@ -0,0 +1,22 @@
+# Test for not skipping processors after running aggregators
+[agent]
+  omit_hostname = true
+  skip_processors_after_aggregators = false
+
+[[inputs.file]]
+  files = ["testcases/aggregators-rerun-processors/input.influx"]
+  data_format = "influx"
+
+[[processors.starlark]]
+  source = '''
+def apply(metric):
+    for k, v in metric.fields.items():
+        if type(v) == "float":
+            metric.fields[k] = v * 10
+    return metric
+'''
+
+[[aggregators.minmax]]
+  period = "1s"
+  drop_original = false
+

agent/testcases/aggregators-skip-processors/expected.out

@@ -0,0 +1,2 @@
+metric value=420
+metric value_min=420,value_max=420

agent/testcases/aggregators-skip-processors/input.influx

@@ -0,0 +1 @@
+metric value=42.0

agent/testcases/aggregators-skip-processors/telegraf.conf

@@ -0,0 +1,22 @@
+# Test for skipping processors after running aggregators
+[agent]
+  omit_hostname = true
+  skip_processors_after_aggregators = true
+
+[[inputs.file]]
+  files = ["testcases/aggregators-skip-processors/input.influx"]
+  data_format = "influx"
+
+[[processors.starlark]]
+  source = '''
+def apply(metric):
+    for k, v in metric.fields.items():
+        if type(v) == "float":
+            metric.fields[k] = v * 10
+    return metric
+'''
+
+[[aggregators.minmax]]
+  period = "1s"
+  drop_original = false
+

agent/testcases/processor-order-appearance/expected.out

@@ -0,0 +1,2 @@
+new_metric_from_starlark,foo=bar baz=42i,timestamp="2023-07-13T12:53:54.197709713Z" 1689252834197709713
+old_metric_from_mock,mood=good value=23i,timestamp="2023-07-13T13:10:34Z" 1689253834000000000

agent/testcases/processor-order-appearance/input.influx

@@ -0,0 +1 @@
+old_metric_from_mock,mood=good value=23i 1689253834000000000

agent/testcases/processor-order-appearance/telegraf.conf

@@ -0,0 +1,26 @@
+# Test for using the appearance order in the file for processor order
+[[inputs.file]]
+  files = ["testcases/processor-order-appearance/input.influx"]
+  data_format = "influx"
+
+[[processors.starlark]]
+  source = '''
+def apply(metric):
+  metrics = []
+
+  m = Metric("new_metric_from_starlark")
+  m.tags["foo"] = "bar"
+  m.fields["baz"] = 42
+  m.time = 1689252834197709713
+  metrics.append(m)
+  metrics.append(metric)
+
+  return metrics
+'''
+
+[[processors.date]]
+  field_key = "timestamp"
+  date_format = "2006-01-02T15:04:05.999999999Z"
+  timezone = "UTC"
+
+

agent/testcases/processor-order-explicit/expected.out

@@ -0,0 +1,2 @@
+new_metric_from_starlark,foo=bar baz=42i,timestamp="2023-07-13T12:53:54.197709713Z" 1689252834197709713
+old_metric_from_mock,mood=good value=23i,timestamp="2023-07-13T13:10:34Z" 1689253834000000000

agent/testcases/processor-order-explicit/input.influx

@@ -0,0 +1 @@
+old_metric_from_mock,mood=good value=23i 1689253834000000000

agent/testcases/processor-order-explicit/telegraf.conf

@@ -0,0 +1,27 @@
+# Test for specifying an explicit processor order
+[[inputs.file]]
+  files = ["testcases/processor-order-explicit/input.influx"]
+  data_format = "influx"
+
+
+[[processors.date]]
+  field_key = "timestamp"
+  date_format = "2006-01-02T15:04:05.999999999Z"
+  timezone = "UTC"
+  order = 2
+
+[[processors.starlark]]
+  source = '''
+def apply(metric):
+  metrics = []
+
+  m = Metric("new_metric_from_starlark")
+  m.tags["foo"] = "bar"
+  m.fields["baz"] = 42
+  m.time = 1689252834197709713
+  metrics.append(m)
+  metrics.append(metric)
+
+  return metrics
+'''
+  order = 1

agent/testcases/processor-order-mixed/expected.out

@@ -0,0 +1,2 @@
+new_metric_from_starlark,foo=bar baz=42i,timestamp="2023-07-13T12:53:54.197709713Z" 1689252834197709713
+old_metric_from_mock,mood=good value=23i,timestamp="2023-07-13T13:10:34Z" 1689253834000000000

agent/testcases/processor-order-mixed/input.influx

@@ -0,0 +1 @@
+old_metric_from_mock,mood=good value=23i 1689253834000000000

agent/testcases/processor-order-mixed/telegraf.conf

@@ -0,0 +1,25 @@
+# Test for using the appearance order in the file for processor order
+[[inputs.file]]
+  files = ["testcases/processor-order-appearance/input.influx"]
+  data_format = "influx"
+
+[[processors.starlark]]
+  source = '''
+def apply(metric):
+  metrics = []
+
+  m = Metric("new_metric_from_starlark")
+  m.tags["foo"] = "bar"
+  m.fields["baz"] = 42
+  m.time = 1689252834197709713
+  metrics.append(m)
+  metrics.append(metric)
+
+  return metrics
+'''
+
+[[processors.date]]
+  field_key = "timestamp"
+  date_format = "2006-01-02T15:04:05.999999999Z"
+  timezone = "UTC"
+  order = 1

agent/testcases/processor-order-no-starlark/expected.out

@@ -0,0 +1,2 @@
+new_metric_from_starlark,foo=bar baz=42i 1689252834197709713
+old_metric_from_mock,mood=good value=23i,timestamp="2023-07-13T13:10:34Z" 1689253834000000000

agent/testcases/processor-order-no-starlark/input.influx

@@ -0,0 +1 @@
+old_metric_from_mock,mood=good value=23i 1689253834000000000

agent/testcases/processor-order-no-starlark/telegraf.conf

@@ -0,0 +1,26 @@
+# Test for using the appearance order in the file for processor order.
+# This will not add the "timestamp" field as the starlark processor runs _after_
+# the date processor.
+[[inputs.file]]
+  files = ["testcases/processor-order-no-starlark/input.influx"]
+  data_format = "influx"
+
+[[processors.date]]
+  field_key = "timestamp"
+  date_format = "2006-01-02T15:04:05.999999999Z"
+  timezone = "UTC"
+
+[[processors.starlark]]
+  source = '''
+def apply(metric):
+  metrics = []
+
+  m = Metric("new_metric_from_starlark")
+  m.tags["foo"] = "bar"
+  m.fields["baz"] = 42
+  m.time = 1689252834197709713
+  metrics.append(m)
+  metrics.append(metric)
+
+  return metrics
+'''

agent/tick.go

@@ -0,0 +1,281 @@
+package agent
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/benbjohnson/clock"
+
+	"github.com/influxdata/telegraf/internal"
+)
+
+type Ticker interface {
+	Elapsed() <-chan time.Time
+	Stop()
+}
+
+// AlignedTicker delivers ticks at aligned times plus an optional jitter.  Each
+// tick is realigned to avoid drift and handle changes to the system clock.
+//
+// The ticks may have an jitter duration applied to them as an random offset to
+// the interval.  However the overall pace of is that of the interval, so on
+// average you will have one collection each interval.
+//
+// The first tick is emitted at the next alignment.
+//
+// Ticks are dropped for slow consumers.
+//
+// The implementation currently does not recalculate until the next tick with
+// no maximum sleep, when using large intervals alignment is not corrected
+// until the next tick.
+type AlignedTicker struct {
+	interval    time.Duration
+	jitter      time.Duration
+	offset      time.Duration
+	minInterval time.Duration
+	ch          chan time.Time
+	cancel      context.CancelFunc
+	wg          sync.WaitGroup
+}
+
+func NewAlignedTicker(now time.Time, interval, jitter, offset time.Duration) *AlignedTicker {
+	t := &AlignedTicker{
+		interval:    interval,
+		jitter:      jitter,
+		offset:      offset,
+		minInterval: interval / 100,
+	}
+	t.start(now, clock.New())
+	return t
+}
+
+func (t *AlignedTicker) start(now time.Time, clk clock.Clock) {
+	t.ch = make(chan time.Time, 1)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.cancel = cancel
+
+	d := t.next(now)
+	timer := clk.Timer(d)
+
+	t.wg.Add(1)
+	go func() {
+		defer t.wg.Done()
+		t.run(ctx, timer)
+	}()
+}
+
+func (t *AlignedTicker) next(now time.Time) time.Duration {
+	// Add minimum interval size to avoid scheduling an interval that is
+	// exceptionally short.  This avoids an issue that can occur where the
+	// previous interval ends slightly early due to very minor clock changes.
+	next := now.Add(t.minInterval)
+
+	next = internal.AlignTime(next, t.interval)
+	d := next.Sub(now)
+	if d == 0 {
+		d = t.interval
+	}
+	d += t.offset
+	d += internal.RandomDuration(t.jitter)
+	return d
+}
+
+func (t *AlignedTicker) run(ctx context.Context, timer *clock.Timer) {
+	for {
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		case now := <-timer.C:
+			select {
+			case t.ch <- now:
+			default:
+			}
+
+			d := t.next(now)
+			timer.Reset(d)
+		}
+	}
+}
+
+func (t *AlignedTicker) Elapsed() <-chan time.Time {
+	return t.ch
+}
+
+func (t *AlignedTicker) Stop() {
+	t.cancel()
+	t.wg.Wait()
+}
+
+// UnalignedTicker delivers ticks at regular but unaligned intervals.  No
+// effort is made to avoid drift.
+//
+// The ticks may have an jitter duration applied to them as an random offset to
+// the interval.  However the overall pace of is that of the interval, so on
+// average you will have one collection each interval.
+//
+// The first tick is emitted immediately.
+//
+// Ticks are dropped for slow consumers.
+type UnalignedTicker struct {
+	interval time.Duration
+	jitter   time.Duration
+	offset   time.Duration
+	ch       chan time.Time
+	cancel   context.CancelFunc
+	wg       sync.WaitGroup
+}
+
+func NewUnalignedTicker(interval, jitter, offset time.Duration) *UnalignedTicker {
+	t := &UnalignedTicker{
+		interval: interval,
+		jitter:   jitter,
+		offset:   offset,
+	}
+	t.start(clock.New())
+	return t
+}
+
+func (t *UnalignedTicker) start(clk clock.Clock) {
+	t.ch = make(chan time.Time, 1)
+	ctx, cancel := context.WithCancel(context.Background())
+	t.cancel = cancel
+
+	ticker := clk.Ticker(t.interval)
+	if t.offset == 0 {
+		// Perform initial trigger to stay backward compatible
+		t.ch <- clk.Now()
+	}
+
+	t.wg.Add(1)
+	go func() {
+		defer t.wg.Done()
+		t.run(ctx, ticker, clk)
+	}()
+}
+
+func sleep(ctx context.Context, duration time.Duration, clk clock.Clock) error {
+	if duration == 0 {
+		return nil
+	}
+
+	t := clk.Timer(duration)
+	select {
+	case <-t.C:
+		return nil
+	case <-ctx.Done():
+		t.Stop()
+		return ctx.Err()
+	}
+}
+
+func (t *UnalignedTicker) run(ctx context.Context, ticker *clock.Ticker, clk clock.Clock) {
+	for {
+		select {
+		case <-ctx.Done():
+			ticker.Stop()
+			return
+		case <-ticker.C:
+			jitter := internal.RandomDuration(t.jitter)
+			err := sleep(ctx, t.offset+jitter, clk)
+			if err != nil {
+				ticker.Stop()
+				return
+			}
+			select {
+			case t.ch <- clk.Now():
+			default:
+			}
+		}
+	}
+}
+
+func (t *UnalignedTicker) InjectTick() {
+	t.ch <- time.Now()
+}
+
+func (t *UnalignedTicker) Elapsed() <-chan time.Time {
+	return t.ch
+}
+
+func (t *UnalignedTicker) Stop() {
+	t.cancel()
+	t.wg.Wait()
+}
+
+// RollingTicker delivers ticks at regular but unaligned intervals.
+//
+// Because the next interval is scheduled based on the interval + jitter, you
+// are guaranteed at least interval seconds without missing a tick and ticks
+// will be evenly scheduled over time.
+//
+// On average you will have one collection each interval + (jitter/2).
+//
+// The first tick is emitted after interval+jitter seconds.
+//
+// Ticks are dropped for slow consumers.
+type RollingTicker struct {
+	interval time.Duration
+	jitter   time.Duration
+	ch       chan time.Time
+	cancel   context.CancelFunc
+	wg       sync.WaitGroup
+}
+
+func NewRollingTicker(interval, jitter time.Duration) *RollingTicker {
+	t := &RollingTicker{
+		interval: interval,
+		jitter:   jitter,
+	}
+	t.start(clock.New())
+	return t
+}
+
+func (t *RollingTicker) start(clk clock.Clock) {
+	t.ch = make(chan time.Time, 1)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.cancel = cancel
+
+	d := t.next()
+	timer := clk.Timer(d)
+
+	t.wg.Add(1)
+	go func() {
+		defer t.wg.Done()
+		t.run(ctx, timer)
+	}()
+}
+
+func (t *RollingTicker) next() time.Duration {
+	return t.interval + internal.RandomDuration(t.jitter)
+}
+
+func (t *RollingTicker) run(ctx context.Context, timer *clock.Timer) {
+	for {
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		case now := <-timer.C:
+			select {
+			case t.ch <- now:
+			default:
+			}
+
+			d := t.next()
+			timer.Reset(d)
+		}
+	}
+}
+
+func (t *RollingTicker) Elapsed() <-chan time.Time {
+	return t.ch
+}
+
+func (t *RollingTicker) Stop() {
+	t.cancel()
+	t.wg.Wait()
+}

agent/tick_test.go

@@ -0,0 +1,395 @@
+package agent
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/benbjohnson/clock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAlignedTicker(t *testing.T) {
+	interval := 10 * time.Second
+	jitter := 0 * time.Second
+	offset := 0 * time.Second
+
+	clk := clock.NewMock()
+	since := clk.Now()
+	until := since.Add(60 * time.Second)
+
+	ticker := &AlignedTicker{
+		interval:    interval,
+		jitter:      jitter,
+		offset:      offset,
+		minInterval: interval / 100,
+	}
+	ticker.start(since, clk)
+	defer ticker.Stop()
+
+	expected := []time.Time{
+		time.Unix(10, 0).UTC(),
+		time.Unix(20, 0).UTC(),
+		time.Unix(30, 0).UTC(),
+		time.Unix(40, 0).UTC(),
+		time.Unix(50, 0).UTC(),
+		time.Unix(60, 0).UTC(),
+	}
+
+	actual := make([]time.Time, 0)
+	clk.Add(10 * time.Second)
+	for !clk.Now().After(until) {
+		tm := <-ticker.Elapsed()
+		actual = append(actual, tm.UTC())
+
+		clk.Add(10 * time.Second)
+	}
+
+	require.Equal(t, expected, actual)
+}
+
+func TestAlignedTickerJitter(t *testing.T) {
+	interval := 10 * time.Second
+	jitter := 5 * time.Second
+	offset := 0 * time.Second
+
+	clk := clock.NewMock()
+	since := clk.Now()
+	until := since.Add(61 * time.Second)
+
+	ticker := &AlignedTicker{
+		interval:    interval,
+		jitter:      jitter,
+		offset:      offset,
+		minInterval: interval / 100,
+	}
+	ticker.start(since, clk)
+	defer ticker.Stop()
+
+	last := since
+	for !clk.Now().After(until) {
+		select {
+		case tm := <-ticker.Elapsed():
+			dur := tm.Sub(last)
+			// 10s interval + 5s jitter + up to 1s late firing.
+			require.LessOrEqual(t, dur, 16*time.Second, "expected elapsed time to be less than 16 seconds, but was %s", dur)
+			require.GreaterOrEqual(t, dur, 5*time.Second, "expected elapsed time to be more than 5 seconds, but was %s", dur)
+			last = last.Add(interval)
+		default:
+		}
+		clk.Add(1 * time.Second)
+	}
+}
+
+func TestAlignedTickerOffset(t *testing.T) {
+	interval := 10 * time.Second
+	jitter := 0 * time.Second
+	offset := 3 * time.Second
+
+	clk := clock.NewMock()
+	since := clk.Now()
+	until := since.Add(61 * time.Second)
+
+	ticker := &AlignedTicker{
+		interval:    interval,
+		jitter:      jitter,
+		offset:      offset,
+		minInterval: interval / 100,
+	}
+	ticker.start(since, clk)
+	defer ticker.Stop()
+
+	expected := []time.Time{
+		time.Unix(13, 0).UTC(),
+		time.Unix(23, 0).UTC(),
+		time.Unix(33, 0).UTC(),
+		time.Unix(43, 0).UTC(),
+		time.Unix(53, 0).UTC(),
+	}
+
+	actual := make([]time.Time, 0)
+	clk.Add(10*time.Second + offset)
+	for !clk.Now().After(until) {
+		tm := <-ticker.Elapsed()
+		actual = append(actual, tm.UTC())
+		clk.Add(10 * time.Second)
+	}
+
+	require.Equal(t, expected, actual)
+}
+
+func TestAlignedTickerMissedTick(t *testing.T) {
+	interval := 10 * time.Second
+	jitter := 0 * time.Second
+	offset := 0 * time.Second
+
+	clk := clock.NewMock()
+	since := clk.Now()
+
+	ticker := &AlignedTicker{
+		interval:    interval,
+		jitter:      jitter,
+		offset:      offset,
+		minInterval: interval / 100,
+	}
+	ticker.start(since, clk)
+	defer ticker.Stop()
+
+	clk.Add(25 * time.Second)
+	tm := <-ticker.Elapsed()
+	require.Equal(t, time.Unix(10, 0).UTC(), tm.UTC())
+	clk.Add(5 * time.Second)
+	tm = <-ticker.Elapsed()
+	require.Equal(t, time.Unix(30, 0).UTC(), tm.UTC())
+}
+
+func TestUnalignedTicker(t *testing.T) {
+	interval := 10 * time.Second
+	jitter := 0 * time.Second
+	offset := 0 * time.Second
+
+	clk := clock.NewMock()
+	clk.Add(1 * time.Second)
+	since := clk.Now()
+	until := since.Add(60 * time.Second)
+
+	ticker := &UnalignedTicker{
+		interval: interval,
+		jitter:   jitter,
+		offset:   offset,
+	}
+	ticker.start(clk)
+	defer ticker.Stop()
+
+	expected := []time.Time{
+		time.Unix(1, 0).UTC(),
+		time.Unix(11, 0).UTC(),
+		time.Unix(21, 0).UTC(),
+		time.Unix(31, 0).UTC(),
+		time.Unix(41, 0).UTC(),
+		time.Unix(51, 0).UTC(),
+		time.Unix(61, 0).UTC(),
+	}
+
+	actual := make([]time.Time, 0)
+	for !clk.Now().After(until) {
+		select {
+		case tm := <-ticker.Elapsed():
+			actual = append(actual, tm.UTC())
+		default:
+		}
+		clk.Add(10 * time.Second)
+	}
+
+	require.Equal(t, expected, actual)
+}
+
+func TestRollingTicker(t *testing.T) {
+	interval := 10 * time.Second
+	jitter := 0 * time.Second
+	offset := 0 * time.Second
+
+	clk := clock.NewMock()
+	clk.Add(1 * time.Second)
+	since := clk.Now()
+	until := since.Add(60 * time.Second)
+
+	ticker := &UnalignedTicker{
+		interval: interval,
+		jitter:   jitter,
+		offset:   offset,
+	}
+	ticker.start(clk)
+	defer ticker.Stop()
+
+	expected := []time.Time{
+		time.Unix(1, 0).UTC(),
+		time.Unix(11, 0).UTC(),
+		time.Unix(21, 0).UTC(),
+		time.Unix(31, 0).UTC(),
+		time.Unix(41, 0).UTC(),
+		time.Unix(51, 0).UTC(),
+		time.Unix(61, 0).UTC(),
+	}
+
+	actual := make([]time.Time, 0)
+	for !clk.Now().After(until) {
+		select {
+		case tm := <-ticker.Elapsed():
+			actual = append(actual, tm.UTC())
+		default:
+		}
+		clk.Add(10 * time.Second)
+	}
+
+	require.Equal(t, expected, actual)
+}
+
+// Simulates running the Ticker for an hour and displays stats about the
+// operation.
+func TestAlignedTickerDistribution(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	interval := 10 * time.Second
+	jitter := 5 * time.Second
+	offset := 0 * time.Second
+
+	clk := clock.NewMock()
+	since := clk.Now()
+
+	ticker := &AlignedTicker{
+		interval:    interval,
+		jitter:      jitter,
+		offset:      offset,
+		minInterval: interval / 100,
+	}
+	ticker.start(since, clk)
+	defer ticker.Stop()
+	dist := simulatedDist(ticker, clk)
+	printDist(dist)
+	require.Less(t, 350, dist.Count)
+	require.True(t, 9 < dist.Mean() && dist.Mean() < 11)
+}
+
+func TestAlignedTickerDistributionWithOffset(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	interval := 10 * time.Second
+	jitter := 5 * time.Second
+	offset := 3 * time.Second
+
+	clk := clock.NewMock()
+	since := clk.Now()
+
+	ticker := &AlignedTicker{
+		interval:    interval,
+		jitter:      jitter,
+		offset:      offset,
+		minInterval: interval / 100,
+	}
+	ticker.start(since, clk)
+	defer ticker.Stop()
+	dist := simulatedDist(ticker, clk)
+	printDist(dist)
+	require.Less(t, 350, dist.Count)
+	require.True(t, 9 < dist.Mean() && dist.Mean() < 11)
+}
+
+// Simulates running the Ticker for an hour and displays stats about the
+// operation.
+func TestUnalignedTickerDistribution(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	interval := 10 * time.Second
+	jitter := 5 * time.Second
+	offset := 0 * time.Second
+
+	clk := clock.NewMock()
+
+	ticker := &UnalignedTicker{
+		interval: interval,
+		jitter:   jitter,
+		offset:   offset,
+	}
+	ticker.start(clk)
+	defer ticker.Stop()
+	dist := simulatedDist(ticker, clk)
+	printDist(dist)
+	require.Less(t, 350, dist.Count)
+	require.True(t, 9 < dist.Mean() && dist.Mean() < 11)
+}
+
+func TestUnalignedTickerDistributionWithOffset(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	interval := 10 * time.Second
+	jitter := 5 * time.Second
+	offset := 3 * time.Second
+
+	clk := clock.NewMock()
+
+	ticker := &UnalignedTicker{
+		interval: interval,
+		jitter:   jitter,
+		offset:   offset,
+	}
+	ticker.start(clk)
+	defer ticker.Stop()
+	dist := simulatedDist(ticker, clk)
+	printDist(dist)
+	require.Less(t, 350, dist.Count)
+	require.True(t, 9 < dist.Mean() && dist.Mean() < 11)
+}
+
+// Simulates running the Ticker for an hour and displays stats about the
+// operation.
+func TestRollingTickerDistribution(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	interval := 10 * time.Second
+	jitter := 5 * time.Second
+
+	clk := clock.NewMock()
+
+	ticker := &RollingTicker{
+		interval: interval,
+		jitter:   jitter,
+	}
+	ticker.start(clk)
+	defer ticker.Stop()
+	dist := simulatedDist(ticker, clk)
+	printDist(dist)
+	require.Less(t, 275, dist.Count)
+	require.True(t, 12 < dist.Mean() && 13 > dist.Mean())
+}
+
+type Distribution struct {
+	Buckets  [60]int
+	Count    int
+	Waittime float64
+}
+
+func (d *Distribution) Mean() float64 {
+	return d.Waittime / float64(d.Count)
+}
+
+func printDist(dist Distribution) {
+	for i, count := range dist.Buckets {
+		fmt.Printf("%2d %s\n", i, strings.Repeat("x", count))
+	}
+	fmt.Printf("Average interval: %f\n", dist.Mean())
+	fmt.Printf("Count: %d\n", dist.Count)
+}
+
+func simulatedDist(ticker Ticker, clk *clock.Mock) Distribution {
+	since := clk.Now()
+	until := since.Add(1 * time.Hour)
+
+	var dist Distribution
+
+	last := clk.Now()
+	for !clk.Now().After(until) {
+		select {
+		case tm := <-ticker.Elapsed():
+			dist.Buckets[tm.Second()]++
+			dist.Count++
+			dist.Waittime += tm.Sub(last).Seconds()
+			last = tm
+		default:
+			clk.Add(1 * time.Second)
+		}
+	}
+
+	return dist
+}