Adding upstream version 1.34.4.

Signed-off-by: Daniel Baumann <daniel@debian.org>

plugins/common/parallel/ordered.go

@@ -0,0 +1,84 @@
+package parallel
+
+import (
+	"sync"
+
+	"github.com/influxdata/telegraf"
+)
+
+type Ordered struct {
+	wg sync.WaitGroup
+	fn func(telegraf.Metric) []telegraf.Metric
+
+	// queue of jobs coming in. Workers pick jobs off this queue for processing
+	workerQueue chan job
+
+	// queue of ordered metrics going out
+	queue chan futureMetric
+}
+
+func NewOrdered(acc telegraf.Accumulator, fn func(telegraf.Metric) []telegraf.Metric, orderedQueueSize, workerCount int) *Ordered {
+	p := &Ordered{
+		fn:          fn,
+		workerQueue: make(chan job, workerCount),
+		queue:       make(chan futureMetric, orderedQueueSize),
+	}
+	p.startWorkers(workerCount)
+	p.wg.Add(1)
+	go func() {
+		p.readQueue(acc)
+		p.wg.Done()
+	}()
+	return p
+}
+
+func (p *Ordered) Enqueue(metric telegraf.Metric) {
+	future := make(futureMetric)
+	p.queue <- future
+
+	// write the future to the worker pool. Order doesn't matter now because the
+	// outgoing p.queue will enforce order regardless of the order the jobs are
+	// completed in
+	p.workerQueue <- job{
+		future: future,
+		metric: metric,
+	}
+}
+
+func (p *Ordered) readQueue(acc telegraf.Accumulator) {
+	// wait for the response from each worker in order
+	for mCh := range p.queue {
+		// allow each worker to write out multiple metrics
+		for metrics := range mCh {
+			for _, m := range metrics {
+				acc.AddMetric(m)
+			}
+		}
+	}
+}
+
+func (p *Ordered) startWorkers(count int) {
+	p.wg.Add(count)
+	for i := 0; i < count; i++ {
+		go func() {
+			for job := range p.workerQueue {
+				job.future <- p.fn(job.metric)
+				close(job.future)
+			}
+			p.wg.Done()
+		}()
+	}
+}
+
+func (p *Ordered) Stop() {
+	close(p.queue)
+	close(p.workerQueue)
+	p.wg.Wait()
+}
+
+type futureMetric chan []telegraf.Metric
+
+type job struct {
+	future futureMetric
+	metric telegraf.Metric
+}

plugins/common/parallel/parallel.go

@@ -0,0 +1,8 @@
+package parallel
+
+import "github.com/influxdata/telegraf"
+
+type Parallel interface {
+	Enqueue(telegraf.Metric)
+	Stop()
+}

plugins/common/parallel/parallel_test.go

@@ -0,0 +1,117 @@
+package parallel_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/metric"
+	"github.com/influxdata/telegraf/plugins/common/parallel"
+	"github.com/influxdata/telegraf/testutil"
+)
+
+func TestOrderedJobsStayOrdered(t *testing.T) {
+	acc := &testutil.Accumulator{}
+
+	p := parallel.NewOrdered(acc, jobFunc, 10000, 10)
+	now := time.Now()
+	for i := 0; i < 20000; i++ {
+		m := metric.New("test",
+			map[string]string{},
+			map[string]interface{}{
+				"val": i,
+			},
+			now,
+		)
+		now = now.Add(1)
+		p.Enqueue(m)
+	}
+	p.Stop()
+
+	i := 0
+	require.Len(t, acc.Metrics, 20000)
+	for _, m := range acc.GetTelegrafMetrics() {
+		v, ok := m.GetField("val")
+		require.True(t, ok)
+		require.EqualValues(t, i, v)
+		i++
+	}
+}
+
+func TestUnorderedJobsDontDropAnyJobs(t *testing.T) {
+	acc := &testutil.Accumulator{}
+
+	p := parallel.NewUnordered(acc, jobFunc, 10)
+
+	now := time.Now()
+
+	expectedTotal := 0
+	for i := 0; i < 20000; i++ {
+		expectedTotal += i
+		m := metric.New("test",
+			map[string]string{},
+			map[string]interface{}{
+				"val": i,
+			},
+			now,
+		)
+		now = now.Add(1)
+		p.Enqueue(m)
+	}
+	p.Stop()
+
+	actualTotal := int64(0)
+	require.Len(t, acc.Metrics, 20000)
+	for _, m := range acc.GetTelegrafMetrics() {
+		v, ok := m.GetField("val")
+		require.True(t, ok)
+		actualTotal += v.(int64)
+	}
+	require.EqualValues(t, expectedTotal, actualTotal)
+}
+
+func BenchmarkOrdered(b *testing.B) {
+	acc := &testutil.Accumulator{}
+
+	p := parallel.NewOrdered(acc, jobFunc, 10000, 10)
+
+	m := metric.New("test",
+		map[string]string{},
+		map[string]interface{}{
+			"val": 1,
+		},
+		time.Now(),
+	)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		p.Enqueue(m)
+	}
+	p.Stop()
+}
+
+func BenchmarkUnordered(b *testing.B) {
+	acc := &testutil.Accumulator{}
+
+	p := parallel.NewUnordered(acc, jobFunc, 10)
+
+	m := metric.New("test",
+		map[string]string{},
+		map[string]interface{}{
+			"val": 1,
+		},
+		time.Now(),
+	)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		p.Enqueue(m)
+	}
+	p.Stop()
+}
+
+func jobFunc(m telegraf.Metric) []telegraf.Metric {
+	return []telegraf.Metric{m}
+}

plugins/common/parallel/unordered.go

@@ -0,0 +1,60 @@
+package parallel
+
+import (
+	"sync"
+
+	"github.com/influxdata/telegraf"
+)
+
+type Unordered struct {
+	wg      sync.WaitGroup
+	acc     telegraf.Accumulator
+	fn      func(telegraf.Metric) []telegraf.Metric
+	inQueue chan telegraf.Metric
+}
+
+func NewUnordered(
+	acc telegraf.Accumulator,
+	fn func(telegraf.Metric) []telegraf.Metric,
+	workerCount int,
+) *Unordered {
+	p := &Unordered{
+		acc:     acc,
+		inQueue: make(chan telegraf.Metric, workerCount),
+		fn:      fn,
+	}
+
+	// start workers
+	p.wg.Add(1)
+	go func() {
+		p.startWorkers(workerCount)
+		p.wg.Done()
+	}()
+
+	return p
+}
+
+func (p *Unordered) startWorkers(count int) {
+	wg := sync.WaitGroup{}
+	wg.Add(count)
+	for i := 0; i < count; i++ {
+		go func() {
+			for metric := range p.inQueue {
+				for _, m := range p.fn(metric) {
+					p.acc.AddMetric(m)
+				}
+			}
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+}
+
+func (p *Unordered) Stop() {
+	close(p.inQueue)
+	p.wg.Wait()
+}
+
+func (p *Unordered) Enqueue(m telegraf.Metric) {
+	p.inQueue <- m
+}