Adding upstream version 1.34.4.

Signed-off-by: Daniel Baumann <daniel@debian.org>

plugins/common/shim/README.md

@@ -0,0 +1,64 @@
+# Telegraf Execd Go Shim
+
+The goal of this _shim_ is to make it trivial to extract an internal input,
+processor, or output plugin from the main Telegraf repo out to a stand-alone
+repo. This allows anyone to build and run it as a separate app using one of the
+execd plugins:
+
+- [inputs.execd](/plugins/inputs/execd)
+- [processors.execd](/plugins/processors/execd)
+- [outputs.execd](/plugins/outputs/execd)
+
+## Steps to externalize a plugin
+
+1. Move the project to an external repo, it's recommended to preserve the path
+  structure, (but not strictly necessary). eg if your plugin was at
+  `plugins/inputs/cpu`, it's recommended that it also be under `plugins/inputs/cpu`
+  in the new repo. For a further example of what this might look like, take a
+  look at [ssoroka/rand](https://github.com/ssoroka/rand) or
+  [danielnelson/telegraf-plugins](https://github.com/danielnelson/telegraf-plugins)
+1. Copy [main.go](./example/cmd/main.go) into your project under the `cmd` folder.
+  This will be the entrypoint to the plugin when run as a stand-alone program, and
+  it will call the shim code for you to make that happen. It's recommended to
+  have only one plugin per repo, as the shim is not designed to run multiple
+  plugins at the same time (it would vastly complicate things).
+1. Edit the main.go file to import your plugin. Within Telegraf this would have
+  been done in an all.go file, but here we don't split the two apart, and the change
+  just goes in the top of main.go. If you skip this step, your plugin will do nothing.
+  eg: `_ "github.com/me/my-plugin-telegraf/plugins/inputs/cpu"`
+1. Optionally add a [plugin.conf](./example/cmd/plugin.conf) for configuration
+  specific to your plugin. Note that this config file **must be separate from the
+  rest of the config for Telegraf, and must not be in a shared directory where
+  Telegraf is expecting to load all configs**. If Telegraf reads this config file
+  it will not know which plugin it relates to. Telegraf instead uses an execd config
+  block to look for this plugin.
+
+## Steps to build and run your plugin
+
+1. Build the cmd/main.go. For my rand project this looks like `go build -o rand cmd/main.go`
+1. If you're building an input, you can test out the binary just by running it.
+  eg `./rand -config plugin.conf`
+  Depending on your polling settings and whether you implemented a service plugin or
+  an input gathering plugin, you may see data right away, or you may have to hit enter
+  first, or wait for your poll duration to elapse, but the metrics will be written to
+  STDOUT. Ctrl-C to end your test.
+  If you're testing a processor or output manually, you can still do this but you
+  will need to feed valid metrics in on STDIN to verify that it is doing what you
+  want. This can be a very valuable debugging technique before hooking it up to
+  Telegraf.
+1. Configure Telegraf to call your new plugin binary. For an input, this would
+  look something like:
+
+```toml
+[[inputs.execd]]
+  command = ["/path/to/rand", "-config", "/path/to/plugin.conf"]
+  signal = "none"
+```
+
+  Refer to the execd plugin readmes for more information.
+
+## Congratulations
+
+You've done it! Consider publishing your plugin to github and open a Pull Request
+back to the Telegraf repo letting us know about the availability of your
+[external plugin](https://github.com/influxdata/telegraf/blob/master/EXTERNAL_PLUGINS.md).

plugins/common/shim/config.go

@@ -0,0 +1,170 @@
+package shim
+
+import (
+	"errors"
+	"fmt"
+	"log" //nolint:depguard // Allow exceptional but valid use of log here.
+	"os"
+
+	"github.com/BurntSushi/toml"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/plugins/outputs"
+	"github.com/influxdata/telegraf/plugins/processors"
+)
+
+type config struct {
+	Inputs     map[string][]toml.Primitive
+	Processors map[string][]toml.Primitive
+	Outputs    map[string][]toml.Primitive
+}
+
+type loadedConfig struct {
+	Input     telegraf.Input
+	Processor telegraf.StreamingProcessor
+	Output    telegraf.Output
+}
+
+// LoadConfig Adds plugins to the shim
+func (s *Shim) LoadConfig(filePath *string) error {
+	conf, err := LoadConfig(filePath)
+	if err != nil {
+		return err
+	}
+	if conf.Input != nil {
+		if err = s.AddInput(conf.Input); err != nil {
+			return fmt.Errorf("failed to add Input: %w", err)
+		}
+	} else if conf.Processor != nil {
+		if err = s.AddStreamingProcessor(conf.Processor); err != nil {
+			return fmt.Errorf("failed to add Processor: %w", err)
+		}
+	} else if conf.Output != nil {
+		if err = s.AddOutput(conf.Output); err != nil {
+			return fmt.Errorf("failed to add Output: %w", err)
+		}
+	}
+	return nil
+}
+
+// LoadConfig loads the config and returns inputs that later need to be loaded.
+func LoadConfig(filePath *string) (loaded loadedConfig, err error) {
+	var data string
+	conf := config{}
+	if filePath != nil && *filePath != "" {
+		b, err := os.ReadFile(*filePath)
+		if err != nil {
+			return loadedConfig{}, err
+		}
+
+		data = expandEnvVars(b)
+	} else {
+		conf = DefaultImportedPlugins()
+	}
+
+	md, err := toml.Decode(data, &conf)
+	if err != nil {
+		return loadedConfig{}, err
+	}
+
+	return createPluginsWithTomlConfig(md, conf)
+}
+
+func expandEnvVars(contents []byte) string {
+	return os.Expand(string(contents), getEnv)
+}
+
+func getEnv(key string) string {
+	v := os.Getenv(key)
+
+	return envVarEscaper.Replace(v)
+}
+
+func createPluginsWithTomlConfig(md toml.MetaData, conf config) (loadedConfig, error) {
+	loadedConf := loadedConfig{}
+
+	for name, primitives := range conf.Inputs {
+		creator, ok := inputs.Inputs[name]
+		if !ok {
+			return loadedConf, errors.New("unknown input " + name)
+		}
+
+		plugin := creator()
+		if len(primitives) > 0 {
+			primitive := primitives[0]
+			if err := md.PrimitiveDecode(primitive, plugin); err != nil {
+				return loadedConf, err
+			}
+		}
+
+		loadedConf.Input = plugin
+		break
+	}
+
+	for name, primitives := range conf.Processors {
+		creator, ok := processors.Processors[name]
+		if !ok {
+			return loadedConf, errors.New("unknown processor " + name)
+		}
+
+		plugin := creator()
+		if len(primitives) > 0 {
+			primitive := primitives[0]
+			var p telegraf.PluginDescriber = plugin
+			if processor, ok := plugin.(processors.HasUnwrap); ok {
+				p = processor.Unwrap()
+			}
+			if err := md.PrimitiveDecode(primitive, p); err != nil {
+				return loadedConf, err
+			}
+		}
+		loadedConf.Processor = plugin
+		break
+	}
+
+	for name, primitives := range conf.Outputs {
+		creator, ok := outputs.Outputs[name]
+		if !ok {
+			return loadedConf, errors.New("unknown output " + name)
+		}
+
+		plugin := creator()
+		if len(primitives) > 0 {
+			primitive := primitives[0]
+			if err := md.PrimitiveDecode(primitive, plugin); err != nil {
+				return loadedConf, err
+			}
+		}
+		loadedConf.Output = plugin
+		break
+	}
+	return loadedConf, nil
+}
+
+// DefaultImportedPlugins defaults to whatever plugins happen to be loaded and
+// have registered themselves with the registry. This makes loading plugins
+// without having to define a config dead easy.
+func DefaultImportedPlugins() config {
+	conf := config{
+		Inputs:     make(map[string][]toml.Primitive, len(inputs.Inputs)),
+		Processors: make(map[string][]toml.Primitive, len(processors.Processors)),
+		Outputs:    make(map[string][]toml.Primitive, len(outputs.Outputs)),
+	}
+	for name := range inputs.Inputs {
+		log.Println("No config found. Loading default config for plugin", name)
+		conf.Inputs[name] = make([]toml.Primitive, 0)
+		return conf
+	}
+	for name := range processors.Processors {
+		log.Println("No config found. Loading default config for plugin", name)
+		conf.Processors[name] = make([]toml.Primitive, 0)
+		return conf
+	}
+	for name := range outputs.Outputs {
+		log.Println("No config found. Loading default config for plugin", name)
+		conf.Outputs[name] = make([]toml.Primitive, 0)
+		return conf
+	}
+	return conf
+}

plugins/common/shim/config_test.go

@@ -0,0 +1,87 @@
+package shim
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/influxdata/telegraf"
+	cfg "github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/plugins/processors"
+)
+
+func TestLoadConfig(t *testing.T) {
+	t.Setenv("SECRET_TOKEN", "xxxxxxxxxx")
+	t.Setenv("SECRET_VALUE", `test"\test`)
+
+	inputs.Add("test", func() telegraf.Input {
+		return &serviceInput{}
+	})
+
+	c := "./testdata/plugin.conf"
+	conf, err := LoadConfig(&c)
+	require.NoError(t, err)
+
+	inp := conf.Input.(*serviceInput)
+
+	require.Equal(t, "awesome name", inp.ServiceName)
+	require.Equal(t, "xxxxxxxxxx", inp.SecretToken)
+	require.Equal(t, `test"\test`, inp.SecretValue)
+}
+
+func TestLoadingSpecialTypes(t *testing.T) {
+	inputs.Add("test", func() telegraf.Input {
+		return &testDurationInput{}
+	})
+
+	c := "./testdata/special.conf"
+	conf, err := LoadConfig(&c)
+	require.NoError(t, err)
+
+	inp := conf.Input.(*testDurationInput)
+
+	require.EqualValues(t, 3*time.Second, inp.Duration)
+	require.EqualValues(t, 3*1000*1000, inp.Size)
+	require.EqualValues(t, 52, inp.Hex)
+}
+
+func TestLoadingProcessorWithConfig(t *testing.T) {
+	proc := &testConfigProcessor{}
+	processors.Add("test_config_load", func() telegraf.Processor {
+		return proc
+	})
+
+	c := "./testdata/processor.conf"
+	_, err := LoadConfig(&c)
+	require.NoError(t, err)
+
+	require.EqualValues(t, "yep", proc.Loaded)
+}
+
+type testDurationInput struct {
+	Duration cfg.Duration `toml:"duration"`
+	Size     cfg.Size     `toml:"size"`
+	Hex      int64        `toml:"hex"`
+}
+
+func (*testDurationInput) SampleConfig() string {
+	return ""
+}
+
+func (*testDurationInput) Gather(telegraf.Accumulator) error {
+	return nil
+}
+
+type testConfigProcessor struct {
+	Loaded string `toml:"loaded"`
+}
+
+func (*testConfigProcessor) SampleConfig() string {
+	return ""
+}
+
+func (*testConfigProcessor) Apply(metrics ...telegraf.Metric) []telegraf.Metric {
+	return metrics
+}

plugins/common/shim/example/cmd/main.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	// TODO: import your plugins
+	_ "github.com/influxdata/tail" // Example external package for showing where you can import your plugins
+
+	"github.com/influxdata/telegraf/plugins/common/shim"
+)
+
+var pollInterval = flag.Duration("poll_interval", 1*time.Second, "how often to send metrics")
+
+var pollIntervalDisabled = flag.Bool(
+	"poll_interval_disabled",
+	false,
+	"set to true to disable polling. You want to use this when you are sending metrics on your own schedule",
+)
+var configFile = flag.String("config", "", "path to the config file for this plugin")
+var err error
+
+// This is designed to be simple; Just change the import above, and you're good.
+//
+// However, if you want to do all your config in code, you can like so:
+//
+// // initialize your plugin with any settings you want
+//
+//	myInput := &mypluginname.MyPlugin{
+//		DefaultSettingHere: 3,
+//	}
+//
+// shim := shim.New()
+//
+// shim.AddInput(myInput)
+//
+// // now the shim.Run() call as below. Note the shim is only intended to run a single plugin.
+func main() {
+	// parse command line options
+	flag.Parse()
+	if *pollIntervalDisabled {
+		*pollInterval = shim.PollIntervalDisabled
+	}
+
+	// create the shim. This is what will run your plugins.
+	shimLayer := shim.New()
+
+	// If no config is specified, all imported plugins are loaded.
+	// otherwise, follow what the config asks for.
+	// Check for settings from a config toml file,
+	// (or just use whatever plugins were imported above)
+	if err = shimLayer.LoadConfig(configFile); err != nil {
+		fmt.Fprintf(os.Stderr, "Err loading input: %s\n", err)
+		os.Exit(1)
+	}
+
+	// run a single plugin until stdin closes, or we receive a termination signal
+	if err = shimLayer.Run(*pollInterval); err != nil {
+		fmt.Fprintf(os.Stderr, "Err: %s\n", err)
+		os.Exit(1)
+	}
+}

plugins/common/shim/example/cmd/plugin.conf

@@ -0,0 +1,2 @@
+[[inputs.my_plugin_name]]
+	value_name = "value"

plugins/common/shim/goshim.go

@@ -0,0 +1,145 @@
+package shim
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/logger"
+	"github.com/influxdata/telegraf/plugins/serializers/influx"
+)
+
+type empty struct{}
+
+var (
+	forever       = 100 * 365 * 24 * time.Hour
+	envVarEscaper = strings.NewReplacer(
+		`"`, `\"`,
+		`\`, `\\`,
+	)
+)
+
+const (
+	// PollIntervalDisabled is used to indicate that you want to disable polling,
+	// as opposed to duration 0 meaning poll constantly.
+	PollIntervalDisabled = time.Duration(0)
+)
+
+// Shim allows you to wrap your inputs and run them as if they were part of Telegraf,
+// except built externally.
+type Shim struct {
+	Input     telegraf.Input
+	Processor telegraf.StreamingProcessor
+	Output    telegraf.Output
+
+	log telegraf.Logger
+
+	// streams
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
+
+	// outgoing metric channel
+	metricCh chan telegraf.Metric
+
+	// input only
+	gatherPromptCh chan empty
+}
+
+// New creates a new shim interface
+func New() *Shim {
+	return &Shim{
+		metricCh: make(chan telegraf.Metric, 1),
+		stdin:    os.Stdin,
+		stdout:   os.Stdout,
+		stderr:   os.Stderr,
+		log:      logger.New("", "", ""),
+	}
+}
+
+func (*Shim) watchForShutdown(cancel context.CancelFunc) {
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		<-quit // user-triggered quit
+		// cancel, but keep looping until the metric channel closes.
+		cancel()
+	}()
+}
+
+// Run the input plugins..
+func (s *Shim) Run(pollInterval time.Duration) error {
+	if s.Input != nil {
+		err := s.RunInput(pollInterval)
+		if err != nil {
+			return fmt.Errorf("running input failed: %w", err)
+		}
+	} else if s.Processor != nil {
+		err := s.RunProcessor()
+		if err != nil {
+			return fmt.Errorf("running processor failed: %w", err)
+		}
+	} else if s.Output != nil {
+		err := s.RunOutput()
+		if err != nil {
+			return fmt.Errorf("running output failed: %w", err)
+		}
+	} else {
+		return errors.New("nothing to run")
+	}
+
+	return nil
+}
+
+func hasQuit(ctx context.Context) bool {
+	return ctx.Err() != nil
+}
+
+func (s *Shim) writeProcessedMetrics() error {
+	serializer := &influx.Serializer{}
+	if err := serializer.Init(); err != nil {
+		return fmt.Errorf("creating serializer failed: %w", err)
+	}
+	for { //nolint:staticcheck // for-select used on purpose
+		select {
+		case m, open := <-s.metricCh:
+			if !open {
+				return nil
+			}
+			b, err := serializer.Serialize(m)
+			if err != nil {
+				m.Reject()
+				return fmt.Errorf("failed to serialize metric: %w", err)
+			}
+			// Write this to stdout
+			_, err = fmt.Fprint(s.stdout, string(b))
+			if err != nil {
+				m.Drop()
+				return fmt.Errorf("failed to write metric: %w", err)
+			}
+			m.Accept()
+		}
+	}
+}
+
+// LogName satisfies the MetricMaker interface
+func (*Shim) LogName() string {
+	return ""
+}
+
+// MakeMetric satisfies the MetricMaker interface
+func (*Shim) MakeMetric(m telegraf.Metric) telegraf.Metric {
+	return m // don't need to do anything to it.
+}
+
+// Log satisfies the MetricMaker interface
+func (s *Shim) Log() telegraf.Logger {
+	return s.log
+}

plugins/common/shim/goshim_test.go

@@ -0,0 +1,78 @@
+package shim
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/logger"
+)
+
+func TestShimSetsUpLogger(t *testing.T) {
+	stderrReader, stderrWriter := io.Pipe()
+	stdinReader, stdinWriter := io.Pipe()
+
+	runErroringInputPlugin(t, 40*time.Second, stdinReader, nil, stderrWriter)
+
+	_, err := stdinWriter.Write([]byte("\n"))
+	require.NoError(t, err)
+
+	r := bufio.NewReader(stderrReader)
+	out, err := r.ReadString('\n')
+	require.NoError(t, err)
+	require.Contains(t, out, "Error in plugin: intentional")
+
+	err = stdinWriter.Close()
+	require.NoError(t, err)
+}
+
+func runErroringInputPlugin(t *testing.T, interval time.Duration, stdin io.Reader, stdout, stderr io.Writer) (processed, exited chan bool) {
+	processed = make(chan bool, 1)
+	exited = make(chan bool, 1)
+	inp := &erroringInput{}
+
+	shim := New()
+	if stdin != nil {
+		shim.stdin = stdin
+	}
+	if stdout != nil {
+		shim.stdout = stdout
+	}
+	if stderr != nil {
+		shim.stderr = stderr
+		logger.RedirectLogging(stderr)
+	}
+
+	require.NoError(t, shim.AddInput(inp))
+	go func(e chan bool) {
+		if err := shim.Run(interval); err != nil {
+			t.Error(err)
+		}
+		e <- true
+	}(exited)
+	return processed, exited
+}
+
+type erroringInput struct {
+}
+
+func (*erroringInput) SampleConfig() string {
+	return ""
+}
+
+func (*erroringInput) Gather(acc telegraf.Accumulator) error {
+	acc.AddError(errors.New("intentional"))
+	return nil
+}
+
+func (*erroringInput) Start(telegraf.Accumulator) error {
+	return nil
+}
+
+func (*erroringInput) Stop() {
+}

plugins/common/shim/input.go

@@ -0,0 +1,116 @@
+package shim
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/agent"
+	"github.com/influxdata/telegraf/models"
+)
+
+// AddInput adds the input to the shim. Later calls to Run() will run this input.
+func (s *Shim) AddInput(input telegraf.Input) error {
+	models.SetLoggerOnPlugin(input, s.Log())
+	if p, ok := input.(telegraf.Initializer); ok {
+		err := p.Init()
+		if err != nil {
+			return fmt.Errorf("failed to init input: %w", err)
+		}
+	}
+
+	s.Input = input
+	return nil
+}
+
+func (s *Shim) RunInput(pollInterval time.Duration) error {
+	// context is used only to close the stdin reader. everything else cascades
+	// from that point and closes cleanly when it's done.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	s.watchForShutdown(cancel)
+
+	acc := agent.NewAccumulator(s, s.metricCh)
+	acc.SetPrecision(time.Nanosecond)
+
+	if serviceInput, ok := s.Input.(telegraf.ServiceInput); ok {
+		if err := serviceInput.Start(acc); err != nil {
+			return fmt.Errorf("failed to start input: %w", err)
+		}
+	}
+	s.gatherPromptCh = make(chan empty, 1)
+	go func() {
+		s.startGathering(ctx, s.Input, acc, pollInterval)
+		if serviceInput, ok := s.Input.(telegraf.ServiceInput); ok {
+			serviceInput.Stop()
+		}
+		// closing the metric channel gracefully stops writing to stdout
+		close(s.metricCh)
+	}()
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		err := s.writeProcessedMetrics()
+		if err != nil {
+			s.log.Warn(err.Error())
+		}
+		wg.Done()
+	}()
+
+	go func() {
+		scanner := bufio.NewScanner(s.stdin)
+		for scanner.Scan() {
+			// push a non-blocking message to trigger metric collection.
+			s.pushCollectMetricsRequest()
+		}
+
+		cancel() // cancel gracefully stops gathering
+	}()
+
+	wg.Wait() // wait for writing to stdout to finish
+	return nil
+}
+
+func (s *Shim) startGathering(ctx context.Context, input telegraf.Input, acc telegraf.Accumulator, pollInterval time.Duration) {
+	if pollInterval == PollIntervalDisabled {
+		pollInterval = forever
+	}
+	t := time.NewTicker(pollInterval)
+	defer t.Stop()
+	for {
+		// give priority to stopping.
+		if hasQuit(ctx) {
+			return
+		}
+		// see what's up
+		select {
+		case <-ctx.Done():
+			return
+		case <-s.gatherPromptCh:
+			if err := input.Gather(acc); err != nil {
+				fmt.Fprintf(s.stderr, "failed to gather metrics: %s\n", err)
+			}
+		case <-t.C:
+			if err := input.Gather(acc); err != nil {
+				fmt.Fprintf(s.stderr, "failed to gather metrics: %s\n", err)
+			}
+		}
+	}
+}
+
+// pushCollectMetricsRequest pushes a non-blocking (nil) message to the
+// gatherPromptCh channel to trigger metric collection.
+// The channel is defined with a buffer of 1, so while it's full, subsequent
+// requests are discarded.
+func (s *Shim) pushCollectMetricsRequest() {
+	// push a message out to each channel to collect metrics. don't block.
+	select {
+	case s.gatherPromptCh <- empty{}:
+	default:
+	}
+}

plugins/common/shim/input_test.go

@@ -0,0 +1,140 @@
+package shim
+
+import (
+	"bufio"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/influxdata/telegraf"
+)
+
+func TestInputShimTimer(t *testing.T) {
+	stdoutReader, stdoutWriter := io.Pipe()
+
+	stdin, _ := io.Pipe() // hold the stdin pipe open
+
+	metricProcessed, _ := runInputPlugin(t, 10*time.Millisecond, stdin, stdoutWriter, nil)
+
+	<-metricProcessed
+	r := bufio.NewReader(stdoutReader)
+	out, err := r.ReadString('\n')
+	require.NoError(t, err)
+	require.Contains(t, out, "\n")
+	metricLine := strings.Split(out, "\n")[0]
+	require.Equal(t, "measurement,tag=tag field=1i 1234000005678", metricLine)
+}
+
+func TestInputShimStdinSignalingWorks(t *testing.T) {
+	stdinReader, stdinWriter := io.Pipe()
+	stdoutReader, stdoutWriter := io.Pipe()
+
+	metricProcessed, exited := runInputPlugin(t, 40*time.Second, stdinReader, stdoutWriter, nil)
+
+	_, err := stdinWriter.Write([]byte("\n"))
+	require.NoError(t, err)
+
+	<-metricProcessed
+
+	r := bufio.NewReader(stdoutReader)
+	out, err := r.ReadString('\n')
+	require.NoError(t, err)
+	require.Equal(t, "measurement,tag=tag field=1i 1234000005678\n", out)
+
+	err = stdinWriter.Close()
+	require.NoError(t, err)
+	go func() {
+		if _, err = io.ReadAll(r); err != nil {
+			t.Error(err)
+		}
+	}()
+	// check that it exits cleanly
+	<-exited
+}
+
+func runInputPlugin(t *testing.T, interval time.Duration, stdin io.Reader, stdout, stderr io.Writer) (processed, exited chan bool) {
+	processed = make(chan bool, 1)
+	exited = make(chan bool, 1)
+	inp := &testInput{
+		metricProcessed: processed,
+	}
+
+	shim := New()
+	if stdin != nil {
+		shim.stdin = stdin
+	}
+	if stdout != nil {
+		shim.stdout = stdout
+	}
+	if stderr != nil {
+		shim.stderr = stderr
+	}
+	err := shim.AddInput(inp)
+	require.NoError(t, err)
+	go func(e chan bool) {
+		if err := shim.Run(interval); err != nil {
+			t.Error(err)
+		}
+		e <- true
+	}(exited)
+	return processed, exited
+}
+
+type testInput struct {
+	metricProcessed chan bool
+}
+
+func (*testInput) SampleConfig() string {
+	return ""
+}
+
+func (i *testInput) Gather(acc telegraf.Accumulator) error {
+	acc.AddFields("measurement",
+		map[string]interface{}{
+			"field": 1,
+		},
+		map[string]string{
+			"tag": "tag",
+		}, time.Unix(1234, 5678))
+	i.metricProcessed <- true
+	return nil
+}
+
+func (*testInput) Start(telegraf.Accumulator) error {
+	return nil
+}
+
+func (*testInput) Stop() {
+}
+
+type serviceInput struct {
+	ServiceName string `toml:"service_name"`
+	SecretToken string `toml:"secret_token"`
+	SecretValue string `toml:"secret_value"`
+}
+
+func (*serviceInput) SampleConfig() string {
+	return ""
+}
+
+func (*serviceInput) Gather(acc telegraf.Accumulator) error {
+	acc.AddFields("measurement",
+		map[string]interface{}{
+			"field": 1,
+		},
+		map[string]string{
+			"tag": "tag",
+		}, time.Unix(1234, 5678))
+
+	return nil
+}
+
+func (*serviceInput) Start(telegraf.Accumulator) error {
+	return nil
+}
+
+func (*serviceInput) Stop() {
+}

plugins/common/shim/output.go

@@ -0,0 +1,54 @@
+package shim
+
+import (
+	"bufio"
+	"fmt"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/models"
+	"github.com/influxdata/telegraf/plugins/parsers/influx"
+)
+
+// AddOutput adds the input to the shim. Later calls to Run() will run this.
+func (s *Shim) AddOutput(output telegraf.Output) error {
+	models.SetLoggerOnPlugin(output, s.Log())
+	if p, ok := output.(telegraf.Initializer); ok {
+		err := p.Init()
+		if err != nil {
+			return fmt.Errorf("failed to init input: %w", err)
+		}
+	}
+
+	s.Output = output
+	return nil
+}
+
+func (s *Shim) RunOutput() error {
+	parser := influx.Parser{}
+	err := parser.Init()
+	if err != nil {
+		return fmt.Errorf("failed to create new parser: %w", err)
+	}
+
+	err = s.Output.Connect()
+	if err != nil {
+		return fmt.Errorf("failed to start processor: %w", err)
+	}
+	defer s.Output.Close()
+
+	var m telegraf.Metric
+
+	scanner := bufio.NewScanner(s.stdin)
+	for scanner.Scan() {
+		m, err = parser.ParseLine(scanner.Text())
+		if err != nil {
+			fmt.Fprintf(s.stderr, "Failed to parse metric: %s\n", err)
+			continue
+		}
+		if err = s.Output.Write([]telegraf.Metric{m}); err != nil {
+			fmt.Fprintf(s.stderr, "Failed to write metric: %s\n", err)
+		}
+	}
+
+	return nil
+}

plugins/common/shim/output_test.go

@@ -0,0 +1,81 @@
+package shim
+
+import (
+	"io"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/metric"
+	"github.com/influxdata/telegraf/plugins/serializers/influx"
+	"github.com/influxdata/telegraf/testutil"
+)
+
+func TestOutputShim(t *testing.T) {
+	o := &testOutput{}
+
+	stdinReader, stdinWriter := io.Pipe()
+
+	s := New()
+	s.stdin = stdinReader
+	err := s.AddOutput(o)
+	require.NoError(t, err)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		if err := s.RunOutput(); err != nil {
+			t.Error(err)
+		}
+		wg.Done()
+	}()
+
+	serializer := &influx.Serializer{}
+	require.NoError(t, serializer.Init())
+
+	m := metric.New("thing",
+		map[string]string{
+			"a": "b",
+		},
+		map[string]interface{}{
+			"v": 1,
+		},
+		time.Now(),
+	)
+	b, err := serializer.Serialize(m)
+	require.NoError(t, err)
+	_, err = stdinWriter.Write(b)
+	require.NoError(t, err)
+	err = stdinWriter.Close()
+	require.NoError(t, err)
+
+	wg.Wait()
+
+	require.Len(t, o.MetricsWritten, 1)
+	mOut := o.MetricsWritten[0]
+
+	testutil.RequireMetricEqual(t, m, mOut)
+}
+
+type testOutput struct {
+	MetricsWritten []telegraf.Metric
+}
+
+func (*testOutput) Connect() error {
+	return nil
+}
+func (*testOutput) Close() error {
+	return nil
+}
+func (o *testOutput) Write(metrics []telegraf.Metric) error {
+	o.MetricsWritten = append(o.MetricsWritten, metrics...)
+	return nil
+}
+
+func (*testOutput) SampleConfig() string {
+	return ""
+}

plugins/common/shim/processor.go

@@ -0,0 +1,81 @@
+package shim
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/agent"
+	"github.com/influxdata/telegraf/models"
+	"github.com/influxdata/telegraf/plugins/parsers/influx"
+	"github.com/influxdata/telegraf/plugins/processors"
+)
+
+// AddProcessor adds the processor to the shim. Later calls to Run() will run this.
+func (s *Shim) AddProcessor(processor telegraf.Processor) error {
+	models.SetLoggerOnPlugin(processor, s.Log())
+	p := processors.NewStreamingProcessorFromProcessor(processor)
+	return s.AddStreamingProcessor(p)
+}
+
+// AddStreamingProcessor adds the processor to the shim. Later calls to Run() will run this.
+func (s *Shim) AddStreamingProcessor(processor telegraf.StreamingProcessor) error {
+	models.SetLoggerOnPlugin(processor, s.Log())
+	if p, ok := processor.(telegraf.Initializer); ok {
+		err := p.Init()
+		if err != nil {
+			return fmt.Errorf("failed to init input: %w", err)
+		}
+	}
+
+	s.Processor = processor
+	return nil
+}
+
+func (s *Shim) RunProcessor() error {
+	acc := agent.NewAccumulator(s, s.metricCh)
+	acc.SetPrecision(time.Nanosecond)
+
+	err := s.Processor.Start(acc)
+	if err != nil {
+		return fmt.Errorf("failed to start processor: %w", err)
+	}
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		err := s.writeProcessedMetrics()
+		if err != nil {
+			s.log.Warn(err.Error())
+		}
+		wg.Done()
+	}()
+
+	parser := influx.NewStreamParser(s.stdin)
+	for {
+		m, err := parser.Next()
+		if err != nil {
+			if errors.Is(err, influx.EOF) {
+				break // stream ended
+			}
+			var parseErr *influx.ParseError
+			if errors.As(err, &parseErr) {
+				fmt.Fprintf(s.stderr, "Failed to parse metric: %s\b", parseErr)
+				continue
+			}
+			fmt.Fprintf(s.stderr, "Failure during reading stdin: %s\b", err)
+			continue
+		}
+
+		if err = s.Processor.Add(m, acc); err != nil {
+			fmt.Fprintf(s.stderr, "Failure during processing metric by processor: %v\b", err)
+		}
+	}
+
+	close(s.metricCh)
+	s.Processor.Stop()
+	wg.Wait()
+	return nil
+}

plugins/common/shim/processor_test.go

@@ -0,0 +1,113 @@
+package shim
+
+import (
+	"bufio"
+	"io"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/metric"
+	"github.com/influxdata/telegraf/plugins/parsers/influx"
+	serializers_influx "github.com/influxdata/telegraf/plugins/serializers/influx"
+)
+
+func TestProcessorShim(t *testing.T) {
+	testSendAndReceive(t, "f1", "fv1")
+}
+
+func TestProcessorShimWithLargerThanDefaultScannerBufferSize(t *testing.T) {
+	letters := []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	b := make([]rune, 0, bufio.MaxScanTokenSize*2)
+	for i := 0; i < bufio.MaxScanTokenSize*2; i++ {
+		b = append(b, letters[rand.Intn(len(letters))])
+	}
+
+	testSendAndReceive(t, "f1", string(b))
+}
+
+func testSendAndReceive(t *testing.T, fieldKey, fieldValue string) {
+	p := &testProcessor{"hi", "mom"}
+
+	stdinReader, stdinWriter := io.Pipe()
+	stdoutReader, stdoutWriter := io.Pipe()
+
+	s := New()
+	// inject test into shim
+	s.stdin = stdinReader
+	s.stdout = stdoutWriter
+	err := s.AddProcessor(p)
+	require.NoError(t, err)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		if err := s.RunProcessor(); err != nil {
+			t.Error(err)
+		}
+		wg.Done()
+	}()
+
+	serializer := &serializers_influx.Serializer{}
+	require.NoError(t, serializer.Init())
+
+	parser := influx.Parser{}
+	require.NoError(t, parser.Init())
+
+	m := metric.New("thing",
+		map[string]string{
+			"a": "b",
+		},
+		map[string]interface{}{
+			"v":      1,
+			fieldKey: fieldValue,
+		},
+		time.Now(),
+	)
+	b, err := serializer.Serialize(m)
+	require.NoError(t, err)
+	_, err = stdinWriter.Write(b)
+	require.NoError(t, err)
+	err = stdinWriter.Close()
+	require.NoError(t, err)
+
+	r := bufio.NewReader(stdoutReader)
+	out, err := r.ReadString('\n')
+	require.NoError(t, err)
+	mOut, err := parser.ParseLine(out)
+	require.NoError(t, err)
+
+	val, ok := mOut.GetTag(p.tagName)
+	require.True(t, ok)
+	require.Equal(t, p.tagValue, val)
+	val2, ok := mOut.Fields()[fieldKey]
+	require.True(t, ok)
+	require.Equal(t, fieldValue, val2)
+	go func() {
+		if _, err = io.ReadAll(r); err != nil {
+			t.Error(err)
+		}
+	}()
+	wg.Wait()
+}
+
+type testProcessor struct {
+	tagName  string
+	tagValue string
+}
+
+func (p *testProcessor) Apply(in ...telegraf.Metric) []telegraf.Metric {
+	for _, m := range in {
+		m.AddTag(p.tagName, p.tagValue)
+	}
+	return in
+}
+
+func (*testProcessor) SampleConfig() string {
+	return ""
+}

plugins/common/shim/testdata/plugin.conf

@@ -0,0 +1,4 @@
+[[inputs.test]]
+	service_name = "awesome name"
+	secret_token = "${SECRET_TOKEN}"
+	secret_value = "$SECRET_VALUE"

plugins/common/shim/testdata/processor.conf

@@ -0,0 +1,2 @@
+[[processors.test_config_load]]
+  loaded = "yep"

plugins/common/shim/testdata/special.conf

@@ -0,0 +1,5 @@
+# testing custom field types
+[[inputs.test]]
+	duration = "3s"
+  size = "3MB"
+  hex = 0x34