Adding upstream version 1.34.4.

Signed-off-by: Daniel Baumann <daniel@debian.org>

plugins/inputs/mqtt_consumer/README.md

@@ -0,0 +1,276 @@
+# MQTT Consumer Input Plugin
+
+This service plugin consumes messages from [MQTT][mqtt] brokers for the
+configured topics in one of the supported [data formats][data_formats].
+
+⭐ Telegraf v0.10.3
+🏷️ messaging
+💻 all
+
+[mqtt]: https://mqtt.org
+[data_formats]: /docs/DATA_FORMATS_INPUT.md
+
+## Service Input <!-- @/docs/includes/service_input.md -->
+
+This plugin is a service input. Normal plugins gather metrics determined by the
+interval setting. Service plugins start a service to listen and wait for
+metrics or events to occur. Service plugins have two key differences from
+normal plugins:
+
+1. The global or plugin specific `interval` setting may not apply
+2. The CLI options of `--test`, `--test-wait`, and `--once` may not produce
+   output for this plugin
+
+## Global configuration options <!-- @/docs/includes/plugin_config.md -->
+
+In addition to the plugin-specific configuration settings, plugins support
+additional global and plugin configuration settings. These settings are used to
+modify metrics, tags, and field or create aliases and configure ordering, etc.
+See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
+
+[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
+
+## Startup error behavior options <!-- @/docs/includes/startup_error_behavior.md -->
+
+In addition to the plugin-specific and global configuration settings the plugin
+supports options for specifying the behavior when experiencing startup errors
+using the `startup_error_behavior` setting. Available values are:
+
+- `error`:  Telegraf with stop and exit in case of startup errors. This is the
+            default behavior.
+- `ignore`: Telegraf will ignore startup errors for this plugin and disables it
+            but continues processing for all other plugins.
+- `retry`:  Telegraf will try to startup the plugin in every gather or write
+            cycle in case of startup errors. The plugin is disabled until
+            the startup succeeds.
+- `probe`:  Telegraf will probe the plugin's function (if possible) and disables the plugin
+            in case probing fails. If the plugin does not support probing, Telegraf will
+            behave as if `ignore` was set instead.
+
+## Secret-store support
+
+This plugin supports secrets from secret-stores for the `username` and
+`password` option.
+See the [secret-store documentation][SECRETSTORE] for more details on how
+to use them.
+
+[SECRETSTORE]: ../../../docs/CONFIGURATION.md#secret-store-secrets
+
+## Configuration
+
+```toml @sample.conf
+# Read metrics from MQTT topic(s)
+[[inputs.mqtt_consumer]]
+  ## Broker URLs for the MQTT server or cluster.  To connect to multiple
+  ## clusters or standalone servers, use a separate plugin instance.
+  ##   example: servers = ["tcp://localhost:1883"]
+  ##            servers = ["ssl://localhost:1883"]
+  ##            servers = ["ws://localhost:1883"]
+  servers = ["tcp://127.0.0.1:1883"]
+
+  ## Topics that will be subscribed to.
+  topics = [
+    "telegraf/host01/cpu",
+    "telegraf/+/mem",
+    "sensors/#",
+  ]
+
+  ## The message topic will be stored in a tag specified by this value.  If set
+  ## to the empty string no topic tag will be created.
+  # topic_tag = "topic"
+
+  ## QoS policy for messages
+  ##   0 = at most once
+  ##   1 = at least once
+  ##   2 = exactly once
+  ##
+  ## When using a QoS of 1 or 2, you should enable persistent_session to allow
+  ## resuming unacknowledged messages.
+  # qos = 0
+
+  ## Connection timeout for initial connection in seconds
+  # connection_timeout = "30s"
+
+  ## Interval and ping timeout for keep-alive messages
+  ## The sum of those options defines when a connection loss is detected.
+  ## Note: The keep-alive interval needs to be greater or equal one second and
+  ## fractions of a second are not supported.
+  # keepalive = "60s"
+  # ping_timeout = "10s"
+
+  ## Max undelivered messages
+  ## This plugin uses tracking metrics, which ensure messages are read to
+  ## outputs before acknowledging them to the original broker to ensure data
+  ## is not lost. This option sets the maximum messages to read from the
+  ## broker that have not been written by an output.
+  ##
+  ## This value needs to be picked with awareness of the agent's
+  ## metric_batch_size value as well. Setting max undelivered messages too high
+  ## can result in a constant stream of data batches to the output. While
+  ## setting it too low may never flush the broker's messages.
+  # max_undelivered_messages = 1000
+
+  ## Persistent session disables clearing of the client session on connection.
+  ## In order for this option to work you must also set client_id to identify
+  ## the client.  To receive messages that arrived while the client is offline,
+  ## also set the qos option to 1 or 2 and don't forget to also set the QoS when
+  ## publishing. Finally, using a persistent session will use the initial
+  ## connection topics and not subscribe to any new topics even after
+  ## reconnecting or restarting without a change in client ID.
+  # persistent_session = false
+
+  ## If unset, a random client ID will be generated.
+  # client_id = ""
+
+  ## Username and password to connect MQTT server.
+  # username = "telegraf"
+  # password = "metricsmetricsmetricsmetrics"
+
+  ## Optional TLS Config
+  # tls_ca = "/etc/telegraf/ca.pem"
+  # tls_cert = "/etc/telegraf/cert.pem"
+  # tls_key = "/etc/telegraf/key.pem"
+  ## Use TLS but skip chain & host verification
+  # insecure_skip_verify = false
+
+  ## Client trace messages
+  ## When set to true, and debug mode enabled in the agent settings, the MQTT
+  ## client's messages are included in telegraf logs. These messages are very
+  ## noisey, but essential for debugging issues.
+  # client_trace = false
+
+  ## Data format to consume.
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "influx"
+
+  ## Enable extracting tag values from MQTT topics
+  ## _ denotes an ignored entry in the topic path,
+  ## # denotes a variable length path element (can only be used once per setting)
+  # [[inputs.mqtt_consumer.topic_parsing]]
+  #   topic = ""
+  #   measurement = ""
+  #   tags = ""
+  #   fields = ""
+  ## Value supported is int, float, unit
+  #   [inputs.mqtt_consumer.topic_parsing.types]
+  #      key = type
+```
+
+## Example Output
+
+```text
+mqtt_consumer,host=pop-os,topic=telegraf/host01/cpu value=45i 1653579140440951943
+mqtt_consumer,host=pop-os,topic=telegraf/host01/cpu value=100i 1653579153147395661
+```
+
+## About Topic Parsing
+
+The MQTT topic as a whole is stored as a tag, but this can be far too coarse to
+be easily used when utilizing the data further down the line. This change allows
+tag values to be extracted from the MQTT topic letting you store the information
+provided in the topic in a meaningful way. An `_` denotes an ignored entry in
+the topic path. Please see the following example.
+
+### Topic Parsing Example
+
+```toml
+[[inputs.mqtt_consumer]]
+  ## Broker URLs for the MQTT server or cluster.  To connect to multiple
+  ## clusters or standalone servers, use a separate plugin instance.
+  ##   example: servers = ["tcp://localhost:1883"]
+  ##            servers = ["ssl://localhost:1883"]
+  ##            servers = ["ws://localhost:1883"]
+  servers = ["tcp://127.0.0.1:1883"]
+
+  ## Topics that will be subscribed to.
+  topics = [
+    "telegraf/+/cpu/23",
+  ]
+
+  ## Data format to consume.
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "value"
+  data_type = "float"
+
+  [[inputs.mqtt_consumer.topic_parsing]]
+    topic = "telegraf/one/cpu/23"
+    measurement = "_/_/measurement/_"
+    tags = "tag/_/_/_"
+    fields = "_/_/_/test"
+    [inputs.mqtt_consumer.topic_parsing.types]
+      test = "int"
+```
+
+Will result in the following metric:
+
+```text
+cpu,host=pop-os,tag=telegraf,topic=telegraf/one/cpu/23 value=45,test=23i 1637014942460689291
+```
+
+## Field Pivoting Example
+
+You can use the pivot processor to rotate single
+valued metrics into a multi field metric.
+For more info check out the pivot processors
+[here][1].
+
+For this example these are the topics:
+
+```text
+/sensors/CLE/v1/device5/temp
+/sensors/CLE/v1/device5/rpm
+/sensors/CLE/v1/device5/ph
+/sensors/CLE/v1/device5/spin
+```
+
+And these are the metrics:
+
+```text
+sensors,site=CLE,version=v1,device_name=device5,field=temp value=390
+sensors,site=CLE,version=v1,device_name=device5,field=rpm value=45.0
+sensors,site=CLE,version=v1,device_name=device5,field=ph value=1.45
+```
+
+Using pivot in the config will rotate the metrics into a multi field metric.
+The config:
+
+```toml
+[[inputs.mqtt_consumer]]
+    ....
+    topics = "/sensors/#"
+    [[inputs.mqtt_consumer.topic_parsing]]
+        measurement = "/measurement/_/_/_/_"
+        tags = "/_/site/version/device_name/field"
+[[processors.pivot]]
+    tag_key = "field"
+    value_key = "value"
+```
+
+Will result in the following metric:
+
+```text
+sensors,site=CLE,version=v1,device_name=device5 temp=390,rpm=45.0,ph=1.45
+```
+
+[1]: <https://github.com/influxdata/telegraf/tree/master/plugins/processors/pivot> "Pivot Processor"
+
+## Metrics
+
+- All measurements are tagged with the incoming topic, ie
+`topic=telegraf/host01/cpu`
+
+- example when [[inputs.mqtt_consumer.topic_parsing]] is set
+
+- when [[inputs.internal]] is set:
+  - payload_size (int): get the cumulative size in bytes that have been received from incoming messages
+  - messages_received (int): count of the number of messages that have been received from mqtt
+
+This will result in the following metric:
+
+```text
+internal_mqtt_consumer host=pop-os version=1.24.0 messages_received=622i payload_size=37942i 1657282270000000000
+```

plugins/inputs/mqtt_consumer/mqtt_consumer.go

@@ -0,0 +1,366 @@
+//go:generate ../../../tools/readme_config_includer/generator
+package mqtt_consumer
+
+import (
+	"context"
+	_ "embed"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	mqtt "github.com/eclipse/paho.mqtt.golang"
+	"github.com/eclipse/paho.mqtt.golang/packets"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/internal"
+	"github.com/influxdata/telegraf/plugins/common/tls"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/selfstat"
+)
+
+//go:embed sample.conf
+var sampleConfig string
+
+var (
+	once sync.Once
+	// 30 Seconds is the default used by paho.mqtt.golang
+	defaultConnectionTimeout      = config.Duration(30 * time.Second)
+	defaultMaxUndeliveredMessages = 1000
+)
+
+type MQTTConsumer struct {
+	Servers                []string             `toml:"servers"`
+	Topics                 []string             `toml:"topics"`
+	TopicTag               *string              `toml:"topic_tag"`
+	TopicParserConfig      []topicParsingConfig `toml:"topic_parsing"`
+	Username               config.Secret        `toml:"username"`
+	Password               config.Secret        `toml:"password"`
+	QoS                    int                  `toml:"qos"`
+	ConnectionTimeout      config.Duration      `toml:"connection_timeout"`
+	KeepAliveInterval      config.Duration      `toml:"keepalive"`
+	PingTimeout            config.Duration      `toml:"ping_timeout"`
+	MaxUndeliveredMessages int                  `toml:"max_undelivered_messages"`
+	PersistentSession      bool                 `toml:"persistent_session"`
+	ClientTrace            bool                 `toml:"client_trace"`
+	ClientID               string               `toml:"client_id"`
+	Log                    telegraf.Logger      `toml:"-"`
+	tls.ClientConfig
+
+	parser        telegraf.Parser
+	clientFactory clientFactory
+	client        client
+	opts          *mqtt.ClientOptions
+	acc           telegraf.TrackingAccumulator
+	sem           semaphore
+	messages      map[telegraf.TrackingID]mqtt.Message
+	messagesMutex sync.Mutex
+	topicTagParse string
+	topicParsers  []*topicParser
+	ctx           context.Context
+	cancel        context.CancelFunc
+	payloadSize   selfstat.Stat
+	messagesRecv  selfstat.Stat
+	wg            sync.WaitGroup
+}
+
+type client interface {
+	Connect() mqtt.Token
+	SubscribeMultiple(filters map[string]byte, callback mqtt.MessageHandler) mqtt.Token
+	AddRoute(topic string, callback mqtt.MessageHandler)
+	Disconnect(quiesce uint)
+	IsConnected() bool
+}
+
+type empty struct{}
+type semaphore chan empty
+type clientFactory func(o *mqtt.ClientOptions) client
+
+func (*MQTTConsumer) SampleConfig() string {
+	return sampleConfig
+}
+
+func (m *MQTTConsumer) Init() error {
+	if m.ClientTrace {
+		log := &mqttLogger{m.Log}
+		mqtt.ERROR = log
+		mqtt.CRITICAL = log
+		mqtt.WARN = log
+		mqtt.DEBUG = log
+	}
+
+	if m.PersistentSession && m.ClientID == "" {
+		return errors.New("persistent_session requires client_id")
+	}
+	if m.QoS > 2 || m.QoS < 0 {
+		return fmt.Errorf("qos value must be 0, 1, or 2: %d", m.QoS)
+	}
+	if time.Duration(m.ConnectionTimeout) < 1*time.Second {
+		return fmt.Errorf("connection_timeout must be greater than 1s: %s", time.Duration(m.ConnectionTimeout))
+	}
+	m.topicTagParse = "topic"
+	if m.TopicTag != nil {
+		m.topicTagParse = *m.TopicTag
+	}
+	opts, err := m.createOpts()
+	if err != nil {
+		return err
+	}
+	m.opts = opts
+	m.messages = make(map[telegraf.TrackingID]mqtt.Message)
+
+	m.topicParsers = make([]*topicParser, 0, len(m.TopicParserConfig))
+	for _, cfg := range m.TopicParserConfig {
+		p, err := cfg.newParser()
+		if err != nil {
+			return fmt.Errorf("config error topic parsing: %w", err)
+		}
+		m.topicParsers = append(m.topicParsers, p)
+	}
+
+	m.payloadSize = selfstat.Register("mqtt_consumer", "payload_size", make(map[string]string))
+	m.messagesRecv = selfstat.Register("mqtt_consumer", "messages_received", make(map[string]string))
+	return nil
+}
+
+func (m *MQTTConsumer) SetParser(parser telegraf.Parser) {
+	m.parser = parser
+}
+
+func (m *MQTTConsumer) Start(acc telegraf.Accumulator) error {
+	m.acc = acc.WithTracking(m.MaxUndeliveredMessages)
+	m.sem = make(semaphore, m.MaxUndeliveredMessages)
+	m.ctx, m.cancel = context.WithCancel(context.Background())
+
+	m.wg.Add(1)
+	go func() {
+		defer m.wg.Done()
+		for {
+			select {
+			case <-m.ctx.Done():
+				return
+			case track := <-m.acc.Delivered():
+				m.onDelivered(track)
+			}
+		}
+	}()
+
+	return m.connect()
+}
+
+func (m *MQTTConsumer) Gather(_ telegraf.Accumulator) error {
+	if !m.client.IsConnected() {
+		m.Log.Debugf("Connecting %v", m.Servers)
+		return m.connect()
+	}
+	return nil
+}
+
+func (m *MQTTConsumer) Stop() {
+	if m.client.IsConnected() {
+		m.Log.Debugf("Disconnecting %v", m.Servers)
+		m.client.Disconnect(200)
+		m.Log.Debugf("Disconnected %v", m.Servers)
+	}
+	if m.cancel != nil {
+		m.cancel()
+	}
+}
+
+func (m *MQTTConsumer) connect() error {
+	m.client = m.clientFactory(m.opts)
+	// AddRoute sets up the function for handling messages.  These need to be
+	// added in case we find a persistent session containing subscriptions so we
+	// know where to dispatch persisted and new messages to.  In the alternate
+	// case that we need to create the subscriptions these will be replaced.
+	for _, topic := range m.Topics {
+		m.client.AddRoute(topic, m.onMessage)
+	}
+	token := m.client.Connect()
+	if token.Wait() && token.Error() != nil {
+		if ct, ok := token.(*mqtt.ConnectToken); ok && ct.ReturnCode() == packets.ErrNetworkError {
+			// Network errors might be retryable, stop the metric-tracking
+			// goroutine and return a retryable error.
+			if m.cancel != nil {
+				m.cancel()
+				m.cancel = nil
+			}
+			return &internal.StartupError{
+				Err:   token.Error(),
+				Retry: true,
+			}
+		}
+		return token.Error()
+	}
+	m.Log.Infof("Connected %v", m.Servers)
+
+	// Persistent sessions should skip subscription if a session is present, as
+	// the subscriptions are stored by the server.
+	type sessionPresent interface {
+		SessionPresent() bool
+	}
+	if t, ok := token.(sessionPresent); ok && t.SessionPresent() {
+		m.Log.Debugf("Session found %v", m.Servers)
+		return nil
+	}
+	topics := make(map[string]byte)
+	for _, topic := range m.Topics {
+		topics[topic] = byte(m.QoS)
+	}
+	subscribeToken := m.client.SubscribeMultiple(topics, m.onMessage)
+	subscribeToken.Wait()
+	if subscribeToken.Error() != nil {
+		m.acc.AddError(fmt.Errorf("subscription error: topics %q: %w", strings.Join(m.Topics[:], ","), subscribeToken.Error()))
+	}
+	return nil
+}
+
+func (m *MQTTConsumer) onConnectionLost(_ mqtt.Client, err error) {
+	// Should already be disconnected, but make doubly sure
+	m.client.Disconnect(5)
+	m.acc.AddError(fmt.Errorf("connection lost: %w", err))
+	m.Log.Debugf("Disconnected %v", m.Servers)
+}
+
+func (m *MQTTConsumer) onDelivered(track telegraf.DeliveryInfo) {
+	<-m.sem
+
+	m.messagesMutex.Lock()
+	defer m.messagesMutex.Unlock()
+
+	msg, ok := m.messages[track.ID()]
+	if !ok {
+		m.Log.Errorf("could not mark message delivered: %d", track.ID())
+		return
+	}
+
+	if track.Delivered() && m.PersistentSession {
+		msg.Ack()
+	}
+
+	delete(m.messages, track.ID())
+}
+
+func (m *MQTTConsumer) onMessage(_ mqtt.Client, msg mqtt.Message) {
+	m.sem <- empty{}
+
+	payloadBytes := len(msg.Payload())
+	m.payloadSize.Incr(int64(payloadBytes))
+	m.messagesRecv.Incr(1)
+
+	metrics, err := m.parser.Parse(msg.Payload())
+	if err != nil || len(metrics) == 0 {
+		if len(metrics) == 0 {
+			once.Do(func() {
+				m.Log.Warn(internal.NoMetricsCreatedMsg)
+			})
+		}
+
+		if m.PersistentSession {
+			msg.Ack()
+		}
+		m.acc.AddError(err)
+		<-m.sem
+		return
+	}
+
+	for _, metric := range metrics {
+		if m.topicTagParse != "" {
+			metric.AddTag(m.topicTagParse, msg.Topic())
+		}
+		for _, p := range m.topicParsers {
+			if err := p.parse(metric, msg.Topic()); err != nil {
+				if m.PersistentSession {
+					msg.Ack()
+				}
+				m.acc.AddError(err)
+				<-m.sem
+				return
+			}
+		}
+	}
+	m.messagesMutex.Lock()
+	id := m.acc.AddTrackingMetricGroup(metrics)
+	m.messages[id] = msg
+	m.messagesMutex.Unlock()
+}
+
+func (m *MQTTConsumer) createOpts() (*mqtt.ClientOptions, error) {
+	opts := mqtt.NewClientOptions()
+	opts.ConnectTimeout = time.Duration(m.ConnectionTimeout)
+	if m.ClientID == "" {
+		randomString, err := internal.RandomString(5)
+		if err != nil {
+			return nil, fmt.Errorf("generating random string for client ID failed: %w", err)
+		}
+		opts.SetClientID("Telegraf-Consumer-" + randomString)
+	} else {
+		opts.SetClientID(m.ClientID)
+	}
+	tlsCfg, err := m.ClientConfig.TLSConfig()
+	if err != nil {
+		return nil, err
+	}
+	if tlsCfg != nil {
+		opts.SetTLSConfig(tlsCfg)
+	}
+	if !m.Username.Empty() {
+		user, err := m.Username.Get()
+		if err != nil {
+			return nil, fmt.Errorf("getting username failed: %w", err)
+		}
+		opts.SetUsername(user.String())
+		user.Destroy()
+	}
+
+	if !m.Password.Empty() {
+		password, err := m.Password.Get()
+		if err != nil {
+			return nil, fmt.Errorf("getting password failed: %w", err)
+		}
+		opts.SetPassword(password.String())
+		password.Destroy()
+	}
+	if len(m.Servers) == 0 {
+		return opts, errors.New("could not get host information")
+	}
+	for _, server := range m.Servers {
+		// Preserve support for host:port style servers; deprecated in Telegraf 1.4.4
+		if !strings.Contains(server, "://") {
+			m.Log.Warnf("Server %q should be updated to use `scheme://host:port` format", server)
+			if tlsCfg == nil {
+				server = "tcp://" + server
+			} else {
+				server = "ssl://" + server
+			}
+		}
+		opts.AddBroker(server)
+	}
+	opts.SetAutoReconnect(false)
+	opts.SetKeepAlive(time.Duration(m.KeepAliveInterval))
+	opts.SetPingTimeout(time.Duration(m.PingTimeout))
+	opts.SetCleanSession(!m.PersistentSession)
+	opts.SetAutoAckDisabled(m.PersistentSession)
+	opts.SetConnectionLostHandler(m.onConnectionLost)
+	return opts, nil
+}
+
+func newMQTTConsumer(factory clientFactory) *MQTTConsumer {
+	return &MQTTConsumer{
+		Servers:                []string{"tcp://127.0.0.1:1883"},
+		MaxUndeliveredMessages: defaultMaxUndeliveredMessages,
+		ConnectionTimeout:      defaultConnectionTimeout,
+		KeepAliveInterval:      config.Duration(60 * time.Second),
+		PingTimeout:            config.Duration(10 * time.Second),
+		clientFactory:          factory,
+	}
+}
+func init() {
+	inputs.Add("mqtt_consumer", func() telegraf.Input {
+		return newMQTTConsumer(func(o *mqtt.ClientOptions) client {
+			return mqtt.NewClient(o)
+		})
+	})
+}

plugins/inputs/mqtt_consumer/mqtt_logger.go

@@ -0,0 +1,19 @@
+package mqtt_consumer
+
+import (
+	"github.com/influxdata/telegraf"
+)
+
+type mqttLogger struct {
+	telegraf.Logger
+}
+
+// Printf implements mqtt.Logger
+func (l mqttLogger) Printf(fmt string, args ...interface{}) {
+	l.Logger.Debugf(fmt, args...)
+}
+
+// Println implements mqtt.Logger
+func (l mqttLogger) Println(args ...interface{}) {
+	l.Logger.Debug(args...)
+}

plugins/inputs/mqtt_consumer/sample.conf

@@ -0,0 +1,97 @@
+# Read metrics from MQTT topic(s)
+[[inputs.mqtt_consumer]]
+  ## Broker URLs for the MQTT server or cluster.  To connect to multiple
+  ## clusters or standalone servers, use a separate plugin instance.
+  ##   example: servers = ["tcp://localhost:1883"]
+  ##            servers = ["ssl://localhost:1883"]
+  ##            servers = ["ws://localhost:1883"]
+  servers = ["tcp://127.0.0.1:1883"]
+
+  ## Topics that will be subscribed to.
+  topics = [
+    "telegraf/host01/cpu",
+    "telegraf/+/mem",
+    "sensors/#",
+  ]
+
+  ## The message topic will be stored in a tag specified by this value.  If set
+  ## to the empty string no topic tag will be created.
+  # topic_tag = "topic"
+
+  ## QoS policy for messages
+  ##   0 = at most once
+  ##   1 = at least once
+  ##   2 = exactly once
+  ##
+  ## When using a QoS of 1 or 2, you should enable persistent_session to allow
+  ## resuming unacknowledged messages.
+  # qos = 0
+
+  ## Connection timeout for initial connection in seconds
+  # connection_timeout = "30s"
+
+  ## Interval and ping timeout for keep-alive messages
+  ## The sum of those options defines when a connection loss is detected.
+  ## Note: The keep-alive interval needs to be greater or equal one second and
+  ## fractions of a second are not supported.
+  # keepalive = "60s"
+  # ping_timeout = "10s"
+
+  ## Max undelivered messages
+  ## This plugin uses tracking metrics, which ensure messages are read to
+  ## outputs before acknowledging them to the original broker to ensure data
+  ## is not lost. This option sets the maximum messages to read from the
+  ## broker that have not been written by an output.
+  ##
+  ## This value needs to be picked with awareness of the agent's
+  ## metric_batch_size value as well. Setting max undelivered messages too high
+  ## can result in a constant stream of data batches to the output. While
+  ## setting it too low may never flush the broker's messages.
+  # max_undelivered_messages = 1000
+
+  ## Persistent session disables clearing of the client session on connection.
+  ## In order for this option to work you must also set client_id to identify
+  ## the client.  To receive messages that arrived while the client is offline,
+  ## also set the qos option to 1 or 2 and don't forget to also set the QoS when
+  ## publishing. Finally, using a persistent session will use the initial
+  ## connection topics and not subscribe to any new topics even after
+  ## reconnecting or restarting without a change in client ID.
+  # persistent_session = false
+
+  ## If unset, a random client ID will be generated.
+  # client_id = ""
+
+  ## Username and password to connect MQTT server.
+  # username = "telegraf"
+  # password = "metricsmetricsmetricsmetrics"
+
+  ## Optional TLS Config
+  # tls_ca = "/etc/telegraf/ca.pem"
+  # tls_cert = "/etc/telegraf/cert.pem"
+  # tls_key = "/etc/telegraf/key.pem"
+  ## Use TLS but skip chain & host verification
+  # insecure_skip_verify = false
+
+  ## Client trace messages
+  ## When set to true, and debug mode enabled in the agent settings, the MQTT
+  ## client's messages are included in telegraf logs. These messages are very
+  ## noisey, but essential for debugging issues.
+  # client_trace = false
+
+  ## Data format to consume.
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "influx"
+
+  ## Enable extracting tag values from MQTT topics
+  ## _ denotes an ignored entry in the topic path,
+  ## # denotes a variable length path element (can only be used once per setting)
+  # [[inputs.mqtt_consumer.topic_parsing]]
+  #   topic = ""
+  #   measurement = ""
+  #   tags = ""
+  #   fields = ""
+  ## Value supported is int, float, unit
+  #   [inputs.mqtt_consumer.topic_parsing.types]
+  #      key = type

plugins/inputs/mqtt_consumer/testdata/mosquitto.conf

@@ -0,0 +1,3 @@
+listener 1883 0.0.0.0
+allow_anonymous true
+connection_messages true

plugins/inputs/mqtt_consumer/topic_parser.go

@@ -0,0 +1,230 @@
+package mqtt_consumer
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/influxdata/telegraf"
+)
+
+type topicParsingConfig struct {
+	Topic       string            `toml:"topic"`
+	Measurement string            `toml:"measurement"`
+	Tags        string            `toml:"tags"`
+	Fields      string            `toml:"fields"`
+	FieldTypes  map[string]string `toml:"types"`
+}
+
+type topicParser struct {
+	topicIndices   map[string]int
+	topicVarLength bool
+	topicMinLength int
+
+	extractMeasurement bool
+	measurementIndex   int
+	tagIndices         map[string]int
+	fieldIndices       map[string]int
+	fieldTypes         map[string]string
+}
+
+func (cfg *topicParsingConfig) newParser() (*topicParser, error) {
+	p := &topicParser{
+		fieldTypes: cfg.FieldTypes,
+	}
+
+	// Build a check list for topic elements
+	var topicMinLength int
+	var topicInvert bool
+	topicParts := strings.Split(cfg.Topic, "/")
+	p.topicIndices = make(map[string]int, len(topicParts))
+	for i, k := range topicParts {
+		switch k {
+		case "+":
+			topicMinLength++
+		case "#":
+			if p.topicVarLength {
+				return nil, errors.New("topic can only contain one hash")
+			}
+			p.topicVarLength = true
+			topicInvert = true
+		default:
+			if !topicInvert {
+				p.topicIndices[k] = i
+			} else {
+				p.topicIndices[k] = i - len(topicParts)
+			}
+			topicMinLength++
+		}
+	}
+
+	// Determine metric name selection
+	var measurementMinLength int
+	var measurementInvert bool
+	measurementParts := strings.Split(cfg.Measurement, "/")
+	for i, k := range measurementParts {
+		if k == "_" || k == "" {
+			measurementMinLength++
+			continue
+		}
+
+		if k == "#" {
+			measurementInvert = true
+			continue
+		}
+
+		if p.extractMeasurement {
+			return nil, errors.New("measurement can only contain one element")
+		}
+
+		if !measurementInvert {
+			p.measurementIndex = i
+		} else {
+			p.measurementIndex = i - len(measurementParts)
+		}
+		p.extractMeasurement = true
+		measurementMinLength++
+	}
+
+	// Determine tag selections
+	var tagMinLength int
+	var tagInvert bool
+	tagParts := strings.Split(cfg.Tags, "/")
+	p.tagIndices = make(map[string]int, len(tagParts))
+	for i, k := range tagParts {
+		if k == "_" || k == "" {
+			tagMinLength++
+			continue
+		}
+		if k == "#" {
+			tagInvert = true
+			continue
+		}
+		if !tagInvert {
+			p.tagIndices[k] = i
+		} else {
+			p.tagIndices[k] = i - len(tagParts)
+		}
+		tagMinLength++
+	}
+
+	// Determine tag selections
+	var fieldMinLength int
+	var fieldInvert bool
+	fieldParts := strings.Split(cfg.Fields, "/")
+	p.fieldIndices = make(map[string]int, len(fieldParts))
+	for i, k := range fieldParts {
+		if k == "_" || k == "" {
+			fieldMinLength++
+			continue
+		}
+		if k == "#" {
+			fieldInvert = true
+			continue
+		}
+		if !fieldInvert {
+			p.fieldIndices[k] = i
+		} else {
+			p.fieldIndices[k] = i - len(fieldParts)
+		}
+		fieldMinLength++
+	}
+
+	if !p.topicVarLength {
+		if measurementMinLength != topicMinLength && p.extractMeasurement {
+			return nil, errors.New("measurement length does not equal topic length")
+		}
+
+		if fieldMinLength != topicMinLength && cfg.Fields != "" {
+			return nil, errors.New("fields length does not equal topic length")
+		}
+
+		if tagMinLength != topicMinLength && cfg.Tags != "" {
+			return nil, errors.New("tags length does not equal topic length")
+		}
+	}
+
+	p.topicMinLength = max(topicMinLength, measurementMinLength, tagMinLength, fieldMinLength)
+
+	return p, nil
+}
+
+func (p *topicParser) parse(metric telegraf.Metric, topic string) error {
+	// Split the actual topic into its elements and check for a match
+	topicParts := strings.Split(topic, "/")
+	if p.topicVarLength && len(topicParts) < p.topicMinLength || !p.topicVarLength && len(topicParts) != p.topicMinLength {
+		return nil
+	}
+	for expected, i := range p.topicIndices {
+		if i >= 0 && topicParts[i] != expected || i < 0 && topicParts[len(topicParts)+i] != expected {
+			return nil
+		}
+	}
+
+	// Extract the measurement name
+	var measurement string
+	if p.extractMeasurement {
+		if p.measurementIndex >= 0 {
+			measurement = topicParts[p.measurementIndex]
+		} else {
+			measurement = topicParts[len(topicParts)+p.measurementIndex]
+		}
+		metric.SetName(measurement)
+	}
+
+	// Extract the tags
+	for k, i := range p.tagIndices {
+		if i >= 0 {
+			metric.AddTag(k, topicParts[i])
+		} else {
+			metric.AddTag(k, topicParts[len(topicParts)+i])
+		}
+	}
+
+	// Extract the fields
+	for k, i := range p.fieldIndices {
+		var raw string
+		if i >= 0 {
+			raw = topicParts[i]
+		} else {
+			raw = topicParts[len(topicParts)+i]
+		}
+		v, err := p.convertToFieldType(raw, k)
+		if err != nil {
+			return err
+		}
+		metric.AddField(k, v)
+	}
+
+	return nil
+}
+
+func (p *topicParser) convertToFieldType(value, key string) (interface{}, error) {
+	// If the user configured inputs.mqtt_consumer.topic.types, check for the desired type
+	desiredType, ok := p.fieldTypes[key]
+	if !ok {
+		return value, nil
+	}
+
+	var v interface{}
+	var err error
+	switch desiredType {
+	case "uint":
+		if v, err = strconv.ParseUint(value, 10, 64); err != nil {
+			return nil, fmt.Errorf("unable to convert field %q to type uint: %w", value, err)
+		}
+	case "int":
+		if v, err = strconv.ParseInt(value, 10, 64); err != nil {
+			return nil, fmt.Errorf("unable to convert field %q to type int: %w", value, err)
+		}
+	case "float":
+		if v, err = strconv.ParseFloat(value, 64); err != nil {
+			return nil, fmt.Errorf("unable to convert field %q to type float: %w", value, err)
+		}
+	default:
+		return nil, fmt.Errorf("converting to the type %s is not supported: use int, uint, or float", desiredType)
+	}
+
+	return v, nil
+}