influx/telegraf
1
0
Fork 0
Code Issues Activity

Adding upstream version 1.34.4.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-05-24 07:26:29 +02:00
parent e393c3af3f
commit 4978089aab
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

349
plugins/outputs/mqtt/README.md Normal file
View file

@ -0,0 +1,349 @@
# MQTT Producer Output Plugin
This plugin writes metrics to a [MQTT broker][mqtt] acting as a MQTT producer.
The plugin supports the MQTT protocols `3.1.1` and `5`.
> [!NOTE]
> In v2.0.12+ of the mosquitto MQTT server, there is a [bug][mosquitto_bug]
> requiring the `keep_alive` value to be set non-zero in Telegraf. Otherwise,
> the server will return with `identifier rejected`.
> As a reference `eclipse/paho.golang` sets the `keep_alive` to 30.
⭐ Telegraf v0.2.0
🏷️ messaging
💻 all
[mqtt]: http://http://mqtt.org/
[mosquitto_bug]: https://github.com/eclipse/mosquitto/issues/2117
## Global configuration options <!-- @/docs/includes/plugin_config.md -->
In addition to the plugin-specific configuration settings, plugins support
additional global and plugin configuration settings. These settings are used to
modify metrics, tags, and field or create aliases and configure ordering, etc.
See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
## Secret-store support
This plugin supports secrets from secret-stores for the `username` and
`password` option.
See the [secret-store documentation][SECRETSTORE] for more details on how
to use them.
[SECRETSTORE]: ../../../docs/CONFIGURATION.md#secret-store-secrets
## Configuration
```toml @sample.conf
# Configuration for MQTT server to send metrics to
[[outputs.mqtt]]
## MQTT Brokers
## The list of brokers should only include the hostname or IP address and the
## port to the broker. This should follow the format `[{scheme}://]{host}:{port}`. For
## example, `localhost:1883` or `mqtt://localhost:1883`.
## Scheme can be any of the following: tcp://, mqtt://, tls://, mqtts://
## non-TLS and TLS servers can not be mix-and-matched.
servers = ["localhost:1883", ] # or ["mqtts://tls.example.com:1883"]
## Protocol can be `3.1.1` or `5`. Default is `3.1.1`
# protocol = "3.1.1"
## MQTT Topic for Producer Messages
## MQTT outputs send metrics to this topic format:
## prefix/{{ .Tag "host" }}/{{ .Name }}/{{ .Tag "tag_key" }}
## (e.g. prefix/web01.example.com/mem/some_tag_value)
## Each path segment accepts either a template placeholder, an environment variable, or a tag key
## of the form `{{.Tag "tag_key_name"}}`. All the functions provided by the Sprig library
## (http://masterminds.github.io/sprig/) are available. Empty path elements as well as special MQTT
## characters (such as `+` or `#`) are invalid to form the topic name and will lead to an error.
## In case a tag is missing in the metric, that path segment omitted for the final topic.
topic = 'telegraf/{{ .Tag "host" }}/{{ .Name }}'
## QoS policy for messages
## The mqtt QoS policy for sending messages.
## See https://www.ibm.com/support/knowledgecenter/en/SSFKSJ_9.0.0/com.ibm.mq.dev.doc/q029090_.htm
## 0 = at most once
## 1 = at least once
## 2 = exactly once
# qos = 2
## Keep Alive
## Defines the maximum length of time that the broker and client may not
## communicate. Defaults to 0 which turns the feature off.
##
## For version v2.0.12 and later mosquitto there is a bug
## (see https://github.com/eclipse/mosquitto/issues/2117), which requires
## this to be non-zero. As a reference eclipse/paho.mqtt.golang defaults to 30.
# keep_alive = 0
## username and password to connect MQTT server.
# username = "telegraf"
# password = "metricsmetricsmetricsmetrics"
## client ID
## The unique client id to connect MQTT server. If this parameter is not set
## then a random ID is generated.
# client_id = ""
## Timeout for write operations. default: 5s
# timeout = "5s"
## Optional TLS Config
# tls_ca = "/etc/telegraf/ca.pem"
# tls_cert = "/etc/telegraf/cert.pem"
# tls_key = "/etc/telegraf/key.pem"
## Use TLS but skip chain & host verification
# insecure_skip_verify = false
## When true, metrics will be sent in one MQTT message per flush. Otherwise,
## metrics are written one metric per MQTT message.
## DEPRECATED: Use layout option instead
# batch = false
## When true, metric will have RETAIN flag set, making broker cache entries until someone
## actually reads it
# retain = false
## Client trace messages
## When set to true, and debug mode enabled in the agent settings, the MQTT
## client's messages are included in telegraf logs. These messages are very
## noisey, but essential for debugging issues.
# client_trace = false
## Layout of the topics published.
## The following choices are available:
## non-batch -- send individual messages, one for each metric
## batch -- send all metric as a single message per MQTT topic
## NOTE: The following options will ignore the 'data_format' option and send single values
## field -- send individual messages for each field, appending its name to the metric topic
## homie-v4 -- send metrics with fields and tags according to the 4.0.0 specs
## see https://homieiot.github.io/specification/
# layout = "non-batch"
## HOMIE specific settings
## The following options provide templates for setting the device name
## and the node-ID for the topics. Both options are MANDATORY and can contain
## {{ .Name }} (metric name), {{ .Tag "key"}} (tag reference to 'key') or
## constant strings. The templates MAY NOT contain slashes!
# homie_device_name = ""
# homie_node_id = ""
## Each data format has its own unique set of configuration options, read
## more about them here:
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
data_format = "influx"
## NOTE: Due to the way TOML is parsed, tables must be at the END of the
## plugin definition, otherwise additional config options are read as part of
## the table
## Optional MQTT 5 publish properties
## These setting only apply if the "protocol" property is set to 5. This must
## be defined at the end of the plugin settings, otherwise TOML will assume
## anything else is part of this table. For more details on publish properties
## see the spec:
## https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Toc3901109
# [outputs.mqtt.v5]
# content_type = ""
# response_topic = ""
# message_expiry = "0s"
# topic_alias = 0
# [outputs.mqtt.v5.user_properties]
# "key1" = "value 1"
# "key2" = "value 2"
```
### `field` layout
This layout will publish one topic per metric __field__, only containing the
value as string. This means that the `data_format` option will be ignored.
For example writing the metrics
```text
modbus,location=main\ building,source=device\ 1,status=ok,type=Machine\ A temperature=21.4,serial\ number="324nlk234r5u9834t",working\ hours=123i,supplied=true 1676522982000000000
modbus,location=main\ building,source=device\ 2,status=offline,type=Machine\ B temperature=25.0,supplied=true 1676522982000000000
```
with configuration
```toml
[[outputs.mqtt]]
topic = 'telegraf/{{ .Name }}/{{ .Tag "source" }}'
layout = "field"
...
```
will result in the following topics and values
```text
telegraf/modbus/device 1/temperature 21.4
telegraf/modbus/device 1/serial number 324nlk234r5u9834t
telegraf/modbus/device 1/supplied true
telegraf/modbus/device 1/working hours 123
telegraf/modbus/device 2/temperature 25
telegraf/modbus/device 2/supplied false
```
__NOTE__: Only fields will be output, tags and the timestamp are omitted. To
also output those, please convert them to fields first.
### `homie-v4` layout
This layout will publish metrics according to the
[Homie v4.0 specification][HomieSpecV4]. Here, the `topic` template will be
used to specify the `device-id` path. The __mandatory__ options
`homie_device_name` will specify the content of the `$name` topic of the device,
while `homie_node_id` will provide a template for the `node-id` part of the
topic. Both options can contain [Go templates][GoTemplates] similar to `topic`
with `{{ .Name }}` referencing the metric name and `{{ .Tag "key"}}` referencing
the tag with the name `key`.
[Sprig](http://masterminds.github.io/sprig/) helper functions are available.
For example writing the metrics
```text
modbus,source=device\ 1,location=main\ building,type=Machine\ A,status=ok temperature=21.4,serial\ number="324nlk234r5u9834t",working\ hours=123i,supplied=true 1676522982000000000
modbus,source=device\ 2,location=main\ building,type=Machine\ B,status=offline supplied=false 1676522982000000000
modbus,source=device\ 2,location=main\ building,type=Machine\ B,status=online supplied=true,Throughput=12345i,Load\ [%]=81.2,account\ no="T3L3GrAf",Temperature=25.38,Voltage=24.1,Current=100 1676542982000000000
```
with configuration
```toml
[[outputs.mqtt]]
topic = 'telegraf/{{ .Name }}'
layout = "homie-v4"
homie_device_name ='{{ .Name }} plugin'
homie_node_id = '{{ .Tag "source" }}'
...
```
will result in the following topics and values
```text
telegraf/modbus/$homie 4.0
telegraf/modbus/$name modbus plugin
telegraf/modbus/$state ready
telegraf/modbus/$nodes device-1
telegraf/modbus/device-1/$name device 1
telegraf/modbus/device-1/$properties location,serial-number,source,status,supplied,temperature,type,working-hours
telegraf/modbus/device-1/location main building
telegraf/modbus/device-1/location/$name location
telegraf/modbus/device-1/location/$datatype string
telegraf/modbus/device-1/status ok
telegraf/modbus/device-1/status/$name status
telegraf/modbus/device-1/status/$datatype string
telegraf/modbus/device-1/type Machine A
telegraf/modbus/device-1/type/$name type
telegraf/modbus/device-1/type/$datatype string
telegraf/modbus/device-1/source device 1
telegraf/modbus/device-1/source/$name source
telegraf/modbus/device-1/source/$datatype string
telegraf/modbus/device-1/temperature 21.4
telegraf/modbus/device-1/temperature/$name temperature
telegraf/modbus/device-1/temperature/$datatype float
telegraf/modbus/device-1/serial-number 324nlk234r5u9834t
telegraf/modbus/device-1/serial-number/$name serial number
telegraf/modbus/device-1/serial-number/$datatype string
telegraf/modbus/device-1/working-hours 123
telegraf/modbus/device-1/working-hours/$name working hours
telegraf/modbus/device-1/working-hours/$datatype integer
telegraf/modbus/device-1/supplied true
telegraf/modbus/device-1/supplied/$name supplied
telegraf/modbus/device-1/supplied/$datatype boolean
telegraf/modbus/$nodes device-1,device-2
telegraf/modbus/device-2/$name device 2
telegraf/modbus/device-2/$properties location,source,status,supplied,type
telegraf/modbus/device-2/location main building
telegraf/modbus/device-2/location/$name location
telegraf/modbus/device-2/location/$datatype string
telegraf/modbus/device-2/status offline
telegraf/modbus/device-2/status/$name status
telegraf/modbus/device-2/status/$datatype string
telegraf/modbus/device-2/type Machine B
telegraf/modbus/device-2/type/$name type
telegraf/modbus/device-2/type/$datatype string
telegraf/modbus/device-2/source device 2
telegraf/modbus/device-2/source/$name source
telegraf/modbus/device-2/source/$datatype string
telegraf/modbus/device-2/supplied false
telegraf/modbus/device-2/supplied/$name supplied
telegraf/modbus/device-2/supplied/$datatype boolean
telegraf/modbus/device-2/$properties account-no,current,load,location,source,status,supplied,temperature,throughput,type,voltage
telegraf/modbus/device-2/location main building
telegraf/modbus/device-2/location/$name location
telegraf/modbus/device-2/location/$datatype string
telegraf/modbus/device-2/status online
telegraf/modbus/device-2/status/$name status
telegraf/modbus/device-2/status/$datatype string
telegraf/modbus/device-2/type Machine B
telegraf/modbus/device-2/type/$name type
telegraf/modbus/device-2/type/$datatype string
telegraf/modbus/device-2/source device 2
telegraf/modbus/device-2/source/$name source
telegraf/modbus/device-2/source/$datatype string
telegraf/modbus/device-2/temperature 25.38
telegraf/modbus/device-2/temperature/$name Temperature
telegraf/modbus/device-2/temperature/$datatype float
telegraf/modbus/device-2/voltage 24.1
telegraf/modbus/device-2/voltage/$name Voltage
telegraf/modbus/device-2/voltage/$datatype float
telegraf/modbus/device-2/current 100
telegraf/modbus/device-2/current/$name Current
telegraf/modbus/device-2/current/$datatype float
telegraf/modbus/device-2/throughput 12345
telegraf/modbus/device-2/throughput/$name Throughput
telegraf/modbus/device-2/throughput/$datatype integer
telegraf/modbus/device-2/load 81.2
telegraf/modbus/device-2/load/$name Load [%]
telegraf/modbus/device-2/load/$datatype float
telegraf/modbus/device-2/account-no T3L3GrAf
telegraf/modbus/device-2/account-no/$name account no
telegraf/modbus/device-2/account-no/$datatype string
telegraf/modbus/device-2/supplied true
telegraf/modbus/device-2/supplied/$name supplied
telegraf/modbus/device-2/supplied/$datatype boolean
```
#### Important notes and limitations
It is important to notice that the __"devices" and "nodes" are dynamically
changing__ in Telegraf as the metrics and their structure is not known a-priori.
As a consequence, the content of both `$nodes` and `$properties` topics are
changing as new `device-id`s, `node-id`s and `properties` (i.e. tags and fields)
appear. Best effort is made to limit the number of changes by keeping a
superset of all devices and nodes seen, however especially during startup those
topics will change more often. Both `topic` and `homie_node_id` should be chosen
in a way to group metrics with identical structure!
Furthermore, __lifecycle management of devices is very limited__! Devices will
only be in `ready` state due to the dynamic nature of Telegraf. Due to
limitations in the MQTT client library, it is not possible to set a "will"
dynamically. In consequence, devices are only marked `lost` when exiting
Telegraf normally and might not change in abnormal aborts.
Note that __all field- and tag-names are automatically converted__ to adhere to
the [Homie topic ID specification][HomieSpecV4TopicIDs]. In that process, the
names are converted to lower-case and forbidden character sequences (everything
not being a lower-case character, digit or hyphen) will be replaces by a hyphen.
Finally, leading and trailing hyphens are removed.
This is important as there is a __risk of name collisions__ between fields and
tags of the same node especially after the conversion to ID. Please __make sure
to avoid those collisions__ as otherwise property topics will be sent multiple
times for the colliding items.
[HomieSpecV4]: https://homieiot.github.io/specification/spec-core-v4_0_0
[GoTemplates]: https://pkg.go.dev/text/template
[HomieSpecV4TopicIDs]: https://homieiot.github.io/specification/#topic-ids

110
plugins/outputs/mqtt/homie.go Normal file
View file

@ -0,0 +1,110 @@
package mqtt
import (
"errors"
"fmt"
"regexp"
"sort"
"strings"
"text/template"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
)
var idRe = regexp.MustCompile(`([^a-z0-9]+)`)
func (m *MQTT) collectHomieDeviceMessages(topic string, metric telegraf.Metric) ([]message, string, error) {
var messages []message
// Check if the device-id is already registered
if _, found := m.homieSeen[topic]; !found {
deviceName, err := homieGenerate(m.homieDeviceNameGenerator, metric)
if err != nil {
return nil, "", fmt.Errorf("generating device name failed: %w", err)
}
messages = append(messages,
message{topic + "/$homie", []byte("4.0")},
message{topic + "/$name", []byte(deviceName)},
message{topic + "/$state", []byte("ready")},
)
m.homieSeen[topic] = make(map[string]bool)
}
// Generate the node-ID from the metric and fixup invalid characters
nodeName, err := homieGenerate(m.homieNodeIDGenerator, metric)
if err != nil {
return nil, "", fmt.Errorf("generating device ID failed: %w", err)
}
nodeID := normalizeID(nodeName)
if !m.homieSeen[topic][nodeID] {
m.homieSeen[topic][nodeID] = true
nodeIDs := make([]string, 0, len(m.homieSeen[topic]))
for id := range m.homieSeen[topic] {
nodeIDs = append(nodeIDs, id)
}
sort.Strings(nodeIDs)
messages = append(messages,
message{topic + "/$nodes", []byte(strings.Join(nodeIDs, ","))},
message{topic + "/" + nodeID + "/$name", []byte(nodeName)},
)
}
properties := make([]string, 0, len(metric.TagList())+len(metric.FieldList()))
for _, tag := range metric.TagList() {
properties = append(properties, normalizeID(tag.Key))
}
for _, field := range metric.FieldList() {
properties = append(properties, normalizeID(field.Key))
}
sort.Strings(properties)
messages = append(messages, message{
topic + "/" + nodeID + "/$properties",
[]byte(strings.Join(properties, ",")),
})
return messages, nodeID, nil
}
func normalizeID(raw string) string {
// IDs in Home can only contain lowercase letters and hyphens
// see https://homieiot.github.io/specification/#topic-ids
id := strings.ToLower(raw)
id = idRe.ReplaceAllString(id, "-")
return strings.Trim(id, "-")
}
func convertType(value interface{}) (val, dtype string, err error) {
v, err := internal.ToString(value)
if err != nil {
return "", "", err
}
switch value.(type) {
case int8, int16, int32, int64, uint8, uint16, uint32, uint64:
return v, "integer", nil
case float32, float64:
return v, "float", nil
case []byte, string, fmt.Stringer:
return v, "string", nil
case bool:
return v, "boolean", nil
}
return "", "", fmt.Errorf("unknown type %T", value)
}
func homieGenerate(t *template.Template, m telegraf.Metric) (string, error) {
var b strings.Builder
if err := t.Execute(&b, m.(telegraf.TemplateMetric)); err != nil {
return "", err
}
result := b.String()
if strings.Contains(result, "/") {
return "", errors.New("cannot contain /")
}
return result, nil
}

312
plugins/outputs/mqtt/mqtt.go Normal file
View file

@ -0,0 +1,312 @@
//go:generate ../../../tools/readme_config_includer/generator
package mqtt
import (
// Blank import to support go:embed compile directive
_ "embed"
"errors"
"fmt"
"regexp"
"sync"
"text/template"
"time"
"github.com/Masterminds/sprig/v3"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/common/mqtt"
"github.com/influxdata/telegraf/plugins/outputs"
)
//go:embed sample.conf
var sampleConfig string
var pluginNameRe = regexp.MustCompile(`({{.*\B)\.PluginName(\b[^}]*}})`)
var hostnameRe = regexp.MustCompile(`({{.*\B)\.Hostname(\b[^}]*}})`)
type message struct {
topic string
payload []byte
}
type MQTT struct {
TopicPrefix string `toml:"topic_prefix" deprecated:"1.25.0;1.35.0;use 'topic' instead"`
Topic string `toml:"topic"`
BatchMessage bool `toml:"batch" deprecated:"1.25.2;1.35.0;use 'layout = \"batch\"' instead"`
Layout string `toml:"layout"`
HomieDeviceName string `toml:"homie_device_name"`
HomieNodeID string `toml:"homie_node_id"`
Log telegraf.Logger `toml:"-"`
mqtt.MqttConfig
client mqtt.Client
serializer telegraf.Serializer
generator *TopicNameGenerator
homieDeviceNameGenerator *template.Template
homieNodeIDGenerator *template.Template
homieSeen map[string]map[string]bool
sync.Mutex
}
func (*MQTT) SampleConfig() string {
return sampleConfig
}
func (m *MQTT) Init() error {
if len(m.Servers) == 0 {
return errors.New("no servers specified")
}
if m.PersistentSession && m.ClientID == "" {
return errors.New("persistent_session requires client_id")
}
if m.QoS > 2 || m.QoS < 0 {
return fmt.Errorf("qos value must be 0, 1, or 2: %d", m.QoS)
}
var err error
m.generator, err = NewTopicNameGenerator(m.TopicPrefix, m.Topic)
if err != nil {
return err
}
switch m.Layout {
case "":
// For backward compatibility
if m.BatchMessage {
m.Layout = "batch"
} else {
m.Layout = "non-batch"
}
case "non-batch", "batch", "field":
case "homie-v4":
if m.HomieDeviceName == "" {
return errors.New("missing 'homie_device_name' option")
}
m.HomieDeviceName = pluginNameRe.ReplaceAllString(m.HomieDeviceName, `$1.Name$2`)
m.homieDeviceNameGenerator, err = template.New("topic_name").Funcs(sprig.TxtFuncMap()).Parse(m.HomieDeviceName)
if err != nil {
return fmt.Errorf("creating device name generator failed: %w", err)
}
if m.HomieNodeID == "" {
return errors.New("missing 'homie_node_id' option")
}
m.HomieNodeID = pluginNameRe.ReplaceAllString(m.HomieNodeID, `$1.Name$2`)
m.homieNodeIDGenerator, err = template.New("topic_name").Funcs(sprig.TxtFuncMap()).Parse(m.HomieNodeID)
if err != nil {
return fmt.Errorf("creating node ID name generator failed: %w", err)
}
default:
return fmt.Errorf("invalid layout %q", m.Layout)
}
return nil
}
func (m *MQTT) Connect() error {
m.Lock()
defer m.Unlock()
m.homieSeen = make(map[string]map[string]bool)
client, err := mqtt.NewClient(&m.MqttConfig)
if err != nil {
return err
}
m.client = client
_, err = m.client.Connect()
return err
}
func (m *MQTT) SetSerializer(serializer telegraf.Serializer) {
m.serializer = serializer
}
func (m *MQTT) Close() error {
// Unregister devices if Homie layout was used. Usually we should do this
// using a "will" message, but this can only be done at connect time where,
// due to the dynamic nature of Telegraf messages, we do not know the topics
// to issue that "will" yet.
if len(m.homieSeen) > 0 {
for topic := range m.homieSeen {
//nolint:errcheck // We will ignore potential errors as we cannot do anything here
m.client.Publish(topic+"/$state", []byte("lost"))
}
// Give the messages some time to settle
time.Sleep(100 * time.Millisecond)
}
return m.client.Close()
}
func (m *MQTT) Write(metrics []telegraf.Metric) error {
m.Lock()
defer m.Unlock()
if len(metrics) == 0 {
return nil
}
// Group the metrics to topics and serialize them
var topicMessages []message
switch m.Layout {
case "batch":
topicMessages = m.collectBatch(metrics)
case "non-batch":
topicMessages = m.collectNonBatch(metrics)
case "field":
topicMessages = m.collectField(metrics)
case "homie-v4":
topicMessages = m.collectHomieV4(metrics)
default:
return fmt.Errorf("unknown layout %q", m.Layout)
}
for _, msg := range topicMessages {
if err := m.client.Publish(msg.topic, msg.payload); err != nil {
// We do receive a timeout error if the remote broker is down,
// so let's retry the metrics in this case and drop them otherwise.
if errors.Is(err, internal.ErrTimeout) {
return fmt.Errorf("could not publish message to MQTT server: %w", err)
}
m.Log.Warnf("Could not publish message to MQTT server: %v", err)
}
}
return nil
}
func (m *MQTT) collectNonBatch(metrics []telegraf.Metric) []message {
collection := make([]message, 0, len(metrics))
for _, metric := range metrics {
topic, err := m.generateTopic(metric)
if err != nil {
m.Log.Warnf("Generating topic name failed: %v", err)
m.Log.Debugf("metric was: %v", metric)
continue
}
buf, err := m.serializer.Serialize(metric)
if err != nil {
m.Log.Warnf("Could not serialize metric for topic %q: %v", topic, err)
m.Log.Debugf("metric was: %v", metric)
continue
}
collection = append(collection, message{topic, buf})
}
return collection
}
func (m *MQTT) collectBatch(metrics []telegraf.Metric) []message {
metricsCollection := make(map[string][]telegraf.Metric)
for _, metric := range metrics {
topic, err := m.generateTopic(metric)
if err != nil {
m.Log.Warnf("Generating topic name failed: %v", err)
m.Log.Debugf("metric was: %v", metric)
continue
}
metricsCollection[topic] = append(metricsCollection[topic], metric)
}
collection := make([]message, 0, len(metricsCollection))
for topic, ms := range metricsCollection {
buf, err := m.serializer.SerializeBatch(ms)
if err != nil {
m.Log.Warnf("Could not serialize metric batch for topic %q: %v", topic, err)
continue
}
collection = append(collection, message{topic, buf})
}
return collection
}
func (m *MQTT) collectField(metrics []telegraf.Metric) []message {
var collection []message
for _, metric := range metrics {
topic, err := m.generateTopic(metric)
if err != nil {
m.Log.Warnf("Generating topic name failed: %v", err)
m.Log.Debugf("metric was: %v", metric)
continue
}
for n, v := range metric.Fields() {
buf, err := internal.ToString(v)
if err != nil {
m.Log.Warnf("Could not serialize metric for topic %q field %q: %v", topic, n, err)
m.Log.Debugf("metric was: %v", metric)
continue
}
collection = append(collection, message{topic + "/" + n, []byte(buf)})
}
}
return collection
}
func (m *MQTT) collectHomieV4(metrics []telegraf.Metric) []message {
var collection []message
for _, metric := range metrics {
topic, err := m.generateTopic(metric)
if err != nil {
m.Log.Warnf("Generating topic name failed: %v", err)
m.Log.Debugf("metric was: %v", metric)
continue
}
msgs, nodeID, err := m.collectHomieDeviceMessages(topic, metric)
if err != nil {
m.Log.Warn(err.Error())
m.Log.Debugf("metric was: %v", metric)
continue
}
path := topic + "/" + nodeID
collection = append(collection, msgs...)
for _, tag := range metric.TagList() {
propID := normalizeID(tag.Key)
collection = append(collection,
message{path + "/" + propID, []byte(tag.Value)},
message{path + "/" + propID + "/$name", []byte(tag.Key)},
message{path + "/" + propID + "/$datatype", []byte("string")},
)
}
for _, field := range metric.FieldList() {
v, dt, err := convertType(field.Value)
if err != nil {
m.Log.Warnf("Could not serialize metric for topic %q field %q: %v", topic, field.Key, err)
m.Log.Debugf("metric was: %v", metric)
continue
}
propID := normalizeID(field.Key)
collection = append(collection,
message{path + "/" + propID, []byte(v)},
message{path + "/" + propID + "/$name", []byte(field.Key)},
message{path + "/" + propID + "/$datatype", []byte(dt)},
)
}
}
return collection
}
func init() {
outputs.Add("mqtt", func() telegraf.Output {
return &MQTT{
MqttConfig: mqtt.MqttConfig{
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
AutoReconnect: true,
},
}
})
}

938
plugins/outputs/mqtt/mqtt_test.go Normal file
View file

@ -0,0 +1,938 @@
package mqtt
import (
"fmt"
"path/filepath"
"sync"
"testing"
"time"
paho "github.com/eclipse/paho.mqtt.golang"
"github.com/stretchr/testify/require"
"github.com/testcontainers/testcontainers-go/wait"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/metric"
"github.com/influxdata/telegraf/plugins/common/mqtt"
"github.com/influxdata/telegraf/plugins/parsers/influx"
serializers_influx "github.com/influxdata/telegraf/plugins/serializers/influx"
"github.com/influxdata/telegraf/testutil"
)
const servicePort = "1883"
func launchTestContainer(t *testing.T) *testutil.Container {
conf, err := filepath.Abs(filepath.Join("testdata", "mosquitto.conf"))
require.NoError(t, err, "missing file mosquitto.conf")
container := testutil.Container{
Image: "eclipse-mosquitto:2",
ExposedPorts: []string{servicePort},
WaitingFor: wait.ForListeningPort(servicePort),
Files: map[string]string{
"/mosquitto/config/mosquitto.conf": conf,
},
}
err = container.Start()
require.NoError(t, err, "failed to start container")
return &container
}
func TestConnectAndWriteIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
container := launchTestContainer(t)
defer container.Terminate()
var url = fmt.Sprintf("%s:%s", container.Address, container.Ports[servicePort])
s := &serializers_influx.Serializer{}
require.NoError(t, s.Init())
m := &MQTT{
MqttConfig: mqtt.MqttConfig{
Servers: []string{url},
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
},
serializer: s,
Log: testutil.Logger{Name: "mqtt-default-integration-test"},
}
// Verify that we can connect to the MQTT broker
require.NoError(t, m.Init())
// Verify that we can connect to the MQTT broker
require.NoError(t, m.Connect())
// Verify that we can successfully write data to the mqtt broker
require.NoError(t, m.Write(testutil.MockMetrics()))
}
func TestConnectAndWriteIntegrationMQTTv3(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
container := launchTestContainer(t)
defer container.Terminate()
var url = fmt.Sprintf("%s:%s", container.Address, container.Ports[servicePort])
s := &serializers_influx.Serializer{}
require.NoError(t, s.Init())
m := &MQTT{
MqttConfig: mqtt.MqttConfig{
Servers: []string{url},
Protocol: "3.1.1",
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
},
serializer: s,
Log: testutil.Logger{Name: "mqttv311-integration-test"},
}
// Verify that we can connect to the MQTT broker
require.NoError(t, m.Init())
// Verify that we can connect to the MQTT broker
require.NoError(t, m.Connect())
// Verify that we can successfully write data to the mqtt broker
require.NoError(t, m.Write(testutil.MockMetrics()))
}
func TestConnectAndWriteIntegrationMQTTv5(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
container := launchTestContainer(t)
defer container.Terminate()
url := fmt.Sprintf("%s:%s", container.Address, container.Ports[servicePort])
s := &serializers_influx.Serializer{}
require.NoError(t, s.Init())
m := &MQTT{
MqttConfig: mqtt.MqttConfig{
Servers: []string{url},
Protocol: "5",
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
},
serializer: s,
Log: testutil.Logger{Name: "mqttv5-integration-test"},
}
// Verify that we can connect to the MQTT broker
require.NoError(t, m.Init())
require.NoError(t, m.Connect())
// Verify that we can successfully write data to the mqtt broker
require.NoError(t, m.Write(testutil.MockMetrics()))
}
func TestIntegrationMQTTv3(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
conf, err := filepath.Abs(filepath.Join("testdata", "mosquitto.conf"))
require.NoError(t, err, "missing file mosquitto.conf")
container := testutil.Container{
Image: "eclipse-mosquitto:2",
ExposedPorts: []string{servicePort},
WaitingFor: wait.ForListeningPort(servicePort),
Files: map[string]string{
"/mosquitto/config/mosquitto.conf": conf,
},
}
require.NoError(t, container.Start(), "failed to start container")
defer container.Terminate()
// Setup the parser / serializer pair
parser := &influx.Parser{}
require.NoError(t, parser.Init())
serializer := &serializers_influx.Serializer{}
require.NoError(t, serializer.Init())
// Setup the plugin
url := fmt.Sprintf("tcp://%s:%s", container.Address, container.Ports[servicePort])
topic := "testv3"
plugin := &MQTT{
MqttConfig: mqtt.MqttConfig{
Servers: []string{url},
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
AutoReconnect: true,
},
Topic: topic + "/{{.Name}}",
Layout: "non-batch",
Log: testutil.Logger{Name: "mqttv3-integration-test"},
}
plugin.SetSerializer(serializer)
require.NoError(t, plugin.Init())
// Prepare the receiver message
var acc testutil.Accumulator
onMessage := createMetricMessageHandler(&acc, parser)
// Startup the plugin and subscribe to the topic
require.NoError(t, plugin.Connect())
defer plugin.Close()
// Add routing for the messages
subscriptionPattern := topic + "/#"
plugin.client.AddRoute(subscriptionPattern, onMessage)
// Subscribe to the topic
topics := map[string]byte{subscriptionPattern: byte(plugin.QoS)}
require.NoError(t, plugin.client.SubscribeMultiple(topics, onMessage))
// Setup and execute the test case
input := make([]telegraf.Metric, 0, 3)
expected := make([]telegraf.Metric, 0, len(input))
for i := 0; i < cap(input); i++ {
name := fmt.Sprintf("test%d", i)
m := testutil.TestMetric(i, name)
input = append(input, m)
e := m.Copy()
e.AddTag("topic", topic+"/"+name)
expected = append(expected, e)
}
require.NoError(t, plugin.Write(input))
// Verify the result
require.Eventually(t, func() bool {
return acc.NMetrics() >= uint64(len(expected))
}, time.Second, 100*time.Millisecond)
require.NoError(t, plugin.Close())
require.Empty(t, acc.Errors)
testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics())
}
func TestMQTTv5Properties(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
container := launchTestContainer(t)
defer container.Terminate()
tests := []struct {
name string
properties *mqtt.PublishProperties
}{
{
name: "no publish properties",
properties: nil,
},
{
name: "content type set",
properties: &mqtt.PublishProperties{ContentType: "text/plain"},
},
{
name: "response topic set",
properties: &mqtt.PublishProperties{ResponseTopic: "test/topic"},
},
{
name: "message expiry set",
properties: &mqtt.PublishProperties{MessageExpiry: config.Duration(10 * time.Minute)},
},
{
name: "topic alias set",
properties: &mqtt.PublishProperties{TopicAlias: new(uint16)},
},
{
name: "user properties set",
properties: &mqtt.PublishProperties{UserProperties: map[string]string{"key": "value"}},
},
}
topic := "testv3"
url := fmt.Sprintf("%s:%s", container.Address, container.Ports[servicePort])
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
plugin := &MQTT{
MqttConfig: mqtt.MqttConfig{
Servers: []string{url},
Protocol: "5",
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
AutoReconnect: true,
},
Topic: topic,
Log: testutil.Logger{Name: "mqttv5-integration-test"},
}
// Setup the metric serializer
serializer := &serializers_influx.Serializer{}
require.NoError(t, serializer.Init())
plugin.SetSerializer(serializer)
// Verify that we can connect to the MQTT broker
require.NoError(t, plugin.Init())
require.NoError(t, plugin.Connect())
// Verify that we can successfully write data to the mqtt broker
require.NoError(t, plugin.Write(testutil.MockMetrics()))
})
}
}
func TestIntegrationMQTTLayoutNonBatch(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
conf, err := filepath.Abs(filepath.Join("testdata", "mosquitto.conf"))
require.NoError(t, err, "missing file mosquitto.conf")
container := testutil.Container{
Image: "eclipse-mosquitto:2",
ExposedPorts: []string{servicePort},
WaitingFor: wait.ForListeningPort(servicePort),
Files: map[string]string{
"/mosquitto/config/mosquitto.conf": conf,
},
}
require.NoError(t, container.Start(), "failed to start container")
defer container.Terminate()
// Setup the parser / serializer pair
parser := &influx.Parser{}
require.NoError(t, parser.Init())
serializer := &serializers_influx.Serializer{}
require.NoError(t, serializer.Init())
// Setup the plugin
url := fmt.Sprintf("tcp://%s:%s", container.Address, container.Ports[servicePort])
topic := "test_nonbatch"
plugin := &MQTT{
MqttConfig: mqtt.MqttConfig{
Servers: []string{url},
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
AutoReconnect: true,
},
Topic: topic + "/{{.Name}}",
Layout: "non-batch",
Log: testutil.Logger{Name: "mqttv3-integration-test"},
}
plugin.SetSerializer(serializer)
require.NoError(t, plugin.Init())
// Prepare the receiver message
var acc testutil.Accumulator
onMessage := createMetricMessageHandler(&acc, parser)
// Startup the plugin and subscribe to the topic
require.NoError(t, plugin.Connect())
defer plugin.Close()
// Add routing for the messages
subscriptionPattern := topic + "/#"
plugin.client.AddRoute(subscriptionPattern, onMessage)
// Subscribe to the topic
topics := map[string]byte{subscriptionPattern: byte(plugin.QoS)}
require.NoError(t, plugin.client.SubscribeMultiple(topics, onMessage))
// Setup and execute the test case
input := make([]telegraf.Metric, 0, 3)
expected := make([]telegraf.Metric, 0, len(input))
for i := 0; i < cap(input); i++ {
name := fmt.Sprintf("test%d", i)
m := metric.New(
name,
map[string]string{"case": "mqtt"},
map[string]interface{}{"value": i},
time.Unix(1676470949, 0),
)
input = append(input, m)
e := m.Copy()
e.AddTag("topic", topic+"/"+name)
expected = append(expected, e)
}
require.NoError(t, plugin.Write(input))
// Verify the result
require.Eventually(t, func() bool {
return acc.NMetrics() >= uint64(len(expected))
}, time.Second, 100*time.Millisecond)
require.NoError(t, plugin.Close())
require.Empty(t, acc.Errors)
testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics())
}
func TestIntegrationMQTTLayoutBatch(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
conf, err := filepath.Abs(filepath.Join("testdata", "mosquitto.conf"))
require.NoError(t, err, "missing file mosquitto.conf")
container := testutil.Container{
Image: "eclipse-mosquitto:2",
ExposedPorts: []string{servicePort},
WaitingFor: wait.ForListeningPort(servicePort),
Files: map[string]string{
"/mosquitto/config/mosquitto.conf": conf,
},
}
require.NoError(t, container.Start(), "failed to start container")
defer container.Terminate()
// Setup the parser / serializer pair
parser := &influx.Parser{}
require.NoError(t, parser.Init())
serializer := &serializers_influx.Serializer{}
require.NoError(t, serializer.Init())
// Setup the plugin
url := fmt.Sprintf("tcp://%s:%s", container.Address, container.Ports[servicePort])
topic := "test_batch"
plugin := &MQTT{
MqttConfig: mqtt.MqttConfig{
Servers: []string{url},
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
AutoReconnect: true,
},
Topic: topic + "/{{.Name}}",
Layout: "batch",
Log: testutil.Logger{Name: "mqttv3-integration-test-"},
}
plugin.SetSerializer(serializer)
require.NoError(t, plugin.Init())
// Prepare the receiver message
var acc testutil.Accumulator
onMessage := createMetricMessageHandler(&acc, parser)
// Startup the plugin and subscribe to the topic
require.NoError(t, plugin.Connect())
defer plugin.Close()
// Add routing for the messages
subscriptionPattern := topic + "/#"
plugin.client.AddRoute(subscriptionPattern, onMessage)
// Subscribe to the topic
topics := map[string]byte{subscriptionPattern: byte(plugin.QoS)}
require.NoError(t, plugin.client.SubscribeMultiple(topics, onMessage))
// Setup and execute the test case
input := make([]telegraf.Metric, 0, 6)
expected := make([]telegraf.Metric, 0, len(input))
for i := 0; i < cap(input); i++ {
name := fmt.Sprintf("test%d", i%3)
m := metric.New(
name,
map[string]string{
"case": "mqtt",
"id": fmt.Sprintf("test%d", i),
},
map[string]interface{}{"value": i},
time.Unix(1676470949, 0),
)
input = append(input, m)
e := m.Copy()
e.AddTag("topic", topic+"/"+name)
expected = append(expected, e)
}
require.NoError(t, plugin.Write(input))
// Verify the result
require.Eventually(t, func() bool {
return acc.NMetrics() >= uint64(len(expected))
}, time.Second, 100*time.Millisecond)
require.NoError(t, plugin.Close())
require.Empty(t, acc.Errors)
testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics(), testutil.SortMetrics())
}
func TestIntegrationMQTTLayoutField(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
conf, err := filepath.Abs(filepath.Join("testdata", "mosquitto.conf"))
require.NoError(t, err, "missing file mosquitto.conf")
container := testutil.Container{
Image: "eclipse-mosquitto:2",
ExposedPorts: []string{servicePort},
WaitingFor: wait.ForListeningPort(servicePort),
Files: map[string]string{
"/mosquitto/config/mosquitto.conf": conf,
},
}
require.NoError(t, container.Start(), "failed to start container")
defer container.Terminate()
// Setup the plugin
url := fmt.Sprintf("tcp://%s:%s", container.Address, container.Ports[servicePort])
topic := "test_field"
plugin := &MQTT{
MqttConfig: mqtt.MqttConfig{
Servers: []string{url},
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
AutoReconnect: true,
},
Topic: topic + `/{{.Name}}/{{.Tag "source"}}`,
Layout: "field",
Log: testutil.Logger{Name: "mqttv3-integration-test-"},
}
require.NoError(t, plugin.Init())
// Startup the plugin and subscribe to the topic
require.NoError(t, plugin.Connect())
defer plugin.Close()
// Prepare the message receiver
var received []message
var mtx sync.Mutex
onMessage := func(_ paho.Client, msg paho.Message) {
mtx.Lock()
defer mtx.Unlock()
received = append(received, message{msg.Topic(), msg.Payload()})
}
// Add routing for the messages
subscriptionPattern := topic + "/#"
plugin.client.AddRoute(subscriptionPattern, onMessage)
// Subscribe to the topic
topics := map[string]byte{subscriptionPattern: byte(plugin.QoS)}
require.NoError(t, plugin.client.SubscribeMultiple(topics, onMessage))
// Setup and execute the test case
input := []telegraf.Metric{
metric.New(
"modbus",
map[string]string{
"source": "device 1",
"type": "Machine A",
"location": "main building",
"status": "ok",
},
map[string]interface{}{
"temperature": 21.4,
"serial number": "324nlk234r5u9834t",
"working hours": 123,
"supplied": true,
},
time.Unix(1676522982, 0),
),
metric.New(
"modbus",
map[string]string{
"source": "device 2",
"type": "Machine B",
"location": "main building",
"status": "offline",
},
map[string]interface{}{
"temperature": 25.0,
"supplied": false,
},
time.Unix(1676522982, 0),
),
}
expected := []string{
topic + "/modbus/device 1/temperature" + " " + "21.4",
topic + "/modbus/device 1/serial number" + " " + "324nlk234r5u9834t",
topic + "/modbus/device 1/supplied" + " " + "true",
topic + "/modbus/device 1/working hours" + " " + "123",
topic + "/modbus/device 2/temperature" + " " + "25",
topic + "/modbus/device 2/supplied" + " " + "false",
}
require.NoError(t, plugin.Write(input))
// Verify the result
require.Eventually(t, func() bool {
mtx.Lock()
defer mtx.Unlock()
return len(received) >= len(expected)
}, time.Second, 100*time.Millisecond)
require.NoError(t, plugin.Close())
actual := make([]string, 0, len(received))
for _, msg := range received {
actual = append(actual, msg.topic+" "+string(msg.payload))
}
require.ElementsMatch(t, expected, actual)
}
func TestIntegrationMQTTLayoutHomieV4(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
conf, err := filepath.Abs(filepath.Join("testdata", "mosquitto.conf"))
require.NoError(t, err, "missing file mosquitto.conf")
container := testutil.Container{
Image: "eclipse-mosquitto:2",
ExposedPorts: []string{servicePort},
WaitingFor: wait.ForListeningPort(servicePort),
Files: map[string]string{
"/mosquitto/config/mosquitto.conf": conf,
},
}
require.NoError(t, container.Start(), "failed to start container")
defer container.Terminate()
// Setup the plugin
url := fmt.Sprintf("tcp://%s:%s", container.Address, container.Ports[servicePort])
topic := "homie"
plugin := &MQTT{
MqttConfig: mqtt.MqttConfig{
Servers: []string{url},
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
AutoReconnect: true,
},
Topic: topic + "/{{.Name}}",
HomieDeviceName: `{{.Name}}`,
HomieNodeID: `{{.Tag "source"}}`,
Layout: "homie-v4",
Log: testutil.Logger{Name: "mqttv3-integration-test-"},
}
require.NoError(t, plugin.Init())
// Startup the plugin and subscribe to the topic
require.NoError(t, plugin.Connect())
defer plugin.Close()
// Prepare the message receiver
var received []message
var mtx sync.Mutex
onMessage := func(_ paho.Client, msg paho.Message) {
mtx.Lock()
defer mtx.Unlock()
received = append(received, message{msg.Topic(), msg.Payload()})
}
// Add routing for the messages
subscriptionPattern := topic + "/#"
plugin.client.AddRoute(subscriptionPattern, onMessage)
// Subscribe to the topic
topics := map[string]byte{subscriptionPattern: byte(plugin.QoS)}
require.NoError(t, plugin.client.SubscribeMultiple(topics, onMessage))
// Setup and execute the test case
input := []telegraf.Metric{
metric.New(
"modbus",
map[string]string{
"source": "device 1",
"type": "Machine A",
"location": "main building",
"status": "ok",
},
map[string]interface{}{
"temperature": 21.4,
"serial number": "324nlk234r5u9834t",
"working hours": 123,
"supplied": true,
},
time.Unix(1676522982, 0),
),
metric.New(
"modbus",
map[string]string{
"source": "device 2",
"type": "Machine B",
"location": "main building",
"status": "offline",
},
map[string]interface{}{
"supplied": false,
},
time.Unix(1676522982, 0),
),
metric.New(
"modbus",
map[string]string{
"source": "device 2",
"type": "Machine B",
"location": "main building",
"status": "online",
"in operation": "yes",
},
map[string]interface{}{
"Temperature": 25.38,
"Voltage": 24.1,
"Current": 100.0,
"Throughput": 12345,
"Load [%]": 81.2,
"account no": "T3L3GrAf",
"supplied": true,
},
time.Unix(1676542982, 0),
),
}
dev1Props := "location,serial-number,source,status,supplied,temperature,type,working-hours"
dev2Props := "account-no,current,in-operation,load,location,source,status,supplied,temperature,"
dev2Props += "throughput,type,voltage"
expected := []string{
topic + "/modbus/$homie" + " " + "4.0",
topic + "/modbus/$name" + " " + "modbus",
topic + "/modbus/$state" + " " + "ready",
topic + "/modbus/$nodes" + " " + "device-1",
topic + "/modbus/device-1/$name" + " " + "device 1",
topic + "/modbus/device-1/$properties" + " " + dev1Props,
topic + "/modbus/device-1/location" + " " + "main building",
topic + "/modbus/device-1/location/$name" + " " + "location",
topic + "/modbus/device-1/location/$datatype" + " " + "string",
topic + "/modbus/device-1/status" + " " + "ok",
topic + "/modbus/device-1/status/$name" + " " + "status",
topic + "/modbus/device-1/status/$datatype" + " " + "string",
topic + "/modbus/device-1/type" + " " + "Machine A",
topic + "/modbus/device-1/type/$name" + " " + "type",
topic + "/modbus/device-1/type/$datatype" + " " + "string",
topic + "/modbus/device-1/source" + " " + "device 1",
topic + "/modbus/device-1/source/$name" + " " + "source",
topic + "/modbus/device-1/source/$datatype" + " " + "string",
topic + "/modbus/device-1/temperature" + " " + "21.4",
topic + "/modbus/device-1/temperature/$name" + " " + "temperature",
topic + "/modbus/device-1/temperature/$datatype" + " " + "float",
topic + "/modbus/device-1/serial-number" + " " + "324nlk234r5u9834t",
topic + "/modbus/device-1/serial-number/$name" + " " + "serial number",
topic + "/modbus/device-1/serial-number/$datatype" + " " + "string",
topic + "/modbus/device-1/working-hours" + " " + "123",
topic + "/modbus/device-1/working-hours/$name" + " " + "working hours",
topic + "/modbus/device-1/working-hours/$datatype" + " " + "integer",
topic + "/modbus/device-1/supplied" + " " + "true",
topic + "/modbus/device-1/supplied/$name" + " " + "supplied",
topic + "/modbus/device-1/supplied/$datatype" + " " + "boolean",
topic + "/modbus/$nodes" + " " + "device-1,device-2",
topic + "/modbus/device-2/$name" + " " + "device 2",
topic + "/modbus/device-2/$properties" + " " + "location,source,status,supplied,type",
topic + "/modbus/device-2/location" + " " + "main building",
topic + "/modbus/device-2/location/$name" + " " + "location",
topic + "/modbus/device-2/location/$datatype" + " " + "string",
topic + "/modbus/device-2/status" + " " + "offline",
topic + "/modbus/device-2/status/$name" + " " + "status",
topic + "/modbus/device-2/status/$datatype" + " " + "string",
topic + "/modbus/device-2/type" + " " + "Machine B",
topic + "/modbus/device-2/type/$name" + " " + "type",
topic + "/modbus/device-2/type/$datatype" + " " + "string",
topic + "/modbus/device-2/source" + " " + "device 2",
topic + "/modbus/device-2/source/$name" + " " + "source",
topic + "/modbus/device-2/source/$datatype" + " " + "string",
topic + "/modbus/device-2/supplied" + " " + "false",
topic + "/modbus/device-2/supplied/$name" + " " + "supplied",
topic + "/modbus/device-2/supplied/$datatype" + " " + "boolean",
topic + "/modbus/device-2/$properties" + " " + dev2Props,
topic + "/modbus/device-2/location" + " " + "main building",
topic + "/modbus/device-2/location/$name" + " " + "location",
topic + "/modbus/device-2/location/$datatype" + " " + "string",
topic + "/modbus/device-2/in-operation" + " " + "yes",
topic + "/modbus/device-2/in-operation/$name" + " " + "in operation",
topic + "/modbus/device-2/in-operation/$datatype" + " " + "string",
topic + "/modbus/device-2/status" + " " + "online",
topic + "/modbus/device-2/status/$name" + " " + "status",
topic + "/modbus/device-2/status/$datatype" + " " + "string",
topic + "/modbus/device-2/type" + " " + "Machine B",
topic + "/modbus/device-2/type/$name" + " " + "type",
topic + "/modbus/device-2/type/$datatype" + " " + "string",
topic + "/modbus/device-2/source" + " " + "device 2",
topic + "/modbus/device-2/source/$name" + " " + "source",
topic + "/modbus/device-2/source/$datatype" + " " + "string",
topic + "/modbus/device-2/temperature" + " " + "25.38",
topic + "/modbus/device-2/temperature/$name" + " " + "Temperature",
topic + "/modbus/device-2/temperature/$datatype" + " " + "float",
topic + "/modbus/device-2/voltage" + " " + "24.1",
topic + "/modbus/device-2/voltage/$name" + " " + "Voltage",
topic + "/modbus/device-2/voltage/$datatype" + " " + "float",
topic + "/modbus/device-2/current" + " " + "100",
topic + "/modbus/device-2/current/$name" + " " + "Current",
topic + "/modbus/device-2/current/$datatype" + " " + "float",
topic + "/modbus/device-2/throughput" + " " + "12345",
topic + "/modbus/device-2/throughput/$name" + " " + "Throughput",
topic + "/modbus/device-2/throughput/$datatype" + " " + "integer",
topic + "/modbus/device-2/load" + " " + "81.2",
topic + "/modbus/device-2/load/$name" + " " + "Load [%]",
topic + "/modbus/device-2/load/$datatype" + " " + "float",
topic + "/modbus/device-2/account-no" + " " + "T3L3GrAf",
topic + "/modbus/device-2/account-no/$name" + " " + "account no",
topic + "/modbus/device-2/account-no/$datatype" + " " + "string",
topic + "/modbus/device-2/supplied" + " " + "true",
topic + "/modbus/device-2/supplied/$name" + " " + "supplied",
topic + "/modbus/device-2/supplied/$datatype" + " " + "boolean",
topic + "/modbus/$state" + " " + "lost",
}
require.NoError(t, plugin.Write(input))
require.NoError(t, plugin.Close())
// Verify the result
require.Eventually(t, func() bool {
mtx.Lock()
defer mtx.Unlock()
return len(received) >= len(expected)
}, time.Second, 100*time.Millisecond)
actual := make([]string, 0, len(received))
for _, msg := range received {
actual = append(actual, msg.topic+" "+string(msg.payload))
}
require.ElementsMatch(t, expected, actual)
}
func createMetricMessageHandler(acc telegraf.Accumulator, parser telegraf.Parser) paho.MessageHandler {
return func(_ paho.Client, msg paho.Message) {
metrics, err := parser.Parse(msg.Payload())
if err != nil {
acc.AddError(err)
return
}
for _, m := range metrics {
m.AddTag("topic", msg.Topic())
acc.AddMetric(m)
}
}
}
func TestMissingServers(t *testing.T) {
plugin := &MQTT{}
require.ErrorContains(t, plugin.Init(), "no servers specified")
}
func TestMQTTTopicGenerationTemplateIsValid(t *testing.T) {
tests := []struct {
name string
topic string
expectedError string
}{
{
name: "a valid pattern is accepted",
topic: "this/is/valid",
expectedError: "",
},
{
name: "an invalid pattern is rejected",
topic: "this/is/#/invalid",
expectedError: "found forbidden character # in the topic name this/is/#/invalid",
},
{
name: "an invalid pattern is rejected",
topic: "this/is/+/invalid",
expectedError: "found forbidden character + in the topic name this/is/+/invalid",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m := &MQTT{
Topic: tt.topic,
MqttConfig: mqtt.MqttConfig{
Servers: []string{"tcp://localhost:1883"},
},
}
err := m.Init()
if tt.expectedError != "" {
require.ErrorContains(t, err, tt.expectedError)
} else {
require.NoError(t, err)
}
})
}
}
func TestGenerateTopicName(t *testing.T) {
s := &serializers_influx.Serializer{}
require.NoError(t, s.Init())
m := &MQTT{
MqttConfig: mqtt.MqttConfig{
Servers: []string{"tcp://localhost:1883"},
KeepAlive: 30,
Timeout: config.Duration(5 * time.Second),
},
serializer: s,
Log: testutil.Logger{},
}
tests := []struct {
name string
pattern string
want string
}{
{
name: "matches default legacy format",
pattern: "telegraf/{{ .Hostname }}/{{ .PluginName }}",
want: "telegraf/hostname/metric-name",
},
{
name: "matches default format",
pattern: `telegraf/{{ .Tag "host" }}/{{ .Name }}`,
want: "telegraf/hostname/metric-name",
},
{
name: "respect hardcoded strings",
pattern: "this/is/a/topic",
want: "this/is/a/topic",
},
{
name: "allows the use of tags",
pattern: "{{ .TopicPrefix }}/{{ .Tag \"tag1\" }}",
want: "prefix/value1",
},
{
name: "uses the plugin name when no pattern is provided",
pattern: "",
want: "metric-name",
},
{
name: "ignores tag when tag does not exists",
pattern: "{{ .TopicPrefix }}/{{ .Tag \"not-a-tag\" }}",
want: "prefix",
},
{
name: "ignores empty forward slashes",
pattern: "double//slashes//are//ignored",
want: "double/slashes/are/ignored",
},
{
name: "preserve leading forward slash",
pattern: "/this/is/a/topic",
want: "/this/is/a/topic",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m.Topic = tt.pattern
m.TopicPrefix = "prefix"
met := metric.New(
"metric-name",
map[string]string{"tag1": "value1", "host": "hostname"},
map[string]interface{}{"value": 123},
time.Date(2022, time.November, 10, 23, 0, 0, 0, time.UTC),
)
require.NoError(t, m.Init())
actual, err := m.generateTopic(met)
require.NoError(t, err)
require.Equal(t, tt.want, actual)
})
}
}

117
plugins/outputs/mqtt/sample.conf Normal file
View file

@ -0,0 +1,117 @@
# Configuration for MQTT server to send metrics to
[[outputs.mqtt]]
## MQTT Brokers
## The list of brokers should only include the hostname or IP address and the
## port to the broker. This should follow the format `[{scheme}://]{host}:{port}`. For
## example, `localhost:1883` or `mqtt://localhost:1883`.
## Scheme can be any of the following: tcp://, mqtt://, tls://, mqtts://
## non-TLS and TLS servers can not be mix-and-matched.
servers = ["localhost:1883", ] # or ["mqtts://tls.example.com:1883"]
## Protocol can be `3.1.1` or `5`. Default is `3.1.1`
# protocol = "3.1.1"
## MQTT Topic for Producer Messages
## MQTT outputs send metrics to this topic format:
## prefix/{{ .Tag "host" }}/{{ .Name }}/{{ .Tag "tag_key" }}
## (e.g. prefix/web01.example.com/mem/some_tag_value)
## Each path segment accepts either a template placeholder, an environment variable, or a tag key
## of the form `{{.Tag "tag_key_name"}}`. All the functions provided by the Sprig library
## (http://masterminds.github.io/sprig/) are available. Empty path elements as well as special MQTT
## characters (such as `+` or `#`) are invalid to form the topic name and will lead to an error.
## In case a tag is missing in the metric, that path segment omitted for the final topic.
topic = 'telegraf/{{ .Tag "host" }}/{{ .Name }}'
## QoS policy for messages
## The mqtt QoS policy for sending messages.
## See https://www.ibm.com/support/knowledgecenter/en/SSFKSJ_9.0.0/com.ibm.mq.dev.doc/q029090_.htm
## 0 = at most once
## 1 = at least once
## 2 = exactly once
# qos = 2
## Keep Alive
## Defines the maximum length of time that the broker and client may not
## communicate. Defaults to 0 which turns the feature off.
##
## For version v2.0.12 and later mosquitto there is a bug
## (see https://github.com/eclipse/mosquitto/issues/2117), which requires
## this to be non-zero. As a reference eclipse/paho.mqtt.golang defaults to 30.
# keep_alive = 0
## username and password to connect MQTT server.
# username = "telegraf"
# password = "metricsmetricsmetricsmetrics"
## client ID
## The unique client id to connect MQTT server. If this parameter is not set
## then a random ID is generated.
# client_id = ""
## Timeout for write operations. default: 5s
# timeout = "5s"
## Optional TLS Config
# tls_ca = "/etc/telegraf/ca.pem"
# tls_cert = "/etc/telegraf/cert.pem"
# tls_key = "/etc/telegraf/key.pem"
## Use TLS but skip chain & host verification
# insecure_skip_verify = false
## When true, metrics will be sent in one MQTT message per flush. Otherwise,
## metrics are written one metric per MQTT message.
## DEPRECATED: Use layout option instead
# batch = false
## When true, metric will have RETAIN flag set, making broker cache entries until someone
## actually reads it
# retain = false
## Client trace messages
## When set to true, and debug mode enabled in the agent settings, the MQTT
## client's messages are included in telegraf logs. These messages are very
## noisey, but essential for debugging issues.
# client_trace = false
## Layout of the topics published.
## The following choices are available:
## non-batch -- send individual messages, one for each metric
## batch -- send all metric as a single message per MQTT topic
## NOTE: The following options will ignore the 'data_format' option and send single values
## field -- send individual messages for each field, appending its name to the metric topic
## homie-v4 -- send metrics with fields and tags according to the 4.0.0 specs
## see https://homieiot.github.io/specification/
# layout = "non-batch"
## HOMIE specific settings
## The following options provide templates for setting the device name
## and the node-ID for the topics. Both options are MANDATORY and can contain
## {{ .Name }} (metric name), {{ .Tag "key"}} (tag reference to 'key') or
## constant strings. The templates MAY NOT contain slashes!
# homie_device_name = ""
# homie_node_id = ""
## Each data format has its own unique set of configuration options, read
## more about them here:
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
data_format = "influx"
## NOTE: Due to the way TOML is parsed, tables must be at the END of the
## plugin definition, otherwise additional config options are read as part of
## the table
## Optional MQTT 5 publish properties
## These setting only apply if the "protocol" property is set to 5. This must
## be defined at the end of the plugin settings, otherwise TOML will assume
## anything else is part of this table. For more details on publish properties
## see the spec:
## https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Toc3901109
# [outputs.mqtt.v5]
# content_type = ""
# response_topic = ""
# message_expiry = "0s"
# topic_alias = 0
# [outputs.mqtt.v5.user_properties]
# "key1" = "value 1"
# "key2" = "value 2"

3
plugins/outputs/mqtt/testdata/mosquitto.conf vendored Normal file
View file

@ -0,0 +1,3 @@
listener 1883 0.0.0.0
allow_anonymous true
connection_messages true

88
plugins/outputs/mqtt/topic_name_generator.go Normal file
View file

@ -0,0 +1,88 @@
package mqtt
import (
"fmt"
"strings"
"text/template"
"time"
"github.com/Masterminds/sprig/v3"
"github.com/influxdata/telegraf"
)
type TopicNameGenerator struct {
TopicPrefix string
metric telegraf.TemplateMetric
template *template.Template
}
func NewTopicNameGenerator(topicPrefix, topic string) (*TopicNameGenerator, error) {
topic = hostnameRe.ReplaceAllString(topic, `$1.Tag "host"$2`)
topic = pluginNameRe.ReplaceAllString(topic, `$1.Name$2`)
tt, err := template.New("topic_name").Funcs(sprig.TxtFuncMap()).Parse(topic)
if err != nil {
return nil, err
}
for _, p := range strings.Split(topic, "/") {
if strings.ContainsAny(p, "#+") {
return nil, fmt.Errorf("found forbidden character %s in the topic name %s", p, topic)
}
}
return &TopicNameGenerator{TopicPrefix: topicPrefix, template: tt}, nil
}
func (t *TopicNameGenerator) Name() string {
return t.metric.Name()
}
func (t *TopicNameGenerator) Tag(key string) string {
return t.metric.Tag(key)
}
func (t *TopicNameGenerator) Field(key string) interface{} {
return t.metric.Field(key)
}
func (t *TopicNameGenerator) Time() time.Time {
return t.metric.Time()
}
func (t *TopicNameGenerator) Tags() map[string]string {
return t.metric.Tags()
}
func (t *TopicNameGenerator) Fields() map[string]interface{} {
return t.metric.Fields()
}
func (t *TopicNameGenerator) String() string {
return t.metric.String()
}
func (m *MQTT) generateTopic(metric telegraf.Metric) (string, error) {
m.generator.metric = metric.(telegraf.TemplateMetric)
// Cannot directly pass TemplateMetric since TopicNameGenerator still contains TopicPrefix (until v1.35.0)
var b strings.Builder
err := m.generator.template.Execute(&b, m.generator)
if err != nil {
return "", err
}
var ts []string
for _, p := range strings.Split(b.String(), "/") {
if p != "" {
ts = append(ts, p)
}
}
topic := strings.Join(ts, "/")
// This is to keep backward compatibility with previous behaviour where the plugin name was always present
if topic == "" {
return metric.Name(), nil
}
if strings.HasPrefix(b.String(), "/") {
topic = "/" + topic
}
return topic, nil
}