influx/telegraf
1
0
Fork 0
Code Issues Activity

Adding upstream version 1.34.4.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-05-24 07:26:29 +02:00
parent e393c3af3f
commit 4978089aab
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

47
plugins/parsers/EXAMPLE_README.md Normal file
View file

@ -0,0 +1,47 @@
# Example
This description explains at a high level what the parser does and provides
links to where additional information about the format can be found.
## Configuration
This section contains the sample configuration for the parser. Since the
configuration for a parser is not have a standalone plugin, use the `file` or
`exec` input as the base config.
```toml
[[inputs.file]]
files = ["example"]
## Data format to consume.
## Each data format has its own unique set of configuration options, read
## more about them here:
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
data_format = "example"
## Describe variables using the standard SampleConfig style.
## https://github.com/influxdata/telegraf/wiki/SampleConfig
example_option = "example_value"
```
### example_option
If an option requires a more expansive explanation than can be included inline
in the sample configuration, it may be described here.
## Metrics
The optional Metrics section contains details about how the parser converts
input data into Telegraf metrics.
## Examples
The optional Examples section can show an example conversion from the input
format using InfluxDB Line Protocol as the reference format.
For line delimited text formats a diff may be appropriate:
```diff
- cpu|host=localhost|source=example.org|value=42
+ cpu,host=localhost,source=example.org value=42
```

1
plugins/parsers/all/all.go Normal file
View file

@ -0,0 +1 @@
package all

5
plugins/parsers/all/avro.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.avro
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/avro" // register plugin

5
plugins/parsers/all/binary.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.binary
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/binary" // register plugin

5
plugins/parsers/all/collectd.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.collectd
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/collectd" // register plugin

5
plugins/parsers/all/csv.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.csv
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/csv" // register plugin

5
plugins/parsers/all/dropwizard.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.dropwizard
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/dropwizard" // register plugin

5
plugins/parsers/all/form_urlencoded.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.form_urlencoded
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/form_urlencoded" // register plugin

5
plugins/parsers/all/graphite.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.graphite
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/graphite" // register plugin

5
plugins/parsers/all/grok.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.grok
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/grok" // register plugin

8
plugins/parsers/all/influx.go Normal file
View file

@ -0,0 +1,8 @@
//go:build !custom || parsers || parsers.influx
package all
import (
_ "github.com/influxdata/telegraf/plugins/parsers/influx" // register plugin
_ "github.com/influxdata/telegraf/plugins/parsers/influx/influx_upstream" // register plugin
)

5
plugins/parsers/all/json.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.json
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/json" // register plugin

5
plugins/parsers/all/json_v2.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.json_v2
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/json_v2" // register plugin

5
plugins/parsers/all/logfmt.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.logfmt
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/logfmt" // register plugin

5
plugins/parsers/all/nagios.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.nagios
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/nagios" // register plugin

5
plugins/parsers/all/openmetrics.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.openmetrics
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/openmetrics" // register plugin

5
plugins/parsers/all/opentsdb.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.opentsdb
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/opentsdb" // register plugin

5
plugins/parsers/all/parquet.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.parquet
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/parquet" // register plugin

5
plugins/parsers/all/prometheus.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.prometheus
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/prometheus" // register plugin

5
plugins/parsers/all/prometheusremotewrite.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.prometheusremotewrite
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/prometheusremotewrite" // register plugin

5
plugins/parsers/all/value.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.value
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/value" // register plugin

5
plugins/parsers/all/wavefront.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.wavefront
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/wavefront" // register plugin

5
plugins/parsers/all/xpath.go Normal file
View file

@ -0,0 +1,5 @@
//go:build !custom || parsers || parsers.xpath
package all
import _ "github.com/influxdata/telegraf/plugins/parsers/xpath" // register plugin

148
plugins/parsers/avro/README.md Normal file
View file

@ -0,0 +1,148 @@
# Avro Parser Plugin
The `Avro` parser creates metrics from a message serialized with Avro.
The message is supposed to be encoded as follows:
| Bytes | Area | Description |
| ----- | ---------- | ------------------------------------------------ |
| 0 | Magic Byte | Confluent serialization format version number. |
| 1-4 | Schema ID | 4-byte schema ID as returned by Schema Registry. |
| 5- | Data | Serialized data. |
The metric name will be set according the following priority:
1. Try to get metric name from the message field if it is set in the
`avro_measurement_field` option.
2. If the name is not determined, then try to get it from
`avro_measurement` option as the static value.
3. If the name is still not determined, then try to get it from the
schema definition in the following format `[schema_namespace.]schema_name`,
where schema namespace is optional and will be added only if it is specified
in the schema definition.
In case if the metric name could not be determined according to these steps
the error will be raised and the message will not be parsed.
## Configuration
```toml
[[inputs.kafka_consumer]]
## Kafka brokers.
brokers = ["localhost:9092"]
## Topics to consume.
topics = ["telegraf"]
## Maximum length of a message to consume, in bytes (default 0/unlimited);
## larger messages are dropped
max_message_len = 1000000
## Avro data format settings
data_format = "avro"
## Avro message format
## Supported values are "binary" (default) and "json"
# avro_format = "binary"
## URL of the schema registry which may contain username and password in the
## form http[s]://[username[:password]@]<host>[:port]
## NOTE: Exactly one of schema registry and schema must be set
avro_schema_registry = "http://localhost:8081"
## Path to the schema registry certificate. Should be specified only if
## required for connection to the schema registry.
# avro_schema_registry_cert = "/etc/telegraf/ca_cert.crt"
## Schema string; exactly one of schema registry and schema must be set
#avro_schema = '''
# {
# "type":"record",
# "name":"Value",
# "namespace":"com.example",
# "fields":[
# {
# "name":"tag",
# "type":"string"
# },
# {
# "name":"field",
# "type":"long"
# },
# {
# "name":"timestamp",
# "type":"long"
# }
# ]
# }
#'''
## Measurement field name; The meauserment name will be taken
## from this field. If not set, determine measurement name
## from the following 'avro_measurement' option
# avro_measurement_field = "field_name"
## Measurement string; if not set, determine measurement name from
## schema (as "<namespace>.<name>")
# avro_measurement = "ratings"
## Avro fields to be used as tags; optional.
# avro_tags = ["CHANNEL", "CLUB_STATUS"]
## Avro fields to be used as fields; if empty, any Avro fields
## detected from the schema, not used as tags, will be used as
## measurement fields.
# avro_fields = ["STARS"]
## Avro fields to be used as timestamp; if empty, current time will
## be used for the measurement timestamp.
# avro_timestamp = ""
## If avro_timestamp is specified, avro_timestamp_format must be set
## to one of 'unix', 'unix_ms', 'unix_us', or 'unix_ns'. It will
## default to 'unix'.
# avro_timestamp_format = "unix"
## Used to separate parts of array structures. As above, the default
## is the empty string, so a=["a", "b"] becomes a0="a", a1="b".
## If this were set to "_", then it would be a_0="a", a_1="b".
# avro_field_separator = "_"
## Define handling of union types. Possible values are:
## flatten -- add type suffix to field name (default)
## nullable -- do not modify field name but discard "null" field values
## any -- do not modify field name and set field value to the received type
# avro_union_mode = "flatten"
## Default values for given tags: optional
# tags = { "application": "hermes", "region": "central" }
```
### `avro_format`
This optional setting specifies the format of the Avro messages. Currently, the
parser supports the `binary` and `json` formats with `binary` being the default.
### `avro_timestamp` and `avro_timestamp_format`
By default the current time at ingestion will be used for all created
metrics; to set the time using the Avro message you can use the
`avro_timestamp` and `avro_timestamp_format` options together to set the
time to a value in the parsed document.
The `avro_timestamp` option specifies the field containing the time
value. If it is not set, the time of record ingestion is used. If it
is set, the field may be any numerical type: notably, it is *not*
constrained to an Avro `long` (int64) (which Avro uses for timestamps in
millisecond or microsecond resolution). However, it must represent the
number of time increments since the Unix epoch (00:00 UTC 1 Jan 1970).
The `avro_timestamp_format` option specifies the precision of the timestamp
field, and, if set, must be one of `unix`, `unix_ms`, `unix_us`, or
`unix_ns`. If `avro_timestamp` is set, `avro_timestamp_format` must be
as well.
## Metrics
One metric is created for each message. The type of each field is
automatically determined based on the schema.

340
plugins/parsers/avro/parser.go Normal file
View file

@ -0,0 +1,340 @@
package avro
import (
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jeremywohl/flatten/v2"
"github.com/linkedin/goavro/v2"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/metric"
"github.com/influxdata/telegraf/plugins/parsers"
)
// If SchemaRegistry is set, we assume that our input will be in
// Confluent Wire Format
// (https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#wire-format) and we will load the schema from the registry.
// If Schema is set, we assume the input will be Avro binary format, without
// an attached schema or schema fingerprint
type Parser struct {
MetricName string `toml:"metric_name"`
SchemaRegistry string `toml:"avro_schema_registry"`
CaCertPath string `toml:"avro_schema_registry_cert"`
Schema string `toml:"avro_schema"`
Format string `toml:"avro_format"`
Measurement string `toml:"avro_measurement"`
MeasurementField string `toml:"avro_measurement_field"`
Tags []string `toml:"avro_tags"`
Fields []string `toml:"avro_fields"`
Timestamp string `toml:"avro_timestamp"`
TimestampFormat string `toml:"avro_timestamp_format"`
FieldSeparator string `toml:"avro_field_separator"`
UnionMode string `toml:"avro_union_mode"`
DefaultTags map[string]string `toml:"tags"`
Log telegraf.Logger `toml:"-"`
registryObj *schemaRegistry
}
func (p *Parser) Init() error {
switch p.Format {
case "":
p.Format = "binary"
case "binary", "json":
// Do nothing as those are valid settings
default:
return fmt.Errorf("unknown 'avro_format' %q", p.Format)
}
switch p.UnionMode {
case "":
p.UnionMode = "flatten"
case "flatten", "nullable", "any":
// Do nothing as those are valid settings
default:
return fmt.Errorf("unknown avro_union_mode %q", p.Format)
}
if (p.Schema == "" && p.SchemaRegistry == "") || (p.Schema != "" && p.SchemaRegistry != "") {
return errors.New("exactly one of 'schema_registry' or 'schema' must be specified")
}
switch p.TimestampFormat {
case "":
p.TimestampFormat = "unix"
case "unix", "unix_ns", "unix_us", "unix_ms":
// Valid values
default:
return fmt.Errorf("invalid timestamp format '%v'", p.TimestampFormat)
}
if p.SchemaRegistry != "" {
registry, err := newSchemaRegistry(p.SchemaRegistry, p.CaCertPath)
if err != nil {
return fmt.Errorf("error connecting to the schema registry %q: %w", p.SchemaRegistry, err)
}
p.registryObj = registry
}
return nil
}
func (p *Parser) Parse(buf []byte) ([]telegraf.Metric, error) {
var schema string
var codec *goavro.Codec
var err error
var message []byte
message = buf[:]
if p.registryObj != nil {
// The input must be Confluent Wire Protocol
if buf[0] != 0 {
return nil, errors.New("first byte is not 0: not Confluent Wire Protocol")
}
schemaID := int(binary.BigEndian.Uint32(buf[1:5]))
schemastruct, err := p.registryObj.getSchemaAndCodec(schemaID)
if err != nil {
return nil, err
}
schema = schemastruct.Schema
codec = schemastruct.Codec
message = buf[5:]
} else {
// Check for single-object encoding
magicBytes := int(binary.BigEndian.Uint16(buf[:2]))
expectedMagic := int(binary.BigEndian.Uint16([]byte("c301")))
if magicBytes == expectedMagic {
message = buf[10:]
// We could in theory validate the fingerprint against
// the schema. Maybe later.
// We would get the fingerprint as int(binary.LittleEndian.Uint64(buf[2:10]))
} // Otherwise we assume bare Avro binary
schema = p.Schema
codec, err = goavro.NewCodec(schema)
if err != nil {
return nil, err
}
}
var native interface{}
switch p.Format {
case "binary":
native, _, err = codec.NativeFromBinary(message)
case "json":
native, _, err = codec.NativeFromTextual(message)
default:
return nil, fmt.Errorf("unknown format %q", p.Format)
}
if err != nil {
return nil, err
}
// Cast to string-to-interface
codecSchema, ok := native.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("native is of unsupported type %T", native)
}
m, err := p.createMetric(codecSchema, schema)
if err != nil {
return nil, err
}
return []telegraf.Metric{m}, nil
}
func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
metrics, err := p.Parse([]byte(line))
if err != nil {
return nil, err
}
if len(metrics) != 1 {
return nil, errors.New("line contains multiple metrics")
}
return metrics[0], nil
}
func (p *Parser) SetDefaultTags(tags map[string]string) {
p.DefaultTags = tags
}
func (p *Parser) flattenField(fldName string, fldVal map[string]interface{}) map[string]interface{} {
// Helper function for the "nullable" and "any" p.UnionModes
// fldVal is a one-item map of string-to-something
ret := make(map[string]interface{})
if p.UnionMode == "nullable" {
_, ok := fldVal["null"]
if ok {
return ret // Return the empty map
}
}
// Otherwise, we just return the value in the fieldname.
// See README.md for an important warning about "any" and "nullable".
for _, v := range fldVal {
ret[fldName] = v
break // Not really needed, since it's a one-item map
}
return ret
}
func (p *Parser) flattenItem(fld string, fldVal interface{}) (map[string]interface{}, error) {
sep := flatten.SeparatorStyle{
Before: "",
Middle: p.FieldSeparator,
After: "",
}
candidate := make(map[string]interface{})
candidate[fld] = fldVal
var flat map[string]interface{}
var err error
// Exactly how we flatten is decided by p.UnionMode
if p.UnionMode == "flatten" {
flat, err = flatten.Flatten(candidate, "", sep)
if err != nil {
return nil, fmt.Errorf("flatten candidate %q failed: %w", candidate, err)
}
} else {
// "nullable" or "any"
typedVal, ok := candidate[fld].(map[string]interface{})
if !ok {
// the "key" is not a string, so ...
// most likely an array? Do the default thing
// and flatten the candidate.
flat, err = flatten.Flatten(candidate, "", sep)
if err != nil {
return nil, fmt.Errorf("flatten candidate %q failed: %w", candidate, err)
}
} else {
flat = p.flattenField(fld, typedVal)
}
}
return flat, nil
}
func (p *Parser) createMetric(data map[string]interface{}, schema string) (telegraf.Metric, error) {
// Tags differ from fields, in that tags are inherently strings.
// fields can be of any type.
fields := make(map[string]interface{})
tags := make(map[string]string)
// Set default tag values
for k, v := range p.DefaultTags {
tags[k] = v
}
// Avro doesn't have a Tag/Field distinction, so we have to tell
// Telegraf which items are our tags.
for _, tag := range p.Tags {
flat, flattenErr := p.flattenItem(tag, data[tag])
if flattenErr != nil {
return nil, fmt.Errorf("flatten tag %q failed: %w", tag, flattenErr)
}
for k, v := range flat {
sTag, stringErr := internal.ToString(v)
if stringErr != nil {
p.Log.Warnf("Could not convert %v to string for tag %q: %v", data[tag], tag, stringErr)
continue
}
tags[k] = sTag
}
}
var fieldList []string
if len(p.Fields) != 0 {
// If you have specified your fields in the config, you
// get what you asked for.
fieldList = p.Fields
} else {
for k := range data {
// Otherwise, that which is not a tag is a field
if _, ok := tags[k]; !ok {
fieldList = append(fieldList, k)
}
}
}
// We need to flatten out our fields. The default (the separator
// string is empty) is equivalent to what streamreactor does.
for _, fld := range fieldList {
flat, err := p.flattenItem(fld, data[fld])
if err != nil {
return nil, fmt.Errorf("flatten field %q failed: %w", fld, err)
}
for k, v := range flat {
fields[k] = v
}
}
var schemaObj map[string]interface{}
if err := json.Unmarshal([]byte(schema), &schemaObj); err != nil {
return nil, fmt.Errorf("unmarshalling schema failed: %w", err)
}
if len(fields) == 0 {
// A telegraf metric needs at least one field.
return nil, errors.New("number of fields is 0; unable to create metric")
}
// If measurement field name is specified in the configuration
// take value from that field and do not include it into fields or tags
name := ""
if p.MeasurementField != "" {
sField := p.MeasurementField
sMetric, err := internal.ToString(data[sField])
if err != nil {
p.Log.Warnf("Could not convert %v to string for metric name %q: %s", data[sField], sField, err.Error())
} else {
name = sMetric
}
}
// Now some fancy stuff to extract the measurement.
// If it's set in the configuration, use that.
if name == "" {
// If field name is not specified or field does not exist and
// metric name set in the configuration, use that.
name = p.Measurement
}
separator := "."
if name == "" {
// Try using the namespace defined in the schema. In case there
// is none, just use the schema's name definition.
nsStr, ok := schemaObj["namespace"].(string)
// namespace is optional
if !ok {
separator = ""
}
nStr, ok := schemaObj["name"].(string)
if !ok {
return nil, fmt.Errorf("could not determine name from schema %s", schema)
}
name = nsStr + separator + nStr
}
// Still don't have a name? Guess we should use the metric name if
// it's set.
if name == "" {
name = p.MetricName
}
// Nothing? Give up.
if name == "" {
return nil, errors.New("could not determine measurement name")
}
var timestamp time.Time
if p.Timestamp != "" {
rawTime := fmt.Sprintf("%v", data[p.Timestamp])
var err error
timestamp, err = internal.ParseTimestamp(p.TimestampFormat, rawTime, nil)
if err != nil {
return nil, fmt.Errorf("could not parse '%s' to '%s'", rawTime, p.TimestampFormat)
}
} else {
timestamp = time.Now()
}
return metric.New(name, tags, fields, timestamp), nil
}
func init() {
parsers.Add("avro",
func(defaultMetricName string) telegraf.Parser {
return &Parser{MetricName: defaultMetricName}
})
}

183
plugins/parsers/avro/parser_test.go Normal file
View file

@ -0,0 +1,183 @@
package avro
import (
"os"
"path/filepath"
"testing"
"github.com/linkedin/goavro/v2"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/plugins/inputs/file"
"github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/testutil"
)
func TestCases(t *testing.T) {
// Get all test-case directories
folders, err := os.ReadDir("testcases")
require.NoError(t, err)
// Make sure testdata contains data
require.NotEmpty(t, folders)
// Set up for file inputs
inputs.Add("file", func() telegraf.Input {
return &file.File{}
})
for _, f := range folders {
fname := f.Name()
testdataPath := filepath.Join("testcases", fname)
configFilename := filepath.Join(testdataPath, "telegraf.conf")
expectedFilename := filepath.Join(testdataPath, "expected.out")
expectedErrorFilename := filepath.Join(testdataPath, "expected.err")
t.Run(fname, func(t *testing.T) {
// Get parser to parse expected output
testdataParser := &influx.Parser{}
require.NoError(t, testdataParser.Init())
var expected []telegraf.Metric
if _, err := os.Stat(expectedFilename); err == nil {
var err error
expected, err = testutil.ParseMetricsFromFile(expectedFilename, testdataParser)
require.NoError(t, err)
}
// Read the expected errors if any
var expectedErrors []string
if _, err := os.Stat(expectedErrorFilename); err == nil {
var err error
expectedErrors, err = testutil.ParseLinesFromFile(expectedErrorFilename)
require.NoError(t, err)
require.NotEmpty(t, expectedErrors)
}
// Set up error catching
var acc testutil.Accumulator
var actualErrors []string
var actual []telegraf.Metric
// Configure the plugin
cfg := config.NewConfig()
err := cfg.LoadConfig(configFilename)
require.NoError(t, err)
for _, input := range cfg.Inputs {
require.NoError(t, input.Init())
if err := input.Gather(&acc); err != nil {
actualErrors = append(actualErrors, err.Error())
}
}
require.ElementsMatch(t, actualErrors, expectedErrors)
actual = acc.GetTelegrafMetrics()
// Process expected metrics and compare with resulting metrics
testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime())
})
}
}
const benchmarkSchema = `
{
"namespace": "com.benchmark",
"name": "benchmark",
"type": "record",
"version": "1",
"fields": [
{"name": "value", "type": "float", "doc": ""},
{"name": "timestamp", "type": "long", "doc": ""},
{"name": "tags_platform", "type": "string", "doc": ""},
{"name": "tags_sdkver", "type": "string", "default": "", "doc": ""},
{"name": "source", "type": "string", "default": "", "doc": ""}
]
}
`
func BenchmarkParsing(b *testing.B) {
plugin := &Parser{
Format: "json",
Measurement: "benchmark",
Tags: []string{"tags_platform", "tags_sdkver", "source"},
Fields: []string{"value"},
Timestamp: "timestamp",
TimestampFormat: "unix",
Schema: benchmarkSchema,
}
require.NoError(b, plugin.Init())
benchmarkData, err := os.ReadFile(filepath.Join("testcases", "benchmark", "message.json"))
require.NoError(b, err)
b.ResetTimer()
for n := 0; n < b.N; n++ {
//nolint:errcheck // Benchmarking so skip the error check to avoid the unnecessary operations
plugin.Parse(benchmarkData)
}
}
func TestBenchmarkDataBinary(t *testing.T) {
plugin := &Parser{
Measurement: "benchmark",
Tags: []string{"tags_platform", "tags_sdkver", "source"},
Fields: []string{"value"},
Timestamp: "timestamp",
TimestampFormat: "unix",
Schema: benchmarkSchema,
}
require.NoError(t, plugin.Init())
benchmarkDir := filepath.Join("testcases", "benchmark")
// Read the expected valued from file
parser := &influx.Parser{}
require.NoError(t, parser.Init())
expected, err := testutil.ParseMetricsFromFile(filepath.Join(benchmarkDir, "expected.out"), parser)
require.NoError(t, err)
// Re-encode the benchmark data from JSON to binary format
jsonData, err := os.ReadFile(filepath.Join(benchmarkDir, "message.json"))
require.NoError(t, err)
codec, err := goavro.NewCodec(benchmarkSchema)
require.NoError(t, err)
native, _, err := codec.NativeFromTextual(jsonData)
require.NoError(t, err)
benchmarkData, err := codec.BinaryFromNative(nil, native)
require.NoError(t, err)
// Do the actual testing
actual, err := plugin.Parse(benchmarkData)
require.NoError(t, err)
testutil.RequireMetricsEqual(t, expected, actual, testutil.SortMetrics())
}
func BenchmarkParsingBinary(b *testing.B) {
plugin := &Parser{
Measurement: "benchmark",
Tags: []string{"tags_platform", "tags_sdkver", "source"},
Fields: []string{"value"},
Timestamp: "timestamp",
TimestampFormat: "unix",
Schema: benchmarkSchema,
}
require.NoError(b, plugin.Init())
// Re-encode the benchmark data from JSON to binary format
jsonData, err := os.ReadFile(filepath.Join("testcases", "benchmark", "message.json"))
require.NoError(b, err)
codec, err := goavro.NewCodec(benchmarkSchema)
require.NoError(b, err)
native, _, err := codec.NativeFromTextual(jsonData)
require.NoError(b, err)
benchmarkData, err := codec.BinaryFromNative(nil, native)
require.NoError(b, err)
for n := 0; n < b.N; n++ {
//nolint:errcheck // Benchmarking so skip the error check to avoid the unnecessary operations
plugin.Parse(benchmarkData)
}
}

134
plugins/parsers/avro/schema_registry.go Normal file
View file

@ -0,0 +1,134 @@
package avro
import (
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"sync"
"time"
"github.com/linkedin/goavro/v2"
)
type schemaAndCodec struct {
Schema string
Codec *goavro.Codec
}
type schemaRegistry struct {
url string
username string
password string
cache map[int]*schemaAndCodec
client *http.Client
mu sync.RWMutex
}
const schemaByID = "%s/schemas/ids/%d"
func newSchemaRegistry(addr, caCertPath string) (*schemaRegistry, error) {
var client *http.Client
var tlsCfg *tls.Config
if caCertPath != "" {
caCert, err := os.ReadFile(caCertPath)
if err != nil {
return nil, err
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
tlsCfg = &tls.Config{
RootCAs: caCertPool,
}
}
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsCfg,
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
},
}
u, err := url.Parse(addr)
if err != nil {
return nil, fmt.Errorf("parsing registry URL failed: %w", err)
}
var username, password string
if u.User != nil {
username = u.User.Username()
password, _ = u.User.Password()
}
registry := &schemaRegistry{
url: u.String(),
username: username,
password: password,
cache: make(map[int]*schemaAndCodec),
client: client,
}
return registry, nil
}
// Helper function to make managing lock easier
func (sr *schemaRegistry) getSchemaAndCodecFromCache(id int) (*schemaAndCodec, error) {
// Read-lock the cache map before access.
sr.mu.RLock()
defer sr.mu.RUnlock()
if v, ok := sr.cache[id]; ok {
return v, nil
}
return nil, fmt.Errorf("schema %d not in cache", id)
}
func (sr *schemaRegistry) getSchemaAndCodec(id int) (*schemaAndCodec, error) {
v, err := sr.getSchemaAndCodecFromCache(id)
if err == nil {
return v, nil
}
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(schemaByID, sr.url, id), nil)
if err != nil {
return nil, err
}
if sr.username != "" {
req.SetBasicAuth(sr.username, sr.password)
}
resp, err := sr.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var jsonResponse map[string]interface{}
if err := json.NewDecoder(resp.Body).Decode(&jsonResponse); err != nil {
return nil, err
}
schema, ok := jsonResponse["schema"]
if !ok {
return nil, errors.New("malformed response from schema registry: no 'schema' key")
}
schemaValue, ok := schema.(string)
if !ok {
return nil, fmt.Errorf("malformed response from schema registry: %v cannot be cast to string", schema)
}
codec, err := goavro.NewCodec(schemaValue)
if err != nil {
return nil, err
}
retval := &schemaAndCodec{Schema: schemaValue, Codec: codec}
// Lock the cache map before update.
sr.mu.Lock()
defer sr.mu.Unlock()
sr.cache[id] = retval
return retval, nil
}

1
plugins/parsers/avro/testcases/bad-timestamp-format/expected.err Normal file
View file

@ -0,0 +1 @@
could not instantiate parser: invalid timestamp format 'unix_ps'

0
plugins/parsers/avro/testcases/bad-timestamp-format/expected.out Normal file
View file

0
plugins/parsers/avro/testcases/bad-timestamp-format/message.avro Normal file
View file

29
plugins/parsers/avro/testcases/bad-timestamp-format/telegraf.conf Normal file
View file

@ -0,0 +1,29 @@
[[ inputs.file ]]
files = ["./testcases/bad-timestamp-format/message.avro"]
data_format = "avro"
avro_measurement = "measurement"
avro_tags = [ "tag" ]
avro_timestamp = "timestamp"
avro_timestamp_format = "unix_ps"
avro_schema = '''
{
"type":"record",
"name":"Value",
"namespace":"com.example",
"fields":[
{
"name":"tag",
"type":"string"
},
{
"name":"field",
"type":"long"
},
{
"name":"timestamp",
"type":"long"
}
]
}
'''

1
plugins/parsers/avro/testcases/benchmark/expected.out Normal file
View file

@ -0,0 +1 @@
benchmark,source=myhost,tags_platform=python,tags_sdkver=3.11.5 value=5.0 1653643421000000000

7
plugins/parsers/avro/testcases/benchmark/message.json Normal file
View file

@ -0,0 +1,7 @@
{
"timestamp": 1653643421,
"value": 5,
"source": "myhost",
"tags_platform": "python",
"tags_sdkver": "3.11.5"
}

25
plugins/parsers/avro/testcases/benchmark/telegraf.conf Normal file
View file

@ -0,0 +1,25 @@
[[ inputs.file ]]
files = ["./testcases/benchmark/message.json"]
data_format = "avro"
avro_format = "json"
avro_measurement = "benchmark"
avro_tags = ["tags_platform", "tags_sdkver", "source"]
avro_fields = ["value"]
avro_timestamp = "timestamp"
avro_timestamp_format = "unix"
avro_schema = '''
{
"namespace": "com.benchmark",
"name": "benchmark",
"type": "record",
"version": "1",
"fields": [
{"name": "value", "type": "float", "doc": ""},
{"name": "timestamp", "type": "long", "doc": ""},
{"name": "tags_platform", "type": "string", "doc": ""},
{"name": "tags_sdkver", "type": "string", "default": "", "doc": ""},
{"name": "source", "type": "string", "default": "", "doc": ""}
]
}
'''

4
plugins/parsers/avro/testcases/config-both/expected.err Normal file
View file

@ -0,0 +1,4 @@
could not instantiate parser: exactly one of 'schema_registry' or 'schema' must be specified

0
plugins/parsers/avro/testcases/config-both/expected.out Normal file
View file

0
plugins/parsers/avro/testcases/config-both/message.avro Normal file
View file

28
plugins/parsers/avro/testcases/config-both/telegraf.conf Normal file
View file

@ -0,0 +1,28 @@
[[ inputs.file ]]
files = ["./testcases/config-both/message.avro"]
data_format = "avro"
avro_measurement = "measurement"
avro_tags = [ "tag" ]
avro_schema_registry = "https://localhost:8081"
avro_schema = '''
{
"type":"record",
"name":"Value",
"namespace":"com.example",
"fields":[
{
"name":"tag",
"type":"string"
},
{
"name":"field",
"type":"long"
},
{
"name":"timestamp",
"type":"long"
}
]
}
'''

2
plugins/parsers/avro/testcases/config-neither/expected.err Normal file
View file

@ -0,0 +1,2 @@
could not instantiate parser: exactly one of 'schema_registry' or 'schema' must be specified

0
plugins/parsers/avro/testcases/config-neither/expected.out Normal file
View file

0
plugins/parsers/avro/testcases/config-neither/message.avro Normal file
View file

5
plugins/parsers/avro/testcases/config-neither/telegraf.conf Normal file
View file

@ -0,0 +1,5 @@
[[ inputs.file ]]
files = ["./testcases/config-neither/message.avro"]
data_format = "avro"
avro_measurement = "measurement"
avro_tags = [ "tag" ]

1
plugins/parsers/avro/testcases/enum/expected.out Normal file
View file

@ -0,0 +1 @@
sensors,name=temperature value_int=42i,status="OK"

7
plugins/parsers/avro/testcases/enum/message.json Normal file
View file

@ -0,0 +1,7 @@
{
"name": "temperature",
"value": {
"int": 42
},
"status": "OK"
}

41
plugins/parsers/avro/testcases/enum/telegraf.conf Normal file
View file

@ -0,0 +1,41 @@
[[ inputs.file ]]
files = ["./testcases/enum/message.json"]
data_format = "avro"
avro_format = "json"
avro_measurement = "sensors"
avro_tags = ["name"]
avro_fields = ["value", "status"]
avro_field_separator = "_"
avro_schema = '''
{
"type": "record",
"name": "Metric",
"fields": [
{
"name": "name",
"type": "string"
},
{
"name": "value",
"type": [
"null",
"int",
"string"
]
},
{
"name": "status",
"type": {
"type": "enum",
"name": "Status",
"symbols": [
"UNKNOWN",
"OK",
"FAILURE"
]
}
}
]
}
'''

1
plugins/parsers/avro/testcases/json-array/expected.out Normal file
View file

@ -0,0 +1 @@
array,name=pi data_0=3,data_1=3.0999999046325684,data_2=3.140000104904175,data_3=3.1410000324249268 1682509200092000

5
plugins/parsers/avro/testcases/json-array/message.json Normal file
View file

@ -0,0 +1,5 @@
{
"statistics_collection_time": 1682509200092,
"data": [ 3, 3.1, 3.14, 3.141 ],
"name": "pi"
}

24
plugins/parsers/avro/testcases/json-array/telegraf.conf Normal file
View file

@ -0,0 +1,24 @@
[[ inputs.file ]]
files = ["./testcases/json-array/message.json"]
data_format = "avro"
avro_format = "json"
avro_measurement = "array"
avro_tags = ["name"]
avro_timestamp = "statistics_collection_time"
avro_timestamp_format = "unix_ms"
avro_fields = ["data"]
avro_field_separator = "_"
avro_schema = '''
{
"namespace": "constants",
"name": "classical",
"type": "record",
"version": "1",
"fields": [
{"name": "name", "type": "string"},
{"name": "data", "type": "array", "items": "float"},
{"name": "statistics_collection_time", "type": "long"}
]
}
'''

1
plugins/parsers/avro/testcases/json-format/expected.out Normal file
View file

@ -0,0 +1 @@
Switch,switch_wwn=10:00:50:EB:1A:0B:84:3A up_time=1166984904i,cpu_utilization=14.0,memory_utilization=20.0 1682509200092000

7
plugins/parsers/avro/testcases/json-format/message.json Normal file
View file

@ -0,0 +1,7 @@
{
"switch_wwn": "10:00:50:EB:1A:0B:84:3A",
"statistics_collection_time": 1682509200092,
"up_time": 1166984904,
"cpu_utilization": 14.0,
"memory_utilization": 20.0
}

25
plugins/parsers/avro/testcases/json-format/telegraf.conf Normal file
View file

@ -0,0 +1,25 @@
[[ inputs.file ]]
files = ["./testcases/json-format/message.json"]
data_format = "avro"
avro_format = "json"
avro_measurement = "Switch"
avro_tags = ["switch_wwn"]
avro_fields = ["up_time", "cpu_utilization", "memory_utilization"]
avro_timestamp = "statistics_collection_time"
avro_timestamp_format = "unix_ms"
avro_schema = '''
{
"namespace": "com.brocade.streaming",
"name": "fibrechannel_switch_statistics",
"type": "record",
"version": "1",
"fields": [
{"name": "switch_wwn", "type": "string", "doc": "WWN of the Physical Switch."},
{"name": "statistics_collection_time", "type": "long", "doc": "Epoch time when statistics is collected."},
{"name": "up_time", "type": "long", "doc": "Switch Up Time (in hundredths of a second)"},
{"name": "cpu_utilization", "type": "float", "default": 0, "doc": "CPU Utilization in %"},
{"name": "memory_utilization", "type": "float", "default": 0, "doc": "Memory Utilization in %"}
]
}
'''

1
plugins/parsers/avro/testcases/measurement_name_from_message/expected.out Normal file
View file

@ -0,0 +1 @@
cpu_load,Server=test_server Value=18.7 1694526986671

1
plugins/parsers/avro/testcases/measurement_name_from_message/message.avro Normal file
View file

@ -0,0 +1 @@
ÞæîšÑbtest_server33333³2@cpu_load

30
plugins/parsers/avro/testcases/measurement_name_from_message/telegraf.conf Normal file
View file

@ -0,0 +1,30 @@
[[ inputs.file ]]
files = ["./testcases/measurement_name_from_message/message.avro"]
data_format = "avro"
avro_measurement_field = "Measurement"
avro_tags = [ "Server" ]
avro_fields = [ "Value" ]
avro_schema = '''
{
"type": "record",
"name": "TestRecord",
"fields": [
{
"name": "ServerTs",
"type": "long"
},
{
"name": "Server",
"type": "string"
},
{
"name": "Value",
"type": "double"
},
{
"name": "Measurement",
"type": "string"
}
]
}
'''

1
plugins/parsers/avro/testcases/no-timestamp-format/expected.out Normal file
View file

@ -0,0 +1 @@
measurement,tag=test_tag field=19i,timestamp=1664296121000000i 1664296121000000

1
plugins/parsers/avro/testcases/no-timestamp-format/message.avro Normal file
View file

@ -0,0 +1 @@
test_tag&€<>¿±äêô

28
plugins/parsers/avro/testcases/no-timestamp-format/telegraf.conf Normal file
View file

@ -0,0 +1,28 @@
[[ inputs.file ]]
files = ["./testcases/no-timestamp-format/message.avro"]
data_format = "avro"
avro_measurement = "measurement"
avro_tags = [ "tag" ]
avro_timestamp = "timestamp"
avro_schema = '''
{
"type":"record",
"name":"Value",
"namespace":"com.example",
"fields":[
{
"name":"tag",
"type":"string"
},
{
"name":"field",
"type":"long"
},
{
"name":"timestamp",
"type":"long"
}
]
}
'''

1
plugins/parsers/avro/testcases/supplied_timestamp/expected.out Normal file
View file

@ -0,0 +1 @@
measurement,tag=test_tag field=19i,timestamp=1664296121000000i 1664296121000000

1
plugins/parsers/avro/testcases/supplied_timestamp/message.avro Normal file
View file

@ -0,0 +1 @@
test_tag&€<>¿±äêô

28
plugins/parsers/avro/testcases/supplied_timestamp/telegraf.conf Normal file
View file

@ -0,0 +1,28 @@
[[ inputs.file ]]
files = ["./testcases/supplied_timestamp/message.avro"]
data_format = "avro"
avro_measurement = "measurement"
avro_tags = [ "tag" ]
avro_timestamp = "timestamp"
avro_timestamp_format = "unix_us"
avro_schema = '''
{
"type":"record",
"name":"Value",
"namespace":"com.example",
"fields":[
{
"name":"tag",
"type":"string"
},
{
"name":"field",
"type":"long"
},
{
"name":"timestamp",
"type":"long"
}
]
}
'''

1
plugins/parsers/avro/testcases/supplied_timestamp_fields_specified/expected.out Normal file
View file

@ -0,0 +1 @@
measurement,tag=test_tag field=19i,timestamp=1664296121000000i 1664296121000000

1
plugins/parsers/avro/testcases/supplied_timestamp_fields_specified/message.avro Normal file
View file

@ -0,0 +1 @@
test_tag&€<>¿±äêô

29
plugins/parsers/avro/testcases/supplied_timestamp_fields_specified/telegraf.conf Normal file
View file

@ -0,0 +1,29 @@
[[ inputs.file ]]
files = ["./testcases/supplied_timestamp_fields_specified/message.avro"]
data_format = "avro"
avro_measurement = "measurement"
avro_tags = [ "tag" ]
avro_fields = [ "field", "timestamp"]
avro_timestamp = "timestamp"
avro_timestamp_format = "unix_us"
avro_schema = '''
{
"type":"record",
"name":"Value",
"namespace":"com.example",
"fields":[
{
"name":"tag",
"type":"string"
},
{
"name":"field",
"type":"long"
},
{
"name":"timestamp",
"type":"long"
}
]
}
'''

1
plugins/parsers/avro/testcases/supplied_timestamp_fields_unspecified/expected.out Normal file
View file

@ -0,0 +1 @@
measurement,tag=test_tag field=19i 1664296121000000

1
plugins/parsers/avro/testcases/supplied_timestamp_fields_unspecified/message.avro Normal file
View file

@ -0,0 +1 @@
test_tag&€<>¿±äêô

23
plugins/parsers/avro/testcases/supplied_timestamp_fields_unspecified/telegraf.conf Normal file
View file

@ -0,0 +1,23 @@
[[ inputs.file ]]
files = ["./testcases/supplied_timestamp_fields_unspecified/message.avro"]
data_format = "avro"
avro_measurement = "measurement"
avro_tags = [ "tag" ]
avro_fields = [ "field" ]
avro_schema = '''
{
"type":"record",
"name":"Value",
"namespace":"com.example",
"fields":[
{
"name":"tag",
"type":"string"
},
{
"name":"field",
"type":"long"
}
]
}
'''

1
plugins/parsers/avro/testcases/union-any/expected.out Normal file
View file

@ -0,0 +1 @@
Switch,switch_wwn=10:00:50:EB:1A:0B:84:3A statistics_collection_time=1682509200092i,up_time=1166984904i,cpu_utilization=11i,memory_utilization=20.0 1682509200092000

11
plugins/parsers/avro/testcases/union-any/message.json Normal file
View file

@ -0,0 +1,11 @@
{
"switch_wwn": "10:00:50:EB:1A:0B:84:3A",
"statistics_collection_time": 1682509200092,
"up_time": 1166984904,
"cpu_utilization": {
"int": 11
},
"memory_utilization": {
"float": 20.0
}
}

26
plugins/parsers/avro/testcases/union-any/telegraf.conf Normal file
View file

@ -0,0 +1,26 @@
[[ inputs.file ]]
files = ["./testcases/union-any/message.json"]
data_format = "avro"
avro_format = "json"
avro_measurement = "Switch"
avro_tags = ["switch_wwn"]
avro_fields = ["up_time", "cpu_utilization", "memory_utilization", "statistics_collection_time"]
avro_timestamp = "statistics_collection_time"
avro_timestamp_format = "unix_ms"
avro_union_mode = "any"
avro_schema = '''
{
"namespace": "com.brocade.streaming",
"name": "fibrechannel_switch_statistics",
"type": "record",
"version": "1",
"fields": [
{"name": "switch_wwn", "type": "string", "doc": "WWN of the Physical Switch."},
{"name": "statistics_collection_time", "type": "long", "doc": "Epoch time when statistics is collected."},
{"name": "up_time", "type": "long", "doc": "Switch Up Time (in hundredths of a second)"},
{"name": "cpu_utilization", "type": ["null", "float", "int"], "default": null, "doc": "CPU Utilization in %"},
{"name": "memory_utilization", "type": ["null", "float"], "doc": "Memory Utilization in %"}
]
}
'''

1
plugins/parsers/avro/testcases/union-array/expected.out Normal file
View file

@ -0,0 +1 @@
array,name=pi data_0=3,data_1=3.0999999046325684,data_2=3.140000104904175,data_3=3.1410000324249268 1682509200092000

5
plugins/parsers/avro/testcases/union-array/message.json Normal file
View file

@ -0,0 +1,5 @@
{
"statistics_collection_time": 1682509200092,
"data": [ 3, 3.1, 3.14, 3.141 ],
"name": "pi"
}

25
plugins/parsers/avro/testcases/union-array/telegraf.conf Normal file
View file

@ -0,0 +1,25 @@
[[ inputs.file ]]
files = ["./testcases/union-array/message.json"]
data_format = "avro"
avro_format = "json"
avro_measurement = "array"
avro_tags = ["name"]
avro_timestamp = "statistics_collection_time"
avro_timestamp_format = "unix_ms"
avro_fields = ["data"]
avro_union_mode = "any"
avro_field_separator = "_"
avro_schema = '''
{
"namespace": "constants",
"name": "classical",
"type": "record",
"version": "1",
"fields": [
{"name": "name", "type": "string"},
{"name": "data", "type": "array", "items": "float"},
{"name": "statistics_collection_time", "type": "long"}
]
}
'''

1
plugins/parsers/avro/testcases/union-nullable-tag/expected.out Normal file
View file

@ -0,0 +1 @@
Switch,switch_wwn=10:00:50:EB:1A:0B:84:3A,some_union_in_a_tag=some_value statistics_collection_time=1682509200092i,up_time=1166984904i,memory_utilization=20.0 1682509200092000

14
plugins/parsers/avro/testcases/union-nullable-tag/message.json Normal file
View file

@ -0,0 +1,14 @@
{
"some_union_in_a_tag": {
"string": "some_value"
},
"switch_wwn": "10:00:50:EB:1A:0B:84:3A",
"statistics_collection_time": 1682509200092,
"up_time": 1166984904,
"cpu_utilization": {
"null": null
},
"memory_utilization": {
"float": 20.0
}
}

27
plugins/parsers/avro/testcases/union-nullable-tag/telegraf.conf Normal file
View file

@ -0,0 +1,27 @@
[[ inputs.file ]]
files = ["./testcases/union-nullable-tag/message.json"]
data_format = "avro"
avro_format = "json"
avro_measurement = "Switch"
avro_tags = ["switch_wwn", "some_union_in_a_tag"]
avro_fields = ["up_time", "cpu_utilization", "memory_utilization", "statistics_collection_time"]
avro_timestamp = "statistics_collection_time"
avro_timestamp_format = "unix_ms"
avro_union_mode = "nullable"
avro_schema = '''
{
"namespace": "com.brocade.streaming",
"name": "fibrechannel_switch_statistics",
"type": "record",
"version": "1",
"fields": [
{"name": "some_union_in_a_tag", "type": ["null", "string"], "default": null, "doc": "Some union that is used in a tag"},
{"name": "switch_wwn", "type": "string", "doc": "WWN of the Physical Switch."},
{"name": "statistics_collection_time", "type": "long", "doc": "Epoch time when statistics is collected."},
{"name": "up_time", "type": "long", "doc": "Switch Up Time (in hundredths of a second)"},
{"name": "cpu_utilization", "type": ["null","float"], "default": null, "doc": "CPU Utilization in %"},
{"name": "memory_utilization", "type": ["null", "float"], "default": null, "doc": "Memory Utilization in %"}
]
}
'''

1
plugins/parsers/avro/testcases/union-nullable/expected.out Normal file
View file

@ -0,0 +1 @@
Switch,switch_wwn=10:00:50:EB:1A:0B:84:3A statistics_collection_time=1682509200092i,up_time=1166984904i,memory_utilization=20.0 1682509200092000

11
plugins/parsers/avro/testcases/union-nullable/message.json Normal file
View file

@ -0,0 +1,11 @@
{
"switch_wwn": "10:00:50:EB:1A:0B:84:3A",
"statistics_collection_time": 1682509200092,
"up_time": 1166984904,
"cpu_utilization": {
"null": null
},
"memory_utilization": {
"float": 20.0
}
}

26
plugins/parsers/avro/testcases/union-nullable/telegraf.conf Normal file
View file

@ -0,0 +1,26 @@
[[ inputs.file ]]
files = ["./testcases/union-nullable/message.json"]
data_format = "avro"
avro_format = "json"
avro_measurement = "Switch"
avro_tags = ["switch_wwn"]
avro_fields = ["up_time", "cpu_utilization", "memory_utilization", "statistics_collection_time"]
avro_timestamp = "statistics_collection_time"
avro_timestamp_format = "unix_ms"
avro_union_mode = "nullable"
avro_schema = '''
{
"namespace": "com.brocade.streaming",
"name": "fibrechannel_switch_statistics",
"type": "record",
"version": "1",
"fields": [
{"name": "switch_wwn", "type": "string", "doc": "WWN of the Physical Switch."},
{"name": "statistics_collection_time", "type": "long", "doc": "Epoch time when statistics is collected."},
{"name": "up_time", "type": "long", "doc": "Switch Up Time (in hundredths of a second)"},
{"name": "cpu_utilization", "type": ["null","float"], "default": null, "doc": "CPU Utilization in %"},
{"name": "memory_utilization", "type": ["null", "float"], "default": null, "doc": "Memory Utilization in %"}
]
}
'''

1
plugins/parsers/avro/testcases/union/expected.out Normal file
View file

@ -0,0 +1 @@
Switch,switch_wwn=10:00:50:EB:1A:0B:84:3A statistics_collection_time=1682509200092i,up_time=1166984904i,memory_utilization_float=20.0 1682509200092000

11
plugins/parsers/avro/testcases/union/message.json Normal file
View file

@ -0,0 +1,11 @@
{
"switch_wwn": "10:00:50:EB:1A:0B:84:3A",
"statistics_collection_time": 1682509200092,
"up_time": 1166984904,
"cpu_utilization": {
"null": null
},
"memory_utilization": {
"float": 20.0
}
}

26
plugins/parsers/avro/testcases/union/telegraf.conf Normal file
View file

@ -0,0 +1,26 @@
[[ inputs.file ]]
files = ["./testcases/union/message.json"]
data_format = "avro"
avro_format = "json"
avro_measurement = "Switch"
avro_tags = ["switch_wwn"]
avro_fields = ["up_time", "cpu_utilization", "memory_utilization", "statistics_collection_time"]
avro_timestamp = "statistics_collection_time"
avro_timestamp_format = "unix_ms"
avro_field_separator = "_"
avro_schema = '''
{
"namespace": "com.brocade.streaming",
"name": "fibrechannel_switch_statistics",
"type": "record",
"version": "1",
"fields": [
{"name": "switch_wwn", "type": "string", "doc": "WWN of the Physical Switch."},
{"name": "statistics_collection_time", "type": "long", "doc": "Epoch time when statistics is collected."},
{"name": "up_time", "type": "long", "doc": "Switch Up Time (in hundredths of a second)"},
{"name": "cpu_utilization", "type": ["null", "float"], "default": null, "doc": "CPU Utilization in %"},
{"name": "memory_utilization", "type": ["null", "float"], "default": null, "doc": "Memory Utilization in %"}
]
}
'''

350
plugins/parsers/binary/README.md Normal file
View file

@ -0,0 +1,350 @@
# Binary Parser Plugin
The `binary` data format parser parses binary protocols into metrics using
user-specified configurations.
## Configuration
```toml
[[inputs.file]]
files = ["example.bin"]
## Data format to consume.
## Each data format has its own unique set of configuration options, read
## more about them here:
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
data_format = "binary"
## Do not error-out if none of the filter expressions below matches.
# allow_no_match = false
## Specify the endianness of the data.
## Available values are "be" (big-endian), "le" (little-endian) and "host",
## where "host" means the same endianness as the machine running Telegraf.
# endianness = "host"
## Interpret input using the specified encoding
## Available values are "none" (raw bytes), "hex" and "base64"
# binary_encoding = "none"
## Multiple parsing sections are allowed
[[inputs.file.binary]]
## Optional: Metric (measurement) name to use if not extracted from the data.
# metric_name = "my_name"
## Definition of the message format and the extracted data.
## Please note that you need to define all elements of the data in the
## correct order with the correct length as the data is parsed in the order
## given.
## An entry can have the following properties:
## name -- Name of the element (e.g. field or tag). Can be omitted
## for special assignments (i.e. time & measurement) or if
## entry is omitted.
## type -- Data-type of the entry. Can be "int8/16/32/64", "uint8/16/32/64",
## "float32/64", "bool" and "string".
## In case of time, this can be any of "unix" (default), "unix_ms", "unix_us",
## "unix_ns" or a valid Golang time format.
## bits -- Length in bits for this entry. If omitted, the length derived from
## the "type" property will be used. For "time" 64-bit will be used
## as default.
## assignment -- Assignment of the gathered data. Can be "measurement", "time",
## "field" or "tag". If omitted "field" is assumed.
## omit -- Omit the given data. If true, the data is skipped and not added
## to the metric. Omitted entries only need a length definition
## via "bits" or "type".
## terminator -- Terminator for dynamic-length strings. Only used for "string" type.
## Valid values are "fixed" (fixed length string given by "bits"),
## "null" (null-terminated string) or a character sequence specified
## as HEX values (e.g. "0x0D0A"). Defaults to "fixed" for strings.
## timezone -- Timezone of "time" entries. Only applies to "time" assignments.
## Can be "utc", "local" or any valid Golang timezone (e.g. "Europe/Berlin")
entries = [
{ type = "string", assignment = "measurement", terminator = "null" },
{ name = "address", type = "uint16", assignment = "tag" },
{ name = "value", type = "float64" },
{ type = "unix", assignment = "time" },
]
## Optional: Filter evaluated before applying the configuration.
## This option can be used to mange multiple configuration specific for
## a certain message type. If no filter is given, the configuration is applied.
# [inputs.file.binary.filter]
# ## Filter message by the exact length in bytes (default: N/A).
# # length = 0
# ## Filter the message by a minimum length in bytes.
# ## Messages longer of equal length will pass.
# # length_min = 0
# ## List of data parts to match.
# ## Only if all selected parts match, the configuration will be
# ## applied. The "offset" is the start of the data to match in bits,
# ## "bits" is the length in bits and "match" is the value to match
# ## against. Non-byte boundaries are supported, data is always right-aligned.
# selection = [
# { offset = 0, bits = 8, match = "0x1F" },
# ]
#
#
```
In this configuration mode, you explicitly specify the field and tags you want
to scrape out of your data.
A configuration can contain multiple `binary` subsections for e.g. the file
plugin to process the binary data multiple times. This can be useful
(together with _filters_) to handle different message types.
__Please note__: The `filter` section needs to be placed __after__ the `entries`
definitions due to TOML constraints as otherwise the entries will be assigned
to the filter section.
### General options and remarks
#### `allow_no_match` (optional)
By specifying `allow_no_match` you allow the parser to silently ignore data
that does not match _any_ given configuration filter. This can be useful if
you only want to collect a subset of the available messages.
#### `endianness` (optional)
This specifies the endianness of the data. If not specified, the parser will
fallback to the "host" endianness, assuming that the message and Telegraf
machine share the same endianness.
Alternatively, you can explicitly specify big-endian format (`"be"`) or
little-endian format (`"le"`).
#### `binary_encoding` (optional)
If this option is not specified or set to `none`, the input data contains the
binary data as raw bytes. This is the default.
If set to `hex`, the input data is interpreted as a string containing
hex-encoded data like `C0 C7 21 A9`. The value is _case insensitive_ and can
handle spaces and prefixes like `0x` or `x`.
If set to `base64` the input data is interpreted as a string containing
padded base64 data `RDLAAA==`.
### Non-byte aligned value extraction
In both, `filter` and `entries` definitions, values can be extracted at non-byte
boundaries. You can for example extract 3-bit starting at bit-offset 8. In those
cases, the result will be masked and shifted such that the resulting byte-value
is _right_ aligned. In case your 3-bit are `101` the resulting byte value is
`0x05`.
This is especially important when specifying the `match` value in the filter
section.
### Entries definitions
The entry array specifies how to dissect the message into the measurement name,
the timestamp, tags and fields.
#### `measurement` specification
When setting the `assignment` to `"measurement"`, the extracted value
will be used as the metric name, overriding other specifications.
The `type` setting is assumed to be `"string"` and can be omitted similar
to the `name` option. See [`string` type handling](#string-type-handling)
for details and further options.
### `time` specification
When setting the `assignment` to `"time"`, the extracted value
will be used as the timestamp of the metric. By default the current
time will be used for all created metrics.
The `type` setting here contains the time-format can be set to `unix`,
`unix_ms`, `unix_us`, `unix_ns`, or an accepted
[Go "reference time"][time const]. Consult the Go [time][time parse]
package for details and additional examples on how to set the time format.
If `type` is omitted the `unix` format is assumed.
For the `unix` format and derivatives, the underlying value is assumed
to be a 64-bit integer. The `bits` setting can be used to specify other
length settings. All other time-formats assume a fixed-length `string`
value to be extracted. The length of the string is automatically
determined using the format setting in `type`.
The `timezone` setting allows to convert the extracted time to the
given value timezone. By default the time will be interpreted as `utc`.
Other valid values are `local`, i.e. the local timezone configured for
the machine, or valid timezone-specification e.g. `Europe/Berlin`.
### `tag` specification
When setting the `assignment` to `"tag"`, the extracted value
will be used as a tag. The `name` setting will be the name of the tag
and the `type` will default to `string`. When specifying other types,
the extracted value will first be interpreted as the given type and
then converted to `string`.
The `bits` setting can be used to specify the length of the data to
extract and is required for fixed-length `string` types.
### `field` specification
When setting the `assignment` to `"field"` or omitting the `assignment`
setting, the extracted value will be used as a field. The `name` setting
is used as the name of the field and the `type` as type of the field value.
The `bits` setting can be used to specify the length of the data to
extract. By default the length corresponding to `type` is used.
Please see the [string](#string-type-handling) and [bool](#bool-type-handling)
specific sections when using those types.
### `string` type handling
Strings are assumed to be fixed-length strings by default. In this case, the
`bits` setting is mandatory to specify the length of the string in _bit_.
To handle dynamic strings, the `terminator` setting can be used to specify
characters to terminate the string. The two named options, `fixed` and `null`
will specify fixed-length and null-terminated strings, respectively.
Any other setting will be interpreted as hexadecimal sequence of bytes
matching the end of the string. The termination-sequence is removed from
the result.
### `bool` type handling
By default `bool` types are assumed to be _one_ bit in length. You can
specify any other length by using the `bits` setting.
When interpreting values as booleans, any zero value will be `false`,
while any non-zero value will result in `true`.
### omitting data
Parts of the data can be omitted by setting `omit = true`. In this case,
you only need to specify the length of the chunk to omit by either using
the `type` or `bits` setting. All other options can be skipped.
### Filter definitions
Filters can be used to match the length or the content of the data against
a specified reference. See the [examples section](#examples) for details.
You can also check multiple parts of the message by specifying multiple
`section` entries for a filter. Each `section` is then matched separately.
All have to match to apply the configuration.
#### `length` and `length_min` options
Using the `length` option, the filter will check if the data to parse has
exactly the given number of _bytes_. Otherwise, the configuration will not
be applied.
Similarly, for `length_min` the data has to have _at least_ the given number
of _bytes_ to generate a match.
#### `selection` list
Selections can be used with or without length constraints to match the content
of the data. Here, the `offset` and `bits` properties will specify the start
and length of the data to check. Both values are in _bit_ allowing for non-byte
aligned value extraction. The extracted data will the be checked against the
given `match` value specified in HEX.
If multiple `selection` entries are specified _all_ of the selections must
match for the configuration to get applied.
## Examples
In the following example, we use a binary protocol with three different messages
in little-endian format
### Message A definition
```text
+--------+------+------+--------+--------+------------+--------------------+--------------------+
| ID | type | len | addr | count | failure | value | timestamp |
+--------+------+------+--------+--------+------------+--------------------+--------------------+
| 0x0201 | 0x0A | 0x18 | 0x7F01 | 0x2A00 | 0x00000000 | 0x6F1283C0CA210940 | 0x10D4DF6200000000 |
+--------+------+------+--------+--------+------------+--------------------+--------------------+
```
### Message B definition
```text
+--------+------+------+------------+
| ID | type | len | value |
+--------+------+------+------------+
| 0x0201 | 0x0B | 0x04 | 0xDEADC0DE |
+--------+------+------+------------+
```
### Message C definition
```text
+--------+------+------+------------+------------+--------------------+
| ID | type | len | value x | value y | timestamp |
+--------+------+------+------------+------------+--------------------+
| 0x0201 | 0x0C | 0x10 | 0x4DF82D40 | 0x5F305C08 | 0x10D4DF6200000000 |
+--------+------+------+------------+------------+--------------------+
```
All messages consists of a 4-byte header containing the _message type_
in the 3rd byte and a message specific body. To parse those messages
you can use the following configuration
```toml
[[inputs.file]]
files = ["messageA.bin", "messageB.bin", "messageC.bin"]
data_format = "binary"
endianness = "le"
[[inputs.file.binary]]
metric_name = "messageA"
entries = [
{ bits = 32, omit = true },
{ name = "address", type = "uint16", assignment = "tag" },
{ name = "count", type = "int16" },
{ name = "failure", type = "bool", bits = 32, assignment = "tag" },
{ name = "value", type = "float64" },
{ type = "unix", assignment = "time" },
]
[inputs.file.binary.filter]
selection = [{ offset = 16, bits = 8, match = "0x0A" }]
[[inputs.file.binary]]
metric_name = "messageB"
entries = [
{ bits = 32, omit = true },
{ name = "value", type = "uint32" },
]
[inputs.file.binary.filter]
selection = [{ offset = 16, bits = 8, match = "0x0B" }]
[[inputs.file.binary]]
metric_name = "messageC"
entries = [
{ bits = 32, omit = true },
{ name = "x", type = "float32" },
{ name = "y", type = "float32" },
{ type = "unix", assignment = "time" },
]
[inputs.file.binary.filter]
selection = [{ offset = 16, bits = 8, match = "0x0C" }]
```
The above configuration has one `[[inputs.file.binary]]` section per
message type and uses a filter in each of those sections to apply
the correct configuration by comparing the 3rd byte (containing
the message type). This will lead to the following output
```text
metricA,address=383,failure=false count=42i,value=3.1415 1658835984000000000
metricB value=3737169374i 1658847037000000000
metricC x=2.718280076980591,y=0.0000000000000000000000000000000006626070178575745 1658835984000000000
```
where `metricB` uses the parsing time as timestamp due to missing
information in the data. The other two metrics use the timestamp
derived from the data.
[time const]: https://golang.org/pkg/time/#pkg-constants
[time parse]: https://golang.org/pkg/time/#Parse

183
plugins/parsers/binary/config.go Normal file
View file

@ -0,0 +1,183 @@
package binary
import (
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/metric"
)
type BinaryPart struct {
Offset uint64 `toml:"offset"`
Bits uint64 `toml:"bits"`
Match string `toml:"match"`
val []byte
}
type Filter struct {
Selection []BinaryPart `toml:"selection"`
LengthMin uint64 `toml:"length_min"`
Length uint64 `toml:"length"`
}
type Config struct {
MetricName string `toml:"metric_name"`
Filter *Filter `toml:"filter"`
Entries []Entry `toml:"entries"`
}
func (c *Config) preprocess(defaultName string) error {
// Preprocess filter part
if c.Filter != nil {
if c.Filter.Length != 0 && c.Filter.LengthMin != 0 {
return errors.New("length and length_min cannot be used together")
}
var length uint64
for i, s := range c.Filter.Selection {
end := (s.Offset + s.Bits) / 8
if (s.Offset+s.Bits)%8 != 0 {
end++
}
if end > length {
length = end
}
var err error
s.val, err = hex.DecodeString(strings.TrimPrefix(s.Match, "0x"))
if err != nil {
return fmt.Errorf("decoding match %d failed: %w", i, err)
}
c.Filter.Selection[i] = s
}
if c.Filter.Length != 0 && length > c.Filter.Length {
return fmt.Errorf("filter length (%d) larger than constraint (%d)", length, c.Filter.Length)
}
if c.Filter.Length == 0 && length > c.Filter.LengthMin {
c.Filter.LengthMin = length
}
}
// Preprocess entries part
var hasField, hasMeasurement bool
defined := make(map[string]bool)
for i, e := range c.Entries {
if err := e.check(); err != nil {
return fmt.Errorf("entry %q (%d): %w", e.Name, i, err)
}
// Store the normalized entry
c.Entries[i] = e
if e.Omit {
continue
}
// Check for duplicate entries
key := e.Assignment + "_" + e.Name
if defined[key] {
return fmt.Errorf("multiple definitions of %q", e.Name)
}
defined[key] = true
hasMeasurement = hasMeasurement || e.Assignment == "measurement"
hasField = hasField || e.Assignment == "field"
}
if !hasMeasurement && c.MetricName == "" {
if defaultName == "" {
return errors.New("no metric name given")
}
c.MetricName = defaultName
}
if !hasField {
return errors.New("no field defined")
}
return nil
}
func (c *Config) matches(in []byte) bool {
// If no filter is given, just match everything
if c.Filter == nil {
return true
}
// Checking length constraints
length := uint64(len(in))
if c.Filter.Length != 0 && length != c.Filter.Length {
return false
}
if c.Filter.LengthMin != 0 && length < c.Filter.LengthMin {
return false
}
// Matching elements
for _, s := range c.Filter.Selection {
data, err := extractPart(in, s.Offset, s.Bits)
if err != nil {
return false
}
if len(data) != len(s.val) {
return false
}
for i, v := range data {
if v != s.val[i] {
return false
}
}
}
return true
}
func (c *Config) collect(in []byte, order binary.ByteOrder, defaultTime time.Time) (telegraf.Metric, error) {
t := defaultTime
name := c.MetricName
tags := make(map[string]string)
fields := make(map[string]interface{})
var offset uint64
for _, e := range c.Entries {
data, n, err := e.extract(in, offset)
if err != nil {
return nil, err
}
offset += n
switch e.Assignment {
case "measurement":
name = convertStringType(data)
case "field":
v, err := e.convertType(data, order)
if err != nil {
return nil, fmt.Errorf("field %q failed: %w", e.Name, err)
}
fields[e.Name] = v
case "tag":
raw, err := e.convertType(data, order)
if err != nil {
return nil, fmt.Errorf("tag %q failed: %w", e.Name, err)
}
v, err := internal.ToString(raw)
if err != nil {
return nil, fmt.Errorf("tag %q failed: %w", e.Name, err)
}
tags[e.Name] = v
case "time":
var err error
t, err = e.convertTimeType(data, order)
if err != nil {
return nil, fmt.Errorf("time failed: %w", err)
}
}
}
return metric.New(name, tags, fields, t), nil
}

310
plugins/parsers/binary/entry.go Normal file
View file

@ -0,0 +1,310 @@
package binary
import (
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"math"
"strings"
"time"
"github.com/influxdata/telegraf/internal"
)
type Entry struct {
Name string `toml:"name"`
Type string `toml:"type"`
Bits uint64 `toml:"bits"`
Omit bool `toml:"omit"`
Terminator string `toml:"terminator"`
Timezone string `toml:"timezone"`
Assignment string `toml:"assignment"`
termination []byte
location *time.Location
}
func (e *Entry) check() error {
// Normalize cases
e.Assignment = strings.ToLower(e.Assignment)
e.Terminator = strings.ToLower(e.Terminator)
if e.Assignment != "time" {
e.Type = strings.ToLower(e.Type)
}
// Handle omitted fields
if e.Omit {
if e.Bits == 0 && e.Type == "" {
return errors.New("neither type nor bits given")
}
if e.Bits == 0 {
bits, err := bitsForType(e.Type)
if err != nil {
return err
}
e.Bits = bits
}
return nil
}
// Set name for global options
if e.Assignment == "measurement" || e.Assignment == "time" {
e.Name = e.Assignment
}
// Check the name
if e.Name == "" {
return errors.New("missing name")
}
// Check the assignment
var defaultType string
switch e.Assignment {
case "measurement":
defaultType = "string"
if e.Type != "string" && e.Type != "" {
return errors.New("'measurement' type has to be 'string'")
}
case "time":
bits := uint64(64)
switch e.Type {
// Make 'unix' the default
case "":
defaultType = "unix"
// Special plugin specific names
case "unix", "unix_ms", "unix_us", "unix_ns":
// Format-specification string formats
default:
bits = uint64(len(e.Type) * 8)
}
if e.Bits == 0 {
e.Bits = bits
}
switch e.Timezone {
case "", "utc":
// Make UTC the default
e.location = time.UTC
case "local":
e.location = time.Local
default:
var err error
e.location, err = time.LoadLocation(e.Timezone)
if err != nil {
return err
}
}
case "tag":
defaultType = "string"
case "", "field":
e.Assignment = "field"
default:
return fmt.Errorf("no assignment for %q", e.Name)
}
// Check type (special type for "time")
switch e.Type {
case "uint8", "int8", "uint16", "int16", "uint32", "int32", "uint64", "int64":
fallthrough
case "float32", "float64":
bits, err := bitsForType(e.Type)
if err != nil {
return err
}
if e.Bits == 0 {
e.Bits = bits
}
if bits < e.Bits {
return fmt.Errorf("type overflow for %q", e.Name)
}
case "bool":
if e.Bits == 0 {
e.Bits = 1
}
case "string":
// Check termination
switch e.Terminator {
case "", "fixed":
e.Terminator = "fixed"
if e.Bits == 0 {
return fmt.Errorf("require 'bits' for fixed-length string for %q", e.Name)
}
case "null":
e.termination = []byte{0}
if e.Bits != 0 {
return fmt.Errorf("cannot use 'bits' and 'null' terminator together for %q", e.Name)
}
default:
if e.Bits != 0 {
return fmt.Errorf("cannot use 'bits' and terminator together for %q", e.Name)
}
var err error
e.termination, err = hex.DecodeString(strings.TrimPrefix(e.Terminator, "0x"))
if err != nil {
return fmt.Errorf("decoding terminator failed for %q: %w", e.Name, err)
}
}
// We can only handle strings that adhere to byte-bounds
if e.Bits%8 != 0 {
return fmt.Errorf("non-byte length for string field %q", e.Name)
}
case "":
if defaultType == "" {
return fmt.Errorf("no type for %q", e.Name)
}
e.Type = defaultType
default:
if e.Assignment != "time" {
return fmt.Errorf("unknown type for %q", e.Name)
}
}
return nil
}
func (e *Entry) extract(in []byte, offset uint64) ([]byte, uint64, error) {
if e.Bits > 0 {
data, err := extractPart(in, offset, e.Bits)
return data, e.Bits, err
}
if e.Type != "string" {
return nil, 0, fmt.Errorf("unexpected entry: %v", e)
}
inbits := uint64(len(in)) * 8
// Read up to the termination
var found bool
var data []byte
var termOffset int
var n uint64
for offset+n+8 <= inbits {
buf, err := extractPart(in, offset+n, 8)
if err != nil {
return nil, 0, err
}
if len(buf) != 1 {
return nil, 0, fmt.Errorf("unexpected length %d", len(buf))
}
data = append(data, buf[0])
n += 8
// Check for terminator
if buf[0] == e.termination[termOffset] {
termOffset++
}
if termOffset == len(e.termination) {
found = true
break
}
}
if !found {
return nil, n, fmt.Errorf("terminator not found for %q", e.Name)
}
// Strip the terminator
return data[:len(data)-len(e.termination)], n, nil
}
func (e *Entry) convertType(in []byte, order binary.ByteOrder) (interface{}, error) {
switch e.Type {
case "uint8", "int8", "uint16", "int16", "uint32", "int32", "float32", "uint64", "int64", "float64":
return convertNumericType(in, e.Type, order)
case "bool":
return convertBoolType(in), nil
case "string":
return convertStringType(in), nil
}
return nil, fmt.Errorf("cannot handle type %q", e.Type)
}
func (e *Entry) convertTimeType(in []byte, order binary.ByteOrder) (time.Time, error) {
factor := int64(1)
switch e.Type {
case "unix":
factor *= 1000
fallthrough
case "unix_ms":
factor *= 1000
fallthrough
case "unix_us":
factor *= 1000
fallthrough
case "unix_ns":
raw, err := convertNumericType(in, "int64", order)
if err != nil {
return time.Unix(0, 0), err
}
v := raw.(int64)
return time.Unix(0, v*factor).In(e.location), nil
}
// We have a format specification (hopefully)
v := convertStringType(in)
return internal.ParseTimestamp(e.Type, v, e.location)
}
func convertStringType(in []byte) string {
return string(in)
}
func convertNumericType(in []byte, t string, order binary.ByteOrder) (interface{}, error) {
bits, err := bitsForType(t)
if err != nil {
return nil, err
}
inlen := uint64(len(in))
expected := bits / 8
if inlen > expected {
// Should never happen
return 0, fmt.Errorf("too many bytes %d vs %d", len(in), expected)
}
// Pad the data if shorter than the datatype length
buf := make([]byte, expected-inlen, expected)
buf = append(buf, in...)
switch t {
case "uint8":
return buf[0], nil
case "int8":
return int8(buf[0]), nil
case "uint16":
return order.Uint16(buf), nil
case "int16":
v := order.Uint16(buf)
return int16(v), nil
case "uint32":
return order.Uint32(buf), nil
case "int32":
v := order.Uint32(buf)
return int32(v), nil
case "uint64":
return order.Uint64(buf), nil
case "int64":
v := order.Uint64(buf)
return int64(v), nil
case "float32":
v := order.Uint32(buf)
return math.Float32frombits(v), nil
case "float64":
v := order.Uint64(buf)
return math.Float64frombits(v), nil
}
return nil, fmt.Errorf("no numeric type %q", t)
}
func convertBoolType(in []byte) bool {
for _, x := range in {
if x != 0 {
return true
}
}
return false
}

44
plugins/parsers/binary/entry_test.go Normal file
View file

@ -0,0 +1,44 @@
package binary
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf/internal"
)
func TestEntryExtract(t *testing.T) {
testdata := []byte{0x01, 0x02, 0x03, 0x04}
e := &Entry{Type: "uint64"}
_, _, err := e.extract(testdata, 0)
require.EqualError(t, err, `unexpected entry: &{ uint64 0 false [] <nil>}`)
}
func TestEntryConvertType(t *testing.T) {
testdata := []byte{0x01, 0x02, 0x03, 0x04}
e := &Entry{Type: "garbage"}
_, err := e.convertType(testdata, internal.HostEndianness)
require.EqualError(t, err, `cannot handle type "garbage"`)
}
func TestEntryConvertTimeType(t *testing.T) {
testdata := []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09}
e := &Entry{Type: "unix_ns", location: time.UTC}
_, err := e.convertTimeType(testdata, internal.HostEndianness)
require.EqualError(t, err, `too many bytes 9 vs 8`)
}
func TestConvertNumericType(t *testing.T) {
testdata := []byte{0x01, 0x02, 0x03, 0x04}
_, err := convertNumericType(testdata, "garbage", internal.HostEndianness)
require.EqualError(t, err, `cannot determine length for type "garbage"`)
_, err = convertNumericType(testdata, "uint8", internal.HostEndianness)
require.EqualError(t, err, `too many bytes 4 vs 1`)
}

213
plugins/parsers/binary/parser.go Normal file
View file

@ -0,0 +1,213 @@
package binary
import (
"encoding/base64"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"strings"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/parsers"
)
type Parser struct {
AllowNoMatch bool `toml:"allow_no_match"`
Endianess string `toml:"endianess" deprecated:"1.27.4;1.35.0;use 'endianness' instead"`
Endianness string `toml:"endianness"`
Configs []Config `toml:"binary"`
HexEncoding bool `toml:"hex_encoding" deprecated:"1.30.0;1.35.0;use 'binary_encoding' instead"`
Encoding string `toml:"binary_encoding"`
Log telegraf.Logger `toml:"-"`
metricName string
defaultTags map[string]string
converter binary.ByteOrder
}
func (p *Parser) Init() error {
// Keep backward compatibility
if p.Endianess != "" && p.Endianness == "" {
p.Endianness = p.Endianess
}
if p.HexEncoding {
if p.Encoding != "" && p.Encoding != "hex" {
return errors.New("conflicting settings between 'hex_encoding' and 'binary_encoding'")
}
p.Encoding = "hex"
}
switch p.Endianness {
case "le":
p.converter = binary.LittleEndian
case "be":
p.converter = binary.BigEndian
case "", "host":
p.converter = internal.HostEndianness
default:
return fmt.Errorf("unknown endianness %q", p.Endianness)
}
switch p.Encoding {
case "", "none", "hex", "base64":
default:
return fmt.Errorf("unknown encoding %q", p.Encoding)
}
// Pre-process the configurations
if len(p.Configs) == 0 {
return errors.New("no configuration given")
}
for i, cfg := range p.Configs {
if err := cfg.preprocess(p.metricName); err != nil {
return fmt.Errorf("config %d invalid: %w", i, err)
}
p.Configs[i] = cfg
}
return nil
}
func (p *Parser) Parse(data []byte) ([]telegraf.Metric, error) {
t := time.Now()
// If the data is encoded in HEX, we need to decode it first
buf := data
switch p.Encoding {
case "hex":
s := strings.TrimPrefix(string(data), "0x")
s = strings.TrimPrefix(s, "x")
s = strings.TrimSpace(s)
s = strings.ReplaceAll(s, " ", "")
s = strings.ReplaceAll(s, "\t", "")
var err error
buf, err = hex.DecodeString(s)
if err != nil {
return nil, fmt.Errorf("decoding hex failed: %w", err)
}
case "base64":
decoder := base64.StdEncoding.WithPadding(base64.StdPadding)
var err error
buf, err = decoder.DecodeString(strings.TrimSpace(string(data)))
if err != nil {
return nil, fmt.Errorf("decoding base64 failed: %w", err)
}
}
matches := 0
metrics := make([]telegraf.Metric, 0)
for i, cfg := range p.Configs {
// Apply the filter and see if we should match this
if !cfg.matches(buf) {
p.Log.Debugf("ignoring data in config %d", i)
continue
}
matches++
// Collect the metric
m, err := cfg.collect(buf, p.converter, t)
if err != nil {
return nil, err
}
metrics = append(metrics, m)
}
if matches == 0 && !p.AllowNoMatch {
return nil, errors.New("no matching configuration")
}
return metrics, nil
}
func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
metrics, err := p.Parse([]byte(line))
if err != nil {
return nil, err
}
switch len(metrics) {
case 0:
return nil, nil
case 1:
return metrics[0], nil
default:
return metrics[0], fmt.Errorf("cannot parse line with multiple (%d) metrics", len(metrics))
}
}
func (p *Parser) SetDefaultTags(tags map[string]string) {
p.defaultTags = tags
}
func init() {
// Register all variants
parsers.Add("binary",
func(defaultMetricName string) telegraf.Parser {
return &Parser{metricName: defaultMetricName}
},
)
}
func extractPart(in []byte, offset, bits uint64) ([]byte, error) {
inLen := uint64(len(in))
start := offset / 8
bitend := offset%8 + bits
length := bitend / 8
if bitend%8 != 0 {
length++
}
if start+length > inLen {
return nil, fmt.Errorf("out-of-bounds @%d with %d bits", offset, bits)
}
var out []byte
out = append(out, in[start:start+length]...)
if offset%8 != 0 {
// Mask the start-byte with the non-aligned bit-mask
startmask := (byte(1) << (8 - offset%8)) - 1
out[0] = out[0] & startmask
}
if bitend%8 == 0 {
// The end is aligned to byte-boundaries
return out, nil
}
shift := 8 - bitend%8
carryshift := bitend % 8
// We need to shift right in case of not ending at a byte boundary
// to make the bits right aligned.
// Carry over the bits from the byte left to fill in...
var carry byte
for i, x := range out {
out[i] = (x >> shift) | carry
carry = x << carryshift
}
if bits%8 == 0 {
// Avoid an empty leading byte
return out[1:], nil
}
return out, nil
}
func bitsForType(t string) (uint64, error) {
switch t {
case "uint8", "int8":
return 8, nil
case "uint16", "int16":
return 16, nil
case "uint32", "int32", "float32":
return 32, nil
case "uint64", "int64", "float64":
return 64, nil
}
return 0, fmt.Errorf("cannot determine length for type %q", t)
}

1614
plugins/parsers/binary/parser_test.go Normal file
View file

File diff suppressed because it is too large Load diff

3
plugins/parsers/binary/testcases/base64_encoding/expected.out Normal file
View file

@ -0,0 +1,3 @@
test value=715
test value=208.5
test value=0.471

1
plugins/parsers/binary/testcases/base64_encoding/messageA.bin Normal file
View file

@ -0,0 +1 @@
RDLAAA==

1
plugins/parsers/binary/testcases/base64_encoding/messageB.bin Normal file
View file

@ -0,0 +1 @@
Q1CAAA==

1
plugins/parsers/binary/testcases/base64_encoding/messageC.bin Normal file
View file

@ -0,0 +1 @@
PvEm6Q==

8
plugins/parsers/binary/testcases/base64_encoding/telegraf.conf Normal file
View file

@ -0,0 +1,8 @@
[[inputs.test]]
files = ["messageA.bin", "messageB.bin", "messageC.bin"]
data_format = "binary"
endianness = "be"
binary_encoding = "base64"
[[inputs.test.binary]]
entries = [{ name = "value", type = "float32" }]

3
plugins/parsers/binary/testcases/hex_encoding/expected.out Normal file
View file

@ -0,0 +1,3 @@
test value=715
test value=208.5
test value=0.471

1
plugins/parsers/binary/testcases/hex_encoding/messageA.bin Normal file
View file

@ -0,0 +1 @@
0x4432c000

1
plugins/parsers/binary/testcases/hex_encoding/messageB.bin Normal file
View file

@ -0,0 +1 @@
0x43508000

1
plugins/parsers/binary/testcases/hex_encoding/messageC.bin Normal file
View file

@ -0,0 +1 @@
0x3ef126e9

8
plugins/parsers/binary/testcases/hex_encoding/telegraf.conf Normal file
View file

@ -0,0 +1,8 @@
[[inputs.test]]
files = ["messageA.bin", "messageB.bin", "messageC.bin"]
data_format = "binary"
endianness = "be"
binary_encoding = "hex"
[[inputs.test.binary]]
entries = [{ name = "value", type = "float32" }]

3
plugins/parsers/binary/testcases/multiple_messages/expected.out Normal file
View file

@ -0,0 +1,3 @@
metricA,address=383,failure=false count=42i,value=3.1415 1658835984000000000
metricB value=3737169374u 1658835984000000000
metricC x=2.718280076980591,y=0.0000000000000000000000000000000006626070178575745 1658835984000000000

BIN
plugins/parsers/binary/testcases/multiple_messages/messageA.bin Normal file
View file

Binary file not shown.

BIN
plugins/parsers/binary/testcases/multiple_messages/messageB.bin Normal file
View file

Binary file not shown.

Some files were not shown because too many files have changed in this diff Show more