Adding upstream version 1.34.4.
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
parent
e393c3af3f
commit
4978089aab
GPG key ID:
FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions
33
plugins/parsers/logfmt/README.md
Normal file
33
plugins/parsers/logfmt/README.md
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# Logfmt Parser Plugin
|
||||
|
||||
The `logfmt` data format parses data in [logfmt] format.
|
||||
|
||||
[logfmt]: https://brandur.org/logfmt
|
||||
|
||||
## Configuration
|
||||
|
||||
```toml
|
||||
[[inputs.file]]
|
||||
files = ["example"]
|
||||
|
||||
## Data format to consume.
|
||||
## Each data format has its own unique set of configuration options, read
|
||||
## more about them here:
|
||||
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
|
||||
data_format = "logfmt"
|
||||
|
||||
## Array of key names which should be collected as tags. Globs accepted.
|
||||
logfmt_tag_keys = ["method","host"]
|
||||
```
|
||||
|
||||
## Metrics
|
||||
|
||||
Each key/value pair in the line is added to a new metric as a field. The type
|
||||
of the field is automatically determined based on the contents of the value.
|
||||
|
||||
## Examples
|
||||
|
||||
```text
|
||||
- method=GET host=example.org ts=2018-07-24T19:43:40.275Z connect=4ms service=8ms status=200 bytes=1653
|
||||
+ logfmt,host=example.org,method=GET ts="2018-07-24T19:43:40.275Z",connect="4ms",service="8ms",status=200i,bytes=1653i
|
||||
```
|
||||
126
plugins/parsers/logfmt/parser.go
Normal file
126
plugins/parsers/logfmt/parser.go
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
package logfmt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-logfmt/logfmt"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/filter"
|
||||
"github.com/influxdata/telegraf/metric"
|
||||
"github.com/influxdata/telegraf/plugins/parsers"
|
||||
)
|
||||
|
||||
var ErrNoMetric = errors.New("no metric in line")
|
||||
|
||||
// Parser decodes logfmt formatted messages into metrics.
|
||||
type Parser struct {
|
||||
TagKeys []string `toml:"logfmt_tag_keys"`
|
||||
DefaultTags map[string]string `toml:"-"`
|
||||
|
||||
metricName string
|
||||
tagFilter filter.Filter
|
||||
}
|
||||
|
||||
// Parse converts a slice of bytes in logfmt format to metrics.
|
||||
func (p *Parser) Parse(b []byte) ([]telegraf.Metric, error) {
|
||||
reader := bytes.NewReader(b)
|
||||
decoder := logfmt.NewDecoder(reader)
|
||||
metrics := make([]telegraf.Metric, 0)
|
||||
for {
|
||||
ok := decoder.ScanRecord()
|
||||
if !ok {
|
||||
err := decoder.Err()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
fields := make(map[string]interface{})
|
||||
tags := make(map[string]string)
|
||||
for decoder.ScanKeyval() {
|
||||
if len(decoder.Value()) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// type conversions
|
||||
value := string(decoder.Value())
|
||||
if p.tagFilter != nil && p.tagFilter.Match(string(decoder.Key())) {
|
||||
tags[string(decoder.Key())] = value
|
||||
} else if iValue, err := strconv.ParseInt(value, 10, 64); err == nil {
|
||||
fields[string(decoder.Key())] = iValue
|
||||
} else if fValue, err := strconv.ParseFloat(value, 64); err == nil {
|
||||
fields[string(decoder.Key())] = fValue
|
||||
} else if bValue, err := strconv.ParseBool(value); err == nil {
|
||||
fields[string(decoder.Key())] = bValue
|
||||
} else {
|
||||
fields[string(decoder.Key())] = value
|
||||
}
|
||||
}
|
||||
if len(fields) == 0 && len(tags) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
m := metric.New(p.metricName, tags, fields, time.Now())
|
||||
|
||||
metrics = append(metrics, m)
|
||||
}
|
||||
p.applyDefaultTags(metrics)
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
// ParseLine converts a single line of text in logfmt format to metrics.
|
||||
func (p *Parser) ParseLine(s string) (telegraf.Metric, error) {
|
||||
metrics, err := p.Parse([]byte(s))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(metrics) < 1 {
|
||||
return nil, ErrNoMetric
|
||||
}
|
||||
return metrics[0], nil
|
||||
}
|
||||
|
||||
// SetDefaultTags adds tags to the metrics outputs of Parse and ParseLine.
|
||||
func (p *Parser) SetDefaultTags(tags map[string]string) {
|
||||
p.DefaultTags = tags
|
||||
}
|
||||
|
||||
func (p *Parser) applyDefaultTags(metrics []telegraf.Metric) {
|
||||
if len(p.DefaultTags) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for _, m := range metrics {
|
||||
for k, v := range p.DefaultTags {
|
||||
if !m.HasTag(k) {
|
||||
m.AddTag(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Parser) Init() error {
|
||||
var err error
|
||||
|
||||
// Compile tag key patterns
|
||||
if p.tagFilter, err = filter.Compile(p.TagKeys); err != nil {
|
||||
return fmt.Errorf("error compiling tag pattern: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Register parser
|
||||
parsers.Add("logfmt",
|
||||
func(defaultMetricName string) telegraf.Parser {
|
||||
return &Parser{metricName: defaultMetricName}
|
||||
},
|
||||
)
|
||||
}
|
||||
334
plugins/parsers/logfmt/parser_test.go
Normal file
334
plugins/parsers/logfmt/parser_test.go
Normal file
|
|
@ -0,0 +1,334 @@
|
|||
package logfmt
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/metric"
|
||||
"github.com/influxdata/telegraf/testutil"
|
||||
)
|
||||
|
||||
func TestParse(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
measurement string
|
||||
bytes []byte
|
||||
want []telegraf.Metric
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "no bytes returns no metrics",
|
||||
},
|
||||
{
|
||||
name: "test without trailing end",
|
||||
bytes: []byte("foo=\"bar\""),
|
||||
measurement: "testlog",
|
||||
want: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"testlog",
|
||||
map[string]string{},
|
||||
map[string]interface{}{
|
||||
"foo": "bar",
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "test with trailing end",
|
||||
bytes: []byte("foo=\"bar\"\n"),
|
||||
measurement: "testlog",
|
||||
want: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"testlog",
|
||||
map[string]string{},
|
||||
map[string]interface{}{
|
||||
"foo": "bar",
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "logfmt parser returns all the fields",
|
||||
bytes: []byte(`ts=2018-07-24T19:43:40.275Z lvl=info msg="http request" method=POST`),
|
||||
measurement: "testlog",
|
||||
want: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"testlog",
|
||||
map[string]string{},
|
||||
map[string]interface{}{
|
||||
"lvl": "info",
|
||||
"msg": "http request",
|
||||
"method": "POST",
|
||||
"ts": "2018-07-24T19:43:40.275Z",
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "logfmt parser parses every line",
|
||||
bytes: []byte(
|
||||
"ts=2018-07-24T19:43:40.275Z lvl=info msg=\"http request\" method=POST\nparent_id=088876RL000 duration=7.45 log_id=09R4e4Rl000",
|
||||
),
|
||||
measurement: "testlog",
|
||||
want: []telegraf.Metric{
|
||||
testutil.MustMetric(
|
||||
"testlog",
|
||||
map[string]string{},
|
||||
map[string]interface{}{
|
||||
"lvl": "info",
|
||||
"msg": "http request",
|
||||
"method": "POST",
|
||||
"ts": "2018-07-24T19:43:40.275Z",
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
testutil.MustMetric(
|
||||
"testlog",
|
||||
map[string]string{},
|
||||
map[string]interface{}{
|
||||
"parent_id": "088876RL000",
|
||||
"duration": 7.45,
|
||||
"log_id": "09R4e4Rl000",
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "keys without = or values are ignored",
|
||||
bytes: []byte(`i am no data.`),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "keys without values are ignored",
|
||||
bytes: []byte(`foo="" bar=`),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "unterminated quote produces error",
|
||||
measurement: "testlog",
|
||||
bytes: []byte(`bar=baz foo="bar`),
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "malformed key",
|
||||
measurement: "testlog",
|
||||
bytes: []byte(`"foo=" bar=baz`),
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
l := Parser{
|
||||
metricName: tt.measurement,
|
||||
}
|
||||
got, err := l.Parse(tt.bytes)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Logfmt.Parse error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
testutil.RequireMetricsEqual(t, tt.want, got, testutil.IgnoreTime())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLine(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
s string
|
||||
measurement string
|
||||
want telegraf.Metric
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "No Metric In line",
|
||||
want: nil,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Log parser fmt returns all fields",
|
||||
measurement: "testlog",
|
||||
s: `ts=2018-07-24T19:43:35.207268Z lvl=5 msg="Write failed" log_id=09R4e4Rl000`,
|
||||
want: testutil.MustMetric(
|
||||
"testlog",
|
||||
map[string]string{},
|
||||
map[string]interface{}{
|
||||
"ts": "2018-07-24T19:43:35.207268Z",
|
||||
"lvl": int64(5),
|
||||
"msg": "Write failed",
|
||||
"log_id": "09R4e4Rl000",
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "ParseLine only returns metrics from first string",
|
||||
measurement: "testlog",
|
||||
s: "ts=2018-07-24T19:43:35.207268Z lvl=5 msg=\"Write failed\" log_id=09R4e4Rl000\nmethod=POST " +
|
||||
"parent_id=088876RL000 duration=7.45 log_id=09R4e4Rl000",
|
||||
want: testutil.MustMetric(
|
||||
"testlog",
|
||||
map[string]string{},
|
||||
map[string]interface{}{
|
||||
"ts": "2018-07-24T19:43:35.207268Z",
|
||||
"lvl": int64(5),
|
||||
"msg": "Write failed",
|
||||
"log_id": "09R4e4Rl000",
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
l := Parser{
|
||||
metricName: tt.measurement,
|
||||
}
|
||||
got, err := l.ParseLine(tt.s)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("Logfmt.Parse error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
testutil.RequireMetricEqual(t, tt.want, got, testutil.IgnoreTime())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTags(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
measurement string
|
||||
tagKeys []string
|
||||
s string
|
||||
want telegraf.Metric
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "logfmt parser returns tags and fields",
|
||||
measurement: "testlog",
|
||||
tagKeys: []string{"lvl"},
|
||||
s: "ts=2018-07-24T19:43:40.275Z lvl=info msg=\"http request\" method=POST",
|
||||
want: testutil.MustMetric(
|
||||
"testlog",
|
||||
map[string]string{
|
||||
"lvl": "info",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"msg": "http request",
|
||||
"method": "POST",
|
||||
"ts": "2018-07-24T19:43:40.275Z",
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "logfmt parser returns no empty metrics",
|
||||
measurement: "testlog",
|
||||
tagKeys: []string{"lvl"},
|
||||
s: "lvl=info",
|
||||
want: testutil.MustMetric(
|
||||
"testlog",
|
||||
map[string]string{
|
||||
"lvl": "info",
|
||||
},
|
||||
map[string]interface{}{},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "logfmt parser returns all keys as tag",
|
||||
measurement: "testlog",
|
||||
tagKeys: []string{"*"},
|
||||
s: "ts=2018-07-24T19:43:40.275Z lvl=info msg=\"http request\" method=POST",
|
||||
want: testutil.MustMetric(
|
||||
"testlog",
|
||||
map[string]string{
|
||||
"lvl": "info",
|
||||
"msg": "http request",
|
||||
"method": "POST",
|
||||
"ts": "2018-07-24T19:43:40.275Z",
|
||||
},
|
||||
map[string]interface{}{},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
l := &Parser{
|
||||
metricName: tt.measurement,
|
||||
DefaultTags: map[string]string{},
|
||||
TagKeys: tt.tagKeys,
|
||||
}
|
||||
require.NoError(t, l.Init())
|
||||
|
||||
got, err := l.ParseLine(tt.s)
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
testutil.RequireMetricEqual(t, tt.want, got, testutil.IgnoreTime())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const benchmarkData = `tags_host=myhost tags_platform=python tags_sdkver=3.11.5 value=5
|
||||
tags_host=myhost tags_platform=python tags_sdkver=3.11.4 value=4
|
||||
`
|
||||
|
||||
func TestBenchmarkData(t *testing.T) {
|
||||
plugin := &Parser{
|
||||
TagKeys: []string{"tags_host", "tags_platform", "tags_sdkver"},
|
||||
}
|
||||
require.NoError(t, plugin.Init())
|
||||
|
||||
expected := []telegraf.Metric{
|
||||
metric.New(
|
||||
"",
|
||||
map[string]string{
|
||||
"tags_host": "myhost",
|
||||
"tags_platform": "python",
|
||||
"tags_sdkver": "3.11.5",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"value": 5,
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
metric.New(
|
||||
"",
|
||||
map[string]string{
|
||||
"tags_host": "myhost",
|
||||
"tags_platform": "python",
|
||||
"tags_sdkver": "3.11.4",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"value": 4,
|
||||
},
|
||||
time.Unix(0, 0),
|
||||
),
|
||||
}
|
||||
|
||||
actual, err := plugin.Parse([]byte(benchmarkData))
|
||||
require.NoError(t, err)
|
||||
testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime(), testutil.SortMetrics())
|
||||
}
|
||||
|
||||
func BenchmarkParsing(b *testing.B) {
|
||||
plugin := &Parser{
|
||||
TagKeys: []string{"tags_host", "tags_platform", "tags_sdkver"},
|
||||
}
|
||||
require.NoError(b, plugin.Init())
|
||||
|
||||
for n := 0; n < b.N; n++ {
|
||||
//nolint:errcheck // Benchmarking so skip the error check to avoid the unnecessary operations
|
||||
plugin.Parse([]byte(benchmarkData))
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue