515 lines
13 KiB
Go
515 lines
13 KiB
Go
|
package json
|
||
|
|
|
||
|
|
import (
|
||
|
|
"encoding/json"
|
||
|
|
"fmt"
|
||
|
|
"math"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/influxdata/toml"
|
||
|
|
"github.com/stretchr/testify/require"
|
||
|
|
|
||
|
|
"github.com/influxdata/telegraf"
|
||
|
|
"github.com/influxdata/telegraf/config"
|
||
|
|
"github.com/influxdata/telegraf/metric"
|
||
|
|
"github.com/influxdata/telegraf/plugins/parsers/influx"
|
||
|
|
"github.com/influxdata/telegraf/plugins/serializers"
|
||
|
|
"github.com/influxdata/telegraf/testutil"
|
||
|
|
)
|
||
|
|
|
||
|
|
func TestSerializeMetricFloat(t *testing.T) {
|
||
|
|
now := time.Now()
|
||
|
|
tags := map[string]string{
|
||
|
|
"cpu": "cpu0",
|
||
|
|
}
|
||
|
|
fields := map[string]interface{}{
|
||
|
|
"usage_idle": float64(91.5),
|
||
|
|
}
|
||
|
|
m := metric.New("cpu", tags, fields, now)
|
||
|
|
|
||
|
|
s := Serializer{}
|
||
|
|
require.NoError(t, s.Init())
|
||
|
|
buf, err := s.Serialize(m)
|
||
|
|
require.NoError(t, err)
|
||
|
|
expS := []byte(fmt.Sprintf(`{"fields":{"usage_idle":91.5},"name":"cpu","tags":{"cpu":"cpu0"},"timestamp":%d}`, now.Unix()) + "\n")
|
||
|
|
require.Equal(t, string(expS), string(buf))
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerialize_TimestampUnits(t *testing.T) {
|
||
|
|
tests := []struct {
|
||
|
|
name string
|
||
|
|
timestampUnits time.Duration
|
||
|
|
timestampFormat string
|
||
|
|
expected string
|
||
|
|
}{
|
||
|
|
{
|
||
|
|
name: "default of 1s",
|
||
|
|
timestampUnits: 0,
|
||
|
|
expected: `{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":1525478795}`,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: "1ns",
|
||
|
|
timestampUnits: 1 * time.Nanosecond,
|
||
|
|
expected: `{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":1525478795123456789}`,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: "1ms",
|
||
|
|
timestampUnits: 1 * time.Millisecond,
|
||
|
|
expected: `{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":1525478795123}`,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: "10ms",
|
||
|
|
timestampUnits: 10 * time.Millisecond,
|
||
|
|
expected: `{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":152547879512}`,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: "15ms is reduced to 10ms",
|
||
|
|
timestampUnits: 15 * time.Millisecond,
|
||
|
|
expected: `{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":152547879512}`,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: "65ms is reduced to 10ms",
|
||
|
|
timestampUnits: 65 * time.Millisecond,
|
||
|
|
expected: `{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":152547879512}`,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: "timestamp format",
|
||
|
|
timestampFormat: "2006-01-02T15:04:05Z07:00",
|
||
|
|
expected: `{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":"2018-05-05T00:06:35Z"}`,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
for _, tt := range tests {
|
||
|
|
t.Run(tt.name, func(t *testing.T) {
|
||
|
|
m := metric.New(
|
||
|
|
"cpu",
|
||
|
|
map[string]string{},
|
||
|
|
map[string]interface{}{
|
||
|
|
"value": 42.0,
|
||
|
|
},
|
||
|
|
time.Unix(1525478795, 123456789),
|
||
|
|
)
|
||
|
|
s := Serializer{
|
||
|
|
TimestampUnits: config.Duration(tt.timestampUnits),
|
||
|
|
TimestampFormat: tt.timestampFormat,
|
||
|
|
}
|
||
|
|
require.NoError(t, s.Init())
|
||
|
|
actual, err := s.Serialize(m)
|
||
|
|
require.NoError(t, err)
|
||
|
|
require.Equal(t, tt.expected+"\n", string(actual))
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeMetricInt(t *testing.T) {
|
||
|
|
now := time.Now()
|
||
|
|
tags := map[string]string{
|
||
|
|
"cpu": "cpu0",
|
||
|
|
}
|
||
|
|
fields := map[string]interface{}{
|
||
|
|
"usage_idle": int64(90),
|
||
|
|
}
|
||
|
|
m := metric.New("cpu", tags, fields, now)
|
||
|
|
|
||
|
|
s := Serializer{}
|
||
|
|
require.NoError(t, s.Init())
|
||
|
|
buf, err := s.Serialize(m)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
expS := []byte(fmt.Sprintf(`{"fields":{"usage_idle":90},"name":"cpu","tags":{"cpu":"cpu0"},"timestamp":%d}`, now.Unix()) + "\n")
|
||
|
|
require.Equal(t, string(expS), string(buf))
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeMetricString(t *testing.T) {
|
||
|
|
now := time.Now()
|
||
|
|
tags := map[string]string{
|
||
|
|
"cpu": "cpu0",
|
||
|
|
}
|
||
|
|
fields := map[string]interface{}{
|
||
|
|
"usage_idle": "foobar",
|
||
|
|
}
|
||
|
|
m := metric.New("cpu", tags, fields, now)
|
||
|
|
|
||
|
|
s := Serializer{}
|
||
|
|
require.NoError(t, s.Init())
|
||
|
|
buf, err := s.Serialize(m)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
expS := []byte(fmt.Sprintf(`{"fields":{"usage_idle":"foobar"},"name":"cpu","tags":{"cpu":"cpu0"},"timestamp":%d}`, now.Unix()) + "\n")
|
||
|
|
require.Equal(t, string(expS), string(buf))
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeMultiFields(t *testing.T) {
|
||
|
|
now := time.Now()
|
||
|
|
tags := map[string]string{
|
||
|
|
"cpu": "cpu0",
|
||
|
|
}
|
||
|
|
fields := map[string]interface{}{
|
||
|
|
"usage_idle": int64(90),
|
||
|
|
"usage_total": 8559615,
|
||
|
|
}
|
||
|
|
m := metric.New("cpu", tags, fields, now)
|
||
|
|
|
||
|
|
s := Serializer{}
|
||
|
|
require.NoError(t, s.Init())
|
||
|
|
buf, err := s.Serialize(m)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
expS := []byte(fmt.Sprintf(`{"fields":{"usage_idle":90,"usage_total":8559615},"name":"cpu","tags":{"cpu":"cpu0"},"timestamp":%d}`, now.Unix()) + "\n")
|
||
|
|
require.Equal(t, string(expS), string(buf))
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeMetricWithEscapes(t *testing.T) {
|
||
|
|
now := time.Now()
|
||
|
|
tags := map[string]string{
|
||
|
|
"cpu tag": "cpu0",
|
||
|
|
}
|
||
|
|
fields := map[string]interface{}{
|
||
|
|
"U,age=Idle": int64(90),
|
||
|
|
}
|
||
|
|
m := metric.New("My CPU", tags, fields, now)
|
||
|
|
|
||
|
|
s := Serializer{}
|
||
|
|
require.NoError(t, s.Init())
|
||
|
|
buf, err := s.Serialize(m)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
expS := []byte(fmt.Sprintf(`{"fields":{"U,age=Idle":90},"name":"My CPU","tags":{"cpu tag":"cpu0"},"timestamp":%d}`, now.Unix()) + "\n")
|
||
|
|
require.Equal(t, string(expS), string(buf))
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeBatch(t *testing.T) {
|
||
|
|
m := metric.New(
|
||
|
|
"cpu",
|
||
|
|
map[string]string{},
|
||
|
|
map[string]interface{}{
|
||
|
|
"value": 42.0,
|
||
|
|
},
|
||
|
|
time.Unix(0, 0),
|
||
|
|
)
|
||
|
|
metrics := []telegraf.Metric{m, m}
|
||
|
|
|
||
|
|
s := Serializer{}
|
||
|
|
require.NoError(t, s.Init())
|
||
|
|
buf, err := s.SerializeBatch(metrics)
|
||
|
|
require.NoError(t, err)
|
||
|
|
require.JSONEq(
|
||
|
|
t,
|
||
|
|
`{"metrics":[{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":0},{"fields":{"value":42},"name":"cpu","tags":{},"timestamp":0}]}`,
|
||
|
|
string(buf),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeBatchSkipInf(t *testing.T) {
|
||
|
|
metrics := []telegraf.Metric{
|
||
|
|
testutil.MustMetric(
|
||
|
|
"cpu",
|
||
|
|
map[string]string{},
|
||
|
|
map[string]interface{}{
|
||
|
|
"inf": math.Inf(1),
|
||
|
|
"time_idle": 42,
|
||
|
|
},
|
||
|
|
time.Unix(0, 0),
|
||
|
|
),
|
||
|
|
}
|
||
|
|
|
||
|
|
s := Serializer{}
|
||
|
|
require.NoError(t, s.Init())
|
||
|
|
buf, err := s.SerializeBatch(metrics)
|
||
|
|
require.NoError(t, err)
|
||
|
|
require.JSONEq(t, `{"metrics":[{"fields":{"time_idle":42},"name":"cpu","tags":{},"timestamp":0}]}`, string(buf))
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeBatchSkipInfAllFields(t *testing.T) {
|
||
|
|
metrics := []telegraf.Metric{
|
||
|
|
testutil.MustMetric(
|
||
|
|
"cpu",
|
||
|
|
map[string]string{},
|
||
|
|
map[string]interface{}{
|
||
|
|
"inf": math.Inf(1),
|
||
|
|
},
|
||
|
|
time.Unix(0, 0),
|
||
|
|
),
|
||
|
|
}
|
||
|
|
|
||
|
|
s := Serializer{}
|
||
|
|
require.NoError(t, s.Init())
|
||
|
|
buf, err := s.SerializeBatch(metrics)
|
||
|
|
require.NoError(t, err)
|
||
|
|
require.JSONEq(t, `{"metrics":[{"fields":{},"name":"cpu","tags":{},"timestamp":0}]}`, string(buf))
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeTransformationNonBatch(t *testing.T) {
|
||
|
|
var tests = []struct {
|
||
|
|
name string
|
||
|
|
filename string
|
||
|
|
}{
|
||
|
|
{
|
||
|
|
name: "non-batch transformation test",
|
||
|
|
filename: "testcases/transformation_single.conf",
|
||
|
|
},
|
||
|
|
}
|
||
|
|
parser := &influx.Parser{}
|
||
|
|
require.NoError(t, parser.Init())
|
||
|
|
|
||
|
|
for _, tt := range tests {
|
||
|
|
t.Run(tt.name, func(t *testing.T) {
|
||
|
|
filename := filepath.FromSlash(tt.filename)
|
||
|
|
cfg, header, err := loadTestConfiguration(filename)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
// Get the input metrics
|
||
|
|
metrics, err := testutil.ParseMetricsFrom(header, "Input:", parser)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
// Get the expectations
|
||
|
|
expectedArray, err := loadJSON(strings.TrimSuffix(filename, ".conf") + "_out.json")
|
||
|
|
require.NoError(t, err)
|
||
|
|
expected := expectedArray.([]interface{})
|
||
|
|
|
||
|
|
// Serialize
|
||
|
|
serializer := Serializer{
|
||
|
|
TimestampUnits: config.Duration(cfg.TimestampUnits),
|
||
|
|
TimestampFormat: cfg.TimestampFormat,
|
||
|
|
Transformation: cfg.Transformation,
|
||
|
|
}
|
||
|
|
require.NoError(t, serializer.Init())
|
||
|
|
|
||
|
|
for i, m := range metrics {
|
||
|
|
buf, err := serializer.Serialize(m)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
// Compare
|
||
|
|
var actual interface{}
|
||
|
|
require.NoError(t, json.Unmarshal(buf, &actual))
|
||
|
|
require.EqualValuesf(t, expected[i], actual, "mismatch in %d", i)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeTransformationBatch(t *testing.T) {
|
||
|
|
var tests = []struct {
|
||
|
|
name string
|
||
|
|
filename string
|
||
|
|
}{
|
||
|
|
{
|
||
|
|
name: "batch transformation test",
|
||
|
|
filename: "testcases/transformation_batch.conf",
|
||
|
|
},
|
||
|
|
}
|
||
|
|
parser := &influx.Parser{}
|
||
|
|
require.NoError(t, parser.Init())
|
||
|
|
|
||
|
|
for _, tt := range tests {
|
||
|
|
t.Run(tt.name, func(t *testing.T) {
|
||
|
|
filename := filepath.FromSlash(tt.filename)
|
||
|
|
cfg, header, err := loadTestConfiguration(filename)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
// Get the input metrics
|
||
|
|
metrics, err := testutil.ParseMetricsFrom(header, "Input:", parser)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
// Get the expectations
|
||
|
|
expected, err := loadJSON(strings.TrimSuffix(filename, ".conf") + "_out.json")
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
// Serialize
|
||
|
|
serializer := Serializer{
|
||
|
|
TimestampUnits: config.Duration(cfg.TimestampUnits),
|
||
|
|
TimestampFormat: cfg.TimestampFormat,
|
||
|
|
Transformation: cfg.Transformation,
|
||
|
|
}
|
||
|
|
require.NoError(t, serializer.Init())
|
||
|
|
|
||
|
|
buf, err := serializer.SerializeBatch(metrics)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
// Compare
|
||
|
|
var actual interface{}
|
||
|
|
require.NoError(t, json.Unmarshal(buf, &actual))
|
||
|
|
require.EqualValues(t, expected, actual)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeTransformationIssue12734(t *testing.T) {
|
||
|
|
input := []telegraf.Metric{
|
||
|
|
metric.New(
|
||
|
|
"data",
|
||
|
|
map[string]string{"key": "a"},
|
||
|
|
map[string]interface{}{"value": 10.1},
|
||
|
|
time.Unix(0, 1676285135457000000),
|
||
|
|
),
|
||
|
|
metric.New(
|
||
|
|
"data",
|
||
|
|
map[string]string{"key": "b"},
|
||
|
|
map[string]interface{}{"value": 20.2},
|
||
|
|
time.Unix(0, 1676285135457000000),
|
||
|
|
),
|
||
|
|
metric.New(
|
||
|
|
"data",
|
||
|
|
map[string]string{"key": "c"},
|
||
|
|
map[string]interface{}{"value": 30.3},
|
||
|
|
time.Unix(0, 1676285135457000000),
|
||
|
|
),
|
||
|
|
}
|
||
|
|
|
||
|
|
transformation := `
|
||
|
|
{
|
||
|
|
"valueRows": metrics{$string(timestamp): fields.value[]} ~> $each(function($v, $k) {
|
||
|
|
{
|
||
|
|
"timestamp": $number($k),
|
||
|
|
"values": $v
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
`
|
||
|
|
|
||
|
|
expected := map[string]interface{}{
|
||
|
|
"valueRows": map[string]interface{}{
|
||
|
|
"timestamp": 1.676285135e+9,
|
||
|
|
"values": []interface{}{10.1, 20.2, 30.3},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
// Setup serializer
|
||
|
|
serializer := Serializer{
|
||
|
|
Transformation: transformation,
|
||
|
|
}
|
||
|
|
require.NoError(t, serializer.Init())
|
||
|
|
|
||
|
|
// Check multiple serializations as issue #12734 shows that the
|
||
|
|
// transformation breaks after the first iteration
|
||
|
|
for i := 1; i <= 3; i++ {
|
||
|
|
buf, err := serializer.SerializeBatch(input)
|
||
|
|
require.NoErrorf(t, err, "broke in iteration %d", i)
|
||
|
|
|
||
|
|
// Compare
|
||
|
|
var actual interface{}
|
||
|
|
require.NoError(t, json.Unmarshal(buf, &actual))
|
||
|
|
require.EqualValuesf(t, expected, actual, "broke in iteration %d", i)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSerializeNesting(t *testing.T) {
|
||
|
|
var tests = []struct {
|
||
|
|
name string
|
||
|
|
filename string
|
||
|
|
out string
|
||
|
|
}{
|
||
|
|
{
|
||
|
|
name: "nested fields include",
|
||
|
|
filename: "testcases/nested_fields_include.conf",
|
||
|
|
out: "testcases/nested_fields_out.json",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: "nested fields exclude",
|
||
|
|
filename: "testcases/nested_fields_exclude.conf",
|
||
|
|
out: "testcases/nested_fields_out.json",
|
||
|
|
},
|
||
|
|
}
|
||
|
|
parser := &influx.Parser{}
|
||
|
|
require.NoError(t, parser.Init())
|
||
|
|
|
||
|
|
for _, tt := range tests {
|
||
|
|
t.Run(tt.name, func(t *testing.T) {
|
||
|
|
filename := filepath.FromSlash(tt.filename)
|
||
|
|
cfg, header, err := loadTestConfiguration(filename)
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
// Get the input metrics
|
||
|
|
metrics, err := testutil.ParseMetricsFrom(header, "Input:", parser)
|
||
|
|
require.NoError(t, err)
|
||
|
|
require.Len(t, metrics, 1)
|
||
|
|
|
||
|
|
// Get the expectations
|
||
|
|
expectedArray, err := loadJSON(tt.out)
|
||
|
|
require.NoError(t, err)
|
||
|
|
expected := expectedArray.(map[string]interface{})
|
||
|
|
|
||
|
|
// Serialize
|
||
|
|
serializer := Serializer{
|
||
|
|
TimestampUnits: config.Duration(cfg.TimestampUnits),
|
||
|
|
TimestampFormat: cfg.TimestampFormat,
|
||
|
|
Transformation: cfg.Transformation,
|
||
|
|
NestedFieldsInclude: cfg.JSONNestedFieldsInclude,
|
||
|
|
NestedFieldsExclude: cfg.JSONNestedFieldsExclude,
|
||
|
|
}
|
||
|
|
require.NoError(t, serializer.Init())
|
||
|
|
|
||
|
|
buf, err := serializer.Serialize(metrics[0])
|
||
|
|
require.NoError(t, err)
|
||
|
|
|
||
|
|
// Compare
|
||
|
|
var actual interface{}
|
||
|
|
require.NoError(t, json.Unmarshal(buf, &actual))
|
||
|
|
require.EqualValues(t, expected, actual)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
type Config struct {
|
||
|
|
TimestampUnits time.Duration `toml:"json_timestamp_units"`
|
||
|
|
TimestampFormat string `toml:"json_timestamp_format"`
|
||
|
|
Transformation string `toml:"json_transformation"`
|
||
|
|
JSONNestedFieldsInclude []string `toml:"json_nested_fields_include"`
|
||
|
|
JSONNestedFieldsExclude []string `toml:"json_nested_fields_exclude"`
|
||
|
|
}
|
||
|
|
|
||
|
|
func loadTestConfiguration(filename string) (*Config, []string, error) {
|
||
|
|
buf, err := os.ReadFile(filename)
|
||
|
|
if err != nil {
|
||
|
|
return nil, nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
header := make([]string, 0)
|
||
|
|
for _, line := range strings.Split(string(buf), "\n") {
|
||
|
|
line = strings.TrimSpace(line)
|
||
|
|
if strings.HasPrefix(line, "#") {
|
||
|
|
header = append(header, line)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
var cfg Config
|
||
|
|
err = toml.Unmarshal(buf, &cfg)
|
||
|
|
return &cfg, header, err
|
||
|
|
}
|
||
|
|
|
||
|
|
func loadJSON(filename string) (interface{}, error) {
|
||
|
|
buf, err := os.ReadFile(filename)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
var data interface{}
|
||
|
|
err = json.Unmarshal(buf, &data)
|
||
|
|
return data, err
|
||
|
|
}
|
||
|
|
|
||
|
|
func BenchmarkSerialize(b *testing.B) {
|
||
|
|
s := &Serializer{}
|
||
|
|
require.NoError(b, s.Init())
|
||
|
|
metrics := serializers.BenchmarkMetrics(b)
|
||
|
|
b.ResetTimer()
|
||
|
|
for i := 0; i < b.N; i++ {
|
||
|
|
_, err := s.Serialize(metrics[i%len(metrics)])
|
||
|
|
require.NoError(b, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func BenchmarkSerializeBatch(b *testing.B) {
|
||
|
|
s := &Serializer{}
|
||
|
|
require.NoError(b, s.Init())
|
||
|
|
m := serializers.BenchmarkMetrics(b)
|
||
|
|
metrics := m[:]
|
||
|
|
b.ResetTimer()
|
||
|
|
for i := 0; i < b.N; i++ {
|
||
|
|
_, err := s.SerializeBatch(metrics)
|
||
|
|
require.NoError(b, err)
|
||
|
|
}
|
||
|
|
}
|