449 lines
11 KiB
Go
449 lines
11 KiB
Go
package prometheusremotewrite
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/gogo/protobuf/jsonpb"
|
|
"github.com/prometheus/prometheus/model/histogram"
|
|
"github.com/prometheus/prometheus/prompb"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/influxdata/telegraf"
|
|
"github.com/influxdata/telegraf/config"
|
|
"github.com/influxdata/telegraf/metric"
|
|
"github.com/influxdata/telegraf/models"
|
|
"github.com/influxdata/telegraf/plugins/parsers/influx"
|
|
"github.com/influxdata/telegraf/testutil"
|
|
test "github.com/influxdata/telegraf/testutil/plugin_input"
|
|
)
|
|
|
|
const (
|
|
testCasesDir = "testcases"
|
|
benchmarkFolder = "benchmark"
|
|
inputFilename = "input.json"
|
|
expectedFilename = "expected_v%d.out"
|
|
configFilename = "telegraf.conf"
|
|
)
|
|
|
|
func TestCases(t *testing.T) {
|
|
// Get all directories in testcases
|
|
folders, err := os.ReadDir(testCasesDir)
|
|
require.NoError(t, err)
|
|
// Make sure testdata contains data
|
|
require.NotEmpty(t, folders)
|
|
|
|
for _, f := range folders {
|
|
if !f.IsDir() {
|
|
continue
|
|
}
|
|
|
|
fname := f.Name()
|
|
testdataPath := filepath.Join(testCasesDir, fname)
|
|
|
|
// Load input data
|
|
inputFilePath := filepath.Join(testdataPath, inputFilename)
|
|
buf, err := os.ReadFile(inputFilePath)
|
|
require.NoError(t, err)
|
|
var writeRequest prompb.WriteRequest
|
|
err = jsonpb.Unmarshal(bytes.NewReader(buf), &writeRequest)
|
|
require.NoError(t, err)
|
|
inputBytes, err := writeRequest.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
versions := []int{1, 2}
|
|
for _, version := range versions {
|
|
t.Run(fmt.Sprintf("%s_v%d", fname, version), func(t *testing.T) {
|
|
// Load parser
|
|
configFilePath := filepath.Join(testdataPath, configFilename)
|
|
cfg := config.NewConfig()
|
|
require.NoError(t, cfg.LoadConfig(configFilePath))
|
|
require.Len(t, cfg.Inputs, 1)
|
|
plugin := cfg.Inputs[0].Input.(*test.Plugin)
|
|
parser := plugin.Parser.(*models.RunningParser).Parser.(*Parser)
|
|
parser.MetricVersion = version
|
|
|
|
// Load expected output
|
|
expectedFilePath := filepath.Join(testdataPath, fmt.Sprintf(expectedFilename, version))
|
|
var expected []telegraf.Metric
|
|
influxParser := &influx.Parser{}
|
|
require.NoError(t, influxParser.Init())
|
|
expected, err := testutil.ParseMetricsFromFile(expectedFilePath, influxParser)
|
|
require.NoError(t, err)
|
|
|
|
// Act and assert
|
|
parsed, err := parser.Parse(inputBytes)
|
|
require.NoError(t, err)
|
|
require.Len(t, parsed, len(expected))
|
|
// Ignore type when comparing, because expected metrics are parsed from influx lines and thus always untyped
|
|
testutil.RequireMetricsEqual(t, expected, parsed, testutil.SortMetrics(), testutil.IgnoreType())
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkParsingMetricVersion1(b *testing.B) {
|
|
parser := &Parser{
|
|
MetricVersion: 1,
|
|
}
|
|
|
|
benchmarkData, err := os.ReadFile(filepath.Join(testCasesDir, benchmarkFolder, inputFilename))
|
|
require.NoError(b, err)
|
|
require.NotEmpty(b, benchmarkData)
|
|
|
|
for n := 0; n < b.N; n++ {
|
|
//nolint:errcheck // Benchmarking so skip the error check to avoid the unnecessary operations
|
|
parser.Parse(benchmarkData)
|
|
}
|
|
}
|
|
|
|
func BenchmarkParsingMetricVersion2(b *testing.B) {
|
|
parser := &Parser{
|
|
MetricVersion: 2,
|
|
}
|
|
|
|
benchmarkData, err := os.ReadFile(filepath.Join(testCasesDir, benchmarkFolder, inputFilename))
|
|
require.NoError(b, err)
|
|
require.NotEmpty(b, benchmarkData)
|
|
|
|
for n := 0; n < b.N; n++ {
|
|
//nolint:errcheck // Benchmarking so skip the error check to avoid the unnecessary operations
|
|
parser.Parse(benchmarkData)
|
|
}
|
|
}
|
|
|
|
func TestParse(t *testing.T) {
|
|
prompbInput := prompb.WriteRequest{
|
|
Timeseries: []prompb.TimeSeries{
|
|
{
|
|
Labels: []prompb.Label{
|
|
{Name: "__name__", Value: "go_gc_duration_seconds"},
|
|
{Name: "quantile", Value: "0.99"},
|
|
},
|
|
Samples: []prompb.Sample{
|
|
{Value: 4.63, Timestamp: time.Date(2020, 4, 1, 0, 0, 0, 0, time.UTC).UnixNano()},
|
|
},
|
|
},
|
|
{
|
|
Labels: []prompb.Label{
|
|
{Name: "__name__", Value: "prometheus_target_interval_length_seconds"},
|
|
{Name: "job", Value: "prometheus"},
|
|
},
|
|
Samples: []prompb.Sample{
|
|
{Value: 14.99, Timestamp: time.Date(2020, 4, 1, 0, 0, 0, 0, time.UTC).UnixNano()},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
inoutBytes, err := prompbInput.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
expected := []telegraf.Metric{
|
|
testutil.MustMetric(
|
|
"prometheus_remote_write",
|
|
map[string]string{
|
|
"quantile": "0.99",
|
|
},
|
|
map[string]interface{}{
|
|
"go_gc_duration_seconds": float64(4.63),
|
|
},
|
|
time.Unix(0, 0),
|
|
),
|
|
testutil.MustMetric(
|
|
"prometheus_remote_write",
|
|
map[string]string{
|
|
"job": "prometheus",
|
|
},
|
|
map[string]interface{}{
|
|
"prometheus_target_interval_length_seconds": float64(14.99),
|
|
},
|
|
time.Unix(0, 0),
|
|
),
|
|
}
|
|
|
|
parser := Parser{
|
|
DefaultTags: map[string]string{},
|
|
}
|
|
|
|
metrics, err := parser.Parse(inoutBytes)
|
|
require.NoError(t, err)
|
|
require.Len(t, metrics, 2)
|
|
testutil.RequireMetricsEqual(t, expected, metrics, testutil.IgnoreTime(), testutil.SortMetrics())
|
|
}
|
|
|
|
func generateTestHistogram(i int) *histogram.Histogram {
|
|
return &histogram.Histogram{
|
|
Count: 12 + uint64(i*9),
|
|
ZeroCount: 2 + uint64(i),
|
|
ZeroThreshold: 0.001,
|
|
Sum: 18.4 * float64(i+1),
|
|
Schema: 1,
|
|
PositiveSpans: []histogram.Span{
|
|
{Offset: 0, Length: 2},
|
|
{Offset: 1, Length: 2},
|
|
},
|
|
PositiveBuckets: []int64{int64(i + 1), 1, -1, 0},
|
|
NegativeSpans: []histogram.Span{
|
|
{Offset: 0, Length: 2},
|
|
{Offset: 1, Length: 2},
|
|
},
|
|
NegativeBuckets: []int64{int64(i + 1), 1, -1, 0},
|
|
}
|
|
}
|
|
|
|
func generateTestFloatHistogram(i int) *histogram.FloatHistogram {
|
|
return &histogram.FloatHistogram{
|
|
Count: 12 + float64(i*9),
|
|
ZeroCount: 2 + float64(i),
|
|
ZeroThreshold: 0.001,
|
|
Sum: 18.4 * float64(i+1),
|
|
Schema: 1,
|
|
PositiveSpans: []histogram.Span{
|
|
{Offset: 0, Length: 2},
|
|
{Offset: 1, Length: 2},
|
|
},
|
|
PositiveBuckets: []float64{float64(i + 1), float64(i + 2), float64(i + 1), float64(i + 1)},
|
|
NegativeSpans: []histogram.Span{
|
|
{Offset: 0, Length: 2},
|
|
{Offset: 1, Length: 2},
|
|
},
|
|
NegativeBuckets: []float64{float64(i + 1), float64(i + 2), float64(i + 1), float64(i + 1)},
|
|
}
|
|
}
|
|
|
|
func TestHistograms(t *testing.T) {
|
|
prompbInput := prompb.WriteRequest{
|
|
Timeseries: []prompb.TimeSeries{
|
|
{
|
|
Labels: []prompb.Label{
|
|
{Name: "__name__", Value: "test_metric_seconds"},
|
|
},
|
|
Histograms: []prompb.Histogram{
|
|
prompb.FromIntHistogram(0, generateTestHistogram(1)),
|
|
},
|
|
},
|
|
{
|
|
Labels: []prompb.Label{
|
|
{Name: "__name__", Value: "test_float_metric_seconds"},
|
|
},
|
|
Histograms: []prompb.Histogram{
|
|
prompb.FromFloatHistogram(0, generateTestFloatHistogram(2)),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
inoutBytes, err := prompbInput.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
expected := []telegraf.Metric{
|
|
testutil.MustMetric(
|
|
"prometheus_remote_write",
|
|
map[string]string{},
|
|
map[string]interface{}{
|
|
"test_metric_seconds_sum": float64(36.8),
|
|
},
|
|
time.Unix(0, 0),
|
|
),
|
|
testutil.MustMetric(
|
|
"prometheus_remote_write",
|
|
map[string]string{},
|
|
map[string]interface{}{
|
|
"test_metric_seconds_count": float64(21),
|
|
},
|
|
time.Unix(0, 0),
|
|
),
|
|
testutil.MustMetric(
|
|
"prometheus_remote_write",
|
|
map[string]string{},
|
|
map[string]interface{}{
|
|
"test_float_metric_seconds_sum": float64(55.199999999999996),
|
|
},
|
|
time.Unix(0, 0),
|
|
),
|
|
testutil.MustMetric(
|
|
"prometheus_remote_write",
|
|
map[string]string{},
|
|
map[string]interface{}{
|
|
"test_float_metric_seconds_count": float64(30),
|
|
},
|
|
time.Unix(0, 0),
|
|
),
|
|
}
|
|
|
|
parser := Parser{
|
|
DefaultTags: map[string]string{},
|
|
}
|
|
metrics, err := parser.Parse(inoutBytes)
|
|
require.NoError(t, err)
|
|
require.Len(t, metrics, 22)
|
|
testutil.RequireMetricsSubset(t, expected, metrics, testutil.IgnoreTime(), testutil.SortMetrics())
|
|
}
|
|
|
|
func TestDefaultTags(t *testing.T) {
|
|
prompbInput := prompb.WriteRequest{
|
|
Timeseries: []prompb.TimeSeries{
|
|
{
|
|
Labels: []prompb.Label{
|
|
{Name: "__name__", Value: "foo"},
|
|
{Name: "__eg__", Value: "bar"},
|
|
},
|
|
Samples: []prompb.Sample{
|
|
{Value: 1, Timestamp: time.Date(2020, 4, 1, 0, 0, 0, 0, time.UTC).UnixNano()},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
inoutBytes, err := prompbInput.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
expected := []telegraf.Metric{
|
|
testutil.MustMetric(
|
|
"prometheus_remote_write",
|
|
map[string]string{
|
|
"defaultTag": "defaultTagValue",
|
|
"__eg__": "bar",
|
|
},
|
|
map[string]interface{}{
|
|
"foo": float64(1),
|
|
},
|
|
time.Unix(0, 0),
|
|
),
|
|
}
|
|
|
|
parser := Parser{
|
|
DefaultTags: map[string]string{
|
|
"defaultTag": "defaultTagValue",
|
|
},
|
|
}
|
|
|
|
metrics, err := parser.Parse(inoutBytes)
|
|
require.NoError(t, err)
|
|
require.Len(t, metrics, 1)
|
|
testutil.RequireMetricsEqual(t, expected, metrics, testutil.IgnoreTime(), testutil.SortMetrics())
|
|
}
|
|
|
|
func TestMetricsWithTimestamp(t *testing.T) {
|
|
testTime := time.Date(2020, time.October, 4, 17, 0, 0, 0, time.UTC)
|
|
testTimeUnix := testTime.UnixNano() / int64(time.Millisecond)
|
|
prompbInput := prompb.WriteRequest{
|
|
Timeseries: []prompb.TimeSeries{
|
|
{
|
|
Labels: []prompb.Label{
|
|
{Name: "__name__", Value: "foo"},
|
|
{Name: "__eg__", Value: "bar"},
|
|
},
|
|
Samples: []prompb.Sample{
|
|
{Value: 1, Timestamp: testTimeUnix},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
inoutBytes, err := prompbInput.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
expected := []telegraf.Metric{
|
|
testutil.MustMetric(
|
|
"prometheus_remote_write",
|
|
map[string]string{
|
|
"__eg__": "bar",
|
|
},
|
|
map[string]interface{}{
|
|
"foo": float64(1),
|
|
},
|
|
testTime,
|
|
),
|
|
}
|
|
parser := Parser{
|
|
DefaultTags: map[string]string{},
|
|
}
|
|
|
|
metrics, err := parser.Parse(inoutBytes)
|
|
require.NoError(t, err)
|
|
require.Len(t, metrics, 1)
|
|
testutil.RequireMetricsEqual(t, expected, metrics, testutil.SortMetrics())
|
|
}
|
|
|
|
var benchmarkData = prompb.WriteRequest{
|
|
Timeseries: []prompb.TimeSeries{
|
|
{
|
|
Labels: []prompb.Label{
|
|
{Name: "__name__", Value: "benchmark_a"},
|
|
{Name: "source", Value: "myhost"},
|
|
{Name: "tags_platform", Value: "python"},
|
|
{Name: "tags_sdkver", Value: "3.11.5"},
|
|
},
|
|
Samples: []prompb.Sample{
|
|
{Value: 5.0, Timestamp: time.Date(2020, 4, 1, 0, 0, 0, 0, time.UTC).UnixMilli()},
|
|
},
|
|
},
|
|
{
|
|
Labels: []prompb.Label{
|
|
{Name: "__name__", Value: "benchmark_b"},
|
|
{Name: "source", Value: "myhost"},
|
|
{Name: "tags_platform", Value: "python"},
|
|
{Name: "tags_sdkver", Value: "3.11.4"},
|
|
},
|
|
Samples: []prompb.Sample{
|
|
{Value: 4.0, Timestamp: time.Date(2020, 4, 1, 0, 0, 0, 0, time.UTC).UnixMilli()},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
func TestBenchmarkData(t *testing.T) {
|
|
expected := []telegraf.Metric{
|
|
metric.New(
|
|
"prometheus_remote_write",
|
|
map[string]string{
|
|
"source": "myhost",
|
|
"tags_platform": "python",
|
|
"tags_sdkver": "3.11.5",
|
|
},
|
|
map[string]interface{}{
|
|
"benchmark_a": 5.0,
|
|
},
|
|
time.Unix(1585699200, 0),
|
|
),
|
|
metric.New(
|
|
"prometheus_remote_write",
|
|
map[string]string{
|
|
"source": "myhost",
|
|
"tags_platform": "python",
|
|
"tags_sdkver": "3.11.4",
|
|
},
|
|
map[string]interface{}{
|
|
"benchmark_b": 4.0,
|
|
},
|
|
time.Unix(1585699200, 0),
|
|
),
|
|
}
|
|
|
|
benchmarkData, err := benchmarkData.Marshal()
|
|
require.NoError(t, err)
|
|
|
|
plugin := &Parser{}
|
|
actual, err := plugin.Parse(benchmarkData)
|
|
require.NoError(t, err)
|
|
testutil.RequireMetricsEqual(t, expected, actual, testutil.SortMetrics())
|
|
}
|
|
|
|
func BenchmarkParsing(b *testing.B) {
|
|
benchmarkData, err := benchmarkData.Marshal()
|
|
require.NoError(b, err)
|
|
|
|
plugin := &Parser{}
|
|
|
|
b.ResetTimer()
|
|
for n := 0; n < b.N; n++ {
|
|
//nolint:errcheck // Benchmarking so skip the error check to avoid the unnecessary operations
|
|
plugin.Parse(benchmarkData)
|
|
}
|
|
}
|