Adding upstream version 1.34.4.
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
parent
e393c3af3f
commit
4978089aab
GPG key ID:
FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions
189
plugins/common/adx/adx.go
Normal file
189
plugins/common/adx/adx.go
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
package adx
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-kusto-go/kusto"
|
||||
kustoerrors "github.com/Azure/azure-kusto-go/kusto/data/errors"
|
||||
"github.com/Azure/azure-kusto-go/kusto/ingest"
|
||||
"github.com/Azure/azure-kusto-go/kusto/kql"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/config"
|
||||
"github.com/influxdata/telegraf/internal"
|
||||
)
|
||||
|
||||
const (
|
||||
TablePerMetric = "tablepermetric"
|
||||
SingleTable = "singletable"
|
||||
// These control the amount of memory we use when ingesting blobs
|
||||
bufferSize = 1 << 20 // 1 MiB
|
||||
maxBuffers = 5
|
||||
ManagedIngestion = "managed"
|
||||
QueuedIngestion = "queued"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Endpoint string `toml:"endpoint_url"`
|
||||
Database string `toml:"database"`
|
||||
Timeout config.Duration `toml:"timeout"`
|
||||
MetricsGrouping string `toml:"metrics_grouping_type"`
|
||||
TableName string `toml:"table_name"`
|
||||
CreateTables bool `toml:"create_tables"`
|
||||
IngestionType string `toml:"ingestion_type"`
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
cfg *Config
|
||||
client *kusto.Client
|
||||
ingestors map[string]ingest.Ingestor
|
||||
logger telegraf.Logger
|
||||
}
|
||||
|
||||
func (cfg *Config) NewClient(app string, log telegraf.Logger) (*Client, error) {
|
||||
if cfg.Endpoint == "" {
|
||||
return nil, errors.New("endpoint configuration cannot be empty")
|
||||
}
|
||||
if cfg.Database == "" {
|
||||
return nil, errors.New("database configuration cannot be empty")
|
||||
}
|
||||
|
||||
cfg.MetricsGrouping = strings.ToLower(cfg.MetricsGrouping)
|
||||
if cfg.MetricsGrouping == SingleTable && cfg.TableName == "" {
|
||||
return nil, errors.New("table name cannot be empty for SingleTable metrics grouping type")
|
||||
}
|
||||
|
||||
if cfg.MetricsGrouping == "" {
|
||||
cfg.MetricsGrouping = TablePerMetric
|
||||
}
|
||||
|
||||
if cfg.MetricsGrouping != SingleTable && cfg.MetricsGrouping != TablePerMetric {
|
||||
return nil, errors.New("metrics grouping type is not valid")
|
||||
}
|
||||
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = config.Duration(20 * time.Second)
|
||||
}
|
||||
|
||||
switch cfg.IngestionType {
|
||||
case "":
|
||||
cfg.IngestionType = QueuedIngestion
|
||||
case ManagedIngestion, QueuedIngestion:
|
||||
// Do nothing as those are valid
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown ingestion type %q", cfg.IngestionType)
|
||||
}
|
||||
|
||||
conn := kusto.NewConnectionStringBuilder(cfg.Endpoint).WithDefaultAzureCredential()
|
||||
conn.SetConnectorDetails("Telegraf", internal.ProductToken(), app, "", false, "")
|
||||
client, err := kusto.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Client{
|
||||
cfg: cfg,
|
||||
ingestors: make(map[string]ingest.Ingestor),
|
||||
logger: log,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Clean up and close the ingestor
|
||||
func (adx *Client) Close() error {
|
||||
var errs []error
|
||||
for _, v := range adx.ingestors {
|
||||
if err := v.Close(); err != nil {
|
||||
// accumulate errors while closing ingestors
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if err := adx.client.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
adx.client = nil
|
||||
adx.ingestors = nil
|
||||
|
||||
if len(errs) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Combine errors into a single object and return the combined error
|
||||
return kustoerrors.GetCombinedError(errs...)
|
||||
}
|
||||
|
||||
func (adx *Client) PushMetrics(format ingest.FileOption, tableName string, metrics []byte) error {
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Duration(adx.cfg.Timeout))
|
||||
defer cancel()
|
||||
metricIngestor, err := adx.getMetricIngestor(ctx, tableName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(metrics)
|
||||
mapping := ingest.IngestionMappingRef(tableName+"_mapping", ingest.JSON)
|
||||
if metricIngestor != nil {
|
||||
if _, err := metricIngestor.FromReader(ctx, reader, format, mapping); err != nil {
|
||||
return fmt.Errorf("sending ingestion request to Azure Data Explorer for table %q failed: %w", tableName, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (adx *Client) getMetricIngestor(ctx context.Context, tableName string) (ingest.Ingestor, error) {
|
||||
if ingestor := adx.ingestors[tableName]; ingestor != nil {
|
||||
return ingestor, nil
|
||||
}
|
||||
|
||||
if adx.cfg.CreateTables {
|
||||
if _, err := adx.client.Mgmt(ctx, adx.cfg.Database, createTableCommand(tableName)); err != nil {
|
||||
return nil, fmt.Errorf("creating table for %q failed: %w", tableName, err)
|
||||
}
|
||||
|
||||
if _, err := adx.client.Mgmt(ctx, adx.cfg.Database, createTableMappingCommand(tableName)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new ingestor client for the table
|
||||
var ingestor ingest.Ingestor
|
||||
var err error
|
||||
switch strings.ToLower(adx.cfg.IngestionType) {
|
||||
case ManagedIngestion:
|
||||
ingestor, err = ingest.NewManaged(adx.client, adx.cfg.Database, tableName)
|
||||
case QueuedIngestion:
|
||||
ingestor, err = ingest.New(adx.client, adx.cfg.Database, tableName, ingest.WithStaticBuffer(bufferSize, maxBuffers))
|
||||
default:
|
||||
return nil, fmt.Errorf(`ingestion_type has to be one of %q or %q`, ManagedIngestion, QueuedIngestion)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating ingestor for %q failed: %w", tableName, err)
|
||||
}
|
||||
adx.ingestors[tableName] = ingestor
|
||||
|
||||
return ingestor, nil
|
||||
}
|
||||
|
||||
func createTableCommand(table string) kusto.Statement {
|
||||
builder := kql.New(`.create-merge table ['`).AddTable(table).AddLiteral(`'] `)
|
||||
builder.AddLiteral(`(['fields']:dynamic, ['name']:string, ['tags']:dynamic, ['timestamp']:datetime);`)
|
||||
|
||||
return builder
|
||||
}
|
||||
|
||||
func createTableMappingCommand(table string) kusto.Statement {
|
||||
builder := kql.New(`.create-or-alter table ['`).AddTable(table).AddLiteral(`'] `)
|
||||
builder.AddLiteral(`ingestion json mapping '`).AddTable(table + "_mapping").AddLiteral(`' `)
|
||||
builder.AddLiteral(`'[{"column":"fields", `)
|
||||
builder.AddLiteral(`"Properties":{"Path":"$[\'fields\']"}},{"column":"name", `)
|
||||
builder.AddLiteral(`"Properties":{"Path":"$[\'name\']"}},{"column":"tags", `)
|
||||
builder.AddLiteral(`"Properties":{"Path":"$[\'tags\']"}},{"column":"timestamp", `)
|
||||
builder.AddLiteral(`"Properties":{"Path":"$[\'timestamp\']"}}]'`)
|
||||
|
||||
return builder
|
||||
}
|
||||
219
plugins/common/adx/adx_test.go
Normal file
219
plugins/common/adx/adx_test.go
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
package adx
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-kusto-go/kusto"
|
||||
"github.com/Azure/azure-kusto-go/kusto/ingest"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/config"
|
||||
serializers_json "github.com/influxdata/telegraf/plugins/serializers/json"
|
||||
"github.com/influxdata/telegraf/testutil"
|
||||
)
|
||||
|
||||
func TestInitBlankEndpointData(t *testing.T) {
|
||||
plugin := Config{
|
||||
Endpoint: "",
|
||||
Database: "mydb",
|
||||
}
|
||||
|
||||
_, err := plugin.NewClient("TestKusto.Telegraf", nil)
|
||||
require.Error(t, err)
|
||||
require.Equal(t, "endpoint configuration cannot be empty", err.Error())
|
||||
}
|
||||
|
||||
func TestQueryConstruction(t *testing.T) {
|
||||
const tableName = "mytable"
|
||||
const expectedCreate = `.create-merge table ['mytable'] (['fields']:dynamic, ['name']:string, ['tags']:dynamic, ['timestamp']:datetime);`
|
||||
const expectedMapping = `` +
|
||||
`.create-or-alter table ['mytable'] ingestion json mapping 'mytable_mapping' '[{"column":"fields", ` +
|
||||
`"Properties":{"Path":"$[\'fields\']"}},{"column":"name", "Properties":{"Path":"$[\'name\']"}},{"column":"tags", ` +
|
||||
`"Properties":{"Path":"$[\'tags\']"}},{"column":"timestamp", "Properties":{"Path":"$[\'timestamp\']"}}]'`
|
||||
require.Equal(t, expectedCreate, createTableCommand(tableName).String())
|
||||
require.Equal(t, expectedMapping, createTableMappingCommand(tableName).String())
|
||||
}
|
||||
|
||||
func TestGetMetricIngestor(t *testing.T) {
|
||||
plugin := Client{
|
||||
logger: testutil.Logger{},
|
||||
client: kusto.NewMockClient(),
|
||||
cfg: &Config{
|
||||
Database: "mydb",
|
||||
IngestionType: QueuedIngestion,
|
||||
},
|
||||
ingestors: map[string]ingest.Ingestor{"test1": &fakeIngestor{}},
|
||||
}
|
||||
|
||||
ingestor, err := plugin.getMetricIngestor(t.Context(), "test1")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, ingestor)
|
||||
}
|
||||
|
||||
func TestGetMetricIngestorNoIngester(t *testing.T) {
|
||||
plugin := Client{
|
||||
logger: testutil.Logger{},
|
||||
client: kusto.NewMockClient(),
|
||||
cfg: &Config{
|
||||
IngestionType: QueuedIngestion,
|
||||
},
|
||||
ingestors: map[string]ingest.Ingestor{"test1": &fakeIngestor{}},
|
||||
}
|
||||
|
||||
ingestor, err := plugin.getMetricIngestor(t.Context(), "test1")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, ingestor)
|
||||
}
|
||||
|
||||
func TestPushMetrics(t *testing.T) {
|
||||
plugin := Client{
|
||||
logger: testutil.Logger{},
|
||||
client: kusto.NewMockClient(),
|
||||
cfg: &Config{
|
||||
Database: "mydb",
|
||||
Endpoint: "https://ingest-test.westus.kusto.windows.net",
|
||||
IngestionType: QueuedIngestion,
|
||||
},
|
||||
ingestors: map[string]ingest.Ingestor{"test1": &fakeIngestor{}},
|
||||
}
|
||||
|
||||
metrics := []byte(`{"fields": {"value": 1}, "name": "test1", "tags": {"tag1": "value1"}, "timestamp": "2021-01-01T00:00:00Z"}`)
|
||||
require.NoError(t, plugin.PushMetrics(ingest.FileFormat(ingest.JSON), "test1", metrics))
|
||||
}
|
||||
|
||||
func TestPushMetricsOutputs(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
inputMetric []telegraf.Metric
|
||||
metricsGrouping string
|
||||
createTables bool
|
||||
ingestionType string
|
||||
}{
|
||||
{
|
||||
name: "Valid metric",
|
||||
inputMetric: testutil.MockMetrics(),
|
||||
createTables: true,
|
||||
metricsGrouping: TablePerMetric,
|
||||
},
|
||||
{
|
||||
name: "Don't create tables'",
|
||||
inputMetric: testutil.MockMetrics(),
|
||||
createTables: false,
|
||||
metricsGrouping: TablePerMetric,
|
||||
},
|
||||
{
|
||||
name: "SingleTable metric grouping type",
|
||||
inputMetric: testutil.MockMetrics(),
|
||||
createTables: true,
|
||||
metricsGrouping: SingleTable,
|
||||
},
|
||||
{
|
||||
name: "Valid metric managed ingestion",
|
||||
inputMetric: testutil.MockMetrics(),
|
||||
createTables: true,
|
||||
metricsGrouping: TablePerMetric,
|
||||
ingestionType: ManagedIngestion,
|
||||
},
|
||||
}
|
||||
var expectedMetric = map[string]interface{}{
|
||||
"metricName": "test1",
|
||||
"fields": map[string]interface{}{
|
||||
"value": 1.0,
|
||||
},
|
||||
"tags": map[string]interface{}{
|
||||
"tag1": "value1",
|
||||
},
|
||||
"timestamp": float64(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC).UnixNano() / int64(time.Second)),
|
||||
}
|
||||
for _, tC := range testCases {
|
||||
t.Run(tC.name, func(t *testing.T) {
|
||||
ingestionType := "queued"
|
||||
if tC.ingestionType != "" {
|
||||
ingestionType = tC.ingestionType
|
||||
}
|
||||
|
||||
serializer := &serializers_json.Serializer{
|
||||
TimestampUnits: config.Duration(time.Nanosecond),
|
||||
TimestampFormat: time.RFC3339Nano,
|
||||
}
|
||||
|
||||
cfg := &Config{
|
||||
Endpoint: "https://someendpoint.kusto.net",
|
||||
Database: "databasename",
|
||||
MetricsGrouping: tC.metricsGrouping,
|
||||
TableName: "test1",
|
||||
CreateTables: tC.createTables,
|
||||
IngestionType: ingestionType,
|
||||
Timeout: config.Duration(20 * time.Second),
|
||||
}
|
||||
client, err := cfg.NewClient("telegraf", &testutil.Logger{})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Inject the ingestor
|
||||
ingestor := &fakeIngestor{}
|
||||
client.ingestors["test1"] = ingestor
|
||||
|
||||
tableMetricGroups := make(map[string][]byte)
|
||||
mockmetrics := testutil.MockMetrics()
|
||||
for _, m := range mockmetrics {
|
||||
metricInBytes, err := serializer.Serialize(m)
|
||||
require.NoError(t, err)
|
||||
tableMetricGroups[m.Name()] = append(tableMetricGroups[m.Name()], metricInBytes...)
|
||||
}
|
||||
|
||||
format := ingest.FileFormat(ingest.JSON)
|
||||
for tableName, tableMetrics := range tableMetricGroups {
|
||||
require.NoError(t, client.PushMetrics(format, tableName, tableMetrics))
|
||||
createdFakeIngestor := ingestor
|
||||
require.EqualValues(t, expectedMetric["metricName"], createdFakeIngestor.actualOutputMetric["name"])
|
||||
require.EqualValues(t, expectedMetric["fields"], createdFakeIngestor.actualOutputMetric["fields"])
|
||||
require.EqualValues(t, expectedMetric["tags"], createdFakeIngestor.actualOutputMetric["tags"])
|
||||
timestampStr := createdFakeIngestor.actualOutputMetric["timestamp"].(string)
|
||||
parsedTime, err := time.Parse(time.RFC3339Nano, timestampStr)
|
||||
parsedTimeFloat := float64(parsedTime.UnixNano()) / 1e9
|
||||
require.NoError(t, err)
|
||||
require.InDelta(t, expectedMetric["timestamp"].(float64), parsedTimeFloat, testutil.DefaultDelta)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlreadyClosed(t *testing.T) {
|
||||
plugin := Client{
|
||||
logger: testutil.Logger{},
|
||||
cfg: &Config{
|
||||
IngestionType: QueuedIngestion,
|
||||
},
|
||||
client: kusto.NewMockClient(),
|
||||
}
|
||||
require.NoError(t, plugin.Close())
|
||||
}
|
||||
|
||||
type fakeIngestor struct {
|
||||
actualOutputMetric map[string]interface{}
|
||||
}
|
||||
|
||||
func (f *fakeIngestor) FromReader(_ context.Context, reader io.Reader, _ ...ingest.FileOption) (*ingest.Result, error) {
|
||||
scanner := bufio.NewScanner(reader)
|
||||
scanner.Scan()
|
||||
firstLine := scanner.Text()
|
||||
err := json.Unmarshal([]byte(firstLine), &f.actualOutputMetric)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ingest.Result{}, nil
|
||||
}
|
||||
|
||||
func (*fakeIngestor) FromFile(_ context.Context, _ string, _ ...ingest.FileOption) (*ingest.Result, error) {
|
||||
return &ingest.Result{}, nil
|
||||
}
|
||||
|
||||
func (*fakeIngestor) Close() error {
|
||||
return nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue