influx/telegraf
1
0
Fork 0
Code Issues Activity

Adding upstream version 1.34.4.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-05-24 07:26:29 +02:00
parent e393c3af3f
commit 4978089aab
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

88
plugins/inputs/fluentd/README.md Normal file
View file

@ -0,0 +1,88 @@
# Fluentd Input Plugin
This plugin gathers internal metrics of a [fluentd][fluentd] instance provided
by fluentd's [monitor agent plugin][monitor_agent]. Data provided
by the `/api/plugin.json` resource, `/api/config.json` is not covered.
> [!IMPORTANT]
> This plugin might produce high-cardinality series as the `plugin_id` value is
> random after each restart of fluentd. You might need to adjust your fluentd
> configuration, in order to reduce series cardinality in case your fluentd
> restarts frequently by adding the `@id` parameter to each plugin.
> See [fluentd's documentation][docs] for details.
⭐ Telegraf v1.4.0
🏷️ server
💻 all
[fluentd]: https://www.fluentd.org/
[monitor_agent]: https://docs.fluentd.org/input/monitor_agent
[docs]: https://docs.fluentd.org/configuration/config-file#common-plugin-parameter
## Global configuration options <!-- @/docs/includes/plugin_config.md -->
In addition to the plugin-specific configuration settings, plugins support
additional global and plugin configuration settings. These settings are used to
modify metrics, tags, and field or create aliases and configure ordering, etc.
See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
## Configuration
```toml @sample.conf
# Read metrics exposed by fluentd in_monitor plugin
[[inputs.fluentd]]
## This plugin reads information exposed by fluentd (using /api/plugins.json endpoint).
##
## Endpoint:
## - only one URI is allowed
## - https is not supported
endpoint = "http://localhost:24220/api/plugins.json"
## Define which plugins have to be excluded (based on "type" field - e.g. monitor_agent)
exclude = [
"monitor_agent",
"dummy",
]
```
## Metrics
### Measurements & Fields
Fields may vary depending on the plugin type
- fluentd
- retry_count (float, unit)
- buffer_queue_length (float, unit)
- buffer_total_queued_size (float, unit)
- rollback_count (float, unit)
- flush_time_count (float, unit)
- slow_flush_count (float, unit)
- emit_count (float, unit)
- emit_records (float, unit)
- emit_size (float, unit)
- write_count (float, unit)
- buffer_stage_length (float, unit)
- buffer_queue_byte_size (float, unit)
- buffer_stage_byte_size (float, unit)
- buffer_available_buffer_space_ratios (float, unit)
### Tags
- All measurements have the following tags:
- plugin_id (unique plugin id)
- plugin_type (type of the plugin e.g. s3)
- plugin_category (plugin category e.g. output)
## Example Output
```text
fluentd,host=T440s,plugin_id=object:9f748c,plugin_category=input,plugin_type=dummy buffer_total_queued_size=0,buffer_queue_length=0,retry_count=0 1492006105000000000
fluentd,plugin_category=input,plugin_type=dummy,host=T440s,plugin_id=object:8da98c buffer_queue_length=0,retry_count=0,buffer_total_queued_size=0 1492006105000000000
fluentd,plugin_id=object:820190,plugin_category=input,plugin_type=monitor_agent,host=T440s retry_count=0,buffer_total_queued_size=0,buffer_queue_length=0 1492006105000000000
fluentd,plugin_id=object:c5e054,plugin_category=output,plugin_type=stdout,host=T440s buffer_queue_length=0,retry_count=0,buffer_total_queued_size=0 1492006105000000000
fluentd,plugin_type=s3,host=T440s,plugin_id=object:bd7a90,plugin_category=output buffer_queue_length=0,retry_count=0,buffer_total_queued_size=0 1492006105000000000
fluentd,plugin_id=output_td, plugin_category=output,plugin_type=tdlog, host=T440s buffer_available_buffer_space_ratios=100,buffer_queue_byte_size=0,buffer_queue_length=0,buffer_stage_byte_size=0,buffer_stage_length=0,buffer_total_queued_size=0,emit_count=0,emit_records=0,flush_time_count=0,retry_count=0,rollback_count=0,slow_flush_count=0,write_count=0 1651474085000000000
```

223
plugins/inputs/fluentd/fluentd.go Normal file
View file

@ -0,0 +1,223 @@
//go:generate ../../../tools/readme_config_includer/generator
package fluentd
import (
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/plugins/inputs"
)
//go:embed sample.conf
var sampleConfig string
const measurement = "fluentd"
type Fluentd struct {
Endpoint string `toml:"endpoint"`
Exclude []string `toml:"exclude"`
client *http.Client
}
type endpointInfo struct {
Payload []pluginData `json:"plugins"`
}
type pluginData struct {
PluginID string `json:"plugin_id"`
PluginType string `json:"type"`
PluginCategory string `json:"plugin_category"`
RetryCount *float64 `json:"retry_count"`
BufferQueueLength *float64 `json:"buffer_queue_length"`
BufferTotalQueuedSize *float64 `json:"buffer_total_queued_size"`
RollbackCount *float64 `json:"rollback_count"`
EmitRecords *float64 `json:"emit_records"`
EmitSize *float64 `json:"emit_size"`
EmitCount *float64 `json:"emit_count"`
WriteCount *float64 `json:"write_count"`
SlowFlushCount *float64 `json:"slow_flush_count"`
FlushTimeCount *float64 `json:"flush_time_count"`
BufferStageLength *float64 `json:"buffer_stage_length"`
BufferStageByteSize *float64 `json:"buffer_stage_byte_size"`
BufferQueueByteSize *float64 `json:"buffer_queue_byte_size"`
AvailBufferSpaceRatios *float64 `json:"buffer_available_buffer_space_ratios"`
}
func (*Fluentd) SampleConfig() string {
return sampleConfig
}
func (h *Fluentd) Gather(acc telegraf.Accumulator) error {
_, err := url.Parse(h.Endpoint)
if err != nil {
return fmt.Errorf("invalid URL %q", h.Endpoint)
}
if h.client == nil {
tr := &http.Transport{
ResponseHeaderTimeout: 3 * time.Second,
}
client := &http.Client{
Transport: tr,
Timeout: 4 * time.Second,
}
h.client = client
}
resp, err := h.client.Get(h.Endpoint)
if err != nil {
return fmt.Errorf("unable to perform HTTP client GET on %q: %w", h.Endpoint, err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("unable to read the HTTP body %q: %w", string(body), err)
}
if resp.StatusCode != http.StatusOK {
return errors.New("http status ok not met")
}
dataPoints, err := parse(body)
if err != nil {
return errors.New("problem with parsing")
}
// Go through all plugins one by one
for _, p := range dataPoints {
skip := false
// Check if this specific type was excluded in configuration
for _, exclude := range h.Exclude {
if exclude == p.PluginType {
skip = true
}
}
// If not, create new metric and add it to Accumulator
if !skip {
tmpFields := make(map[string]interface{})
tmpTags := map[string]string{
"plugin_id": p.PluginID,
"plugin_category": p.PluginCategory,
"plugin_type": p.PluginType,
}
if p.BufferQueueLength != nil {
tmpFields["buffer_queue_length"] = *p.BufferQueueLength
}
if p.RetryCount != nil {
tmpFields["retry_count"] = *p.RetryCount
}
if p.BufferTotalQueuedSize != nil {
tmpFields["buffer_total_queued_size"] = *p.BufferTotalQueuedSize
}
if p.RollbackCount != nil {
tmpFields["rollback_count"] = *p.RollbackCount
}
if p.EmitRecords != nil {
tmpFields["emit_records"] = *p.EmitRecords
}
if p.EmitCount != nil {
tmpFields["emit_count"] = *p.EmitCount
}
if p.EmitSize != nil {
tmpFields["emit_size"] = *p.EmitSize
}
if p.WriteCount != nil {
tmpFields["write_count"] = *p.WriteCount
}
if p.SlowFlushCount != nil {
tmpFields["slow_flush_count"] = *p.SlowFlushCount
}
if p.FlushTimeCount != nil {
tmpFields["flush_time_count"] = *p.FlushTimeCount
}
if p.BufferStageLength != nil {
tmpFields["buffer_stage_length"] = *p.BufferStageLength
}
if p.BufferStageByteSize != nil {
tmpFields["buffer_stage_byte_size"] = *p.BufferStageByteSize
}
if p.BufferQueueByteSize != nil {
tmpFields["buffer_queue_byte_size"] = *p.BufferQueueByteSize
}
if p.AvailBufferSpaceRatios != nil {
tmpFields["buffer_available_buffer_space_ratios"] = *p.AvailBufferSpaceRatios
}
if p.BufferQueueLength != nil ||
p.RetryCount != nil ||
p.BufferTotalQueuedSize != nil ||
p.EmitCount != nil ||
p.EmitRecords != nil ||
p.EmitSize != nil ||
p.WriteCount != nil ||
p.FlushTimeCount != nil ||
p.SlowFlushCount != nil ||
p.RollbackCount != nil ||
p.BufferStageLength != nil ||
p.BufferStageByteSize != nil ||
p.BufferQueueByteSize != nil ||
p.AvailBufferSpaceRatios != nil {
acc.AddFields(measurement, tmpFields, tmpTags)
}
}
}
return nil
}
// parse JSON from fluentd Endpoint
// Parameters:
//
// data: unprocessed json received from endpoint
//
// Returns:
//
// pluginData: slice that contains parsed plugins
// error: error that may have occurred
func parse(data []byte) (datapointArray []pluginData, err error) {
var endpointData endpointInfo
if err = json.Unmarshal(data, &endpointData); err != nil {
err = errors.New("processing JSON structure")
return nil, err
}
datapointArray = append(datapointArray, endpointData.Payload...)
return datapointArray, err
}
func init() {
inputs.Add("fluentd", func() telegraf.Input { return &Fluentd{} })
}

243
plugins/inputs/fluentd/fluentd_test.go Normal file
View file

@ -0,0 +1,243 @@
package fluentd
import (
"fmt"
"net"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf/testutil"
)
// sampleJSON from fluentd version '0.14.9'
const sampleJSON = `
{
"plugins": [
{
"plugin_id": "object:f48698",
"plugin_category": "input",
"type": "dummy",
"config": {
"@type": "dummy",
"@log_level": "info",
"tag": "stdout.page.node",
"rate": "",
"dummy": "{\"hello\":\"world_from_first_dummy\"}",
"auto_increment_key": "id1"
},
"output_plugin": false,
"retry_count": null
},
{
"plugin_id": "object:e27138",
"plugin_category": "input",
"type": "dummy",
"config": {
"@type": "dummy",
"@log_level": "info",
"tag": "stdout.superproject.supercontainer",
"rate": "",
"dummy": "{\"hello\":\"world_from_second_dummy\"}",
"auto_increment_key": "id1"
},
"output_plugin": false,
"retry_count": null
},
{
"plugin_id": "object:d74060",
"plugin_category": "input",
"type": "monitor_agent",
"config": {
"@type": "monitor_agent",
"@log_level": "error",
"bind": "0.0.0.0",
"port": "24220"
},
"output_plugin": false,
"retry_count": null
},
{
"plugin_id": "object:11a5e2c",
"plugin_category": "output",
"type": "stdout",
"config": {
"@type": "stdout"
},
"output_plugin": true,
"retry_count": 0
},
{
"plugin_id": "object:11237ec",
"plugin_category": "output",
"type": "s3",
"config": {
"@type": "s3",
"@log_level": "info",
"aws_key_id": "xxxxxx",
"aws_sec_key": "xxxxxx",
"s3_bucket": "bucket",
"s3_endpoint": "http://mock:4567",
"path": "logs/%Y%m%d_%H/${tag[1]}/",
"time_slice_format": "%M",
"s3_object_key_format": "%{path}%{time_slice}_%{hostname}_%{index}_%{hex_random}.%{file_extension}",
"store_as": "gzip"
},
"output_plugin": true,
"buffer_queue_length": 0,
"retry_count": 0,
"buffer_total_queued_size": 0
},
{
"plugin_id": "object:output_td_1",
"plugin_category": "output",
"type": "tdlog",
"config": {
"@type": "tdlog",
"@id": "output_td",
"apikey": "xxxxxx",
"auto_create_table": ""
},
"output_plugin": true,
"buffer_queue_length": 0,
"buffer_total_queued_size": 0,
"retry_count": 0,
"emit_records": 0,
"emit_size": 0,
"emit_count": 0,
"write_count": 0,
"rollback_count": 0,
"slow_flush_count": 0,
"flush_time_count": 0,
"buffer_stage_length": 0,
"buffer_stage_byte_size": 0,
"buffer_queue_byte_size": 0,
"buffer_available_buffer_space_ratios": 0
},
{
"plugin_id": "object:output_td_2",
"plugin_category": "output",
"type": "tdlog",
"config": {
"@type": "tdlog",
"@id": "output_td",
"apikey": "xxxxxx",
"auto_create_table": ""
},
"output_plugin": true,
"buffer_queue_length": 0,
"buffer_total_queued_size": 0,
"retry_count": 0,
"rollback_count": 0,
"emit_records": 0,
"slow_flush_count": 0,
"buffer_available_buffer_space_ratios": 0
}
]
}
`
var (
zero float64
expectedOutput = []pluginData{
// {"object:f48698", "dummy", "input", nil, nil, nil},
// {"object:e27138", "dummy", "input", nil, nil, nil},
// {"object:d74060", "monitor_agent", "input", nil, nil, nil},
{"object:11a5e2c", "stdout", "output", &zero, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil},
{"object:11237ec", "s3", "output", &zero, &zero, &zero, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil},
{"object:output_td_1", "tdlog", "output", &zero, &zero, &zero, &zero, &zero, &zero, &zero, &zero, &zero, &zero, &zero, &zero, &zero, &zero},
{"object:output_td_2", "tdlog", "output", &zero, &zero, &zero, &zero, &zero, nil, nil, nil, &zero, nil, nil, nil, nil, &zero},
}
fluentdTest = &Fluentd{
Endpoint: "http://localhost:8081",
}
)
func Test_parse(t *testing.T) {
t.Log("Testing parser function")
t.Logf("JSON (%s) ", sampleJSON)
_, err := parse([]byte(sampleJSON))
if err != nil {
t.Error(err)
}
}
func Test_Gather(t *testing.T) {
t.Logf("Start HTTP mock (%s) with sampleJSON", fluentdTest.Endpoint)
ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
if _, err := fmt.Fprintf(w, "%s", sampleJSON); err != nil {
w.WriteHeader(http.StatusInternalServerError)
t.Error(err)
return
}
}))
requestURL, err := url.Parse(fluentdTest.Endpoint)
require.NoError(t, err)
require.NotNil(t, requestURL)
ts.Listener, err = net.Listen("tcp", fmt.Sprintf("%s:%s", requestURL.Hostname(), requestURL.Port()))
require.NoError(t, err)
ts.Start()
defer ts.Close()
var acc testutil.Accumulator
err = fluentdTest.Gather(&acc)
if err != nil {
t.Error(err)
}
if !acc.HasMeasurement("fluentd") {
t.Errorf("acc.HasMeasurement: expected fluentd")
}
require.Equal(t, expectedOutput[0].PluginID, acc.Metrics[0].Tags["plugin_id"])
require.Equal(t, expectedOutput[0].PluginType, acc.Metrics[0].Tags["plugin_type"])
require.Equal(t, expectedOutput[0].PluginCategory, acc.Metrics[0].Tags["plugin_category"])
require.InDelta(t, *expectedOutput[0].RetryCount, acc.Metrics[0].Fields["retry_count"], testutil.DefaultDelta)
require.Equal(t, expectedOutput[1].PluginID, acc.Metrics[1].Tags["plugin_id"])
require.Equal(t, expectedOutput[1].PluginType, acc.Metrics[1].Tags["plugin_type"])
require.Equal(t, expectedOutput[1].PluginCategory, acc.Metrics[1].Tags["plugin_category"])
require.InDelta(t, *expectedOutput[1].RetryCount, acc.Metrics[1].Fields["retry_count"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[1].BufferQueueLength, acc.Metrics[1].Fields["buffer_queue_length"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[1].BufferTotalQueuedSize, acc.Metrics[1].Fields["buffer_total_queued_size"], testutil.DefaultDelta)
require.Equal(t, expectedOutput[2].PluginID, acc.Metrics[2].Tags["plugin_id"])
require.Equal(t, expectedOutput[2].PluginType, acc.Metrics[2].Tags["plugin_type"])
require.Equal(t, expectedOutput[2].PluginCategory, acc.Metrics[2].Tags["plugin_category"])
require.InDelta(t, *expectedOutput[2].RetryCount, acc.Metrics[2].Fields["retry_count"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].BufferQueueLength, acc.Metrics[2].Fields["buffer_queue_length"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].BufferTotalQueuedSize, acc.Metrics[2].Fields["buffer_total_queued_size"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].EmitRecords, acc.Metrics[2].Fields["emit_records"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].EmitSize, acc.Metrics[2].Fields["emit_size"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].EmitCount, acc.Metrics[2].Fields["emit_count"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].RollbackCount, acc.Metrics[2].Fields["rollback_count"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].SlowFlushCount, acc.Metrics[2].Fields["slow_flush_count"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].WriteCount, acc.Metrics[2].Fields["write_count"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].FlushTimeCount, acc.Metrics[2].Fields["flush_time_count"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].BufferStageLength, acc.Metrics[2].Fields["buffer_stage_length"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].BufferStageByteSize, acc.Metrics[2].Fields["buffer_stage_byte_size"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].BufferQueueByteSize, acc.Metrics[2].Fields["buffer_queue_byte_size"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[2].AvailBufferSpaceRatios, acc.Metrics[2].Fields["buffer_available_buffer_space_ratios"], testutil.DefaultDelta)
require.Equal(t, expectedOutput[3].PluginID, acc.Metrics[3].Tags["plugin_id"])
require.Equal(t, expectedOutput[3].PluginType, acc.Metrics[3].Tags["plugin_type"])
require.Equal(t, expectedOutput[3].PluginCategory, acc.Metrics[3].Tags["plugin_category"])
require.InDelta(t, *expectedOutput[3].RetryCount, acc.Metrics[3].Fields["retry_count"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[3].BufferQueueLength, acc.Metrics[3].Fields["buffer_queue_length"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[3].BufferTotalQueuedSize, acc.Metrics[3].Fields["buffer_total_queued_size"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[3].EmitRecords, acc.Metrics[3].Fields["emit_records"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[3].RollbackCount, acc.Metrics[3].Fields["rollback_count"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[3].SlowFlushCount, acc.Metrics[3].Fields["slow_flush_count"], testutil.DefaultDelta)
require.InDelta(t, *expectedOutput[3].AvailBufferSpaceRatios, acc.Metrics[3].Fields["buffer_available_buffer_space_ratios"], testutil.DefaultDelta)
}

14
plugins/inputs/fluentd/sample.conf Normal file
View file

@ -0,0 +1,14 @@
# Read metrics exposed by fluentd in_monitor plugin
[[inputs.fluentd]]
## This plugin reads information exposed by fluentd (using /api/plugins.json endpoint).
##
## Endpoint:
## - only one URI is allowed
## - https is not supported
endpoint = "http://localhost:24220/api/plugins.json"
## Define which plugins have to be excluded (based on "type" field - e.g. monitor_agent)
exclude = [
"monitor_agent",
"dummy",
]