influx/telegraf
1
0
Fork 0
Code Issues Activity

Adding upstream version 1.34.4.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-05-24 07:26:29 +02:00
parent e393c3af3f
commit 4978089aab
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

123
plugins/outputs/opentsdb/README.md Normal file
View file

@ -0,0 +1,123 @@
# OpenTSDB Output Plugin
This plugin writes metrics to an [OpenTSDB][opentsdb] instance using either
the telnet or HTTP mode. Using the HTTP API is recommended since OpenTSDB 2.0.
⭐ Telegraf v0.1.9
🏷️ datastore
💻 all
[opentsdb]: http://opentsdb.net/
## Global configuration options <!-- @/docs/includes/plugin_config.md -->
In addition to the plugin-specific configuration settings, plugins support
additional global and plugin configuration settings. These settings are used to
modify metrics, tags, and field or create aliases and configure ordering, etc.
See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
## Configuration
```toml @sample.conf
# Configuration for OpenTSDB server to send metrics to
[[outputs.opentsdb]]
## prefix for metrics keys
prefix = "my.specific.prefix."
## DNS name of the OpenTSDB server
## Using "opentsdb.example.com" or "tcp://opentsdb.example.com" will use the
## telnet API. "http://opentsdb.example.com" will use the Http API.
host = "opentsdb.example.com"
## Port of the OpenTSDB server
port = 4242
## Number of data points to send to OpenTSDB in Http requests.
## Not used with telnet API.
http_batch_size = 50
## URI Path for Http requests to OpenTSDB.
## Used in cases where OpenTSDB is located behind a reverse proxy.
http_path = "/api/put"
## Debug true - Prints OpenTSDB communication
debug = false
## Separator separates measurement name from field
separator = "_"
```
## Transfer "Protocol" in the telnet mode
The expected input from OpenTSDB is specified in the following way:
```text
put <metric> <timestamp> <value> <tagk1=tagv1[ tagk2=tagv2 ...tagkN=tagvN]>
```
The telegraf output plugin adds an optional prefix to the metric keys so that a
subamount can be selected.
```text
put <[prefix.]metric> <timestamp> <value> <tagk1=tagv1[ tagk2=tagv2 ...tagkN=tagvN]>
```
### Example
```text
put nine.telegraf.system_load1 1441910356 0.430000 dc=homeoffice host=irimame scope=green
put nine.telegraf.system_load5 1441910356 0.580000 dc=homeoffice host=irimame scope=green
put nine.telegraf.system_load15 1441910356 0.730000 dc=homeoffice host=irimame scope=green
put nine.telegraf.system_uptime 1441910356 3655970.000000 dc=homeoffice host=irimame scope=green
put nine.telegraf.system_uptime_format 1441910356 dc=homeoffice host=irimame scope=green
put nine.telegraf.mem_total 1441910356 4145426432 dc=homeoffice host=irimame scope=green
...
put nine.telegraf.io_write_bytes 1441910366 0 dc=homeoffice host=irimame name=vda2 scope=green
put nine.telegraf.io_read_time 1441910366 0 dc=homeoffice host=irimame name=vda2 scope=green
put nine.telegraf.io_write_time 1441910366 0 dc=homeoffice host=irimame name=vda2 scope=green
put nine.telegraf.io_io_time 1441910366 0 dc=homeoffice host=irimame name=vda2 scope=green
put nine.telegraf.ping_packets_transmitted 1441910366 dc=homeoffice host=irimame scope=green url=www.google.com
put nine.telegraf.ping_packets_received 1441910366 dc=homeoffice host=irimame scope=green url=www.google.com
put nine.telegraf.ping_percent_packet_loss 1441910366 0.000000 dc=homeoffice host=irimame scope=green url=www.google.com
put nine.telegraf.ping_average_response_ms 1441910366 24.006000 dc=homeoffice host=irimame scope=green url=www.google.com
...
```
The OpenTSDB telnet interface can be simulated with this reader:
```go
// opentsdb_telnet_mode_mock.go
package main
import (
"io"
"log"
"net"
"os"
)
func main() {
l, err := net.Listen("tcp", "localhost:4242")
if err != nil {
log.Fatal(err)
}
defer l.Close()
for {
conn, err := l.Accept()
if err != nil {
log.Fatal(err)
}
go func(c net.Conn) {
defer c.Close()
io.Copy(os.Stdout, c)
}(conn)
}
}
```
## Allowed values for metrics
OpenTSDB allows `integers` and `floats` as input values

259
plugins/outputs/opentsdb/opentsdb.go Normal file
View file

@ -0,0 +1,259 @@
//go:generate ../../../tools/readme_config_includer/generator
package opentsdb
import (
_ "embed"
"errors"
"fmt"
"math"
"net"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/plugins/outputs"
)
//go:embed sample.conf
var sampleConfig string
var (
allowedChars = regexp.MustCompile(`[^a-zA-Z0-9-_./\p{L}]`)
hyphenChars = strings.NewReplacer(
"@", "-",
"*", "-",
`%`, "-",
"#", "-",
"$", "-")
defaultHTTPPath = "/api/put"
defaultSeparator = "_"
)
type OpenTSDB struct {
Prefix string `toml:"prefix"`
Host string `toml:"host"`
Port int `toml:"port"`
HTTPBatchSize int `toml:"http_batch_size"`
HTTPPath string `toml:"http_path"`
Debug bool `toml:"debug"`
Separator string `toml:"separator"`
Log telegraf.Logger `toml:"-"`
}
func ToLineFormat(tags map[string]string) string {
tagsArray := make([]string, 0, len(tags))
for k, v := range tags {
tagsArray = append(tagsArray, fmt.Sprintf("%s=%s", k, v))
}
sort.Strings(tagsArray)
return strings.Join(tagsArray, " ")
}
func (*OpenTSDB) SampleConfig() string {
return sampleConfig
}
func (o *OpenTSDB) Connect() error {
if !strings.HasPrefix(o.Host, "http") && !strings.HasPrefix(o.Host, "tcp") {
o.Host = "tcp://" + o.Host
}
// Test Connection to OpenTSDB Server
u, err := url.Parse(o.Host)
if err != nil {
return fmt.Errorf("error in parsing host url: %w", err)
}
uri := fmt.Sprintf("%s:%d", u.Host, o.Port)
tcpAddr, err := net.ResolveTCPAddr("tcp", uri)
if err != nil {
return fmt.Errorf("failed to resolve TCP address: %w", err)
}
connection, err := net.DialTCP("tcp", nil, tcpAddr)
if err != nil {
return fmt.Errorf("failed to connect to OpenTSDB: %w", err)
}
defer connection.Close()
return nil
}
func (o *OpenTSDB) Write(metrics []telegraf.Metric) error {
if len(metrics) == 0 {
return nil
}
u, err := url.Parse(o.Host)
if err != nil {
return fmt.Errorf("error in parsing host url: %w", err)
}
if u.Scheme == "" || u.Scheme == "tcp" {
return o.WriteTelnet(metrics, u)
} else if u.Scheme == "http" || u.Scheme == "https" {
return o.WriteHTTP(metrics, u)
}
return errors.New("unknown scheme in host parameter")
}
func (o *OpenTSDB) WriteHTTP(metrics []telegraf.Metric, u *url.URL) error {
http := openTSDBHttp{
Host: u.Host,
Port: o.Port,
Scheme: u.Scheme,
User: u.User,
BatchSize: o.HTTPBatchSize,
Path: o.HTTPPath,
Debug: o.Debug,
log: o.Log,
}
for _, m := range metrics {
now := m.Time().UnixNano() / 1000000000
tags := cleanTags(m.Tags())
for fieldName, value := range m.Fields() {
switch fv := value.(type) {
case int64:
case uint64:
case float64:
// JSON does not support these special values
if math.IsNaN(fv) || math.IsInf(fv, 0) {
continue
}
default:
o.Log.Debugf("OpenTSDB does not support metric value: [%s] of type [%T].", value, value)
continue
}
metric := &HTTPMetric{
Metric: sanitize(fmt.Sprintf("%s%s%s%s",
o.Prefix, m.Name(), o.Separator, fieldName)),
Tags: tags,
Timestamp: now,
Value: value,
}
if err := http.sendDataPoint(metric); err != nil {
return err
}
}
}
return http.flush()
}
func (o *OpenTSDB) WriteTelnet(metrics []telegraf.Metric, u *url.URL) error {
// Send Data with telnet / socket communication
uri := fmt.Sprintf("%s:%d", u.Host, o.Port)
tcpAddr, err := net.ResolveTCPAddr("tcp", uri)
if err != nil {
return fmt.Errorf("failed to resolve TCP address: %w", err)
}
connection, err := net.DialTCP("tcp", nil, tcpAddr)
if err != nil {
return fmt.Errorf("failed to connect to OpenTSDB: %w", err)
}
defer connection.Close()
for _, m := range metrics {
now := m.Time().UnixNano() / 1000000000
tags := ToLineFormat(cleanTags(m.Tags()))
for fieldName, value := range m.Fields() {
switch fv := value.(type) {
case int64:
case uint64:
case float64:
// JSON does not support these special values
if math.IsNaN(fv) || math.IsInf(fv, 0) {
continue
}
default:
o.Log.Debugf("OpenTSDB does not support metric value: [%s] of type [%T].", value, value)
continue
}
metricValue, buildError := buildValue(value)
if buildError != nil {
o.Log.Errorf("OpenTSDB: %s", buildError.Error())
continue
}
messageLine := fmt.Sprintf("put %s %v %s %s\n",
sanitize(fmt.Sprintf("%s%s%s%s", o.Prefix, m.Name(), o.Separator, fieldName)),
now, metricValue, tags)
_, err = connection.Write([]byte(messageLine))
if err != nil {
return fmt.Errorf("telnet writing error: %w", err)
}
}
}
return nil
}
func cleanTags(tags map[string]string) map[string]string {
tagSet := make(map[string]string, len(tags))
for k, v := range tags {
val := sanitize(v)
if val != "" {
tagSet[sanitize(k)] = val
}
}
return tagSet
}
func buildValue(v interface{}) (string, error) {
var retv string
switch p := v.(type) {
case int64:
retv = IntToString(p)
case uint64:
retv = UIntToString(p)
case float64:
retv = FloatToString(p)
default:
return retv, fmt.Errorf("unexpected type %T with value %v for OpenTSDB", v, v)
}
return retv, nil
}
func IntToString(inputNum int64) string {
return strconv.FormatInt(inputNum, 10)
}
func UIntToString(inputNum uint64) string {
return strconv.FormatUint(inputNum, 10)
}
func FloatToString(inputNum float64) string {
return strconv.FormatFloat(inputNum, 'f', 6, 64)
}
func (*OpenTSDB) Close() error {
return nil
}
func sanitize(value string) string {
// Apply special hyphenation rules to preserve backwards compatibility
value = hyphenChars.Replace(value)
// Replace any remaining illegal chars
return allowedChars.ReplaceAllLiteralString(value, "_")
}
func init() {
outputs.Add("opentsdb", func() telegraf.Output {
return &OpenTSDB{
HTTPPath: defaultHTTPPath,
Separator: defaultSeparator,
}
})
}

187
plugins/outputs/opentsdb/opentsdb_http.go Normal file
View file

@ -0,0 +1,187 @@
package opentsdb
import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httputil"
"net/url"
"github.com/influxdata/telegraf"
)
type HTTPMetric struct {
Metric string `json:"metric"`
Timestamp int64 `json:"timestamp"`
Value interface{} `json:"value"`
Tags map[string]string `json:"tags"`
}
type openTSDBHttp struct {
Host string
Port int
Scheme string
User *url.Userinfo
BatchSize int
Path string
Debug bool
log telegraf.Logger
metricCounter int
body requestBody
}
type requestBody struct {
b bytes.Buffer
g *gzip.Writer
dbgB bytes.Buffer
w io.Writer
enc *json.Encoder
empty bool
}
func (r *requestBody) reset(debug bool) {
r.b.Reset()
r.dbgB.Reset()
if r.g == nil {
r.g = gzip.NewWriter(&r.b)
} else {
r.g.Reset(&r.b)
}
if debug {
r.w = io.MultiWriter(r.g, &r.dbgB)
} else {
r.w = r.g
}
r.enc = json.NewEncoder(r.w)
//nolint:errcheck // unable to propagate error
io.WriteString(r.w, "[")
r.empty = true
}
func (r *requestBody) addMetric(metric *HTTPMetric) error {
if !r.empty {
if _, err := io.WriteString(r.w, ","); err != nil {
return err
}
}
if err := r.enc.Encode(metric); err != nil {
return fmt.Errorf("metric serialization error %w", err)
}
r.empty = false
return nil
}
func (r *requestBody) close() error {
if _, err := io.WriteString(r.w, "]"); err != nil {
return err
}
if err := r.g.Close(); err != nil {
return fmt.Errorf("error when closing gzip writer: %w", err)
}
return nil
}
func (o *openTSDBHttp) sendDataPoint(metric *HTTPMetric) error {
if o.metricCounter == 0 {
o.body.reset(o.Debug)
}
if err := o.body.addMetric(metric); err != nil {
return err
}
o.metricCounter++
if o.metricCounter == o.BatchSize {
if err := o.flush(); err != nil {
return err
}
o.metricCounter = 0
}
return nil
}
func (o *openTSDBHttp) flush() error {
if o.metricCounter == 0 {
return nil
}
if err := o.body.close(); err != nil {
return err
}
u := url.URL{
Scheme: o.Scheme,
User: o.User,
Host: fmt.Sprintf("%s:%d", o.Host, o.Port),
Path: o.Path,
}
if o.Debug {
u.RawQuery = "details"
}
req, err := http.NewRequest("POST", u.String(), &o.body.b)
if err != nil {
return fmt.Errorf("error when building request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "gzip")
if o.Debug {
dump, err := httputil.DumpRequestOut(req, false)
if err != nil {
return fmt.Errorf("error when dumping request: %w", err)
}
fmt.Printf("Sending metrics:\n%s", dump)
fmt.Printf("Body:\n%s\n\n", o.body.dbgB.String())
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("error when sending metrics: %w", err)
}
defer resp.Body.Close()
if o.Debug {
dump, err := httputil.DumpResponse(resp, true)
if err != nil {
return fmt.Errorf("error when dumping response: %w", err)
}
fmt.Printf("Received response\n%s\n\n", dump)
} else {
// Important so http client reuse connection for next request if need be.
//nolint:errcheck // cannot fail with io.Discard
io.Copy(io.Discard, resp.Body)
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
if resp.StatusCode < 400 || resp.StatusCode > 499 {
return fmt.Errorf("error sending metrics (status %d)", resp.StatusCode)
}
o.log.Errorf("Received %d status code. Dropping metrics to avoid overflowing buffer.", resp.StatusCode)
}
return nil
}

203
plugins/outputs/opentsdb/opentsdb_test.go Normal file
View file

@ -0,0 +1,203 @@
package opentsdb
import (
"fmt"
"net"
"net/http"
"net/http/httptest"
"net/url"
"reflect"
"strconv"
"testing"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/testutil"
)
func TestCleanTags(t *testing.T) {
var tagtests = []struct {
ptIn map[string]string
outTags map[string]string
}{
{
map[string]string{"one": "two", "three": "four"},
map[string]string{"one": "two", "three": "four"},
},
{
map[string]string{"aaa": "bbb"},
map[string]string{"aaa": "bbb"},
},
{
map[string]string{"Sp%ci@l Chars[": "g$t repl#ce)d"},
map[string]string{"Sp-ci-l_Chars_": "g-t_repl-ce_d"},
},
{
map[string]string{"μnicodε_letters": "okαy"},
map[string]string{"μnicodε_letters": "okαy"},
},
{
map[string]string{"n☺": "emojies☠"},
map[string]string{"n_": "emojies_"},
},
{
map[string]string{},
map[string]string{},
},
}
for _, tt := range tagtests {
tags := cleanTags(tt.ptIn)
if !reflect.DeepEqual(tags, tt.outTags) {
t.Errorf("\nexpected %+v\ngot %+v\n", tt.outTags, tags)
}
}
}
func TestBuildTagsTelnet(t *testing.T) {
var tagtests = []struct {
ptIn map[string]string
outTags string
}{
{
map[string]string{"one": "two", "three": "four"},
"one=two three=four",
},
{
map[string]string{"aaa": "bbb"},
"aaa=bbb",
},
{
map[string]string{"one": "two", "aaa": "bbb"},
"aaa=bbb one=two",
},
{
map[string]string{},
"",
},
}
for _, tt := range tagtests {
tags := ToLineFormat(tt.ptIn)
if !reflect.DeepEqual(tags, tt.outTags) {
t.Errorf("\nexpected %+v\ngot %+v\n", tt.outTags, tags)
}
}
}
func TestSanitize(t *testing.T) {
tests := []struct {
name string
value string
expected string
}{
{
name: "Ascii letters and numbers allowed",
value: "ascii 123",
expected: "ascii_123",
},
{
name: "Allowed punct",
value: "-_./",
expected: "-_./",
},
{
name: "Special conversions to hyphen",
value: "@*%#$!",
expected: "-----_",
},
{
name: "Unicode Letters allowed",
value: "μnicodε_letters",
expected: "μnicodε_letters",
},
{
name: "Other Unicode not allowed",
value: "“☢”",
expected: "___",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual := sanitize(tt.value)
require.Equal(t, tt.expected, actual)
})
}
}
func BenchmarkHttpSend(b *testing.B) {
const batchSize = 50
const metricsCount = 4 * batchSize
metrics := make([]telegraf.Metric, 0, metricsCount)
for i := 0; i < metricsCount; i++ {
metrics = append(metrics, testutil.TestMetric(1.0))
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
fmt.Fprintln(w, "{}")
}))
defer ts.Close()
u, err := url.Parse(ts.URL)
if err != nil {
panic(err)
}
_, p, err := net.SplitHostPort(u.Host)
require.NoError(b, err)
port, err := strconv.Atoi(p)
if err != nil {
panic(err)
}
o := &OpenTSDB{
Host: ts.URL,
Port: port,
Prefix: "",
HTTPBatchSize: batchSize,
HTTPPath: "/api/put",
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
//nolint:errcheck // skip error check for benchmarking
o.Write(metrics)
}
}
func TestWriteIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
t.Skip("Skip as OpenTSDB not running")
o := &OpenTSDB{
Host: testutil.GetLocalHost(),
Port: 4242,
Prefix: "prefix.test.",
}
// Verify that we can connect to the OpenTSDB instance
err := o.Connect()
require.NoError(t, err)
// Verify that we can successfully write data to OpenTSDB
err = o.Write(testutil.MockMetrics())
require.NoError(t, err)
// Verify positive and negative test cases of writing data
metrics := testutil.MockMetrics()
metrics = append(metrics,
testutil.TestMetric(float64(1.0), "justametric.float"),
testutil.TestMetric(int64(123456789), "justametric.int"),
testutil.TestMetric(uint64(123456789012345), "justametric.uint"),
testutil.TestMetric("Lorem Ipsum", "justametric.string"),
testutil.TestMetric(float64(42.0), "justametric.anotherfloat"),
testutil.TestMetric(float64(42.0), "metric w/ specialchars"),
)
err = o.Write(metrics)
require.NoError(t, err)
}

26
plugins/outputs/opentsdb/sample.conf Normal file
View file

@ -0,0 +1,26 @@
# Configuration for OpenTSDB server to send metrics to
[[outputs.opentsdb]]
## prefix for metrics keys
prefix = "my.specific.prefix."
## DNS name of the OpenTSDB server
## Using "opentsdb.example.com" or "tcp://opentsdb.example.com" will use the
## telnet API. "http://opentsdb.example.com" will use the Http API.
host = "opentsdb.example.com"
## Port of the OpenTSDB server
port = 4242
## Number of data points to send to OpenTSDB in Http requests.
## Not used with telnet API.
http_batch_size = 50
## URI Path for Http requests to OpenTSDB.
## Used in cases where OpenTSDB is located behind a reverse proxy.
http_path = "/api/put"
## Debug true - Prints OpenTSDB communication
debug = false
## Separator separates measurement name from field
separator = "_"