influx/telegraf
1
0
Fork 0
Code Issues Activity

Adding upstream version 1.34.4.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-05-24 07:26:29 +02:00
parent e393c3af3f
commit 4978089aab
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

129
plugins/outputs/influxdb/README.md Normal file
View file

@ -0,0 +1,129 @@
# InfluxDB v1.x Output Plugin
This plugin writes metrics to a [InfluxDB v1.x][influxdb_v1] instance via
HTTP or UDP protocol.
⭐ Telegraf v0.1.1
🏷️ datastore
💻 all
[influxdb_v1]: https://docs.influxdata.com/influxdb/v1
## Global configuration options <!-- @/docs/includes/plugin_config.md -->
In addition to the plugin-specific configuration settings, plugins support
additional global and plugin configuration settings. These settings are used to
modify metrics, tags, and field or create aliases and configure ordering, etc.
See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
## Secret-store support
This plugin supports secrets from secret-stores for the `username` and
`password` option.
See the [secret-store documentation][SECRETSTORE] for more details on how
to use them.
[SECRETSTORE]: ../../../docs/CONFIGURATION.md#secret-store-secrets
## Configuration
```toml @sample.conf
# Configuration for sending metrics to InfluxDB
[[outputs.influxdb]]
## The full HTTP or UDP URL for your InfluxDB instance.
##
## Multiple URLs can be specified for a single cluster, only ONE of the
## urls will be written to each interval.
# urls = ["unix:///var/run/influxdb.sock"]
# urls = ["udp://127.0.0.1:8089"]
# urls = ["http://127.0.0.1:8086"]
## Local address to bind when connecting to the server
## If empty or not set, the local address is automatically chosen.
# local_address = ""
## The target database for metrics; will be created as needed.
## For UDP url endpoint database needs to be configured on server side.
# database = "telegraf"
## The value of this tag will be used to determine the database. If this
## tag is not set the 'database' option is used as the default.
# database_tag = ""
## If true, the 'database_tag' will not be included in the written metric.
# exclude_database_tag = false
## If true, no CREATE DATABASE queries will be sent. Set to true when using
## Telegraf with a user without permissions to create databases or when the
## database already exists.
# skip_database_creation = false
## Name of existing retention policy to write to. Empty string writes to
## the default retention policy. Only takes effect when using HTTP.
# retention_policy = ""
## The value of this tag will be used to determine the retention policy. If this
## tag is not set the 'retention_policy' option is used as the default.
# retention_policy_tag = ""
## If true, the 'retention_policy_tag' will not be included in the written metric.
# exclude_retention_policy_tag = false
## Write consistency (clusters only), can be: "any", "one", "quorum", "all".
## Only takes effect when using HTTP.
# write_consistency = "any"
## Timeout for HTTP messages.
# timeout = "5s"
## HTTP Basic Auth
# username = "telegraf"
# password = "metricsmetricsmetricsmetrics"
## HTTP User-Agent
# user_agent = "telegraf"
## UDP payload size is the maximum packet size to send.
# udp_payload = "512B"
## Optional TLS Config for use on HTTP connections.
# tls_ca = "/etc/telegraf/ca.pem"
# tls_cert = "/etc/telegraf/cert.pem"
# tls_key = "/etc/telegraf/key.pem"
## Use TLS but skip chain & host verification
# insecure_skip_verify = false
## HTTP Proxy override, if unset values the standard proxy environment
## variables are consulted to determine which proxy, if any, should be used.
# http_proxy = "http://corporate.proxy:3128"
## Additional HTTP headers
# http_headers = {"X-Special-Header" = "Special-Value"}
## HTTP Content-Encoding for write request body, can be set to "gzip" to
## compress body or "identity" to apply no encoding.
# content_encoding = "gzip"
## When true, Telegraf will output unsigned integers as unsigned values,
## i.e.: "42u". You will need a version of InfluxDB supporting unsigned
## integer values. Enabling this option will result in field type errors if
## existing data has been written.
# influx_uint_support = false
## When true, Telegraf will omit the timestamp on data to allow InfluxDB
## to set the timestamp of the data during ingestion. This is generally NOT
## what you want as it can lead to data points captured at different times
## getting omitted due to similar data.
# influx_omit_timestamp = false
```
To send every metrics into multiple influxdb,
define additional `[[outputs.influxdb]]` section with new `urls`.
## Metrics
Reference the [influx serializer][] for details about metric production.
[influx serializer]: /plugins/serializers/influx/README.md#Metrics

594
plugins/outputs/influxdb/http.go Normal file
View file

@ -0,0 +1,594 @@
package influxdb
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/serializers/influx"
)
const (
defaultRequestTimeout = time.Second * 5
defaultDatabase = "telegraf"
errStringDatabaseNotFound = "database not found"
errStringRetentionPolicyNotFound = "retention policy not found"
errStringHintedHandoffNotEmpty = "hinted handoff queue not empty"
errStringPartialWrite = "partial write"
errStringPointsBeyondRP = "points beyond retention policy"
errStringUnableToParse = "unable to parse"
)
var (
// Escape an identifier in InfluxQL.
escapeIdentifier = strings.NewReplacer(
"\n", `\n`,
`\`, `\\`,
`"`, `\"`,
)
)
// APIError is a general error reported by the InfluxDB server
type APIError struct {
StatusCode int
Title string
Description string
}
func (e APIError) Error() string {
if e.Description != "" {
return fmt.Sprintf("%s: %s", e.Title, e.Description)
}
return e.Title
}
type DatabaseNotFoundError struct {
APIError
Database string
}
// queryResponseError is the response body from the /query endpoint
type queryResponseError struct {
Results []queryResult `json:"results"`
}
type queryResult struct {
Err string `json:"error,omitempty"`
}
func (r queryResponseError) Error() string {
if len(r.Results) > 0 {
return r.Results[0].Err
}
return ""
}
// writeResponseError is the response body from the /write endpoint
type writeResponseError struct {
Err string `json:"error,omitempty"`
}
func (r writeResponseError) Error() string {
return r.Err
}
type HTTPConfig struct {
URL *url.URL
LocalAddr *net.TCPAddr
UserAgent string
Timeout time.Duration
Username config.Secret
Password config.Secret
TLSConfig *tls.Config
Proxy *url.URL
Headers map[string]string
ContentEncoding string
Database string
DatabaseTag string
ExcludeDatabaseTag bool
RetentionPolicy string
RetentionPolicyTag string
ExcludeRetentionPolicyTag bool
Consistency string
SkipDatabaseCreation bool
InfluxUintSupport bool `toml:"influx_uint_support"`
Serializer *influx.Serializer
Log telegraf.Logger
}
type httpClient struct {
client *http.Client
config HTTPConfig
// Tracks that the 'create database` statement was executed for the
// database. An attempt to create the database is made each time a new
// database is encountered in the database_tag and after a "database not
// found" error occurs.
createDatabaseExecuted map[string]bool
log telegraf.Logger
}
func NewHTTPClient(cfg HTTPConfig) (*httpClient, error) {
if cfg.URL == nil {
return nil, ErrMissingURL
}
if cfg.Database == "" {
cfg.Database = defaultDatabase
}
if cfg.Timeout == 0 {
cfg.Timeout = defaultRequestTimeout
}
userAgent := cfg.UserAgent
if userAgent == "" {
userAgent = internal.ProductToken()
}
if cfg.Headers == nil {
cfg.Headers = make(map[string]string)
}
cfg.Headers["User-Agent"] = userAgent
for k, v := range cfg.Headers {
cfg.Headers[k] = v
}
var proxy func(*http.Request) (*url.URL, error)
if cfg.Proxy != nil {
proxy = http.ProxyURL(cfg.Proxy)
} else {
proxy = http.ProxyFromEnvironment
}
if cfg.Serializer == nil {
cfg.Serializer = &influx.Serializer{}
if err := cfg.Serializer.Init(); err != nil {
return nil, err
}
}
var transport *http.Transport
switch cfg.URL.Scheme {
case "http", "https":
var dialerFunc func(ctx context.Context, network, addr string) (net.Conn, error)
if cfg.LocalAddr != nil {
dialer := &net.Dialer{LocalAddr: cfg.LocalAddr}
dialerFunc = dialer.DialContext
}
transport = &http.Transport{
Proxy: proxy,
TLSClientConfig: cfg.TLSConfig,
DialContext: dialerFunc,
}
case "unix":
transport = &http.Transport{
Dial: func(_, _ string) (net.Conn, error) {
return net.DialTimeout(
cfg.URL.Scheme,
cfg.URL.Path,
defaultRequestTimeout,
)
},
}
default:
return nil, fmt.Errorf("unsupported scheme %q", cfg.URL.Scheme)
}
client := &httpClient{
client: &http.Client{
Timeout: cfg.Timeout,
Transport: transport,
},
createDatabaseExecuted: make(map[string]bool),
config: cfg,
log: cfg.Log,
}
return client, nil
}
// URL returns the origin URL that this client connects too.
func (c *httpClient) URL() string {
return c.config.URL.String()
}
// Database returns the default database that this client connects too.
func (c *httpClient) Database() string {
return c.config.Database
}
// CreateDatabase attempts to create a new database in the InfluxDB server.
// Note that some names are not allowed by the server, notably those with
// non-printable characters or slashes.
func (c *httpClient) CreateDatabase(ctx context.Context, database string) error {
//nolint:gocritic // sprintfQuotedString - "%s" used by purpose, string escaping is done by special function
query := fmt.Sprintf(`CREATE DATABASE "%s"`, escapeIdentifier.Replace(database))
req, err := c.makeQueryRequest(query)
if err != nil {
return err
}
resp, err := c.client.Do(req.WithContext(ctx))
if err != nil {
internal.OnClientError(c.client, err)
return err
}
defer resp.Body.Close()
body, err := validateResponse(resp.Body)
// Check for poorly formatted response (can't be decoded)
if err != nil {
return &APIError{
StatusCode: resp.StatusCode,
Title: resp.Status,
Description: "An error response was received while attempting to create the following database: " + database + ". Error: " + err.Error(),
}
}
queryResp := &queryResponseError{}
dec := json.NewDecoder(body)
err = dec.Decode(queryResp)
if err != nil {
if resp.StatusCode == 200 {
return nil
}
return &APIError{
StatusCode: resp.StatusCode,
Title: resp.Status,
}
}
// Even with a 200 status code there can be an error in the response body.
// If there is also no error string then the operation was successful.
if resp.StatusCode == http.StatusOK && queryResp.Error() == "" {
c.createDatabaseExecuted[database] = true
return nil
}
// Don't attempt to recreate the database after a 403 Forbidden error.
// This behavior exists only to maintain backwards compatibility.
if resp.StatusCode == http.StatusForbidden {
c.createDatabaseExecuted[database] = true
}
return &APIError{
StatusCode: resp.StatusCode,
Title: resp.Status,
Description: queryResp.Error(),
}
}
type dbrp struct {
Database string
RetentionPolicy string
}
// Write sends the metrics to InfluxDB
func (c *httpClient) Write(ctx context.Context, metrics []telegraf.Metric) error {
// If these options are not used, we can skip in plugin batching and send
// the full batch in a single request.
if c.config.DatabaseTag == "" && c.config.RetentionPolicyTag == "" {
return c.writeBatch(ctx, c.config.Database, c.config.RetentionPolicy, metrics)
}
batches := make(map[dbrp][]telegraf.Metric)
for _, metric := range metrics {
db, ok := metric.GetTag(c.config.DatabaseTag)
if !ok {
db = c.config.Database
}
rp, ok := metric.GetTag(c.config.RetentionPolicyTag)
if !ok {
rp = c.config.RetentionPolicy
}
dbrp := dbrp{
Database: db,
RetentionPolicy: rp,
}
if c.config.ExcludeDatabaseTag || c.config.ExcludeRetentionPolicyTag {
// Avoid modifying the metric in case we need to retry the request.
metric = metric.Copy()
metric.Accept()
if c.config.ExcludeDatabaseTag {
metric.RemoveTag(c.config.DatabaseTag)
}
if c.config.ExcludeRetentionPolicyTag {
metric.RemoveTag(c.config.RetentionPolicyTag)
}
}
batches[dbrp] = append(batches[dbrp], metric)
}
for dbrp, batch := range batches {
if !c.config.SkipDatabaseCreation && !c.createDatabaseExecuted[dbrp.Database] {
err := c.CreateDatabase(ctx, dbrp.Database)
if err != nil {
c.log.Warnf("When writing to [%s]: database %q creation failed: %v",
c.config.URL, dbrp.Database, err)
}
}
err := c.writeBatch(ctx, dbrp.Database, dbrp.RetentionPolicy, batch)
if err != nil {
return err
}
}
return nil
}
func (c *httpClient) writeBatch(ctx context.Context, db, rp string, metrics []telegraf.Metric) error {
loc, err := makeWriteURL(c.config.URL, db, rp, c.config.Consistency)
if err != nil {
return fmt.Errorf("failed making write url: %w", err)
}
reader := c.requestBodyReader(metrics)
defer reader.Close()
req, err := c.makeWriteRequest(loc, reader)
if err != nil {
return fmt.Errorf("failed making write req: %w", err)
}
resp, err := c.client.Do(req.WithContext(ctx))
if err != nil {
internal.OnClientError(c.client, err)
return fmt.Errorf("failed doing req: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNoContent {
return nil
}
body, err := validateResponse(resp.Body)
// Check for poorly formatted response that can't be decoded
if err != nil {
return &APIError{
StatusCode: resp.StatusCode,
Title: resp.Status,
Description: "An error response was received while attempting to write metrics. Error: " + err.Error(),
}
}
writeResp := &writeResponseError{}
dec := json.NewDecoder(body)
var desc string
err = dec.Decode(writeResp)
if err == nil {
desc = writeResp.Err
}
if strings.Contains(desc, errStringDatabaseNotFound) {
return &DatabaseNotFoundError{
APIError: APIError{
StatusCode: resp.StatusCode,
Title: resp.Status,
Description: desc,
},
Database: db,
}
}
// checks for any 4xx code and drops metric and retrying will not make the request work
if len(resp.Status) > 0 && resp.Status[0] == '4' {
c.log.Errorf("E! [outputs.influxdb] Failed to write metric (will be dropped: %s): %s\n", resp.Status, desc)
return nil
}
// This error handles if there is an invalid or missing retention policy
if strings.Contains(desc, errStringRetentionPolicyNotFound) {
c.log.Errorf("When writing to [%s]: received error %v", c.URL(), desc)
return nil
}
// This "error" is an informational message about the state of the
// InfluxDB cluster.
if strings.Contains(desc, errStringHintedHandoffNotEmpty) {
return nil
}
// Points beyond retention policy is returned when points are immediately
// discarded for being older than the retention policy. Usually this not
// a cause for concern and we don't want to retry.
if strings.Contains(desc, errStringPointsBeyondRP) {
c.log.Warnf("When writing to [%s]: received error %v",
c.URL(), desc)
return nil
}
// Other partial write errors, such as "field type conflict", are not
// correctable at this point and so the point is dropped instead of
// retrying.
if strings.Contains(desc, errStringPartialWrite) {
c.log.Errorf("When writing to [%s]: received error %v; discarding points",
c.URL(), desc)
return nil
}
// This error indicates a bug in either Telegraf line protocol
// serialization, retries would not be successful.
if strings.Contains(desc, errStringUnableToParse) {
c.log.Errorf("When writing to [%s]: received error %v; discarding points",
c.URL(), desc)
return nil
}
return &APIError{
StatusCode: resp.StatusCode,
Title: resp.Status,
Description: desc,
}
}
func (c *httpClient) makeQueryRequest(query string) (*http.Request, error) {
queryURL, err := makeQueryURL(c.config.URL)
if err != nil {
return nil, err
}
params := url.Values{}
params.Set("q", query)
form := strings.NewReader(params.Encode())
req, err := http.NewRequest("POST", queryURL, form)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
if err := c.addHeaders(req); err != nil {
return nil, err
}
return req, err
}
func (c *httpClient) makeWriteRequest(address string, body io.Reader) (*http.Request, error) {
var err error
req, err := http.NewRequest("POST", address, body)
if err != nil {
return nil, fmt.Errorf("failed creating new request: %w", err)
}
req.Header.Set("Content-Type", "text/plain; charset=utf-8")
if err := c.addHeaders(req); err != nil {
return nil, err
}
if c.config.ContentEncoding == "gzip" {
req.Header.Set("Content-Encoding", "gzip")
}
return req, nil
}
// requestBodyReader warp io.Reader from influx.NewReader to io.ReadCloser, which is useful to fast close the write
// side of the connection in case of error
func (c *httpClient) requestBodyReader(metrics []telegraf.Metric) io.ReadCloser {
reader := influx.NewReader(metrics, c.config.Serializer)
if c.config.ContentEncoding == "gzip" {
return internal.CompressWithGzip(reader)
}
return io.NopCloser(reader)
}
func (c *httpClient) addHeaders(req *http.Request) error {
if !c.config.Username.Empty() || !c.config.Password.Empty() {
username, err := c.config.Username.Get()
if err != nil {
return fmt.Errorf("getting username failed: %w", err)
}
password, err := c.config.Password.Get()
if err != nil {
username.Destroy()
return fmt.Errorf("getting password failed: %w", err)
}
req.SetBasicAuth(username.String(), password.String())
username.Destroy()
password.Destroy()
}
for header, value := range c.config.Headers {
if strings.EqualFold(header, "host") {
req.Host = value
} else {
req.Header.Set(header, value)
}
}
return nil
}
func validateResponse(response io.ReadCloser) (io.ReadCloser, error) {
bodyBytes, err := io.ReadAll(response)
if err != nil {
return nil, err
}
defer response.Close()
originalResponse := io.NopCloser(bytes.NewBuffer(bodyBytes))
// Empty response is valid.
if response == http.NoBody || len(bodyBytes) == 0 || bodyBytes == nil {
return originalResponse, nil
}
if valid := json.Valid(bodyBytes); !valid {
err = errors.New(string(bodyBytes))
}
return originalResponse, err
}
func makeWriteURL(loc *url.URL, db, rp, consistency string) (string, error) {
params := url.Values{}
params.Set("db", db)
if rp != "" {
params.Set("rp", rp)
}
if consistency != "one" && consistency != "" {
params.Set("consistency", consistency)
}
u := *loc
switch u.Scheme {
case "unix":
u.Scheme = "http"
u.Host = "127.0.0.1"
u.Path = "/write"
case "http", "https":
u.Path = path.Join(u.Path, "write")
default:
return "", fmt.Errorf("unsupported scheme: %q", loc.Scheme)
}
u.RawQuery = params.Encode()
return u.String(), nil
}
func makeQueryURL(loc *url.URL) (string, error) {
u := *loc
switch u.Scheme {
case "unix":
u.Scheme = "http"
u.Host = "127.0.0.1"
u.Path = "/query"
case "http", "https":
u.Path = path.Join(u.Path, "query")
default:
return "", fmt.Errorf("unsupported scheme: %q", loc.Scheme)
}
return u.String(), nil
}
func (c *httpClient) Close() {
c.client.CloseIdleConnections()
}

1258
plugins/outputs/influxdb/http_test.go Normal file
View file

File diff suppressed because it is too large Load diff

296
plugins/outputs/influxdb/influxdb.go Normal file
View file

@ -0,0 +1,296 @@
//go:generate ../../../tools/readme_config_includer/generator
package influxdb
import (
"context"
_ "embed"
"errors"
"fmt"
"math/rand"
"net"
"net/url"
"strconv"
"strings"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/plugins/common/tls"
"github.com/influxdata/telegraf/plugins/outputs"
"github.com/influxdata/telegraf/plugins/serializers/influx"
)
//go:embed sample.conf
var sampleConfig string
var (
defaultURL = "http://localhost:8086"
ErrMissingURL = errors.New("missing URL")
)
type Client interface {
Write(context.Context, []telegraf.Metric) error
CreateDatabase(ctx context.Context, database string) error
Database() string
URL() string
Close()
}
// InfluxDB struct is the primary data structure for the plugin
type InfluxDB struct {
URLs []string `toml:"urls"`
LocalAddr string `toml:"local_address"`
Username config.Secret `toml:"username"`
Password config.Secret `toml:"password"`
Database string `toml:"database"`
DatabaseTag string `toml:"database_tag"`
ExcludeDatabaseTag bool `toml:"exclude_database_tag"`
RetentionPolicy string `toml:"retention_policy"`
RetentionPolicyTag string `toml:"retention_policy_tag"`
ExcludeRetentionPolicyTag bool `toml:"exclude_retention_policy_tag"`
UserAgent string `toml:"user_agent"`
WriteConsistency string `toml:"write_consistency"`
Timeout config.Duration `toml:"timeout"`
UDPPayload config.Size `toml:"udp_payload"`
HTTPProxy string `toml:"http_proxy"`
HTTPHeaders map[string]string `toml:"http_headers"`
ContentEncoding string `toml:"content_encoding"`
SkipDatabaseCreation bool `toml:"skip_database_creation"`
InfluxUintSupport bool `toml:"influx_uint_support"`
OmitTimestamp bool `toml:"influx_omit_timestamp"`
Precision string `toml:"precision" deprecated:"1.0.0;1.35.0;option is ignored"`
Log telegraf.Logger `toml:"-"`
tls.ClientConfig
clients []Client
CreateHTTPClientF func(config *HTTPConfig) (Client, error)
CreateUDPClientF func(config *UDPConfig) (Client, error)
}
func (*InfluxDB) SampleConfig() string {
return sampleConfig
}
func (i *InfluxDB) Connect() error {
ctx := context.Background()
if len(i.URLs) == 0 {
i.URLs = []string{defaultURL}
}
for _, u := range i.URLs {
parts, err := url.Parse(u)
if err != nil {
return fmt.Errorf("error parsing url [%q]: %w", u, err)
}
var proxy *url.URL
if len(i.HTTPProxy) > 0 {
proxy, err = url.Parse(i.HTTPProxy)
if err != nil {
return fmt.Errorf("error parsing proxy_url [%s]: %w", i.HTTPProxy, err)
}
}
var localIP *net.IPAddr
var localPort int
if i.LocalAddr != "" {
var err error
// Resolve the local address into IP address and the given port if any
addr, sPort, err := net.SplitHostPort(i.LocalAddr)
if err != nil {
if !strings.Contains(err.Error(), "missing port") {
return fmt.Errorf("invalid local address: %w", err)
}
addr = i.LocalAddr
}
localIP, err = net.ResolveIPAddr("ip", addr)
if err != nil {
return fmt.Errorf("cannot resolve local address: %w", err)
}
if sPort != "" {
p, err := strconv.ParseUint(sPort, 10, 16)
if err != nil {
return fmt.Errorf("invalid port: %w", err)
}
localPort = int(p)
}
}
switch parts.Scheme {
case "udp", "udp4", "udp6":
var c Client
var err error
if i.LocalAddr == "" {
c, err = i.udpClient(parts, nil)
} else {
c, err = i.udpClient(parts, &net.UDPAddr{IP: localIP.IP, Port: localPort, Zone: localIP.Zone})
}
if err != nil {
return err
}
i.clients = append(i.clients, c)
case "http", "https", "unix":
var c Client
var err error
if i.LocalAddr == "" {
c, err = i.httpClient(ctx, parts, nil, proxy)
} else {
c, err = i.httpClient(ctx, parts, &net.TCPAddr{IP: localIP.IP, Port: localPort, Zone: localIP.Zone}, proxy)
}
if err != nil {
return err
}
i.clients = append(i.clients, c)
default:
return fmt.Errorf("unsupported scheme [%q]: %q", u, parts.Scheme)
}
}
return nil
}
func (i *InfluxDB) Close() error {
for _, client := range i.clients {
client.Close()
}
return nil
}
// Write sends metrics to one of the configured servers, logging each
// unsuccessful. If all servers fail, return an error.
func (i *InfluxDB) Write(metrics []telegraf.Metric) error {
ctx := context.Background()
allErrorsAreDatabaseNotFoundErrors := true
var err error
p := rand.Perm(len(i.clients))
for _, n := range p {
client := i.clients[n]
err = client.Write(ctx, metrics)
if err == nil {
return nil
}
i.Log.Errorf("When writing to [%s]: %v", client.URL(), err)
var apiError *DatabaseNotFoundError
if errors.As(err, &apiError) {
if i.SkipDatabaseCreation {
continue
}
// retry control
// error so the write is retried
if err := client.CreateDatabase(ctx, apiError.Database); err == nil {
return errors.New("database created; retry write")
}
i.Log.Errorf("When writing to [%s]: database %q not found and failed to recreate", client.URL(), apiError.Database)
} else {
allErrorsAreDatabaseNotFoundErrors = false
}
}
if allErrorsAreDatabaseNotFoundErrors {
// return nil because we should not be retrying this
return nil
}
return errors.New("could not write any address")
}
func (i *InfluxDB) udpClient(address *url.URL, localAddr *net.UDPAddr) (Client, error) {
serializer := &influx.Serializer{
UintSupport: i.InfluxUintSupport,
OmitTimestamp: i.OmitTimestamp,
}
if err := serializer.Init(); err != nil {
return nil, err
}
udpConfig := &UDPConfig{
URL: address,
LocalAddr: localAddr,
MaxPayloadSize: int(i.UDPPayload),
Serializer: serializer,
Log: i.Log,
}
c, err := i.CreateUDPClientF(udpConfig)
if err != nil {
return nil, fmt.Errorf("error creating UDP client [%s]: %w", address, err)
}
return c, nil
}
func (i *InfluxDB) httpClient(ctx context.Context, address *url.URL, localAddr *net.TCPAddr, proxy *url.URL) (Client, error) {
tlsConfig, err := i.ClientConfig.TLSConfig()
if err != nil {
return nil, err
}
serializer := &influx.Serializer{
UintSupport: i.InfluxUintSupport,
OmitTimestamp: i.OmitTimestamp,
}
if err := serializer.Init(); err != nil {
return nil, err
}
httpConfig := &HTTPConfig{
URL: address,
LocalAddr: localAddr,
Timeout: time.Duration(i.Timeout),
TLSConfig: tlsConfig,
UserAgent: i.UserAgent,
Username: i.Username,
Password: i.Password,
Proxy: proxy,
ContentEncoding: i.ContentEncoding,
Headers: i.HTTPHeaders,
Database: i.Database,
DatabaseTag: i.DatabaseTag,
ExcludeDatabaseTag: i.ExcludeDatabaseTag,
SkipDatabaseCreation: i.SkipDatabaseCreation,
RetentionPolicy: i.RetentionPolicy,
RetentionPolicyTag: i.RetentionPolicyTag,
ExcludeRetentionPolicyTag: i.ExcludeRetentionPolicyTag,
Consistency: i.WriteConsistency,
Serializer: serializer,
Log: i.Log,
}
c, err := i.CreateHTTPClientF(httpConfig)
if err != nil {
return nil, fmt.Errorf("error creating HTTP client [%s]: %w", address, err)
}
if !i.SkipDatabaseCreation {
err = c.CreateDatabase(ctx, c.Database())
if err != nil {
i.Log.Warnf("When writing to [%s]: database %q creation failed: %v",
c.URL(), c.Database(), err)
}
}
return c, nil
}
func init() {
outputs.Add("influxdb", func() telegraf.Output {
return &InfluxDB{
Timeout: config.Duration(time.Second * 5),
CreateHTTPClientF: func(config *HTTPConfig) (Client, error) {
return NewHTTPClient(*config)
},
CreateUDPClientF: func(config *UDPConfig) (Client, error) {
return NewUDPClient(*config)
},
ContentEncoding: "gzip",
}
})
}

555
plugins/outputs/influxdb/influxdb_test.go Normal file
View file

@ -0,0 +1,555 @@
package influxdb_test
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/metric"
"github.com/influxdata/telegraf/plugins/common/tls"
"github.com/influxdata/telegraf/plugins/outputs/influxdb"
"github.com/influxdata/telegraf/testutil"
)
type MockClient struct {
URLF func() string
WriteF func() error
CreateDatabaseF func() error
DatabaseF func() string
CloseF func()
log telegraf.Logger
}
func (c *MockClient) URL() string {
return c.URLF()
}
func (c *MockClient) Write(context.Context, []telegraf.Metric) error {
return c.WriteF()
}
func (c *MockClient) CreateDatabase(context.Context, string) error {
return c.CreateDatabaseF()
}
func (c *MockClient) Database() string {
return c.DatabaseF()
}
func (c *MockClient) Close() {
c.CloseF()
}
func (c *MockClient) SetLogger(log telegraf.Logger) {
c.log = log
}
func TestDeprecatedURLSupport(t *testing.T) {
var actual *influxdb.UDPConfig
output := influxdb.InfluxDB{
URLs: []string{"udp://localhost:8089"},
CreateUDPClientF: func(config *influxdb.UDPConfig) (influxdb.Client, error) {
actual = config
return &MockClient{}, nil
},
}
output.Log = testutil.Logger{}
err := output.Connect()
require.NoError(t, err)
require.Equal(t, "udp://localhost:8089", actual.URL.String())
}
func TestDefaultURL(t *testing.T) {
var actual *influxdb.HTTPConfig
output := influxdb.InfluxDB{
CreateHTTPClientF: func(config *influxdb.HTTPConfig) (influxdb.Client, error) {
actual = config
return &MockClient{
DatabaseF: func() string {
return "telegraf"
},
CreateDatabaseF: func() error {
return nil
},
}, nil
},
}
output.Log = testutil.Logger{}
err := output.Connect()
require.NoError(t, err)
require.Equal(t, "http://localhost:8086", actual.URL.String())
}
func TestConnectUDPConfig(t *testing.T) {
var actual *influxdb.UDPConfig
output := influxdb.InfluxDB{
URLs: []string{"udp://localhost:8089"},
UDPPayload: config.Size(42),
CreateUDPClientF: func(config *influxdb.UDPConfig) (influxdb.Client, error) {
actual = config
return &MockClient{}, nil
},
}
output.Log = testutil.Logger{}
err := output.Connect()
require.NoError(t, err)
require.Equal(t, "udp://localhost:8089", actual.URL.String())
require.Equal(t, 42, actual.MaxPayloadSize)
require.NotNil(t, actual.Serializer)
}
func TestConnectHTTPConfig(t *testing.T) {
var actual *influxdb.HTTPConfig
output := influxdb.InfluxDB{
URLs: []string{"http://localhost:8086"},
Database: "telegraf",
RetentionPolicy: "default",
WriteConsistency: "any",
Timeout: config.Duration(5 * time.Second),
Username: config.NewSecret([]byte("guy")),
Password: config.NewSecret([]byte("smiley")),
UserAgent: "telegraf",
HTTPProxy: "http://localhost:8086",
HTTPHeaders: map[string]string{
"x": "y",
},
ContentEncoding: "gzip",
ClientConfig: tls.ClientConfig{
InsecureSkipVerify: true,
},
CreateHTTPClientF: func(config *influxdb.HTTPConfig) (influxdb.Client, error) {
actual = config
return &MockClient{
DatabaseF: func() string {
return "telegraf"
},
CreateDatabaseF: func() error {
return nil
},
}, nil
},
}
output.Log = testutil.Logger{}
err := output.Connect()
require.NoError(t, err)
require.Equal(t, output.URLs[0], actual.URL.String())
require.Equal(t, output.UserAgent, actual.UserAgent)
require.Equal(t, time.Duration(output.Timeout), actual.Timeout)
require.Equal(t, output.Username, actual.Username)
require.Equal(t, output.Password, actual.Password)
require.Equal(t, output.HTTPProxy, actual.Proxy.String())
require.Equal(t, output.HTTPHeaders, actual.Headers)
require.Equal(t, output.ContentEncoding, actual.ContentEncoding)
require.Equal(t, output.Database, actual.Database)
require.Equal(t, output.RetentionPolicy, actual.RetentionPolicy)
require.Equal(t, output.WriteConsistency, actual.Consistency)
require.NotNil(t, actual.TLSConfig)
require.NotNil(t, actual.Serializer)
require.Equal(t, output.Database, actual.Database)
}
func TestWriteRecreateDatabaseIfDatabaseNotFound(t *testing.T) {
output := influxdb.InfluxDB{
URLs: []string{"http://localhost:8086"},
CreateHTTPClientF: func(*influxdb.HTTPConfig) (influxdb.Client, error) {
return &MockClient{
DatabaseF: func() string {
return "telegraf"
},
CreateDatabaseF: func() error {
return nil
},
WriteF: func() error {
return &influxdb.DatabaseNotFoundError{
APIError: influxdb.APIError{
StatusCode: http.StatusNotFound,
Title: "404 Not Found",
Description: `database not found "telegraf"`,
},
}
},
URLF: func() string {
return "http://localhost:8086"
},
}, nil
},
}
output.Log = testutil.Logger{}
err := output.Connect()
require.NoError(t, err)
m := metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"value": 42.0,
},
time.Unix(0, 0),
)
metrics := []telegraf.Metric{m}
err = output.Write(metrics)
// We only have one URL, so we expect an error
require.Error(t, err)
}
func TestInfluxDBLocalAddress(t *testing.T) {
output := influxdb.InfluxDB{
URLs: []string{"http://localhost:8086"},
LocalAddr: "localhost",
CreateHTTPClientF: func(_ *influxdb.HTTPConfig) (influxdb.Client, error) {
return &MockClient{
DatabaseF: func() string {
return "telegraf"
},
CreateDatabaseF: func() error {
return nil
},
}, nil
},
}
require.NoError(t, output.Connect())
}
func BenchmarkWrite1k(b *testing.B) {
batchsize := 1000
// Setup a test server
ts := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}),
)
defer ts.Close()
// Setup plugin and connect
plugin := &influxdb.InfluxDB{
URLs: []string{"http://" + ts.Listener.Addr().String()},
Username: config.NewSecret([]byte("user")),
Password: config.NewSecret([]byte("secret")),
Database: "my_database",
Timeout: config.Duration(time.Second * 5),
Log: &testutil.Logger{},
CreateHTTPClientF: func(config *influxdb.HTTPConfig) (influxdb.Client, error) {
return influxdb.NewHTTPClient(*config)
},
CreateUDPClientF: func(config *influxdb.UDPConfig) (influxdb.Client, error) {
return influxdb.NewUDPClient(*config)
},
ContentEncoding: "gzip",
SkipDatabaseCreation: true,
}
require.NoError(b, plugin.Connect())
defer plugin.Close()
metrics := make([]telegraf.Metric, 0, batchsize)
for i := range batchsize {
metrics = append(metrics, metric.New(
"cpu",
map[string]string{
"database": "foo",
},
map[string]interface{}{
"value": float64(i),
},
time.Unix(0, 0),
))
}
// Benchmark the writing
b.ResetTimer()
for b.Loop() {
require.NoError(b, plugin.Write(metrics))
}
b.ReportMetric(float64(batchsize*b.N)/b.Elapsed().Seconds(), "metrics/s")
}
func BenchmarkWrite5k(b *testing.B) {
batchsize := 5000
// Setup a test server
ts := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}),
)
defer ts.Close()
// Setup plugin and connect
plugin := &influxdb.InfluxDB{
URLs: []string{"http://" + ts.Listener.Addr().String()},
Username: config.NewSecret([]byte("user")),
Password: config.NewSecret([]byte("secret")),
Database: "my_database",
Timeout: config.Duration(time.Second * 5),
Log: &testutil.Logger{},
CreateHTTPClientF: func(config *influxdb.HTTPConfig) (influxdb.Client, error) {
return influxdb.NewHTTPClient(*config)
},
CreateUDPClientF: func(config *influxdb.UDPConfig) (influxdb.Client, error) {
return influxdb.NewUDPClient(*config)
},
ContentEncoding: "gzip",
SkipDatabaseCreation: true,
}
require.NoError(b, plugin.Connect())
defer plugin.Close()
metrics := make([]telegraf.Metric, 0, batchsize)
for i := range batchsize {
metrics = append(metrics, metric.New(
"cpu",
map[string]string{
"database": "foo",
},
map[string]interface{}{
"value": float64(i),
},
time.Unix(0, 0),
))
}
// Benchmark the writing
b.ResetTimer()
for b.Loop() {
require.NoError(b, plugin.Write(metrics))
}
b.ReportMetric(float64(batchsize*b.N)/b.Elapsed().Seconds(), "metrics/s")
}
func BenchmarkWrite10k(b *testing.B) {
batchsize := 10000
// Setup a test server
ts := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}),
)
defer ts.Close()
// Setup plugin and connect
plugin := &influxdb.InfluxDB{
URLs: []string{"http://" + ts.Listener.Addr().String()},
Username: config.NewSecret([]byte("user")),
Password: config.NewSecret([]byte("secret")),
Database: "my_database",
Timeout: config.Duration(time.Second * 5),
Log: &testutil.Logger{},
CreateHTTPClientF: func(config *influxdb.HTTPConfig) (influxdb.Client, error) {
return influxdb.NewHTTPClient(*config)
},
CreateUDPClientF: func(config *influxdb.UDPConfig) (influxdb.Client, error) {
return influxdb.NewUDPClient(*config)
},
ContentEncoding: "gzip",
SkipDatabaseCreation: true,
}
require.NoError(b, plugin.Connect())
defer plugin.Close()
metrics := make([]telegraf.Metric, 0, batchsize)
for i := range batchsize {
metrics = append(metrics, metric.New(
"cpu",
map[string]string{
"database": "foo",
},
map[string]interface{}{
"value": float64(i),
},
time.Unix(0, 0),
))
}
// Benchmark the writing
b.ResetTimer()
for b.Loop() {
require.NoError(b, plugin.Write(metrics))
}
b.ReportMetric(float64(batchsize*b.N)/b.Elapsed().Seconds(), "metrics/s")
}
func BenchmarkWrite25k(b *testing.B) {
batchsize := 25000
// Setup a test server
ts := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}),
)
defer ts.Close()
// Setup plugin and connect
plugin := &influxdb.InfluxDB{
URLs: []string{"http://" + ts.Listener.Addr().String()},
Username: config.NewSecret([]byte("user")),
Password: config.NewSecret([]byte("secret")),
Database: "my_database",
Timeout: config.Duration(time.Second * 5),
Log: &testutil.Logger{},
CreateHTTPClientF: func(config *influxdb.HTTPConfig) (influxdb.Client, error) {
return influxdb.NewHTTPClient(*config)
},
CreateUDPClientF: func(config *influxdb.UDPConfig) (influxdb.Client, error) {
return influxdb.NewUDPClient(*config)
},
ContentEncoding: "gzip",
SkipDatabaseCreation: true,
}
require.NoError(b, plugin.Connect())
defer plugin.Close()
metrics := make([]telegraf.Metric, 0, batchsize)
for i := range batchsize {
metrics = append(metrics, metric.New(
"cpu",
map[string]string{
"database": "foo",
},
map[string]interface{}{
"value": float64(i),
},
time.Unix(0, 0),
))
}
// Benchmark the writing
b.ResetTimer()
for b.Loop() {
require.NoError(b, plugin.Write(metrics))
}
b.ReportMetric(float64(batchsize*b.N)/b.Elapsed().Seconds(), "metrics/s")
}
func BenchmarkWrite50k(b *testing.B) {
batchsize := 50000
// Setup a test server
ts := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}),
)
defer ts.Close()
// Setup plugin and connect
plugin := &influxdb.InfluxDB{
URLs: []string{"http://" + ts.Listener.Addr().String()},
Username: config.NewSecret([]byte("user")),
Password: config.NewSecret([]byte("secret")),
Database: "my_database",
Timeout: config.Duration(time.Second * 5),
Log: &testutil.Logger{},
CreateHTTPClientF: func(config *influxdb.HTTPConfig) (influxdb.Client, error) {
return influxdb.NewHTTPClient(*config)
},
CreateUDPClientF: func(config *influxdb.UDPConfig) (influxdb.Client, error) {
return influxdb.NewUDPClient(*config)
},
ContentEncoding: "gzip",
SkipDatabaseCreation: true,
}
require.NoError(b, plugin.Connect())
defer plugin.Close()
metrics := make([]telegraf.Metric, 0, batchsize)
for i := range batchsize {
metrics = append(metrics, metric.New(
"cpu",
map[string]string{
"database": "foo",
},
map[string]interface{}{
"value": float64(i),
},
time.Unix(0, 0),
))
}
// Benchmark the writing
b.ResetTimer()
for b.Loop() {
require.NoError(b, plugin.Write(metrics))
}
b.ReportMetric(float64(batchsize*b.N)/b.Elapsed().Seconds(), "metrics/s")
}
func BenchmarkWrite100k(b *testing.B) {
batchsize := 100000
// Setup a test server
ts := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
}),
)
defer ts.Close()
// Setup plugin and connect
plugin := &influxdb.InfluxDB{
URLs: []string{"http://" + ts.Listener.Addr().String()},
Username: config.NewSecret([]byte("user")),
Password: config.NewSecret([]byte("secret")),
Database: "my_database",
Timeout: config.Duration(time.Second * 5),
Log: &testutil.Logger{},
CreateHTTPClientF: func(config *influxdb.HTTPConfig) (influxdb.Client, error) {
return influxdb.NewHTTPClient(*config)
},
CreateUDPClientF: func(config *influxdb.UDPConfig) (influxdb.Client, error) {
return influxdb.NewUDPClient(*config)
},
ContentEncoding: "gzip",
SkipDatabaseCreation: true,
}
require.NoError(b, plugin.Connect())
defer plugin.Close()
metrics := make([]telegraf.Metric, 0, batchsize)
for i := range batchsize {
metrics = append(metrics, metric.New(
"cpu",
map[string]string{
"database": "foo",
},
map[string]interface{}{
"value": float64(i),
},
time.Unix(0, 0),
))
}
// Benchmark the writing
b.ResetTimer()
for b.Loop() {
require.NoError(b, plugin.Write(metrics))
}
b.ReportMetric(float64(batchsize*b.N)/b.Elapsed().Seconds(), "metrics/s")
}

87
plugins/outputs/influxdb/sample.conf Normal file
View file

@ -0,0 +1,87 @@
# Configuration for sending metrics to InfluxDB
[[outputs.influxdb]]
## The full HTTP or UDP URL for your InfluxDB instance.
##
## Multiple URLs can be specified for a single cluster, only ONE of the
## urls will be written to each interval.
# urls = ["unix:///var/run/influxdb.sock"]
# urls = ["udp://127.0.0.1:8089"]
# urls = ["http://127.0.0.1:8086"]
## Local address to bind when connecting to the server
## If empty or not set, the local address is automatically chosen.
# local_address = ""
## The target database for metrics; will be created as needed.
## For UDP url endpoint database needs to be configured on server side.
# database = "telegraf"
## The value of this tag will be used to determine the database. If this
## tag is not set the 'database' option is used as the default.
# database_tag = ""
## If true, the 'database_tag' will not be included in the written metric.
# exclude_database_tag = false
## If true, no CREATE DATABASE queries will be sent. Set to true when using
## Telegraf with a user without permissions to create databases or when the
## database already exists.
# skip_database_creation = false
## Name of existing retention policy to write to. Empty string writes to
## the default retention policy. Only takes effect when using HTTP.
# retention_policy = ""
## The value of this tag will be used to determine the retention policy. If this
## tag is not set the 'retention_policy' option is used as the default.
# retention_policy_tag = ""
## If true, the 'retention_policy_tag' will not be included in the written metric.
# exclude_retention_policy_tag = false
## Write consistency (clusters only), can be: "any", "one", "quorum", "all".
## Only takes effect when using HTTP.
# write_consistency = "any"
## Timeout for HTTP messages.
# timeout = "5s"
## HTTP Basic Auth
# username = "telegraf"
# password = "metricsmetricsmetricsmetrics"
## HTTP User-Agent
# user_agent = "telegraf"
## UDP payload size is the maximum packet size to send.
# udp_payload = "512B"
## Optional TLS Config for use on HTTP connections.
# tls_ca = "/etc/telegraf/ca.pem"
# tls_cert = "/etc/telegraf/cert.pem"
# tls_key = "/etc/telegraf/key.pem"
## Use TLS but skip chain & host verification
# insecure_skip_verify = false
## HTTP Proxy override, if unset values the standard proxy environment
## variables are consulted to determine which proxy, if any, should be used.
# http_proxy = "http://corporate.proxy:3128"
## Additional HTTP headers
# http_headers = {"X-Special-Header" = "Special-Value"}
## HTTP Content-Encoding for write request body, can be set to "gzip" to
## compress body or "identity" to apply no encoding.
# content_encoding = "gzip"
## When true, Telegraf will output unsigned integers as unsigned values,
## i.e.: "42u". You will need a version of InfluxDB supporting unsigned
## integer values. Enabling this option will result in field type errors if
## existing data has been written.
# influx_uint_support = false
## When true, Telegraf will omit the timestamp on data to allow InfluxDB
## to set the timestamp of the data during ingestion. This is generally NOT
## what you want as it can lead to data points captured at different times
## getting omitted due to similar data.
# influx_omit_timestamp = false

145
plugins/outputs/influxdb/udp.go Normal file
View file

@ -0,0 +1,145 @@
package influxdb
import (
"bufio"
"bytes"
"context"
"fmt"
"net"
"net/url"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/plugins/serializers/influx"
)
const (
// DefaultMaxPayloadSize is the maximum length of the UDP data payload
DefaultMaxPayloadSize = 512
)
type Dialer interface {
DialContext(ctx context.Context, network, address string) (Conn, error)
}
type Conn interface {
Write(b []byte) (int, error)
Close() error
}
type UDPConfig struct {
MaxPayloadSize int
URL *url.URL
LocalAddr *net.UDPAddr
Serializer *influx.Serializer
Dialer Dialer
Log telegraf.Logger
}
func NewUDPClient(config UDPConfig) (*udpClient, error) {
if config.URL == nil {
return nil, ErrMissingURL
}
size := config.MaxPayloadSize
if size == 0 {
size = DefaultMaxPayloadSize
}
serializer := config.Serializer
if serializer == nil {
serializer = &influx.Serializer{}
if err := serializer.Init(); err != nil {
return nil, err
}
}
serializer.MaxLineBytes = size
dialer := config.Dialer
if dialer == nil {
dialer = &netDialer{net.Dialer{LocalAddr: config.LocalAddr}}
}
client := &udpClient{
url: config.URL,
serializer: serializer,
dialer: dialer,
log: config.Log,
}
return client, nil
}
type udpClient struct {
conn Conn
dialer Dialer
serializer *influx.Serializer
url *url.URL
log telegraf.Logger
}
func (c *udpClient) URL() string {
return c.url.String()
}
func (*udpClient) Database() string {
return ""
}
func (c *udpClient) Write(ctx context.Context, metrics []telegraf.Metric) error {
if c.conn == nil {
conn, err := c.dialer.DialContext(ctx, c.url.Scheme, c.url.Host)
if err != nil {
return fmt.Errorf("error dialing address [%s]: %w", c.url, err)
}
c.conn = conn
}
for _, metric := range metrics {
octets, err := c.serializer.Serialize(metric)
if err != nil {
// Since we are serializing multiple metrics, don't fail the
// entire batch just because of one unserializable metric.
c.log.Errorf("When writing to [%s] could not serialize metric: %v",
c.URL(), err)
continue
}
scanner := bufio.NewScanner(bytes.NewReader(octets))
scanner.Split(scanLines)
for scanner.Scan() {
_, err = c.conn.Write(scanner.Bytes())
}
if err != nil {
_ = c.conn.Close()
c.conn = nil
return err
}
}
return nil
}
func (*udpClient) CreateDatabase(_ context.Context, _ string) error {
return nil
}
type netDialer struct {
net.Dialer
}
func (d *netDialer) DialContext(ctx context.Context, network, address string) (Conn, error) {
return d.Dialer.DialContext(ctx, network, address)
}
func scanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := bytes.IndexByte(data, '\n'); i >= 0 {
// We have a full newline-terminated line.
return i + 1, data[0 : i+1], nil
}
return 0, nil, nil
}
func (*udpClient) Close() {
}

270
plugins/outputs/influxdb/udp_test.go Normal file
View file

@ -0,0 +1,270 @@
package influxdb_test
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"net"
"net/url"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/metric"
"github.com/influxdata/telegraf/plugins/outputs/influxdb"
"github.com/influxdata/telegraf/testutil"
)
var (
metricString = "cpu value=42 0\n"
)
func getMetric() telegraf.Metric {
m := metric.New(
"cpu",
map[string]string{},
map[string]interface{}{
"value": 42.0,
},
time.Unix(0, 0),
)
return m
}
func getURL() *url.URL {
u, err := url.Parse("udp://localhost:0")
if err != nil {
panic(err)
}
return u
}
type MockConn struct {
WriteF func(b []byte) (n int, err error)
CloseF func() error
}
func (c *MockConn) Write(b []byte) (n int, err error) {
return c.WriteF(b)
}
func (c *MockConn) Close() error {
return c.CloseF()
}
type MockDialer struct {
DialContextF func() (influxdb.Conn, error)
}
func (d *MockDialer) DialContext(_ context.Context, _, _ string) (influxdb.Conn, error) {
return d.DialContextF()
}
func TestUDP_NewUDPClientNoURL(t *testing.T) {
config := influxdb.UDPConfig{}
_, err := influxdb.NewUDPClient(config)
require.Equal(t, err, influxdb.ErrMissingURL)
}
func TestUDP_URL(t *testing.T) {
u := getURL()
config := influxdb.UDPConfig{
URL: u,
}
client, err := influxdb.NewUDPClient(config)
require.NoError(t, err)
require.Equal(t, u.String(), client.URL())
}
func TestUDP_Simple(t *testing.T) {
var buffer bytes.Buffer
config := influxdb.UDPConfig{
URL: getURL(),
Dialer: &MockDialer{
DialContextF: func() (influxdb.Conn, error) {
conn := &MockConn{
WriteF: func(b []byte) (n int, err error) {
buffer.Write(b)
return 0, nil
},
}
return conn, nil
},
},
}
client, err := influxdb.NewUDPClient(config)
require.NoError(t, err)
err = client.Write(t.Context(), []telegraf.Metric{
getMetric(),
getMetric(),
})
require.NoError(t, err)
require.Equal(t, metricString+metricString, buffer.String())
}
func TestUDP_DialError(t *testing.T) {
u, err := url.Parse("invalid://127.0.0.1:9999")
require.NoError(t, err)
config := influxdb.UDPConfig{
URL: u,
Dialer: &MockDialer{
DialContextF: func() (influxdb.Conn, error) {
return nil, errors.New(`unsupported scheme [invalid://localhost:9999]: "invalid"`)
},
},
}
client, err := influxdb.NewUDPClient(config)
require.NoError(t, err)
err = client.Write(t.Context(), []telegraf.Metric{getMetric()})
require.Error(t, err)
}
func TestUDP_WriteError(t *testing.T) {
closed := false
config := influxdb.UDPConfig{
URL: getURL(),
Dialer: &MockDialer{
DialContextF: func() (influxdb.Conn, error) {
conn := &MockConn{
WriteF: func(_ []byte) (n int, err error) {
return 0, errors.New("write udp 127.0.0.1:52190->127.0.0.1:9999: write: connection refused")
},
CloseF: func() error {
closed = true
return nil
},
}
return conn, nil
},
},
}
client, err := influxdb.NewUDPClient(config)
require.NoError(t, err)
err = client.Write(t.Context(), []telegraf.Metric{getMetric()})
require.Error(t, err)
require.True(t, closed)
}
func TestUDP_ErrorLogging(t *testing.T) {
tests := []struct {
name string
config influxdb.UDPConfig
metrics []telegraf.Metric
logContains string
}{
{
name: "logs need more space",
config: influxdb.UDPConfig{
MaxPayloadSize: 1,
URL: getURL(),
Dialer: &MockDialer{
DialContextF: func() (influxdb.Conn, error) {
conn := &MockConn{}
return conn, nil
},
},
Log: testutil.Logger{},
},
metrics: []telegraf.Metric{getMetric()},
logContains: `could not serialize metric: "cpu": need more space`,
},
{
name: "logs series name",
config: influxdb.UDPConfig{
URL: getURL(),
Dialer: &MockDialer{
DialContextF: func() (influxdb.Conn, error) {
conn := &MockConn{}
return conn, nil
},
},
Log: testutil.Logger{},
},
metrics: []telegraf.Metric{
func() telegraf.Metric {
m := metric.New(
"cpu",
map[string]string{
"host": "example.org",
},
map[string]interface{}{},
time.Unix(0, 0),
)
return m
}(),
},
logContains: `could not serialize metric: "cpu,host=example.org": no serializable fields`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var b bytes.Buffer
log.SetOutput(&b)
client, err := influxdb.NewUDPClient(tt.config)
require.NoError(t, err)
err = client.Write(t.Context(), tt.metrics)
require.NoError(t, err)
require.Contains(t, b.String(), tt.logContains)
})
}
}
func TestUDP_WriteWithRealConn(t *testing.T) {
conn, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
metrics := []telegraf.Metric{
getMetric(),
getMetric(),
}
buf := make([]byte, 200)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
var total int
for range metrics {
n, _, err := conn.ReadFrom(buf[total:])
if err != nil {
break
}
total += n
}
buf = buf[:total]
}()
addr := conn.LocalAddr()
u, err := url.Parse(fmt.Sprintf("%s://%s", addr.Network(), addr))
require.NoError(t, err)
config := influxdb.UDPConfig{
URL: u,
}
client, err := influxdb.NewUDPClient(config)
require.NoError(t, err)
err = client.Write(t.Context(), metrics)
require.NoError(t, err)
wg.Wait()
require.Equal(t, metricString+metricString, string(buf))
}