influx/telegraf
1
0
Fork 0
Code Issues Activity

Adding upstream version 1.34.4.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-05-24 07:26:29 +02:00
parent e393c3af3f
commit 4978089aab
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

73
plugins/inputs/nsq_consumer/README.md Normal file
View file

@ -0,0 +1,73 @@
# NSQ Consumer Input Plugin
This service plugin consumes messages from [NSQ][nsq] realtime distributed
messaging platform brokers in one of the supported [data formats][data_formats].
⭐ Telegraf v0.10.1
🏷️ messaging
💻 all
[nsq]: https://nsq.io/
[data_formats]: /docs/DATA_FORMATS_INPUT.md
## Service Input <!-- @/docs/includes/service_input.md -->
This plugin is a service input. Normal plugins gather metrics determined by the
interval setting. Service plugins start a service to listen and wait for
metrics or events to occur. Service plugins have two key differences from
normal plugins:
1. The global or plugin specific `interval` setting may not apply
2. The CLI options of `--test`, `--test-wait`, and `--once` may not produce
output for this plugin
## Global configuration options <!-- @/docs/includes/plugin_config.md -->
In addition to the plugin-specific configuration settings, plugins support
additional global and plugin configuration settings. These settings are used to
modify metrics, tags, and field or create aliases and configure ordering, etc.
See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
## Configuration
```toml @sample.conf
# Read metrics from NSQD topic(s)
[[inputs.nsq_consumer]]
## An array representing the NSQD TCP HTTP Endpoints
nsqd = ["localhost:4150"]
## An array representing the NSQLookupd HTTP Endpoints
nsqlookupd = ["localhost:4161"]
topic = "telegraf"
channel = "consumer"
max_in_flight = 100
## Max undelivered messages
## This plugin uses tracking metrics, which ensure messages are read to
## outputs before acknowledging them to the original broker to ensure data
## is not lost. This option sets the maximum messages to read from the
## broker that have not been written by an output.
##
## This value needs to be picked with awareness of the agent's
## metric_batch_size value as well. Setting max undelivered messages too high
## can result in a constant stream of data batches to the output. While
## setting it too low may never flush the broker's messages.
# max_undelivered_messages = 1000
## Data format to consume.
## Each data format has its own unique set of configuration options, read
## more about them here:
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
data_format = "influx"
```
## Metrics
The plugin accepts arbitrary input and parses it according to the `data_format`
setting. There is no predefined metric format.
## Example Output
There is no predefined metric format, so output depends on plugin input.

197
plugins/inputs/nsq_consumer/nsq_consumer.go Normal file
View file

@ -0,0 +1,197 @@
//go:generate ../../../tools/readme_config_includer/generator
package nsq_consumer
import (
"context"
_ "embed"
"errors"
"sync"
"github.com/nsqio/go-nsq"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/plugins/inputs"
)
//go:embed sample.conf
var sampleConfig string
const (
defaultMaxUndeliveredMessages = 1000
)
type NSQConsumer struct {
Server string `toml:"server" deprecated:"1.5.0;1.35.0;use 'nsqd' instead"`
Nsqd []string `toml:"nsqd"`
Nsqlookupd []string `toml:"nsqlookupd"`
Topic string `toml:"topic"`
Channel string `toml:"channel"`
MaxInFlight int `toml:"max_in_flight"`
MaxUndeliveredMessages int `toml:"max_undelivered_messages"`
Log telegraf.Logger `toml:"-"`
parser telegraf.Parser
consumer *nsq.Consumer
mu sync.Mutex
messages map[telegraf.TrackingID]*nsq.Message
wg sync.WaitGroup
cancel context.CancelFunc
}
type (
empty struct{}
semaphore chan empty
)
type logger struct {
log telegraf.Logger
}
// Output writes log messages from the NSQ library to the Telegraf logger.
func (l *logger) Output(_ int, s string) error {
l.log.Debug(s)
return nil
}
func (*NSQConsumer) SampleConfig() string {
return sampleConfig
}
func (n *NSQConsumer) Init() error {
// For backward compatibility
if n.Server != "" {
n.Nsqd = append(n.Nsqd, n.Server)
}
// Check if we have anything to connect to
if len(n.Nsqlookupd) == 0 && len(n.Nsqd) == 0 {
return errors.New("either 'nsqd' or 'nsqlookupd' needs to be specified")
}
return nil
}
// SetParser takes the data_format from the config and finds the right parser for that format
func (n *NSQConsumer) SetParser(parser telegraf.Parser) {
n.parser = parser
}
func (n *NSQConsumer) Start(ac telegraf.Accumulator) error {
acc := ac.WithTracking(n.MaxUndeliveredMessages)
sem := make(semaphore, n.MaxUndeliveredMessages)
n.messages = make(map[telegraf.TrackingID]*nsq.Message, n.MaxUndeliveredMessages)
ctx, cancel := context.WithCancel(context.Background())
n.cancel = cancel
if err := n.connect(); err != nil {
return err
}
n.consumer.SetLogger(&logger{log: n.Log}, nsq.LogLevelInfo)
n.consumer.AddHandler(nsq.HandlerFunc(func(message *nsq.Message) error {
metrics, err := n.parser.Parse(message.Body)
if err != nil {
acc.AddError(err)
// Remove the message from the queue
message.Finish()
return nil
}
if len(metrics) == 0 {
message.Finish()
return nil
}
select {
case <-ctx.Done():
return ctx.Err()
case sem <- empty{}:
break
}
n.mu.Lock()
id := acc.AddTrackingMetricGroup(metrics)
n.messages[id] = message
n.mu.Unlock()
message.DisableAutoResponse()
return nil
}))
if len(n.Nsqlookupd) > 0 {
err := n.consumer.ConnectToNSQLookupds(n.Nsqlookupd)
if err != nil && !errors.Is(err, nsq.ErrAlreadyConnected) {
return err
}
}
if len(n.Nsqd) > 0 {
err := n.consumer.ConnectToNSQDs(n.Nsqd)
if err != nil && !errors.Is(err, nsq.ErrAlreadyConnected) {
return err
}
}
n.wg.Add(1)
go func() {
defer n.wg.Done()
n.onDelivery(ctx, acc, sem)
}()
return nil
}
func (*NSQConsumer) Gather(telegraf.Accumulator) error {
return nil
}
func (n *NSQConsumer) Stop() {
n.cancel()
n.wg.Wait()
n.consumer.Stop()
<-n.consumer.StopChan
}
func (n *NSQConsumer) onDelivery(ctx context.Context, acc telegraf.TrackingAccumulator, sem semaphore) {
for {
select {
case <-ctx.Done():
return
case info := <-acc.Delivered():
n.mu.Lock()
msg, ok := n.messages[info.ID()]
if !ok {
n.mu.Unlock()
continue
}
<-sem
delete(n.messages, info.ID())
n.mu.Unlock()
if info.Delivered() {
msg.Finish()
} else {
msg.Requeue(-1)
}
}
}
}
func (n *NSQConsumer) connect() error {
if n.consumer == nil {
config := nsq.NewConfig()
config.MaxInFlight = n.MaxInFlight
consumer, err := nsq.NewConsumer(n.Topic, n.Channel, config)
if err != nil {
return err
}
n.consumer = consumer
}
return nil
}
func init() {
inputs.Add("nsq_consumer", func() telegraf.Input {
return &NSQConsumer{
MaxUndeliveredMessages: defaultMaxUndeliveredMessages,
}
})
}

248
plugins/inputs/nsq_consumer/nsq_consumer_test.go Normal file
View file

@ -0,0 +1,248 @@
package nsq_consumer
import (
"bufio"
"bytes"
"encoding/binary"
"io"
"log"
"net"
"strconv"
"testing"
"time"
"github.com/nsqio/go-nsq"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/testutil"
)
// This test is modeled after the kafka consumer integration test
func TestReadsMetricsFromNSQ(t *testing.T) {
msgID := nsq.MessageID{'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 's', 'd', 'f', 'g', 'h'}
msg := nsq.NewMessage(msgID, []byte("cpu_load_short,direction=in,host=server01,region=us-west value=23422.0 1422568543702900257\n"))
frameMsg, err := frameMessage(msg)
require.NoError(t, err)
script := []instruction{
// SUB
{0, nsq.FrameTypeResponse, []byte("OK")},
// IDENTIFY
{0, nsq.FrameTypeResponse, []byte("OK")},
{20 * time.Millisecond, nsq.FrameTypeMessage, frameMsg},
// needed to exit test
{100 * time.Millisecond, -1, []byte("exit")},
}
addr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:4155")
require.NoError(t, err)
newMockNSQD(t, script, addr.String())
consumer := &NSQConsumer{
Log: testutil.Logger{},
Topic: "telegraf",
Channel: "consume",
MaxInFlight: 1,
MaxUndeliveredMessages: defaultMaxUndeliveredMessages,
Nsqd: []string{"127.0.0.1:4155"},
}
require.NoError(t, consumer.Init())
p := &influx.Parser{}
require.NoError(t, p.Init())
consumer.SetParser(p)
var acc testutil.Accumulator
require.Empty(t, acc.Metrics, "There should not be any points")
require.NoError(t, consumer.Start(&acc))
waitForPoint(&acc, t)
require.Len(t, acc.Metrics, 1, "No points found in accumulator, expected 1")
point := acc.Metrics[0]
require.Equal(t, "cpu_load_short", point.Measurement)
require.Equal(t, map[string]interface{}{"value": 23422.0}, point.Fields)
require.Equal(t, map[string]string{
"host": "server01",
"direction": "in",
"region": "us-west",
}, point.Tags)
require.Equal(t, time.Unix(0, 1422568543702900257).Unix(), point.Time.Unix())
}
// Waits for the metric that was sent to the kafka broker to arrive at the kafka
// consumer
func waitForPoint(acc *testutil.Accumulator, t *testing.T) {
// Give the kafka container up to 2 seconds to get the point to the consumer
ticker := time.NewTicker(5 * time.Millisecond)
defer ticker.Stop()
counter := 0
//nolint:staticcheck // for-select used on purpose
for {
select {
case <-ticker.C:
counter++
if counter > 1000 {
t.Fatal("Waited for 5s, point never arrived to consumer")
} else if acc.NFields() == 1 {
return
}
}
}
}
func newMockNSQD(t *testing.T, script []instruction, addr string) *mockNSQD {
n := &mockNSQD{
script: script,
exitChan: make(chan int),
}
tcpListener, err := net.Listen("tcp", addr)
require.NoError(t, err, "listen (%s) failed", n.tcpAddr.String())
n.tcpListener = tcpListener
n.tcpAddr = tcpListener.Addr().(*net.TCPAddr)
go n.listen()
return n
}
// The code below allows us to mock the interactions with nsqd. This is taken from:
// https://github.com/nsqio/go-nsq/blob/master/mock_test.go
type instruction struct {
delay time.Duration
frameType int32
body []byte
}
type mockNSQD struct {
script []instruction
got [][]byte
tcpAddr *net.TCPAddr
tcpListener net.Listener
exitChan chan int
}
func (n *mockNSQD) listen() {
for {
conn, err := n.tcpListener.Accept()
if err != nil {
break
}
go n.handle(conn)
}
close(n.exitChan)
}
func (n *mockNSQD) handle(conn net.Conn) {
var idx int
buf := make([]byte, 4)
_, err := io.ReadFull(conn, buf)
if err != nil {
//nolint:revive // log.Fatalf called intentionally
log.Fatalf("ERROR: failed to read protocol version - %s", err)
}
readChan := make(chan []byte)
readDoneChan := make(chan int)
scriptTime := time.After(n.script[0].delay)
rdr := bufio.NewReader(conn)
go func() {
for {
line, err := rdr.ReadBytes('\n')
if err != nil {
return
}
// trim the '\n'
line = line[:len(line)-1]
readChan <- line
<-readDoneChan
}
}()
var rdyCount int
for idx < len(n.script) {
select {
case line := <-readChan:
n.got = append(n.got, line)
params := bytes.Split(line, []byte(" "))
switch {
case bytes.Equal(params[0], []byte("IDENTIFY")):
l := make([]byte, 4)
_, err := io.ReadFull(rdr, l)
if err != nil {
log.Print(err.Error())
goto exit
}
size := int32(binary.BigEndian.Uint32(l))
b := make([]byte, size)
_, err = io.ReadFull(rdr, b)
if err != nil {
log.Print(err.Error())
goto exit
}
case bytes.Equal(params[0], []byte("RDY")):
rdy, err := strconv.Atoi(string(params[1]))
if err != nil {
log.Print(err.Error())
goto exit
}
rdyCount = rdy
case bytes.Equal(params[0], []byte("FIN")):
case bytes.Equal(params[0], []byte("REQ")):
}
readDoneChan <- 1
case <-scriptTime:
inst := n.script[idx]
if bytes.Equal(inst.body, []byte("exit")) {
goto exit
}
if inst.frameType == nsq.FrameTypeMessage {
if rdyCount == 0 {
scriptTime = time.After(n.script[idx+1].delay)
continue
}
rdyCount--
}
buf := framedResponse(inst.frameType, inst.body)
_, err = conn.Write(buf)
if err != nil {
log.Print(err.Error())
goto exit
}
scriptTime = time.After(n.script[idx+1].delay)
idx++
}
}
exit:
n.tcpListener.Close()
conn.Close()
}
func framedResponse(frameType int32, data []byte) []byte {
var w bytes.Buffer
beBuf := make([]byte, 4)
size := uint32(len(data)) + 4
binary.BigEndian.PutUint32(beBuf, size)
w.Write(beBuf)
binary.BigEndian.PutUint32(beBuf, uint32(frameType))
w.Write(beBuf)
w.Write(data)
return w.Bytes()
}
func frameMessage(m *nsq.Message) ([]byte, error) {
var b bytes.Buffer
_, err := m.WriteTo(&b)
return b.Bytes(), err
}

28
plugins/inputs/nsq_consumer/sample.conf Normal file
View file

@ -0,0 +1,28 @@
# Read metrics from NSQD topic(s)
[[inputs.nsq_consumer]]
## An array representing the NSQD TCP HTTP Endpoints
nsqd = ["localhost:4150"]
## An array representing the NSQLookupd HTTP Endpoints
nsqlookupd = ["localhost:4161"]
topic = "telegraf"
channel = "consumer"
max_in_flight = 100
## Max undelivered messages
## This plugin uses tracking metrics, which ensure messages are read to
## outputs before acknowledging them to the original broker to ensure data
## is not lost. This option sets the maximum messages to read from the
## broker that have not been written by an output.
##
## This value needs to be picked with awareness of the agent's
## metric_batch_size value as well. Setting max undelivered messages too high
## can result in a constant stream of data batches to the output. While
## setting it too low may never flush the broker's messages.
# max_undelivered_messages = 1000
## Data format to consume.
## Each data format has its own unique set of configuration options, read
## more about them here:
## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
data_format = "influx"