Adding upstream version 1.34.4.
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
parent
e393c3af3f
commit
4978089aab
GPG key ID:
FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions
286
plugins/inputs/cloud_pubsub_push/cloud_pubsub_push.go
Normal file
286
plugins/inputs/cloud_pubsub_push/cloud_pubsub_push.go
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
//go:generate ../../../tools/readme_config_includer/generator
|
||||
package cloud_pubsub_push
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
_ "embed"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/influxdata/telegraf"
|
||||
"github.com/influxdata/telegraf/config"
|
||||
"github.com/influxdata/telegraf/internal"
|
||||
common_tls "github.com/influxdata/telegraf/plugins/common/tls"
|
||||
"github.com/influxdata/telegraf/plugins/inputs"
|
||||
)
|
||||
|
||||
//go:embed sample.conf
|
||||
var sampleConfig string
|
||||
|
||||
var once sync.Once
|
||||
|
||||
// defaultMaxBodySize is the default maximum request body size, in bytes.
|
||||
// if the request body is over this size, we will return an HTTP 413 error.
|
||||
const (
|
||||
// 500 MB
|
||||
defaultMaxBodySize = 500 * 1024 * 1024
|
||||
defaultMaxUndeliveredMessages = 1000
|
||||
)
|
||||
|
||||
type PubSubPush struct {
|
||||
ServiceAddress string
|
||||
Token string
|
||||
Path string
|
||||
ReadTimeout config.Duration
|
||||
WriteTimeout config.Duration
|
||||
MaxBodySize config.Size
|
||||
AddMeta bool
|
||||
Log telegraf.Logger
|
||||
|
||||
MaxUndeliveredMessages int `toml:"max_undelivered_messages"`
|
||||
|
||||
common_tls.ServerConfig
|
||||
telegraf.Parser
|
||||
|
||||
server *http.Server
|
||||
acc telegraf.TrackingAccumulator
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg *sync.WaitGroup
|
||||
mu *sync.Mutex
|
||||
|
||||
undelivered map[telegraf.TrackingID]chan bool
|
||||
sem chan struct{}
|
||||
}
|
||||
|
||||
// message defines the structure of a Google Pub/Sub message.
|
||||
type message struct {
|
||||
Atts map[string]string `json:"attributes"`
|
||||
Data string `json:"data"` // Data is base64 encoded data
|
||||
}
|
||||
|
||||
// payload is the received Google Pub/Sub data. (https://cloud.google.com/pubsub/docs/push)
|
||||
type payload struct {
|
||||
Msg message `json:"message"`
|
||||
Subscription string `json:"subscription"`
|
||||
}
|
||||
|
||||
func (*PubSubPush) SampleConfig() string {
|
||||
return sampleConfig
|
||||
}
|
||||
|
||||
func (p *PubSubPush) SetParser(parser telegraf.Parser) {
|
||||
p.Parser = parser
|
||||
}
|
||||
|
||||
// Start starts the http listener service.
|
||||
func (p *PubSubPush) Start(acc telegraf.Accumulator) error {
|
||||
if p.MaxBodySize == 0 {
|
||||
p.MaxBodySize = config.Size(defaultMaxBodySize)
|
||||
}
|
||||
|
||||
if p.ReadTimeout < config.Duration(time.Second) {
|
||||
p.ReadTimeout = config.Duration(time.Second * 10)
|
||||
}
|
||||
if p.WriteTimeout < config.Duration(time.Second) {
|
||||
p.WriteTimeout = config.Duration(time.Second * 10)
|
||||
}
|
||||
|
||||
tlsConf, err := p.ServerConfig.TLSConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p.server = &http.Server{
|
||||
Addr: p.ServiceAddress,
|
||||
Handler: http.TimeoutHandler(p, time.Duration(p.WriteTimeout), "timed out processing metric"),
|
||||
ReadTimeout: time.Duration(p.ReadTimeout),
|
||||
TLSConfig: tlsConf,
|
||||
}
|
||||
|
||||
p.ctx, p.cancel = context.WithCancel(context.Background())
|
||||
p.wg = &sync.WaitGroup{}
|
||||
p.acc = acc.WithTracking(p.MaxUndeliveredMessages)
|
||||
p.sem = make(chan struct{}, p.MaxUndeliveredMessages)
|
||||
p.undelivered = make(map[telegraf.TrackingID]chan bool)
|
||||
p.mu = &sync.Mutex{}
|
||||
|
||||
p.wg.Add(1)
|
||||
go func() {
|
||||
defer p.wg.Done()
|
||||
p.receiveDelivered()
|
||||
}()
|
||||
|
||||
p.wg.Add(1)
|
||||
go func() {
|
||||
defer p.wg.Done()
|
||||
if tlsConf != nil {
|
||||
if err := p.server.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
|
||||
p.Log.Errorf("listening and serving TLS failed: %v", err)
|
||||
}
|
||||
} else {
|
||||
if err := p.server.ListenAndServe(); err != nil {
|
||||
p.Log.Errorf("listening and serving TLS failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*PubSubPush) Gather(telegraf.Accumulator) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop cleans up all resources
|
||||
func (p *PubSubPush) Stop() {
|
||||
p.cancel()
|
||||
p.server.Shutdown(p.ctx) //nolint:errcheck // we cannot do anything if the shutdown fails
|
||||
p.wg.Wait()
|
||||
}
|
||||
|
||||
func (p *PubSubPush) ServeHTTP(res http.ResponseWriter, req *http.Request) {
|
||||
if req.URL.Path == p.Path {
|
||||
p.authenticateIfSet(p.serveWrite, res, req)
|
||||
} else {
|
||||
p.authenticateIfSet(http.NotFound, res, req)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PubSubPush) serveWrite(res http.ResponseWriter, req *http.Request) {
|
||||
select {
|
||||
case <-req.Context().Done():
|
||||
res.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
case <-p.ctx.Done():
|
||||
res.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
case p.sem <- struct{}{}:
|
||||
break
|
||||
}
|
||||
|
||||
// Check that the content length is not too large for us to handle.
|
||||
if req.ContentLength > int64(p.MaxBodySize) {
|
||||
res.WriteHeader(http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
|
||||
if req.Method != http.MethodPost {
|
||||
res.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
body := http.MaxBytesReader(res, req.Body, int64(p.MaxBodySize))
|
||||
bytes, err := io.ReadAll(body)
|
||||
if err != nil {
|
||||
res.WriteHeader(http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
|
||||
var payload payload
|
||||
if err = json.Unmarshal(bytes, &payload); err != nil {
|
||||
p.Log.Errorf("Error decoding payload %s", err.Error())
|
||||
res.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
sDec, err := base64.StdEncoding.DecodeString(payload.Msg.Data)
|
||||
if err != nil {
|
||||
p.Log.Errorf("Base64-decode failed %s", err.Error())
|
||||
res.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
metrics, err := p.Parse(sDec)
|
||||
if err != nil {
|
||||
p.Log.Debug(err.Error())
|
||||
res.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if len(metrics) == 0 {
|
||||
once.Do(func() {
|
||||
p.Log.Debug(internal.NoMetricsCreatedMsg)
|
||||
})
|
||||
}
|
||||
|
||||
if p.AddMeta {
|
||||
for i := range metrics {
|
||||
for k, v := range payload.Msg.Atts {
|
||||
metrics[i].AddTag(k, v)
|
||||
}
|
||||
metrics[i].AddTag("subscription", payload.Subscription)
|
||||
}
|
||||
}
|
||||
|
||||
ch := make(chan bool, 1)
|
||||
p.mu.Lock()
|
||||
p.undelivered[p.acc.AddTrackingMetricGroup(metrics)] = ch
|
||||
p.mu.Unlock()
|
||||
|
||||
select {
|
||||
case <-req.Context().Done():
|
||||
res.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
case success := <-ch:
|
||||
if success {
|
||||
res.WriteHeader(http.StatusNoContent)
|
||||
} else {
|
||||
res.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PubSubPush) receiveDelivered() {
|
||||
for {
|
||||
select {
|
||||
case <-p.ctx.Done():
|
||||
return
|
||||
case info := <-p.acc.Delivered():
|
||||
<-p.sem
|
||||
|
||||
p.mu.Lock()
|
||||
ch, ok := p.undelivered[info.ID()]
|
||||
if !ok {
|
||||
p.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
|
||||
delete(p.undelivered, info.ID())
|
||||
p.mu.Unlock()
|
||||
|
||||
if info.Delivered() {
|
||||
ch <- true
|
||||
} else {
|
||||
ch <- false
|
||||
p.Log.Debug("Metric group failed to process")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PubSubPush) authenticateIfSet(handler http.HandlerFunc, res http.ResponseWriter, req *http.Request) {
|
||||
if p.Token != "" {
|
||||
if subtle.ConstantTimeCompare([]byte(req.FormValue("token")), []byte(p.Token)) != 1 {
|
||||
http.Error(res, "Unauthorized.", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
handler(res, req)
|
||||
}
|
||||
|
||||
func init() {
|
||||
inputs.Add("cloud_pubsub_push", func() telegraf.Input {
|
||||
return &PubSubPush{
|
||||
ServiceAddress: ":8080",
|
||||
Path: "/",
|
||||
MaxUndeliveredMessages: defaultMaxUndeliveredMessages,
|
||||
}
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue