1
0
Fork 0

Adding upstream version 1.34.4.

Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-05-24 07:26:29 +02:00
parent e393c3af3f
commit 4978089aab
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions

View file

@ -0,0 +1,190 @@
# LeoFS Input Plugin
This plugin gathers metrics of the [LEO filesystem][leofs] services
_LeoGateway_, _LeoManager_, and _LeoStorage_ via SNMP. Check the
[LeoFS system monitoring documentation][docs] for details.
⭐ Telegraf v0.1.5
🏷️ network, server
💻 all
[leofs]: https://leo-project.net/leofs/
[docs]: https://leo-project.net/leofs/docs/admin/system_admin/monitoring/
## Global configuration options <!-- @/docs/includes/plugin_config.md -->
In addition to the plugin-specific configuration settings, plugins support
additional global and plugin configuration settings. These settings are used to
modify metrics, tags, and field or create aliases and configure ordering, etc.
See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
## Configuration
```toml @sample.conf
# Read metrics from a LeoFS Server via SNMP
[[inputs.leofs]]
## An array of URLs of the form:
## host [ ":" port]
servers = ["127.0.0.1:4010"]
```
## Metrics
### Statistics specific to the internals of LeoManager
#### Erlang VM of LeoManager
- 1 min Statistics
- num_of_processes
- total_memory_usage
- system_memory_usage
- processes_memory_usage
- ets_memory_usage
- used_allocated_memory
- allocated_memory
- 5 min Statistics
- num_of_processes_5min
- total_memory_usage_5min
- system_memory_usage_5min
- processes_memory_usage_5min
- ets_memory_usage_5min
- used_allocated_memory_5min
- allocated_memory_5min
### Statistics specific to the internals of LeoStorage
### Erlang VM of LeoStorage
- 1 min Statistics
- num_of_processes
- total_memory_usage
- system_memory_usage
- processes_memory_usage
- ets_memory_usage
- used_allocated_memory
- allocated_memory
- 5 min Statistics
- num_of_processes_5min
- total_memory_usage_5min
- system_memory_usage_5min
- processes_memory_usage_5min
- ets_memory_usage_5min
- used_allocated_memory_5min
- allocated_memory_5min
### Total Number of Requests for LeoStorage
- 1 min Statistics
- num_of_writes
- num_of_reads
- num_of_deletes
- 5 min Statistics
- num_of_writes_5min
- num_of_reads_5min
- num_of_deletes_5min
#### Total Number of Objects and Total Size of Objects
- num_of_active_objects
- total_objects
- total_size_of_active_objects
- total_size
#### Total Number of MQ Messages
- num_of_replication_messages,
- num_of_sync-vnode_messages,
- num_of_rebalance_messages,
- mq_num_of_msg_recovery_node
- mq_num_of_msg_deletion_dir
- mq_num_of_msg_async_deletion_dir
- mq_num_of_msg_req_deletion_dir
- mq_mdcr_num_of_msg_req_comp_metadata
- mq_mdcr_num_of_msg_req_sync_obj
Note: The following items are available since LeoFS v1.4.0:
- mq_num_of_msg_recovery_node
- mq_num_of_msg_deletion_dir
- mq_num_of_msg_async_deletion_dir
- mq_num_of_msg_req_deletion_dir
- mq_mdcr_num_of_msg_req_comp_metadata
- mq_mdcr_num_of_msg_req_sync_obj
#### Data Compaction
- comp_state
- comp_last_start_datetime
- comp_last_end_datetime
- comp_num_of_pending_targets
- comp_num_of_ongoing_targets
- comp_num_of_out_of_targets
Note: The all items are available since LeoFS v1.4.0.
### Statistics specific to the internals of LeoGateway
#### Erlang VM of LeoGateway
- 1 min Statistics
- num_of_processes
- total_memory_usage
- system_memory_usage
- processes_memory_usage
- ets_memory_usage
- used_allocated_memory
- allocated_memory
- 5 min Statistics
- num_of_processes_5min
- total_memory_usage_5min
- system_memory_usage_5min
- processes_memory_usage_5min
- ets_memory_usage_5min
- used_allocated_memory_5min
- allocated_memory_5min
#### Total Number of Requests for LeoGateway
- 1 min Statistics
- num_of_writes
- num_of_reads
- num_of_deletes
- 5 min Statistics
- num_of_writes_5min
- num_of_reads_5min
- num_of_deletes_5min
#### Object Cache
- count_of_cache-hit
- count_of_cache-miss
- total_of_files
- total_cached_size
### Tags
All measurements have the following tags:
- node
## Example Output
### LeoManager
```text
leofs,host=manager_0,node=manager_0@127.0.0.1 allocated_memory=78255445,allocated_memory_5min=78159025,ets_memory_usage=4611900,ets_memory_usage_5min=4632599,num_of_processes=223,num_of_processes_5min=223,processes_memory_usage=20201316,processes_memory_usage_5min=20186559,system_memory_usage=37172701,system_memory_usage_5min=37189213,total_memory_usage=57373373,total_memory_usage_5min=57374653,used_allocated_memory=67,used_allocated_memory_5min=67 1524105758000000000
```
### LeoStorage
```text
leofs,host=storage_0,node=storage_0@127.0.0.1 allocated_memory=63504384,allocated_memory_5min=0,comp_last_end_datetime=0,comp_last_start_datetime=0,comp_num_of_ongoing_targets=0,comp_num_of_out_of_targets=0,comp_num_of_pending_targets=8,comp_state=0,ets_memory_usage=3877824,ets_memory_usage_5min=0,mq_mdcr_num_of_msg_req_comp_metadata=0,mq_mdcr_num_of_msg_req_sync_obj=0,mq_num_of_msg_async_deletion_dir=0,mq_num_of_msg_deletion_dir=0,mq_num_of_msg_recovery_node=0,mq_num_of_msg_req_deletion_dir=0,num_of_active_objects=70,num_of_deletes=0,num_of_deletes_5min=0,num_of_processes=577,num_of_processes_5min=0,num_of_reads=1,num_of_reads_5min=0,num_of_rebalance_messages=0,num_of_replication_messages=0,num_of_sync-vnode_messages=0,num_of_writes=70,num_of_writes_5min=0,processes_memory_usage=20029464,processes_memory_usage_5min=0,system_memory_usage=25900472,system_memory_usage_5min=0,total_memory_usage=45920987,total_memory_usage_5min=0,total_objects=70,total_size=2,total_size_of_active_objects=2,used_allocated_memory=69,used_allocated_memory_5min=0 1524529826000000000
```
### LeoGateway
```text
leofs,host=gateway_0,node=gateway_0@127.0.0.1 allocated_memory=87941120,allocated_memory_5min=88067672,count_of_cache-hit=0,count_of_cache-miss=0,ets_memory_usage=4843497,ets_memory_usage_5min=4841574,num_of_deletes=0,num_of_deletes_5min=0,num_of_processes=555,num_of_processes_5min=555,num_of_reads=0,num_of_reads_5min=0,num_of_writes=0,num_of_writes_5min=0,processes_memory_usage=17388052,processes_memory_usage_5min=17413928,system_memory_usage=49531263,system_memory_usage_5min=49577819,total_cached_size=0,total_memory_usage=66917393,total_memory_usage_5min=66989469,total_of_files=0,used_allocated_memory=69,used_allocated_memory_5min=69 1524105894000000000
```

View file

@ -0,0 +1,249 @@
//go:generate ../../../tools/readme_config_includer/generator
package leofs
import (
"bufio"
_ "embed"
"errors"
"fmt"
"os/exec"
"strconv"
"strings"
"sync"
"time"
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/plugins/inputs"
)
//go:embed sample.conf
var sampleConfig string
const (
oid = ".1.3.6.1.4.1.35450"
// For Manager Master
defaultEndpoint = "127.0.0.1:4020"
)
type serverType int
const (
serverTypeManagerMaster serverType = iota
serverTypeManagerSlave
serverTypeStorage
serverTypeGateway
)
type LeoFS struct {
Servers []string `toml:"servers"`
}
var keyMapping = map[serverType][]string{
serverTypeManagerMaster: {
"num_of_processes",
"total_memory_usage",
"system_memory_usage",
"processes_memory_usage",
"ets_memory_usage",
"num_of_processes_5min",
"total_memory_usage_5min",
"system_memory_usage_5min",
"processes_memory_usage_5min",
"ets_memory_usage_5min",
"used_allocated_memory",
"allocated_memory",
"used_allocated_memory_5min",
"allocated_memory_5min",
},
serverTypeManagerSlave: {
"num_of_processes",
"total_memory_usage",
"system_memory_usage",
"processes_memory_usage",
"ets_memory_usage",
"num_of_processes_5min",
"total_memory_usage_5min",
"system_memory_usage_5min",
"processes_memory_usage_5min",
"ets_memory_usage_5min",
"used_allocated_memory",
"allocated_memory",
"used_allocated_memory_5min",
"allocated_memory_5min",
},
serverTypeStorage: {
"num_of_processes",
"total_memory_usage",
"system_memory_usage",
"processes_memory_usage",
"ets_memory_usage",
"num_of_processes_5min",
"total_memory_usage_5min",
"system_memory_usage_5min",
"processes_memory_usage_5min",
"ets_memory_usage_5min",
"num_of_writes",
"num_of_reads",
"num_of_deletes",
"num_of_writes_5min",
"num_of_reads_5min",
"num_of_deletes_5min",
"num_of_active_objects",
"total_objects",
"total_size_of_active_objects",
"total_size",
"num_of_replication_messages",
"num_of_sync-vnode_messages",
"num_of_rebalance_messages",
"used_allocated_memory",
"allocated_memory",
"used_allocated_memory_5min",
"allocated_memory_5min",
// following items are since LeoFS v1.4.0
"mq_num_of_msg_recovery_node",
"mq_num_of_msg_deletion_dir",
"mq_num_of_msg_async_deletion_dir",
"mq_num_of_msg_req_deletion_dir",
"mq_mdcr_num_of_msg_req_comp_metadata",
"mq_mdcr_num_of_msg_req_sync_obj",
"comp_state",
"comp_last_start_datetime",
"comp_last_end_datetime",
"comp_num_of_pending_targets",
"comp_num_of_ongoing_targets",
"comp_num_of_out_of_targets",
},
serverTypeGateway: {
"num_of_processes",
"total_memory_usage",
"system_memory_usage",
"processes_memory_usage",
"ets_memory_usage",
"num_of_processes_5min",
"total_memory_usage_5min",
"system_memory_usage_5min",
"processes_memory_usage_5min",
"ets_memory_usage_5min",
"num_of_writes",
"num_of_reads",
"num_of_deletes",
"num_of_writes_5min",
"num_of_reads_5min",
"num_of_deletes_5min",
"count_of_cache-hit",
"count_of_cache-miss",
"total_of_files",
"total_cached_size",
"used_allocated_memory",
"allocated_memory",
"used_allocated_memory_5min",
"allocated_memory_5min",
},
}
var serverTypeMapping = map[string]serverType{
"4020": serverTypeManagerMaster,
"4021": serverTypeManagerSlave,
"4010": serverTypeStorage,
"4011": serverTypeStorage,
"4012": serverTypeStorage,
"4013": serverTypeStorage,
"4000": serverTypeGateway,
"4001": serverTypeGateway,
}
func (*LeoFS) SampleConfig() string {
return sampleConfig
}
func (l *LeoFS) Gather(acc telegraf.Accumulator) error {
if len(l.Servers) == 0 {
return gatherServer(defaultEndpoint, serverTypeManagerMaster, acc)
}
var wg sync.WaitGroup
for _, endpoint := range l.Servers {
results := strings.Split(endpoint, ":")
port := "4020"
if len(results) > 2 {
acc.AddError(fmt.Errorf("unable to parse address %q", endpoint))
continue
} else if len(results) == 2 {
_, err := strconv.Atoi(results[1])
if err != nil {
acc.AddError(fmt.Errorf("unable to parse port from %q", endpoint))
continue
}
port = results[1]
}
st, ok := serverTypeMapping[port]
if !ok {
st = serverTypeStorage
}
wg.Add(1)
go func(endpoint string, st serverType) {
defer wg.Done()
acc.AddError(gatherServer(endpoint, st, acc))
}(endpoint, st)
}
wg.Wait()
return nil
}
func gatherServer(endpoint string, serverType serverType, acc telegraf.Accumulator) error {
cmd := exec.Command("snmpwalk", "-v2c", "-cpublic", "-On", endpoint, oid)
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
if err := cmd.Start(); err != nil {
return err
}
defer internal.WaitTimeout(cmd, time.Second*5) //nolint:errcheck // ignore the returned error as we cannot do anything about it anyway
scanner := bufio.NewScanner(stdout)
if !scanner.Scan() {
return errors.New("unable to retrieve the node name")
}
nodeName, err := retrieveTokenAfterColon(scanner.Text())
if err != nil {
return err
}
nodeNameTrimmed := strings.Trim(nodeName, "\"")
tags := map[string]string{
"node": nodeNameTrimmed,
}
i := 0
fields := make(map[string]interface{})
for scanner.Scan() {
key := keyMapping[serverType][i]
val, err := retrieveTokenAfterColon(scanner.Text())
if err != nil {
return err
}
fVal, err := strconv.ParseFloat(val, 64)
if err != nil {
return fmt.Errorf("unable to parse the value %q: %w", val, err)
}
fields[key] = fVal
i++
}
acc.AddFields("leofs", fields, tags)
return nil
}
func retrieveTokenAfterColon(line string) (string, error) {
tokens := strings.Split(line, ":")
if len(tokens) != 2 {
return "", fmt.Errorf("':' not found in the line:%s", line)
}
return strings.TrimSpace(tokens[1]), nil
}
func init() {
inputs.Add("leofs", func() telegraf.Input {
return &LeoFS{}
})
}

View file

@ -0,0 +1,195 @@
package leofs
import (
"os"
"os/exec"
"path/filepath"
"runtime"
"testing"
"github.com/stretchr/testify/require"
"github.com/influxdata/telegraf/testutil"
)
var fakeSNMP4Manager = `
package main
import "fmt"
const output = ` + "`" + `.1.3.6.1.4.1.35450.15.1.0 = STRING: "manager_888@127.0.0.1"
.1.3.6.1.4.1.35450.15.2.0 = Gauge32: 186
.1.3.6.1.4.1.35450.15.3.0 = Gauge32: 46235519
.1.3.6.1.4.1.35450.15.4.0 = Gauge32: 32168525
.1.3.6.1.4.1.35450.15.5.0 = Gauge32: 14066068
.1.3.6.1.4.1.35450.15.6.0 = Gauge32: 5512968
.1.3.6.1.4.1.35450.15.7.0 = Gauge32: 186
.1.3.6.1.4.1.35450.15.8.0 = Gauge32: 46269006
.1.3.6.1.4.1.35450.15.9.0 = Gauge32: 32202867
.1.3.6.1.4.1.35450.15.10.0 = Gauge32: 14064995
.1.3.6.1.4.1.35450.15.11.0 = Gauge32: 5492634
.1.3.6.1.4.1.35450.15.12.0 = Gauge32: 60
.1.3.6.1.4.1.35450.15.13.0 = Gauge32: 43515904
.1.3.6.1.4.1.35450.15.14.0 = Gauge32: 60
.1.3.6.1.4.1.35450.15.15.0 = Gauge32: 43533983` + "`" +
`
func main() {
fmt.Println(output)
}
`
var fakeSNMP4Storage = `
package main
import "fmt"
const output = ` + "`" + `.1.3.6.1.4.1.35450.56.1.0 = STRING: "storage_0@127.0.0.1"
.1.3.6.1.4.1.35450.56.2.0 = Gauge32: 512
.1.3.6.1.4.1.35450.56.3.0 = Gauge32: 38126307
.1.3.6.1.4.1.35450.56.4.0 = Gauge32: 22308716
.1.3.6.1.4.1.35450.56.5.0 = Gauge32: 15816448
.1.3.6.1.4.1.35450.56.6.0 = Gauge32: 5232008
.1.3.6.1.4.1.35450.56.7.0 = Gauge32: 512
.1.3.6.1.4.1.35450.56.8.0 = Gauge32: 38113176
.1.3.6.1.4.1.35450.56.9.0 = Gauge32: 22313398
.1.3.6.1.4.1.35450.56.10.0 = Gauge32: 15798779
.1.3.6.1.4.1.35450.56.11.0 = Gauge32: 5237315
.1.3.6.1.4.1.35450.56.12.0 = Gauge32: 191
.1.3.6.1.4.1.35450.56.13.0 = Gauge32: 824
.1.3.6.1.4.1.35450.56.14.0 = Gauge32: 0
.1.3.6.1.4.1.35450.56.15.0 = Gauge32: 50105
.1.3.6.1.4.1.35450.56.16.0 = Gauge32: 196654
.1.3.6.1.4.1.35450.56.17.0 = Gauge32: 0
.1.3.6.1.4.1.35450.56.18.0 = Gauge32: 2052
.1.3.6.1.4.1.35450.56.19.0 = Gauge32: 50296
.1.3.6.1.4.1.35450.56.20.0 = Gauge32: 35
.1.3.6.1.4.1.35450.56.21.0 = Gauge32: 898
.1.3.6.1.4.1.35450.56.22.0 = Gauge32: 0
.1.3.6.1.4.1.35450.56.23.0 = Gauge32: 0
.1.3.6.1.4.1.35450.56.24.0 = Gauge32: 0
.1.3.6.1.4.1.35450.56.31.0 = Gauge32: 51
.1.3.6.1.4.1.35450.56.32.0 = Gauge32: 53219328
.1.3.6.1.4.1.35450.56.33.0 = Gauge32: 51
.1.3.6.1.4.1.35450.56.34.0 = Gauge32: 53351083
.1.3.6.1.4.1.35450.56.41.0 = Gauge32: 101
.1.3.6.1.4.1.35450.56.42.0 = Gauge32: 216
.1.3.6.1.4.1.35450.56.43.0 = Gauge32: 313
.1.3.6.1.4.1.35450.56.44.0 = Gauge32: 421
.1.3.6.1.4.1.35450.56.45.0 = Gauge32: 597
.1.3.6.1.4.1.35450.56.46.0 = Gauge32: 628
.1.3.6.1.4.1.35450.56.51.0 = Gauge32: 1
.1.3.6.1.4.1.35450.56.52.0 = Gauge32: 1522154118
.1.3.6.1.4.1.35450.56.53.0 = Gauge32: 1522196496
.1.3.6.1.4.1.35450.56.54.0 = Gauge32: 1
.1.3.6.1.4.1.35450.56.55.0 = Gauge32: 7
.1.3.6.1.4.1.35450.56.56.0 = Gauge32: 0` + "`" +
`
func main() {
fmt.Println(output)
}
`
var fakeSNMP4Gateway = `
package main
import "fmt"
const output = ` + "`" + `.1.3.6.1.4.1.35450.34.1.0 = STRING: "gateway_0@127.0.0.1"
.1.3.6.1.4.1.35450.34.2.0 = Gauge32: 465
.1.3.6.1.4.1.35450.34.3.0 = Gauge32: 61676335
.1.3.6.1.4.1.35450.34.4.0 = Gauge32: 46890415
.1.3.6.1.4.1.35450.34.5.0 = Gauge32: 14785011
.1.3.6.1.4.1.35450.34.6.0 = Gauge32: 5578855
.1.3.6.1.4.1.35450.34.7.0 = Gauge32: 465
.1.3.6.1.4.1.35450.34.8.0 = Gauge32: 61644426
.1.3.6.1.4.1.35450.34.9.0 = Gauge32: 46880358
.1.3.6.1.4.1.35450.34.10.0 = Gauge32: 14763002
.1.3.6.1.4.1.35450.34.11.0 = Gauge32: 5582125
.1.3.6.1.4.1.35450.34.12.0 = Gauge32: 191
.1.3.6.1.4.1.35450.34.13.0 = Gauge32: 827
.1.3.6.1.4.1.35450.34.14.0 = Gauge32: 0
.1.3.6.1.4.1.35450.34.15.0 = Gauge32: 50105
.1.3.6.1.4.1.35450.34.16.0 = Gauge32: 196650
.1.3.6.1.4.1.35450.34.17.0 = Gauge32: 0
.1.3.6.1.4.1.35450.34.18.0 = Gauge32: 30256
.1.3.6.1.4.1.35450.34.19.0 = Gauge32: 532158
.1.3.6.1.4.1.35450.34.20.0 = Gauge32: 34
.1.3.6.1.4.1.35450.34.21.0 = Gauge32: 1
.1.3.6.1.4.1.35450.34.31.0 = Gauge32: 53
.1.3.6.1.4.1.35450.34.32.0 = Gauge32: 55050240
.1.3.6.1.4.1.35450.34.33.0 = Gauge32: 53
.1.3.6.1.4.1.35450.34.34.0 = Gauge32: 55186538` + "`" +
`
func main() {
fmt.Println(output)
}
`
func testMain(t *testing.T, code, endpoint string, serverType serverType) {
executable := "snmpwalk"
if runtime.GOOS == "windows" {
executable = "snmpwalk.exe"
}
// Build the fake snmpwalk for test
src := filepath.Join(t.TempDir(), "test.go")
require.NoError(t, os.WriteFile(src, []byte(code), 0600))
defer os.Remove(src)
require.NoError(t, exec.Command("go", "build", "-o", executable, src).Run())
defer os.Remove("./" + executable)
currentWorkingDirectory, err := os.Getwd()
require.NoError(t, err)
// Refer to the fake snmpwalk
t.Setenv("PATH", currentWorkingDirectory)
l := &LeoFS{
Servers: []string{endpoint},
}
var acc testutil.Accumulator
acc.SetDebug(true)
err = acc.GatherError(l.Gather)
require.NoError(t, err)
floatMetrics := keyMapping[serverType]
for _, metric := range floatMetrics {
require.True(t, acc.HasFloatField("leofs", metric), metric)
}
}
func TestLeoFSManagerMasterMetricsIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
testMain(t, fakeSNMP4Manager, "localhost:4020", serverTypeManagerMaster)
}
func TestLeoFSManagerSlaveMetricsIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
testMain(t, fakeSNMP4Manager, "localhost:4021", serverTypeManagerSlave)
}
func TestLeoFSStorageMetricsIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
testMain(t, fakeSNMP4Storage, "localhost:4010", serverTypeStorage)
}
func TestLeoFSGatewayMetricsIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
testMain(t, fakeSNMP4Gateway, "localhost:4000", serverTypeGateway)
}

View file

@ -0,0 +1,5 @@
# Read metrics from a LeoFS Server via SNMP
[[inputs.leofs]]
## An array of URLs of the form:
## host [ ":" port]
servers = ["127.0.0.1:4010"]