Adding upstream version 1.34.4.
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
parent
e393c3af3f
commit
4978089aab
GPG key ID:
FBB4F0E80A80222F
4963 changed files with 677545 additions and 0 deletions
475
plugins/inputs/tail/multiline_test.go
Normal file
475
plugins/inputs/tail/multiline_test.go
Normal file
|
|
@ -0,0 +1,475 @@
|
|||
package tail
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/influxdata/telegraf/config"
|
||||
)
|
||||
|
||||
func TestMultilineConfigOK(t *testing.T) {
|
||||
c := &multilineConfig{
|
||||
Pattern: ".*",
|
||||
MatchWhichLine: previous,
|
||||
}
|
||||
|
||||
_, err := c.newMultiline()
|
||||
|
||||
require.NoError(t, err, "Configuration was OK.")
|
||||
}
|
||||
|
||||
func TestMultilineConfigError(t *testing.T) {
|
||||
c := &multilineConfig{
|
||||
Pattern: "\xA0",
|
||||
MatchWhichLine: previous,
|
||||
}
|
||||
|
||||
_, err := c.newMultiline()
|
||||
|
||||
require.Error(t, err, "The pattern was invalid")
|
||||
}
|
||||
|
||||
func TestMultilineConfigTimeoutSpecified(t *testing.T) {
|
||||
duration := config.Duration(10 * time.Second)
|
||||
c := &multilineConfig{
|
||||
Pattern: ".*",
|
||||
MatchWhichLine: previous,
|
||||
Timeout: &duration,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err, "Configuration was OK.")
|
||||
|
||||
require.Equal(t, duration, *m.config.Timeout)
|
||||
}
|
||||
|
||||
func TestMultilineConfigDefaultTimeout(t *testing.T) {
|
||||
duration := config.Duration(5 * time.Second)
|
||||
c := &multilineConfig{
|
||||
Pattern: ".*",
|
||||
MatchWhichLine: previous,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err, "Configuration was OK.")
|
||||
|
||||
require.Equal(t, duration, *m.config.Timeout)
|
||||
}
|
||||
|
||||
func TestMultilineIsEnabled(t *testing.T) {
|
||||
c := &multilineConfig{
|
||||
Pattern: ".*",
|
||||
MatchWhichLine: previous,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err, "Configuration was OK.")
|
||||
|
||||
isEnabled := m.isEnabled()
|
||||
|
||||
require.True(t, isEnabled, "Should have been enabled")
|
||||
}
|
||||
|
||||
func TestMultilineIsDisabled(t *testing.T) {
|
||||
c := &multilineConfig{
|
||||
MatchWhichLine: previous,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err, "Configuration was OK.")
|
||||
|
||||
isEnabled := m.isEnabled()
|
||||
|
||||
require.False(t, isEnabled, "Should have been disabled")
|
||||
}
|
||||
|
||||
func TestMultilineFlushEmpty(t *testing.T) {
|
||||
var buffer bytes.Buffer
|
||||
text := flush(&buffer)
|
||||
|
||||
require.Empty(t, text)
|
||||
}
|
||||
|
||||
func TestMultilineFlush(t *testing.T) {
|
||||
var buffer bytes.Buffer
|
||||
buffer.WriteString("foo")
|
||||
|
||||
text := flush(&buffer)
|
||||
require.Equal(t, "foo", text)
|
||||
require.Zero(t, buffer.Len())
|
||||
}
|
||||
|
||||
func TestMultiLineProcessLinePrevious(t *testing.T) {
|
||||
c := &multilineConfig{
|
||||
Pattern: "^=>",
|
||||
MatchWhichLine: previous,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err, "Configuration was OK.")
|
||||
var buffer bytes.Buffer
|
||||
|
||||
text := m.processLine("1", &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("=>2", &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("=>3", &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("4", &buffer)
|
||||
require.Equal(t, "1=>2=>3", text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("5", &buffer)
|
||||
require.Equal(t, "4", text)
|
||||
require.Equal(t, "5", buffer.String())
|
||||
}
|
||||
|
||||
func TestMultiLineProcessLineNext(t *testing.T) {
|
||||
c := &multilineConfig{
|
||||
Pattern: "=>$",
|
||||
MatchWhichLine: next,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err, "Configuration was OK.")
|
||||
var buffer bytes.Buffer
|
||||
|
||||
text := m.processLine("1=>", &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("2=>", &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("3=>", &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("4", &buffer)
|
||||
require.Equal(t, "1=>2=>3=>4", text)
|
||||
require.Zero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("5", &buffer)
|
||||
require.Equal(t, "5", text)
|
||||
require.Zero(t, buffer.Len())
|
||||
}
|
||||
|
||||
func TestMultiLineMatchStringWithInvertMatchFalse(t *testing.T) {
|
||||
c := &multilineConfig{
|
||||
Pattern: "=>$",
|
||||
MatchWhichLine: next,
|
||||
InvertMatch: false,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err, "Configuration was OK.")
|
||||
|
||||
matches1 := m.matchString("t=>")
|
||||
matches2 := m.matchString("t")
|
||||
|
||||
require.True(t, matches1)
|
||||
require.False(t, matches2)
|
||||
}
|
||||
|
||||
func TestMultiLineMatchStringWithInvertTrue(t *testing.T) {
|
||||
c := &multilineConfig{
|
||||
Pattern: "=>$",
|
||||
MatchWhichLine: next,
|
||||
InvertMatch: true,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err, "Configuration was OK.")
|
||||
|
||||
matches1 := m.matchString("t=>")
|
||||
matches2 := m.matchString("t")
|
||||
|
||||
require.False(t, matches1)
|
||||
require.True(t, matches2)
|
||||
}
|
||||
|
||||
func TestMultilineWhat(t *testing.T) {
|
||||
var w1 multilineMatchWhichLine
|
||||
require.NoError(t, w1.UnmarshalTOML([]byte(`"previous"`)))
|
||||
require.Equal(t, previous, w1)
|
||||
|
||||
var w2 multilineMatchWhichLine
|
||||
require.NoError(t, w2.UnmarshalTOML([]byte(`previous`)))
|
||||
require.Equal(t, previous, w2)
|
||||
|
||||
var w3 multilineMatchWhichLine
|
||||
require.NoError(t, w3.UnmarshalTOML([]byte(`'previous'`)))
|
||||
require.Equal(t, previous, w3)
|
||||
|
||||
var w4 multilineMatchWhichLine
|
||||
require.NoError(t, w4.UnmarshalTOML([]byte(`"next"`)))
|
||||
require.Equal(t, next, w4)
|
||||
|
||||
var w5 multilineMatchWhichLine
|
||||
require.NoError(t, w5.UnmarshalTOML([]byte(`next`)))
|
||||
require.Equal(t, next, w5)
|
||||
|
||||
var w6 multilineMatchWhichLine
|
||||
require.NoError(t, w6.UnmarshalTOML([]byte(`'next'`)))
|
||||
require.Equal(t, next, w6)
|
||||
|
||||
var w7 multilineMatchWhichLine
|
||||
require.Error(t, w7.UnmarshalTOML([]byte(`nope`)))
|
||||
require.Equal(t, multilineMatchWhichLine(-1), w7)
|
||||
}
|
||||
|
||||
func TestMultilineQuoted(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
quotation string
|
||||
quote string
|
||||
filename string
|
||||
}{
|
||||
{
|
||||
name: "single-quotes",
|
||||
quotation: "single-quotes",
|
||||
quote: `'`,
|
||||
filename: "multiline_quoted_single.csv",
|
||||
},
|
||||
{
|
||||
name: "double-quotes",
|
||||
quotation: "double-quotes",
|
||||
quote: `"`,
|
||||
filename: "multiline_quoted_double.csv",
|
||||
},
|
||||
{
|
||||
name: "backticks",
|
||||
quotation: "backticks",
|
||||
quote: "`",
|
||||
filename: "multiline_quoted_backticks.csv",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
expected := []string{
|
||||
`1660819827410,1,some text without quotes,A`,
|
||||
fmt.Sprintf("1660819827411,1,%ssome text all quoted%s,A", tt.quote, tt.quote),
|
||||
fmt.Sprintf("1660819827412,1,%ssome text all quoted\nbut wrapped%s,A", tt.quote, tt.quote),
|
||||
fmt.Sprintf("1660819827420,2,some text with %squotes%s,B", tt.quote, tt.quote),
|
||||
"1660819827430,3,some text with 'multiple \"quotes\" in `one` line',C",
|
||||
fmt.Sprintf("1660819827440,4,some multiline text with %squotes\n", tt.quote) +
|
||||
fmt.Sprintf("spanning \\%smultiple\\%s\n", tt.quote, tt.quote) +
|
||||
fmt.Sprintf("lines%s but do not %send\ndirectly%s,D", tt.quote, tt.quote, tt.quote),
|
||||
fmt.Sprintf("1660819827450,5,all of %sthis%s should %sbasically%s work...,E", tt.quote, tt.quote, tt.quote, tt.quote),
|
||||
}
|
||||
|
||||
c := &multilineConfig{
|
||||
MatchWhichLine: next,
|
||||
Quotation: tt.quotation,
|
||||
PreserveNewline: true,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := os.Open(filepath.Join("testdata", tt.filename))
|
||||
require.NoError(t, err)
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
|
||||
var buffer bytes.Buffer
|
||||
var result []string
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
text := m.processLine(line, &buffer)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
result = append(result, text)
|
||||
}
|
||||
if text := flush(&buffer); text != "" {
|
||||
result = append(result, text)
|
||||
}
|
||||
|
||||
require.EqualValues(t, expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultilineQuotedError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
quotation string
|
||||
quote string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "messed up quoting",
|
||||
filename: "multiline_quoted_messed_up.csv",
|
||||
quotation: "single-quotes",
|
||||
quote: `'`,
|
||||
expected: []string{
|
||||
"1660819827410,1,some text without quotes,A",
|
||||
"1660819827411,1,'some text all quoted,A\n1660819827412,1,'some text all quoted",
|
||||
"but wrapped,A"},
|
||||
},
|
||||
{
|
||||
name: "missing closing quote",
|
||||
filename: "multiline_quoted_missing_close.csv",
|
||||
quotation: "single-quotes",
|
||||
quote: `'`,
|
||||
expected: []string{"1660819827411,2,'some text all quoted,B\n1660819827410,1,some text without quotes,A"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := &multilineConfig{
|
||||
MatchWhichLine: next,
|
||||
Quotation: tt.quotation,
|
||||
PreserveNewline: true,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := os.Open(filepath.Join("testdata", tt.filename))
|
||||
require.NoError(t, err)
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
|
||||
var buffer bytes.Buffer
|
||||
var result []string
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
text := m.processLine(line, &buffer)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
result = append(result, text)
|
||||
}
|
||||
if text := flush(&buffer); text != "" {
|
||||
result = append(result, text)
|
||||
}
|
||||
|
||||
require.EqualValues(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultilineNewline(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
cfg *multilineConfig
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "do not preserve newline",
|
||||
cfg: &multilineConfig{
|
||||
Pattern: `\[[0-9]{2}/[A-Za-z]{3}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} \+[0-9]{4}\]`,
|
||||
InvertMatch: true,
|
||||
},
|
||||
filename: "test_multiline.log",
|
||||
expected: []string{
|
||||
`[04/Jun/2016:12:41:45 +0100] DEBUG HelloExample: This is debug`,
|
||||
`[04/Jun/2016:12:41:48 +0100] INFO HelloExample: This is info`,
|
||||
"[04/Jun/2016:12:41:46 +0100] ERROR HelloExample: Sorry, something wrong! " +
|
||||
"java.lang.ArithmeticException: / by zero" +
|
||||
"\tat com.foo.HelloExample2.divide(HelloExample2.java:24)" +
|
||||
"\tat com.foo.HelloExample2.main(HelloExample2.java:14)",
|
||||
`[04/Jun/2016:12:41:48 +0100] WARN HelloExample: This is warn`,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "preserve newline",
|
||||
cfg: &multilineConfig{
|
||||
Pattern: `\[[0-9]{2}/[A-Za-z]{3}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} \+[0-9]{4}\]`,
|
||||
InvertMatch: true,
|
||||
PreserveNewline: true,
|
||||
},
|
||||
filename: "test_multiline.log",
|
||||
expected: []string{
|
||||
`[04/Jun/2016:12:41:45 +0100] DEBUG HelloExample: This is debug`,
|
||||
`[04/Jun/2016:12:41:48 +0100] INFO HelloExample: This is info`,
|
||||
`[04/Jun/2016:12:41:46 +0100] ERROR HelloExample: Sorry, something wrong!` + ` ` + `
|
||||
java.lang.ArithmeticException: / by zero
|
||||
at com.foo.HelloExample2.divide(HelloExample2.java:24)
|
||||
at com.foo.HelloExample2.main(HelloExample2.java:14)`,
|
||||
`[04/Jun/2016:12:41:48 +0100] WARN HelloExample: This is warn`,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
m, err := tt.cfg.newMultiline()
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := os.Open(filepath.Join("testdata", tt.filename))
|
||||
require.NoError(t, err)
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
|
||||
var buffer bytes.Buffer
|
||||
var result []string
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
text := m.processLine(line, &buffer)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
result = append(result, text)
|
||||
}
|
||||
if text := flush(&buffer); text != "" {
|
||||
result = append(result, text)
|
||||
}
|
||||
|
||||
require.EqualValues(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiLineQuotedAndPattern(t *testing.T) {
|
||||
c := &multilineConfig{
|
||||
Pattern: "=>$",
|
||||
MatchWhichLine: next,
|
||||
Quotation: "double-quotes",
|
||||
PreserveNewline: true,
|
||||
}
|
||||
m, err := c.newMultiline()
|
||||
require.NoError(t, err, "Configuration was OK.")
|
||||
var buffer bytes.Buffer
|
||||
|
||||
text := m.processLine("1=>", &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("2=>", &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine(`"a quoted`, &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine(`multiline string"=>`, &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("3=>", &buffer)
|
||||
require.Empty(t, text)
|
||||
require.NotZero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("4", &buffer)
|
||||
require.Equal(t, "1=>\n2=>\n\"a quoted\nmultiline string\"=>\n3=>\n4", text)
|
||||
require.Zero(t, buffer.Len())
|
||||
|
||||
text = m.processLine("5", &buffer)
|
||||
require.Equal(t, "5", text)
|
||||
require.Zero(t, buffer.Len())
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue