475 lines
12 KiB
Go
475 lines
12 KiB
Go
package tail
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/influxdata/telegraf/config"
|
|
)
|
|
|
|
func TestMultilineConfigOK(t *testing.T) {
|
|
c := &multilineConfig{
|
|
Pattern: ".*",
|
|
MatchWhichLine: previous,
|
|
}
|
|
|
|
_, err := c.newMultiline()
|
|
|
|
require.NoError(t, err, "Configuration was OK.")
|
|
}
|
|
|
|
func TestMultilineConfigError(t *testing.T) {
|
|
c := &multilineConfig{
|
|
Pattern: "\xA0",
|
|
MatchWhichLine: previous,
|
|
}
|
|
|
|
_, err := c.newMultiline()
|
|
|
|
require.Error(t, err, "The pattern was invalid")
|
|
}
|
|
|
|
func TestMultilineConfigTimeoutSpecified(t *testing.T) {
|
|
duration := config.Duration(10 * time.Second)
|
|
c := &multilineConfig{
|
|
Pattern: ".*",
|
|
MatchWhichLine: previous,
|
|
Timeout: &duration,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err, "Configuration was OK.")
|
|
|
|
require.Equal(t, duration, *m.config.Timeout)
|
|
}
|
|
|
|
func TestMultilineConfigDefaultTimeout(t *testing.T) {
|
|
duration := config.Duration(5 * time.Second)
|
|
c := &multilineConfig{
|
|
Pattern: ".*",
|
|
MatchWhichLine: previous,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err, "Configuration was OK.")
|
|
|
|
require.Equal(t, duration, *m.config.Timeout)
|
|
}
|
|
|
|
func TestMultilineIsEnabled(t *testing.T) {
|
|
c := &multilineConfig{
|
|
Pattern: ".*",
|
|
MatchWhichLine: previous,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err, "Configuration was OK.")
|
|
|
|
isEnabled := m.isEnabled()
|
|
|
|
require.True(t, isEnabled, "Should have been enabled")
|
|
}
|
|
|
|
func TestMultilineIsDisabled(t *testing.T) {
|
|
c := &multilineConfig{
|
|
MatchWhichLine: previous,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err, "Configuration was OK.")
|
|
|
|
isEnabled := m.isEnabled()
|
|
|
|
require.False(t, isEnabled, "Should have been disabled")
|
|
}
|
|
|
|
func TestMultilineFlushEmpty(t *testing.T) {
|
|
var buffer bytes.Buffer
|
|
text := flush(&buffer)
|
|
|
|
require.Empty(t, text)
|
|
}
|
|
|
|
func TestMultilineFlush(t *testing.T) {
|
|
var buffer bytes.Buffer
|
|
buffer.WriteString("foo")
|
|
|
|
text := flush(&buffer)
|
|
require.Equal(t, "foo", text)
|
|
require.Zero(t, buffer.Len())
|
|
}
|
|
|
|
func TestMultiLineProcessLinePrevious(t *testing.T) {
|
|
c := &multilineConfig{
|
|
Pattern: "^=>",
|
|
MatchWhichLine: previous,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err, "Configuration was OK.")
|
|
var buffer bytes.Buffer
|
|
|
|
text := m.processLine("1", &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine("=>2", &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine("=>3", &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine("4", &buffer)
|
|
require.Equal(t, "1=>2=>3", text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine("5", &buffer)
|
|
require.Equal(t, "4", text)
|
|
require.Equal(t, "5", buffer.String())
|
|
}
|
|
|
|
func TestMultiLineProcessLineNext(t *testing.T) {
|
|
c := &multilineConfig{
|
|
Pattern: "=>$",
|
|
MatchWhichLine: next,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err, "Configuration was OK.")
|
|
var buffer bytes.Buffer
|
|
|
|
text := m.processLine("1=>", &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine("2=>", &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine("3=>", &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine("4", &buffer)
|
|
require.Equal(t, "1=>2=>3=>4", text)
|
|
require.Zero(t, buffer.Len())
|
|
|
|
text = m.processLine("5", &buffer)
|
|
require.Equal(t, "5", text)
|
|
require.Zero(t, buffer.Len())
|
|
}
|
|
|
|
func TestMultiLineMatchStringWithInvertMatchFalse(t *testing.T) {
|
|
c := &multilineConfig{
|
|
Pattern: "=>$",
|
|
MatchWhichLine: next,
|
|
InvertMatch: false,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err, "Configuration was OK.")
|
|
|
|
matches1 := m.matchString("t=>")
|
|
matches2 := m.matchString("t")
|
|
|
|
require.True(t, matches1)
|
|
require.False(t, matches2)
|
|
}
|
|
|
|
func TestMultiLineMatchStringWithInvertTrue(t *testing.T) {
|
|
c := &multilineConfig{
|
|
Pattern: "=>$",
|
|
MatchWhichLine: next,
|
|
InvertMatch: true,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err, "Configuration was OK.")
|
|
|
|
matches1 := m.matchString("t=>")
|
|
matches2 := m.matchString("t")
|
|
|
|
require.False(t, matches1)
|
|
require.True(t, matches2)
|
|
}
|
|
|
|
func TestMultilineWhat(t *testing.T) {
|
|
var w1 multilineMatchWhichLine
|
|
require.NoError(t, w1.UnmarshalTOML([]byte(`"previous"`)))
|
|
require.Equal(t, previous, w1)
|
|
|
|
var w2 multilineMatchWhichLine
|
|
require.NoError(t, w2.UnmarshalTOML([]byte(`previous`)))
|
|
require.Equal(t, previous, w2)
|
|
|
|
var w3 multilineMatchWhichLine
|
|
require.NoError(t, w3.UnmarshalTOML([]byte(`'previous'`)))
|
|
require.Equal(t, previous, w3)
|
|
|
|
var w4 multilineMatchWhichLine
|
|
require.NoError(t, w4.UnmarshalTOML([]byte(`"next"`)))
|
|
require.Equal(t, next, w4)
|
|
|
|
var w5 multilineMatchWhichLine
|
|
require.NoError(t, w5.UnmarshalTOML([]byte(`next`)))
|
|
require.Equal(t, next, w5)
|
|
|
|
var w6 multilineMatchWhichLine
|
|
require.NoError(t, w6.UnmarshalTOML([]byte(`'next'`)))
|
|
require.Equal(t, next, w6)
|
|
|
|
var w7 multilineMatchWhichLine
|
|
require.Error(t, w7.UnmarshalTOML([]byte(`nope`)))
|
|
require.Equal(t, multilineMatchWhichLine(-1), w7)
|
|
}
|
|
|
|
func TestMultilineQuoted(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
quotation string
|
|
quote string
|
|
filename string
|
|
}{
|
|
{
|
|
name: "single-quotes",
|
|
quotation: "single-quotes",
|
|
quote: `'`,
|
|
filename: "multiline_quoted_single.csv",
|
|
},
|
|
{
|
|
name: "double-quotes",
|
|
quotation: "double-quotes",
|
|
quote: `"`,
|
|
filename: "multiline_quoted_double.csv",
|
|
},
|
|
{
|
|
name: "backticks",
|
|
quotation: "backticks",
|
|
quote: "`",
|
|
filename: "multiline_quoted_backticks.csv",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
expected := []string{
|
|
`1660819827410,1,some text without quotes,A`,
|
|
fmt.Sprintf("1660819827411,1,%ssome text all quoted%s,A", tt.quote, tt.quote),
|
|
fmt.Sprintf("1660819827412,1,%ssome text all quoted\nbut wrapped%s,A", tt.quote, tt.quote),
|
|
fmt.Sprintf("1660819827420,2,some text with %squotes%s,B", tt.quote, tt.quote),
|
|
"1660819827430,3,some text with 'multiple \"quotes\" in `one` line',C",
|
|
fmt.Sprintf("1660819827440,4,some multiline text with %squotes\n", tt.quote) +
|
|
fmt.Sprintf("spanning \\%smultiple\\%s\n", tt.quote, tt.quote) +
|
|
fmt.Sprintf("lines%s but do not %send\ndirectly%s,D", tt.quote, tt.quote, tt.quote),
|
|
fmt.Sprintf("1660819827450,5,all of %sthis%s should %sbasically%s work...,E", tt.quote, tt.quote, tt.quote, tt.quote),
|
|
}
|
|
|
|
c := &multilineConfig{
|
|
MatchWhichLine: next,
|
|
Quotation: tt.quotation,
|
|
PreserveNewline: true,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err)
|
|
|
|
f, err := os.Open(filepath.Join("testdata", tt.filename))
|
|
require.NoError(t, err)
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
|
|
var buffer bytes.Buffer
|
|
var result []string
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
|
|
text := m.processLine(line, &buffer)
|
|
if text == "" {
|
|
continue
|
|
}
|
|
result = append(result, text)
|
|
}
|
|
if text := flush(&buffer); text != "" {
|
|
result = append(result, text)
|
|
}
|
|
|
|
require.EqualValues(t, expected, result)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMultilineQuotedError(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
filename string
|
|
quotation string
|
|
quote string
|
|
expected []string
|
|
}{
|
|
{
|
|
name: "messed up quoting",
|
|
filename: "multiline_quoted_messed_up.csv",
|
|
quotation: "single-quotes",
|
|
quote: `'`,
|
|
expected: []string{
|
|
"1660819827410,1,some text without quotes,A",
|
|
"1660819827411,1,'some text all quoted,A\n1660819827412,1,'some text all quoted",
|
|
"but wrapped,A"},
|
|
},
|
|
{
|
|
name: "missing closing quote",
|
|
filename: "multiline_quoted_missing_close.csv",
|
|
quotation: "single-quotes",
|
|
quote: `'`,
|
|
expected: []string{"1660819827411,2,'some text all quoted,B\n1660819827410,1,some text without quotes,A"},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
c := &multilineConfig{
|
|
MatchWhichLine: next,
|
|
Quotation: tt.quotation,
|
|
PreserveNewline: true,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err)
|
|
|
|
f, err := os.Open(filepath.Join("testdata", tt.filename))
|
|
require.NoError(t, err)
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
|
|
var buffer bytes.Buffer
|
|
var result []string
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
|
|
text := m.processLine(line, &buffer)
|
|
if text == "" {
|
|
continue
|
|
}
|
|
result = append(result, text)
|
|
}
|
|
if text := flush(&buffer); text != "" {
|
|
result = append(result, text)
|
|
}
|
|
|
|
require.EqualValues(t, tt.expected, result)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMultilineNewline(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
filename string
|
|
cfg *multilineConfig
|
|
expected []string
|
|
}{
|
|
{
|
|
name: "do not preserve newline",
|
|
cfg: &multilineConfig{
|
|
Pattern: `\[[0-9]{2}/[A-Za-z]{3}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} \+[0-9]{4}\]`,
|
|
InvertMatch: true,
|
|
},
|
|
filename: "test_multiline.log",
|
|
expected: []string{
|
|
`[04/Jun/2016:12:41:45 +0100] DEBUG HelloExample: This is debug`,
|
|
`[04/Jun/2016:12:41:48 +0100] INFO HelloExample: This is info`,
|
|
"[04/Jun/2016:12:41:46 +0100] ERROR HelloExample: Sorry, something wrong! " +
|
|
"java.lang.ArithmeticException: / by zero" +
|
|
"\tat com.foo.HelloExample2.divide(HelloExample2.java:24)" +
|
|
"\tat com.foo.HelloExample2.main(HelloExample2.java:14)",
|
|
`[04/Jun/2016:12:41:48 +0100] WARN HelloExample: This is warn`,
|
|
},
|
|
},
|
|
{
|
|
name: "preserve newline",
|
|
cfg: &multilineConfig{
|
|
Pattern: `\[[0-9]{2}/[A-Za-z]{3}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} \+[0-9]{4}\]`,
|
|
InvertMatch: true,
|
|
PreserveNewline: true,
|
|
},
|
|
filename: "test_multiline.log",
|
|
expected: []string{
|
|
`[04/Jun/2016:12:41:45 +0100] DEBUG HelloExample: This is debug`,
|
|
`[04/Jun/2016:12:41:48 +0100] INFO HelloExample: This is info`,
|
|
`[04/Jun/2016:12:41:46 +0100] ERROR HelloExample: Sorry, something wrong!` + ` ` + `
|
|
java.lang.ArithmeticException: / by zero
|
|
at com.foo.HelloExample2.divide(HelloExample2.java:24)
|
|
at com.foo.HelloExample2.main(HelloExample2.java:14)`,
|
|
`[04/Jun/2016:12:41:48 +0100] WARN HelloExample: This is warn`,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
m, err := tt.cfg.newMultiline()
|
|
require.NoError(t, err)
|
|
|
|
f, err := os.Open(filepath.Join("testdata", tt.filename))
|
|
require.NoError(t, err)
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
|
|
var buffer bytes.Buffer
|
|
var result []string
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
|
|
text := m.processLine(line, &buffer)
|
|
if text == "" {
|
|
continue
|
|
}
|
|
result = append(result, text)
|
|
}
|
|
if text := flush(&buffer); text != "" {
|
|
result = append(result, text)
|
|
}
|
|
|
|
require.EqualValues(t, tt.expected, result)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMultiLineQuotedAndPattern(t *testing.T) {
|
|
c := &multilineConfig{
|
|
Pattern: "=>$",
|
|
MatchWhichLine: next,
|
|
Quotation: "double-quotes",
|
|
PreserveNewline: true,
|
|
}
|
|
m, err := c.newMultiline()
|
|
require.NoError(t, err, "Configuration was OK.")
|
|
var buffer bytes.Buffer
|
|
|
|
text := m.processLine("1=>", &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine("2=>", &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine(`"a quoted`, &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine(`multiline string"=>`, &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine("3=>", &buffer)
|
|
require.Empty(t, text)
|
|
require.NotZero(t, buffer.Len())
|
|
|
|
text = m.processLine("4", &buffer)
|
|
require.Equal(t, "1=>\n2=>\n\"a quoted\nmultiline string\"=>\n3=>\n4", text)
|
|
require.Zero(t, buffer.Len())
|
|
|
|
text = m.processLine("5", &buffer)
|
|
require.Equal(t, "5", text)
|
|
require.Zero(t, buffer.Len())
|
|
}
|