beszel/golang-github-pocketbase-pocketbase
1
0
Fork 0
Code Issues Activity

Adding upstream version 0.28.1.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-05-22 10:57:38 +02:00
parent 88f1d47ab6
commit e28c88ef14
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
933 changed files with 194711 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

303
tools/tokenizer/tokenizer_test.go Normal file
View file

@ -0,0 +1,303 @@
package tokenizer
import (
"io"
"strings"
"testing"
)
func TestFactories(t *testing.T) {
expectedContent := "test"
scenarios := []struct {
name string
tk *Tokenizer
}{
{
"New()",
New(strings.NewReader(expectedContent)),
},
{
"NewFromString()",
NewFromString(expectedContent),
},
{
"NewFromBytes()",
NewFromBytes([]byte(expectedContent)),
},
}
for _, s := range scenarios {
t.Run(s.name, func(t *testing.T) {
content, _ := s.tk.r.ReadString(0)
if content != expectedContent {
t.Fatalf("Expected reader with content %q, got %q", expectedContent, content)
}
if s.tk.keepSeparator != false {
t.Fatal("Expected keepSeparator false, got true")
}
if s.tk.ignoreParenthesis != false {
t.Fatal("Expected ignoreParenthesis false, got true")
}
if len(s.tk.separators) != len(DefaultSeparators) {
t.Fatalf("Expected \n%v, \ngot \n%v", DefaultSeparators, s.tk.separators)
}
for _, r := range s.tk.separators {
exists := false
for _, def := range s.tk.separators {
if r == def {
exists = true
break
}
}
if !exists {
t.Fatalf("Unexpected sepator %s", string(r))
}
}
})
}
}
func TestScan(t *testing.T) {
tk := NewFromString("abc, 123.456, (abc)")
expectedTokens := []string{"abc", "123.456", "(abc)"}
for _, token := range expectedTokens {
result, err := tk.Scan()
if err != nil {
t.Fatalf("Expected token %q, got error %v", token, err)
}
if result != token {
t.Fatalf("Expected token %q, got error %v", token, result)
}
}
// scan the last character
token, err := tk.Scan()
if err != io.EOF {
t.Fatalf("Expected EOF error, got %v", err)
}
if token != "" || err != io.EOF {
t.Fatalf("Expected empty token, got %q", token)
}
}
func TestScanAll(t *testing.T) {
scenarios := []struct {
name string
content string
separators []rune
keepSeparator bool
keepEmptyTokens bool
ignoreParenthesis bool
expectError bool
expectTokens []string
}{
{
name: "empty string",
content: "",
separators: DefaultSeparators,
keepSeparator: false,
keepEmptyTokens: false,
ignoreParenthesis: false,
expectError: false,
expectTokens: nil,
},
{
name: "unbalanced parenthesis",
content: `(a,b() c`,
separators: DefaultSeparators,
keepSeparator: false,
keepEmptyTokens: false,
ignoreParenthesis: false,
expectError: true,
expectTokens: []string{},
},
{
name: "unmatching quotes",
content: `'asd"`,
separators: DefaultSeparators,
keepSeparator: false,
keepEmptyTokens: false,
ignoreParenthesis: false,
expectError: true,
expectTokens: []string{},
},
{
name: "no separators",
content: `a, b, c, d, e 123, "abc"`,
separators: nil,
keepSeparator: false,
keepEmptyTokens: false,
ignoreParenthesis: false,
expectError: false,
expectTokens: []string{`a, b, c, d, e 123, "abc"`},
},
{
name: "default separators",
content: `a, b , c , d e , "a,b, c " , ,, , (123, 456)
`,
separators: DefaultSeparators,
keepSeparator: false,
keepEmptyTokens: false,
ignoreParenthesis: false,
expectError: false,
expectTokens: []string{
"a",
"b",
"c",
"d e",
`"a,b, c "`,
`(123, 456)`,
},
},
{
name: "keep separators",
content: `a, b, c, d e, "a,b, c ", (123, 456)`,
separators: []rune{',', ' '}, // the space should be removed from the cutset
keepSeparator: true,
keepEmptyTokens: true,
ignoreParenthesis: false,
expectError: false,
expectTokens: []string{
"a,",
" ",
"b,",
" ",
"c,",
" ",
"d ",
" ",
"e,",
" ",
`"a,b, c ",`,
`(123, 456)`,
},
},
{
name: "custom separators",
content: `a | b c d &(e + f) & "g & h" & & &`,
separators: []rune{'|', '&'},
keepSeparator: false,
keepEmptyTokens: false,
ignoreParenthesis: false,
expectError: false,
expectTokens: []string{
"a",
"b c d",
"(e + f)",
`"g & h"`,
},
},
{
name: "ignoring parenthesis",
content: `a, b, (c,d)`,
separators: DefaultSeparators,
keepSeparator: false,
keepEmptyTokens: false,
ignoreParenthesis: true,
expectError: false,
expectTokens: []string{
"a",
"b",
"(c",
"d)",
},
},
{
name: "keep empty tokens",
content: `a, b, (c, d), ,, , e, , f`,
separators: DefaultSeparators,
keepSeparator: false,
keepEmptyTokens: true,
ignoreParenthesis: false,
expectError: false,
expectTokens: []string{
"a",
"b",
"(c, d)",
"",
"",
"",
"e",
"",
"f",
},
},
}
for _, s := range scenarios {
t.Run(s.name, func(t *testing.T) {
tk := NewFromString(s.content)
tk.Separators(s.separators...)
tk.KeepSeparator(s.keepSeparator)
tk.KeepEmptyTokens(s.keepEmptyTokens)
tk.IgnoreParenthesis(s.ignoreParenthesis)
tokens, err := tk.ScanAll()
hasErr := err != nil
if hasErr != s.expectError {
t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err)
}
if len(tokens) != len(s.expectTokens) {
t.Fatalf("Expected \n%v (%d), \ngot \n%v (%d)", s.expectTokens, len(s.expectTokens), tokens, len(tokens))
}
for _, tok := range tokens {
exists := false
for _, def := range s.expectTokens {
if tok == def {
exists = true
break
}
}
if !exists {
t.Fatalf("Unexpected token %q", tok)
}
}
})
}
}
func TestTrimCutset(t *testing.T) {
scenarios := []struct {
name string
separators []rune
expectedCutset string
}{
{
"default factory separators",
nil,
"\t\n\v\f\r \u0085\u00a0",
},
{
"custom separators",
[]rune{'\t', ' ', '\r', ','},
"\n\v\f\u0085\u00a0",
},
}
for _, s := range scenarios {
t.Run(s.name, func(t *testing.T) {
tk := NewFromString("")
if len(s.separators) > 0 {
tk.Separators(s.separators...)
}
if tk.trimCutset != s.expectedCutset {
t.Fatalf("Expected cutset %q, got %q", s.expectedCutset, tk.trimCutset)
}
})
}
}