package fexpr import ( "fmt" "testing" ) func TestNewScanner(t *testing.T) { s := NewScanner([]byte("test")) data := string(s.data) if data != "test" { t.Errorf("Expected the scanner reader data to be %q, got %q", "test", data) } } func TestScannerScan(t *testing.T) { type output struct { error bool print string } testScenarios := []struct { text string expects []output }{ // whitespace {" ", []output{{false, "{ whitespace }"}}}, {"test 123", []output{{false, "{ identifier test}"}, {false, "{ whitespace }"}, {false, "{ number 123}"}}}, // identifier {`test`, []output{{false, `{ identifier test}`}}}, {`@`, []output{{true, `{ identifier @}`}}}, {`test:`, []output{{true, `{ identifier test:}`}}}, {`test.`, []output{{true, `{ identifier test.}`}}}, {`@test.123:c`, []output{{false, `{ identifier @test.123:c}`}}}, {`_test_a.123`, []output{{false, `{ identifier _test_a.123}`}}}, {`#test.123:456`, []output{{false, `{ identifier #test.123:456}`}}}, {`.test.123`, []output{{true, `{ unexpected .}`}, {false, `{ identifier test.123}`}}}, {`:test.123`, []output{{true, `{ unexpected :}`}, {false, `{ identifier test.123}`}}}, {`test#@`, []output{{false, `{ identifier test}`}, {true, `{ identifier #}`}, {true, `{ identifier @}`}}}, {`test'`, []output{{false, `{ identifier test}`}, {true, `{ text '}`}}}, {`test"d`, []output{{false, `{ identifier test}`}, {true, `{ text "d}`}}}, // number {`123`, []output{{false, `{ number 123}`}}}, {`-123`, []output{{false, `{ number -123}`}}}, {`-123.456`, []output{{false, `{ number -123.456}`}}}, {`123.456`, []output{{false, `{ number 123.456}`}}}, {`12.34.56`, []output{{false, `{ number 12.34}`}, {true, `{ unexpected .}`}, {false, `{ number 56}`}}}, {`.123`, []output{{true, `{ unexpected .}`}, {false, `{ number 123}`}}}, {`- 123`, []output{{true, `{ number -}`}, {false, `{ whitespace }`}, {false, `{ number 123}`}}}, {`12-3`, []output{{false, `{ number 12}`}, {false, `{ number -3}`}}}, {`123.abc`, []output{{true, `{ number 123.}`}, {false, `{ identifier abc}`}}}, // text {`""`, []output{{false, `{ text }`}}}, {`''`, []output{{false, `{ text }`}}}, {`'test'`, []output{{false, `{ text test}`}}}, {`'te\'st'`, []output{{false, `{ text te'st}`}}}, {`"te\"st"`, []output{{false, `{ text te"st}`}}}, {`"tes@#,;!@#%^'\"t"`, []output{{false, `{ text tes@#,;!@#%^'"t}`}}}, {`'tes@#,;!@#%^\'"t'`, []output{{false, `{ text tes@#,;!@#%^'"t}`}}}, {`"test`, []output{{true, `{ text "test}`}}}, {`'test`, []output{{true, `{ text 'test}`}}}, {`'АБЦ`, []output{{true, `{ text 'АБЦ}`}}}, // join types {`&&||`, []output{{true, `{ join &&||}`}}}, {`&& ||`, []output{{false, `{ join &&}`}, {false, `{ whitespace }`}, {false, `{ join ||}`}}}, {`'||test&&'&&123`, []output{{false, `{ text ||test&&}`}, {false, `{ join &&}`}, {false, `{ number 123}`}}}, // expression signs {`=!=`, []output{{true, `{ sign =!=}`}}}, {`= != ~ !~ > >= < <= ?= ?!= ?~ ?!~ ?> ?>= ?< ?<=`, []output{ {false, `{ sign =}`}, {false, `{ whitespace }`}, {false, `{ sign !=}`}, {false, `{ whitespace }`}, {false, `{ sign ~}`}, {false, `{ whitespace }`}, {false, `{ sign !~}`}, {false, `{ whitespace }`}, {false, `{ sign >}`}, {false, `{ whitespace }`}, {false, `{ sign >=}`}, {false, `{ whitespace }`}, {false, `{ sign <}`}, {false, `{ whitespace }`}, {false, `{ sign <=}`}, {false, `{ whitespace }`}, {false, `{ sign ?=}`}, {false, `{ whitespace }`}, {false, `{ sign ?!=}`}, {false, `{ whitespace }`}, {false, `{ sign ?~}`}, {false, `{ whitespace }`}, {false, `{ sign ?!~}`}, {false, `{ whitespace }`}, {false, `{ sign ?>}`}, {false, `{ whitespace }`}, {false, `{ sign ?>=}`}, {false, `{ whitespace }`}, {false, `{ sign ?<}`}, {false, `{ whitespace }`}, {false, `{ sign ?<=}`}, }}, // comments {`/ test`, []output{{true, `{ comment }`}, {false, `{ identifier test}`}}}, {`/ / test`, []output{{true, `{ comment }`}, {true, `{ comment }`}, {false, `{ identifier test}`}}}, {`//`, []output{{false, `{ comment }`}}}, {`//test`, []output{{false, `{ comment test}`}}}, {`// test`, []output{{false, `{ comment test}`}}}, {`// test1 //test2 `, []output{{false, `{ comment test1 //test2}`}}}, {`///test`, []output{{false, `{ comment /test}`}}}, // funcs {`test()`, []output{{false, `{[] function test}`}}}, {`test(a, b`, []output{{true, `{[{ identifier a} { identifier b}] function test}`}}}, {`@test:abc()`, []output{{false, `{[] function @test:abc}`}}}, {`test( a )`, []output{{false, `{[{ identifier a}] function test}`}}}, // with whitespaces {`test(a, b)`, []output{{false, `{[{ identifier a} { identifier b}] function test}`}}}, {`test(a, b, )`, []output{{false, `{[{ identifier a} { identifier b}] function test}`}}}, // single trailing comma {`test(a,,)`, []output{{true, `{[{ identifier a}] function test}`}, {true, `{ unexpected )}`}}}, // unexpected trailing commas {`test(a,,,b)`, []output{{true, `{[{ identifier a}] function test}`}, {true, `{ unexpected ,}`}, {false, `{ identifier b}`}, {true, `{ unexpected )}`}}}, // unexpected mid-args commas {`test( @test.a.b:test , 123, "ab)c", 'd,ce', false)`, []output{{false, `{[{ identifier @test.a.b:test} { number 123} { text ab)c} { text d,ce} { identifier false}] function test}`}}}, {"test(a //test)", []output{{true, `{[{ identifier a}] function test}`}}}, // invalid simple comment {"test(a //test\n)", []output{{false, `{[{ identifier a}] function test}`}}}, // valid simple comment {"test(a, //test\n, b)", []output{{true, `{[{ identifier a}] function test}`}, {false, `{ whitespace }`}, {false, `{ identifier b}`}, {true, `{ unexpected )}`}}}, {"test(a, //test\n b)", []output{{false, `{[{ identifier a} { identifier b}] function test}`}}}, {"test(a, test(test(b), c), d)", []output{{false, `{[{ identifier a} {[{[{ identifier b}] function test} { identifier c}] function test} { identifier d}] function test}`}}}, // max funcs depth {"a(b(c(1)))", []output{{false, `{[{[{[{ number 1}] function c}] function b}] function a}`}}}, {"a(b(c(d(1))))", []output{{true, `{[] function a}`}, {false, `{ number 1}`}, {true, `{ unexpected )}`}, {true, `{ unexpected )}`}, {true, `{ unexpected )}`}, {true, `{ unexpected )}`}}}, // groups/parenthesis {`a)`, []output{{false, `{ identifier a}`}, {true, `{ unexpected )}`}}}, {`(a b c`, []output{{true, `{ group a b c}`}}}, {`(a b c)`, []output{{false, `{ group a b c}`}}}, {`((a b c))`, []output{{false, `{ group (a b c)}`}}}, {`((a )b c))`, []output{{false, `{ group (a )b c}`}, {true, `{ unexpected )}`}}}, {`("ab)("c)`, []output{{false, `{ group "ab)("c}`}}}, {`("ab)(c)`, []output{{true, `{ group "ab)(c)}`}}}, {`( func(1, 2, 3, func(4)) a b c )`, []output{{false, `{ group func(1, 2, 3, func(4)) a b c }`}}}, } for _, scenario := range testScenarios { t.Run(scenario.text, func(t *testing.T) { s := NewScanner([]byte(scenario.text)) // scan the text tokens for j, expect := range scenario.expects { token, err := s.Scan() hasErr := err != nil if expect.error != hasErr { t.Errorf("[%d] Expected hasErr %v, got %v: %v (%v)", j, expect.error, hasErr, err, token) } tokenPrint := fmt.Sprintf("%v", token) if tokenPrint != expect.print { t.Errorf("[%d] Expected token %s, got %s", j, expect.print, tokenPrint) } } // the last remaining token should be the eof lastToken, err := s.Scan() if err != nil || lastToken.Type != TokenEOF { t.Fatalf("Expected EOF token, got %v (%v)", lastToken, err) } }) } }