You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
311 lines
9.8 KiB
311 lines
9.8 KiB
// Copyright 2017 The Bazel Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package syntax
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"go/build"
|
|
"io/ioutil"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func scan(src interface{}) (tokens string, err error) {
|
|
sc, err := newScanner("foo.star", src, false)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
defer sc.recover(&err)
|
|
|
|
var buf bytes.Buffer
|
|
var val tokenValue
|
|
for {
|
|
tok := sc.nextToken(&val)
|
|
|
|
if buf.Len() > 0 {
|
|
buf.WriteByte(' ')
|
|
}
|
|
switch tok {
|
|
case EOF:
|
|
buf.WriteString("EOF")
|
|
case IDENT:
|
|
buf.WriteString(val.raw)
|
|
case INT:
|
|
if val.bigInt != nil {
|
|
fmt.Fprintf(&buf, "%d", val.bigInt)
|
|
} else {
|
|
fmt.Fprintf(&buf, "%d", val.int)
|
|
}
|
|
case FLOAT:
|
|
fmt.Fprintf(&buf, "%e", val.float)
|
|
case STRING, BYTES:
|
|
buf.WriteString(Quote(val.string, tok == BYTES))
|
|
default:
|
|
buf.WriteString(tok.String())
|
|
}
|
|
if tok == EOF {
|
|
break
|
|
}
|
|
}
|
|
return buf.String(), nil
|
|
}
|
|
|
|
func TestScanner(t *testing.T) {
|
|
for _, test := range []struct {
|
|
input, want string
|
|
}{
|
|
{``, "EOF"},
|
|
{`123`, "123 EOF"},
|
|
{`x.y`, "x . y EOF"},
|
|
{`chocolate.éclair`, `chocolate . éclair EOF`},
|
|
{`123 "foo" hello x.y`, `123 "foo" hello x . y EOF`},
|
|
{`print(x)`, "print ( x ) EOF"},
|
|
{`print(x); print(y)`, "print ( x ) ; print ( y ) EOF"},
|
|
{"\nprint(\n1\n)\n", "print ( 1 ) newline EOF"}, // final \n is at toplevel on non-blank line => token
|
|
{`/ // /= //= ///=`, "/ // /= //= // /= EOF"},
|
|
{`# hello
|
|
print(x)`, "print ( x ) EOF"},
|
|
{`# hello
|
|
print(1)
|
|
cc_binary(name="foo")
|
|
def f(x):
|
|
return x+1
|
|
print(1)
|
|
`,
|
|
`print ( 1 ) newline ` +
|
|
`cc_binary ( name = "foo" ) newline ` +
|
|
`def f ( x ) : newline ` +
|
|
`indent return x + 1 newline ` +
|
|
`outdent print ( 1 ) newline ` +
|
|
`EOF`},
|
|
// EOF should act line an implicit newline.
|
|
{`def f(): pass`,
|
|
"def f ( ) : pass EOF"},
|
|
{`def f():
|
|
pass`,
|
|
"def f ( ) : newline indent pass newline outdent EOF"},
|
|
{`def f():
|
|
pass
|
|
# oops`,
|
|
"def f ( ) : newline indent pass newline outdent EOF"},
|
|
{`def f():
|
|
pass \
|
|
`,
|
|
"def f ( ) : newline indent pass newline outdent EOF"},
|
|
{`def f():
|
|
pass
|
|
`,
|
|
"def f ( ) : newline indent pass newline outdent EOF"},
|
|
{`pass
|
|
|
|
|
|
pass`, "pass newline pass EOF"}, // consecutive newlines are consolidated
|
|
{`def f():
|
|
pass
|
|
`, "def f ( ) : newline indent pass newline outdent EOF"},
|
|
{`def f():
|
|
pass
|
|
` + "\n", "def f ( ) : newline indent pass newline outdent EOF"},
|
|
{"pass", "pass EOF"},
|
|
{"pass\n", "pass newline EOF"},
|
|
{"pass\n ", "pass newline EOF"},
|
|
{"pass\n \n", "pass newline EOF"},
|
|
{"if x:\n pass\n ", "if x : newline indent pass newline outdent EOF"},
|
|
{`x = 1 + \
|
|
2`, `x = 1 + 2 EOF`},
|
|
{`x = 'a\nb'`, `x = "a\nb" EOF`},
|
|
{`x = r'a\nb'`, `x = "a\\nb" EOF`},
|
|
{"x = 'a\\\nb'", `x = "ab" EOF`},
|
|
{`x = '\''`, `x = "'" EOF`},
|
|
{`x = "\""`, `x = "\"" EOF`},
|
|
{`x = r'\''`, `x = "\\'" EOF`},
|
|
{`x = '''\''''`, `x = "'" EOF`},
|
|
{`x = r'''\''''`, `x = "\\'" EOF`},
|
|
{`x = ''''a'b'c'''`, `x = "'a'b'c" EOF`},
|
|
{"x = '''a\nb'''", `x = "a\nb" EOF`},
|
|
{"x = '''a\rb'''", `x = "a\nb" EOF`},
|
|
{"x = '''a\r\nb'''", `x = "a\nb" EOF`},
|
|
{"x = '''a\n\rb'''", `x = "a\n\nb" EOF`},
|
|
{"x = r'a\\\nb'", `x = "a\\\nb" EOF`},
|
|
{"x = r'a\\\rb'", `x = "a\\\nb" EOF`},
|
|
{"x = r'a\\\r\nb'", `x = "a\\\nb" EOF`},
|
|
{"a\rb", `a newline b EOF`},
|
|
{"a\nb", `a newline b EOF`},
|
|
{"a\r\nb", `a newline b EOF`},
|
|
{"a\n\nb", `a newline b EOF`},
|
|
// numbers
|
|
{"0", `0 EOF`},
|
|
{"00", `0 EOF`},
|
|
{"0.", `0.000000e+00 EOF`},
|
|
{"0.e1", `0.000000e+00 EOF`},
|
|
{".0", `0.000000e+00 EOF`},
|
|
{"0.0", `0.000000e+00 EOF`},
|
|
{".e1", `. e1 EOF`},
|
|
{"1", `1 EOF`},
|
|
{"1.", `1.000000e+00 EOF`},
|
|
{".1", `1.000000e-01 EOF`},
|
|
{".1e1", `1.000000e+00 EOF`},
|
|
{".1e+1", `1.000000e+00 EOF`},
|
|
{".1e-1", `1.000000e-02 EOF`},
|
|
{"1e1", `1.000000e+01 EOF`},
|
|
{"1e+1", `1.000000e+01 EOF`},
|
|
{"1e-1", `1.000000e-01 EOF`},
|
|
{"123", `123 EOF`},
|
|
{"123e45", `1.230000e+47 EOF`},
|
|
{"999999999999999999999999999999999999999999999999999", `999999999999999999999999999999999999999999999999999 EOF`},
|
|
{"12345678901234567890", `12345678901234567890 EOF`},
|
|
// hex
|
|
{"0xA", `10 EOF`},
|
|
{"0xAAG", `170 G EOF`},
|
|
{"0xG", `foo.star:1:1: invalid hex literal`},
|
|
{"0XA", `10 EOF`},
|
|
{"0XG", `foo.star:1:1: invalid hex literal`},
|
|
{"0xA.", `10 . EOF`},
|
|
{"0xA.e1", `10 . e1 EOF`},
|
|
{"0x12345678deadbeef12345678", `5634002672576678570168178296 EOF`},
|
|
// binary
|
|
{"0b1010", `10 EOF`},
|
|
{"0B111101", `61 EOF`},
|
|
{"0b3", `foo.star:1:3: invalid binary literal`},
|
|
{"0b1010201", `10 201 EOF`},
|
|
{"0b1010.01", `10 1.000000e-02 EOF`},
|
|
{"0b0000", `0 EOF`},
|
|
// octal
|
|
{"0o123", `83 EOF`},
|
|
{"0o12834", `10 834 EOF`},
|
|
{"0o12934", `10 934 EOF`},
|
|
{"0o12934.", `10 9.340000e+02 EOF`},
|
|
{"0o12934.1", `10 9.341000e+02 EOF`},
|
|
{"0o12934e1", `10 9.340000e+03 EOF`},
|
|
{"0o123.", `83 . EOF`},
|
|
{"0o123.1", `83 1.000000e-01 EOF`},
|
|
{"0123", `foo.star:1:5: obsolete form of octal literal; use 0o123`},
|
|
{"012834", `foo.star:1:1: invalid int literal`},
|
|
{"012934", `foo.star:1:1: invalid int literal`},
|
|
{"i = 012934", `foo.star:1:5: invalid int literal`},
|
|
// octal escapes in string literals
|
|
{`"\037"`, `"\x1f" EOF`},
|
|
{`"\377"`, `foo.star:1:1: non-ASCII octal escape \377 (use \u00FF for the UTF-8 encoding of U+00FF)`},
|
|
{`"\378"`, `"\x1f8" EOF`}, // = '\37' + '8'
|
|
{`"\400"`, `foo.star:1:1: non-ASCII octal escape \400`}, // unlike Python 2 and 3
|
|
// hex escapes
|
|
{`"\x00\x20\x09\x41\x7e\x7f"`, `"\x00 \tA~\x7f" EOF`}, // DEL is non-printable
|
|
{`"\x80"`, `foo.star:1:1: non-ASCII hex escape`},
|
|
{`"\xff"`, `foo.star:1:1: non-ASCII hex escape`},
|
|
{`"\xFf"`, `foo.star:1:1: non-ASCII hex escape`},
|
|
{`"\xF"`, `foo.star:1:1: truncated escape sequence \xF`},
|
|
{`"\x"`, `foo.star:1:1: truncated escape sequence \x`},
|
|
{`"\xfg"`, `foo.star:1:1: invalid escape sequence \xfg`},
|
|
// Unicode escapes
|
|
// \uXXXX
|
|
{`"\u0400"`, `"Ѐ" EOF`},
|
|
{`"\u100"`, `foo.star:1:1: truncated escape sequence \u100`},
|
|
{`"\u04000"`, `"Ѐ0" EOF`}, // = U+0400 + '0'
|
|
{`"\u100g"`, `foo.star:1:1: invalid escape sequence \u100g`},
|
|
{`"\u4E16"`, `"世" EOF`},
|
|
{`"\udc00"`, `foo.star:1:1: invalid Unicode code point U+DC00`}, // surrogate
|
|
// \UXXXXXXXX
|
|
{`"\U00000400"`, `"Ѐ" EOF`},
|
|
{`"\U0000400"`, `foo.star:1:1: truncated escape sequence \U0000400`},
|
|
{`"\U000004000"`, `"Ѐ0" EOF`}, // = U+0400 + '0'
|
|
{`"\U1000000g"`, `foo.star:1:1: invalid escape sequence \U1000000g`},
|
|
{`"\U0010FFFF"`, `"\U0010ffff" EOF`},
|
|
{`"\U00110000"`, `foo.star:1:1: code point out of range: \U00110000 (max \U00110000)`},
|
|
{`"\U0001F63F"`, `"😿" EOF`},
|
|
{`"\U0000dc00"`, `foo.star:1:1: invalid Unicode code point U+DC00`}, // surrogate
|
|
|
|
// backslash escapes
|
|
// As in Go, a backslash must escape something.
|
|
// (Python started issuing a deprecation warning in 3.6.)
|
|
{`"foo\(bar"`, `foo.star:1:1: invalid escape sequence \(`},
|
|
{`"\+"`, `foo.star:1:1: invalid escape sequence \+`},
|
|
{`"\w"`, `foo.star:1:1: invalid escape sequence \w`},
|
|
{`"\""`, `"\"" EOF`},
|
|
{`"\'"`, `"'" EOF`},
|
|
{`'\w'`, `foo.star:1:1: invalid escape sequence \w`},
|
|
{`'\''`, `"'" EOF`},
|
|
{`'\"'`, `"\"" EOF`},
|
|
{`"""\w"""`, `foo.star:1:1: invalid escape sequence \w`},
|
|
{`"""\""""`, `"\"" EOF`},
|
|
{`"""\'"""`, `"'" EOF`},
|
|
{`'''\w'''`, `foo.star:1:1: invalid escape sequence \w`},
|
|
{`'''\''''`, `"'" EOF`},
|
|
{`'''\"'''`, `"\"" EOF`},
|
|
{`r"\w"`, `"\\w" EOF`},
|
|
{`r"\""`, `"\\\"" EOF`},
|
|
{`r"\'"`, `"\\'" EOF`},
|
|
{`r'\w'`, `"\\w" EOF`},
|
|
{`r'\''`, `"\\'" EOF`},
|
|
{`r'\"'`, `"\\\"" EOF`},
|
|
{`'a\zb'`, `foo.star:1:1: invalid escape sequence \z`},
|
|
{`"\o123"`, `foo.star:1:1: invalid escape sequence \o`},
|
|
// bytes literals (where they differ from text strings)
|
|
{`b"AЀ世😿"`, `b"AЀ世😿`}, // 1-4 byte encodings, literal
|
|
{`b"\x41\u0400\u4e16\U0001F63F"`, `b"AЀ世😿"`}, // same, as escapes
|
|
{`b"\377\378\x80\xff\xFf"`, `b"\xff\x1f8\x80\xff\xff" EOF`}, // hex/oct escapes allow non-ASCII
|
|
{`b"\400"`, `foo.star:1:2: invalid escape sequence \400`},
|
|
{`b"\udc00"`, `foo.star:1:2: invalid Unicode code point U+DC00`}, // (same as string)
|
|
// floats starting with octal digits
|
|
{"012934.", `1.293400e+04 EOF`},
|
|
{"012934.1", `1.293410e+04 EOF`},
|
|
{"012934e1", `1.293400e+05 EOF`},
|
|
{"0123.", `1.230000e+02 EOF`},
|
|
{"0123.1", `1.231000e+02 EOF`},
|
|
// github.com/google/skylark/issues/16
|
|
{"x ! 0", "foo.star:1:3: unexpected input character '!'"},
|
|
// github.com/google/starlark-go/issues/80
|
|
{"([{<>}])", "( [ { < > } ] ) EOF"},
|
|
{"f();", "f ( ) ; EOF"},
|
|
// github.com/google/starlark-go/issues/104
|
|
{"def f():\n if x:\n pass\n ", `def f ( ) : newline indent if x : newline indent pass newline outdent outdent EOF`},
|
|
{`while cond: pass`, "while cond : pass EOF"},
|
|
// github.com/google/starlark-go/issues/107
|
|
{"~= ~= 5", "~ = ~ = 5 EOF"},
|
|
{"0in", "0 in EOF"},
|
|
{"0or", "foo.star:1:3: invalid octal literal"},
|
|
{"6in", "6 in EOF"},
|
|
{"6or", "6 or EOF"},
|
|
} {
|
|
got, err := scan(test.input)
|
|
if err != nil {
|
|
got = err.(Error).Error()
|
|
}
|
|
// Prefix match allows us to truncate errors in expecations.
|
|
// Success cases all end in EOF.
|
|
if !strings.HasPrefix(got, test.want) {
|
|
t.Errorf("scan `%s` = [%s], want [%s]", test.input, got, test.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// dataFile is the same as starlarktest.DataFile.
|
|
// We make a copy to avoid a dependency cycle.
|
|
var dataFile = func(pkgdir, filename string) string {
|
|
return filepath.Join(build.Default.GOPATH, "src/go.starlark.net", pkgdir, filename)
|
|
}
|
|
|
|
func BenchmarkScan(b *testing.B) {
|
|
filename := dataFile("syntax", "testdata/scan.star")
|
|
b.StopTimer()
|
|
data, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
b.StartTimer()
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
sc, err := newScanner(filename, data, false)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
var val tokenValue
|
|
for sc.nextToken(&val) != EOF {
|
|
}
|
|
}
|
|
}
|