1// Copyright 2017 The Bazel Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package syntax_test 6 7import ( 8 "bytes" 9 "fmt" 10 "reflect" 11 "strings" 12 "testing" 13 14 "github.com/google/skylark/internal/chunkedfile" 15 "github.com/google/skylark/skylarktest" 16 "github.com/google/skylark/syntax" 17) 18 19func TestExprParseTrees(t *testing.T) { 20 for _, test := range []struct { 21 input, want string 22 }{ 23 {`print(1)`, 24 `(CallExpr Fn=print Args=(1))`}, 25 {"print(1)\n", 26 `(CallExpr Fn=print Args=(1))`}, 27 {`x + 1`, 28 `(BinaryExpr X=x Op=+ Y=1)`}, 29 {`[x for x in y]`, 30 `(Comprehension Body=x Clauses=((ForClause Vars=x X=y)))`}, 31 {`[x for x in (a if b else c)]`, 32 `(Comprehension Body=x Clauses=((ForClause Vars=x X=(ParenExpr X=(CondExpr Cond=b True=a False=c)))))`}, 33 {`x[i].f(42)`, 34 `(CallExpr Fn=(DotExpr X=(IndexExpr X=x Y=i) Name=f) Args=(42))`}, 35 {`x.f()`, 36 `(CallExpr Fn=(DotExpr X=x Name=f))`}, 37 {`x+y*z`, 38 `(BinaryExpr X=x Op=+ Y=(BinaryExpr X=y Op=* Y=z))`}, 39 {`x%y-z`, 40 `(BinaryExpr X=(BinaryExpr X=x Op=% Y=y) Op=- Y=z)`}, 41 {`a + b not in c`, 42 `(BinaryExpr X=(BinaryExpr X=a Op=+ Y=b) Op=not in Y=c)`}, 43 {`lambda x, *args, **kwargs: None`, 44 `(LambdaExpr Function=(Function Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=((ReturnStmt Result=None))))`}, 45 {`{"one": 1}`, 46 `(DictExpr List=((DictEntry Key="one" Value=1)))`}, 47 {`a[i]`, 48 `(IndexExpr X=a Y=i)`}, 49 {`a[i:]`, 50 `(SliceExpr X=a Lo=i)`}, 51 {`a[:j]`, 52 `(SliceExpr X=a Hi=j)`}, 53 {`a[::]`, 54 `(SliceExpr X=a)`}, 55 {`a[::k]`, 56 `(SliceExpr X=a Step=k)`}, 57 {`[]`, 58 `(ListExpr)`}, 59 {`[1]`, 60 `(ListExpr List=(1))`}, 61 {`[1,]`, 62 `(ListExpr List=(1))`}, 63 {`[1, 2]`, 64 `(ListExpr List=(1 2))`}, 65 {`()`, 66 `(TupleExpr)`}, 67 {`(4,)`, 68 `(ParenExpr X=(TupleExpr List=(4)))`}, 69 {`(4)`, 70 `(ParenExpr X=4)`}, 71 {`(4, 5)`, 72 `(ParenExpr X=(TupleExpr List=(4 5)))`}, 73 {`{}`, 74 `(DictExpr)`}, 75 {`{"a": 1}`, 76 `(DictExpr List=((DictEntry Key="a" Value=1)))`}, 77 {`{"a": 1,}`, 78 `(DictExpr List=((DictEntry Key="a" Value=1)))`}, 79 {`{"a": 1, "b": 2}`, 80 `(DictExpr List=((DictEntry Key="a" Value=1) (DictEntry Key="b" Value=2)))`}, 81 {`{x: y for (x, y) in z}`, 82 `(Comprehension Curly Body=(DictEntry Key=x Value=y) Clauses=((ForClause Vars=(ParenExpr X=(TupleExpr List=(x y))) X=z)))`}, 83 {`{x: y for a in b if c}`, 84 `(Comprehension Curly Body=(DictEntry Key=x Value=y) Clauses=((ForClause Vars=a X=b) (IfClause Cond=c)))`}, 85 {`-1 + +2`, 86 `(BinaryExpr X=(UnaryExpr Op=- X=1) Op=+ Y=(UnaryExpr Op=+ X=2))`}, 87 {`"foo" + "bar"`, 88 `(BinaryExpr X="foo" Op=+ Y="bar")`}, 89 {`-1 * 2`, // prec(unary -) > prec(binary *) 90 `(BinaryExpr X=(UnaryExpr Op=- X=1) Op=* Y=2)`}, 91 {`-x[i]`, // prec(unary -) < prec(x[i]) 92 `(UnaryExpr Op=- X=(IndexExpr X=x Y=i))`}, 93 {`a | b & c | d`, // prec(|) < prec(&) 94 `(BinaryExpr X=(BinaryExpr X=a Op=| Y=(BinaryExpr X=b Op=& Y=c)) Op=| Y=d)`}, 95 {`a or b and c or d`, 96 `(BinaryExpr X=(BinaryExpr X=a Op=or Y=(BinaryExpr X=b Op=and Y=c)) Op=or Y=d)`}, 97 {`a and b or c and d`, 98 `(BinaryExpr X=(BinaryExpr X=a Op=and Y=b) Op=or Y=(BinaryExpr X=c Op=and Y=d))`}, 99 {`f(1, x=y)`, 100 `(CallExpr Fn=f Args=(1 (BinaryExpr X=x Op== Y=y)))`}, 101 {`f(*args, **kwargs)`, 102 `(CallExpr Fn=f Args=((UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)))`}, 103 {`a if b else c`, 104 `(CondExpr Cond=b True=a False=c)`}, 105 {`a and not b`, 106 `(BinaryExpr X=a Op=and Y=(UnaryExpr Op=not X=b))`}, 107 {`[e for x in y if cond1 if cond2]`, 108 `(Comprehension Body=e Clauses=((ForClause Vars=x X=y) (IfClause Cond=cond1) (IfClause Cond=cond2)))`}, // github.com/google/skylark issue 53 109 } { 110 e, err := syntax.ParseExpr("foo.sky", test.input, 0) 111 if err != nil { 112 t.Errorf("parse `%s` failed: %v", test.input, stripPos(err)) 113 continue 114 } 115 if got := treeString(e); test.want != got { 116 117 t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) 118 } 119 } 120} 121 122func TestStmtParseTrees(t *testing.T) { 123 for _, test := range []struct { 124 input, want string 125 }{ 126 {`print(1)`, 127 `(ExprStmt X=(CallExpr Fn=print Args=(1)))`}, 128 {`return 1, 2`, 129 `(ReturnStmt Result=(TupleExpr List=(1 2)))`}, 130 {`return`, 131 `(ReturnStmt)`}, 132 {`for i in "abc": break`, 133 `(ForStmt Vars=i X="abc" Body=((BranchStmt Token=break)))`}, 134 {`for i in "abc": continue`, 135 `(ForStmt Vars=i X="abc" Body=((BranchStmt Token=continue)))`}, 136 {`for x, y in z: pass`, 137 `(ForStmt Vars=(TupleExpr List=(x y)) X=z Body=((BranchStmt Token=pass)))`}, 138 {`if True: pass`, 139 `(IfStmt Cond=True True=((BranchStmt Token=pass)))`}, 140 {`if True: break`, 141 `(IfStmt Cond=True True=((BranchStmt Token=break)))`}, 142 {`if True: continue`, 143 `(IfStmt Cond=True True=((BranchStmt Token=continue)))`}, 144 {`if True: pass 145else: 146 pass`, 147 `(IfStmt Cond=True True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))`}, 148 {"if a: pass\nelif b: pass\nelse: pass", 149 `(IfStmt Cond=a True=((BranchStmt Token=pass)) False=((IfStmt Cond=b True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))))`}, 150 {`x, y = 1, 2`, 151 `(AssignStmt Op== LHS=(TupleExpr List=(x y)) RHS=(TupleExpr List=(1 2)))`}, 152 {`x[i] = 1`, 153 `(AssignStmt Op== LHS=(IndexExpr X=x Y=i) RHS=1)`}, 154 {`x.f = 1`, 155 `(AssignStmt Op== LHS=(DotExpr X=x Name=f) RHS=1)`}, 156 {`(x, y) = 1`, 157 `(AssignStmt Op== LHS=(ParenExpr X=(TupleExpr List=(x y))) RHS=1)`}, 158 {`load("", "a", b="c")`, 159 `(LoadStmt Module="" From=(a c) To=(a b))`}, 160 {`if True: load("", "a", b="c")`, // load needn't be at toplevel 161 `(IfStmt Cond=True True=((LoadStmt Module="" From=(a c) To=(a b))))`}, 162 {`def f(x, *args, **kwargs): 163 pass`, 164 `(DefStmt Name=f Function=(Function Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=((BranchStmt Token=pass))))`}, 165 {`def f(**kwargs, *args): pass`, 166 `(DefStmt Name=f Function=(Function Params=((UnaryExpr Op=** X=kwargs) (UnaryExpr Op=* X=args)) Body=((BranchStmt Token=pass))))`}, 167 {`def f(a, b, c=d): pass`, 168 `(DefStmt Name=f Function=(Function Params=(a b (BinaryExpr X=c Op== Y=d)) Body=((BranchStmt Token=pass))))`}, 169 {`def f(a, b=c, d): pass`, 170 `(DefStmt Name=f Function=(Function Params=(a (BinaryExpr X=b Op== Y=c) d) Body=((BranchStmt Token=pass))))`}, // TODO(adonovan): fix this 171 {`def f(): 172 def g(): 173 pass 174 pass 175def h(): 176 pass`, 177 `(DefStmt Name=f Function=(Function Body=((DefStmt Name=g Function=(Function Body=((BranchStmt Token=pass)))) (BranchStmt Token=pass))))`}, 178 } { 179 f, err := syntax.Parse("foo.sky", test.input, 0) 180 if err != nil { 181 t.Errorf("parse `%s` failed: %v", test.input, stripPos(err)) 182 continue 183 } 184 if got := treeString(f.Stmts[0]); test.want != got { 185 t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) 186 } 187 } 188} 189 190// TestFileParseTrees tests sequences of statements, and particularly 191// handling of indentation, newlines, line continuations, and blank lines. 192func TestFileParseTrees(t *testing.T) { 193 for _, test := range []struct { 194 input, want string 195 }{ 196 {`x = 1 197print(x)`, 198 `(AssignStmt Op== LHS=x RHS=1) 199(ExprStmt X=(CallExpr Fn=print Args=(x)))`}, 200 {"if cond:\n\tpass", 201 `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`}, 202 {"if cond:\n\tpass\nelse:\n\tpass", 203 `(IfStmt Cond=cond True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))`}, 204 {`def f(): 205 pass 206pass 207 208pass`, 209 `(DefStmt Name=f Function=(Function Body=((BranchStmt Token=pass)))) 210(BranchStmt Token=pass) 211(BranchStmt Token=pass)`}, 212 {`pass; pass`, 213 `(BranchStmt Token=pass) 214(BranchStmt Token=pass)`}, 215 {"pass\npass", 216 `(BranchStmt Token=pass) 217(BranchStmt Token=pass)`}, 218 {"pass\n\npass", 219 `(BranchStmt Token=pass) 220(BranchStmt Token=pass)`}, 221 {`x = (1 + 2222)`, 223 `(AssignStmt Op== LHS=x RHS=(ParenExpr X=(BinaryExpr X=1 Op=+ Y=2)))`}, 224 {`x = 1 \ 225+ 2`, 226 `(AssignStmt Op== LHS=x RHS=(BinaryExpr X=1 Op=+ Y=2))`}, 227 } { 228 f, err := syntax.Parse("foo.sky", test.input, 0) 229 if err != nil { 230 t.Errorf("parse `%s` failed: %v", test.input, stripPos(err)) 231 continue 232 } 233 var buf bytes.Buffer 234 for i, stmt := range f.Stmts { 235 if i > 0 { 236 buf.WriteByte('\n') 237 } 238 writeTree(&buf, reflect.ValueOf(stmt)) 239 } 240 if got := buf.String(); test.want != got { 241 t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) 242 } 243 } 244} 245 246func stripPos(err error) string { 247 s := err.Error() 248 if i := strings.Index(s, ": "); i >= 0 { 249 s = s[i+len(": "):] // strip file:line:col 250 } 251 return s 252} 253 254// treeString prints a syntax node as a parenthesized tree. 255// Idents are printed as foo and Literals as "foo" or 42. 256// Structs are printed as (type name=value ...). 257// Only non-empty fields are shown. 258func treeString(n syntax.Node) string { 259 var buf bytes.Buffer 260 writeTree(&buf, reflect.ValueOf(n)) 261 return buf.String() 262} 263 264func writeTree(out *bytes.Buffer, x reflect.Value) { 265 switch x.Kind() { 266 case reflect.String, reflect.Int, reflect.Bool: 267 fmt.Fprintf(out, "%v", x.Interface()) 268 case reflect.Ptr, reflect.Interface: 269 if elem := x.Elem(); elem.Kind() == 0 { 270 out.WriteString("nil") 271 } else { 272 writeTree(out, elem) 273 } 274 case reflect.Struct: 275 switch v := x.Interface().(type) { 276 case syntax.Literal: 277 if v.Token == syntax.STRING { 278 fmt.Fprintf(out, "%q", v.Value) 279 } else if v.Token == syntax.INT { 280 fmt.Fprintf(out, "%d", v.Value) 281 } 282 return 283 case syntax.Ident: 284 out.WriteString(v.Name) 285 return 286 } 287 fmt.Fprintf(out, "(%s", strings.TrimPrefix(x.Type().String(), "syntax.")) 288 for i, n := 0, x.NumField(); i < n; i++ { 289 f := x.Field(i) 290 if f.Type() == reflect.TypeOf(syntax.Position{}) { 291 continue // skip positions 292 } 293 name := x.Type().Field(i).Name 294 if name == "commentsRef" { 295 continue // skip comments fields 296 } 297 if f.Type() == reflect.TypeOf(syntax.Token(0)) { 298 fmt.Fprintf(out, " %s=%s", name, f.Interface()) 299 continue 300 } 301 302 switch f.Kind() { 303 case reflect.Slice: 304 if n := f.Len(); n > 0 { 305 fmt.Fprintf(out, " %s=(", name) 306 for i := 0; i < n; i++ { 307 if i > 0 { 308 out.WriteByte(' ') 309 } 310 writeTree(out, f.Index(i)) 311 } 312 out.WriteByte(')') 313 } 314 continue 315 case reflect.Ptr, reflect.Interface: 316 if f.IsNil() { 317 continue 318 } 319 case reflect.Bool: 320 if f.Bool() { 321 fmt.Fprintf(out, " %s", name) 322 } 323 continue 324 } 325 fmt.Fprintf(out, " %s=", name) 326 writeTree(out, f) 327 } 328 fmt.Fprintf(out, ")") 329 default: 330 fmt.Fprintf(out, "%T", x.Interface()) 331 } 332} 333 334func TestParseErrors(t *testing.T) { 335 filename := skylarktest.DataFile("skylark/syntax", "testdata/errors.sky") 336 for _, chunk := range chunkedfile.Read(filename, t) { 337 _, err := syntax.Parse(filename, chunk.Source, 0) 338 switch err := err.(type) { 339 case nil: 340 // ok 341 case syntax.Error: 342 chunk.GotError(int(err.Pos.Line), err.Msg) 343 default: 344 t.Error(err) 345 } 346 chunk.Done() 347 } 348} 349 350func TestWalk(t *testing.T) { 351 const src = ` 352for x in y: 353 if x: 354 pass 355 else: 356 f([2*x for x in "abc"]) 357` 358 // TODO(adonovan): test that it finds all syntax.Nodes 359 // (compare against a reflect-based implementation). 360 // TODO(adonovan): test that the result of f is used to prune 361 // the descent. 362 f, err := syntax.Parse("hello.go", src, 0) 363 if err != nil { 364 t.Fatal(err) 365 } 366 367 var buf bytes.Buffer 368 var depth int 369 syntax.Walk(f, func(n syntax.Node) bool { 370 if n == nil { 371 depth-- 372 return true 373 } 374 fmt.Fprintf(&buf, "%s%s\n", 375 strings.Repeat(" ", depth), 376 strings.TrimPrefix(reflect.TypeOf(n).String(), "*syntax.")) 377 depth++ 378 return true 379 }) 380 got := buf.String() 381 want := ` 382File 383 ForStmt 384 Ident 385 Ident 386 IfStmt 387 Ident 388 BranchStmt 389 ExprStmt 390 CallExpr 391 Ident 392 Comprehension 393 ForClause 394 Ident 395 Literal 396 BinaryExpr 397 Literal 398 Ident` 399 got = strings.TrimSpace(got) 400 want = strings.TrimSpace(want) 401 if got != want { 402 t.Errorf("got %s, want %s", got, want) 403 } 404} 405