1--- go1.10.1/src/go/parser/parser.go 2018-03-29 06:29:01.000000000 +0200 2+++ parser.go 2018-04-09 21:20:52.453627706 +0200 3@@ -19,16 +19,18 @@ 4 import ( 5 "fmt" 6 "go/ast" 7- "go/scanner" 8 "go/token" 9 "strconv" 10 "strings" 11 "unicode" 12+ 13+ "github.com/cosmos72/gomacro/scanner" 14+ mt "github.com/cosmos72/gomacro/token" 15 ) 16 17 // The parser structure holds the parser's internal state. 18 type parser struct { 19- file *token.File 20+ file *mt.File 21 errors scanner.ErrorList 22 scanner scanner.Scanner 23 24@@ -42,6 +44,9 @@ 25 leadComment *ast.CommentGroup // last lead comment 26 lineComment *ast.CommentGroup // last line comment 27 28+ tok0 token.Token // patch: Previous token 29+ specialChar rune // patch: prefix for quote operators ' ` , ,@ 30+ 31 // Next token 32 pos token.Pos // token position 33 tok token.Token // one token look-ahead 34@@ -70,17 +75,51 @@ 35 targetStack [][]*ast.Ident // stack of unresolved labels 36 } 37 38-func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode Mode) { 39- p.file = fset.AddFile(filename, -1, len(src)) 40+func (p *parser) init(fset *mt.FileSet, filename string, lineOffset int, src []byte, mode Mode) { 41+ // Explicitly initialize all private fields since a parser may be reused. 42+ if fset == nil { 43+ fset = mt.NewFileSet() 44+ } 45+ p.file = fset.AddFile(filename, -1, len(src), lineOffset) 46+ p.errors = nil 47+ 48 var m scanner.Mode 49 if mode&ParseComments != 0 { 50 m = scanner.ScanComments 51 } 52+ if p.specialChar == '\x00' { 53+ p.specialChar = '~' 54+ } 55 eh := func(pos token.Position, msg string) { p.errors.Add(pos, msg) } 56- p.scanner.Init(p.file, src, eh, m) 57+ p.scanner.Init(p.file, src, eh, m, p.specialChar) 58 59 p.mode = mode 60 p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently) 61+ p.indent = 0 62+ 63+ p.comments = nil 64+ p.leadComment = nil 65+ p.lineComment = nil 66+ 67+ p.pos = token.NoPos 68+ p.tok = token.ILLEGAL 69+ p.lit = "" 70+ 71+ p.syncPos = token.NoPos 72+ p.syncCnt = 0 73+ 74+ p.exprLev = 0 75+ p.inRhs = false 76+ 77+ p.topScope = nil 78+ p.openScope() 79+ p.pkgScope = p.topScope 80+ 81+ p.unresolved = nil 82+ p.imports = nil 83+ 84+ p.labelScope = nil 85+ p.targetStack = nil 86 87 p.next() 88 } 89@@ -243,11 +282,13 @@ 90 // very first token (!p.pos.IsValid()) is not initialized 91 // (it is token.ILLEGAL), so don't print it . 92 if p.trace && p.pos.IsValid() { 93- s := p.tok.String() 94+ s := mt.String(p.tok) // patch: support macro-related keywords 95 switch { 96 case p.tok.IsLiteral(): 97 p.printTrace(s, p.lit) 98- case p.tok.IsOperator(), p.tok.IsKeyword(): 99+ case p.tok.IsOperator(), p.tok.IsKeyword(), 100+ mt.IsMacroKeyword(p.tok): // patch: support macro-related keywords 101+ 102 p.printTrace("\"" + s + "\"") 103 default: 104 p.printTrace(s) 105@@ -317,6 +358,7 @@ 106 p.leadComment = nil 107 p.lineComment = nil 108 prev := p.pos 109+ p.tok0 = p.tok 110 p.next0() 111 112 if p.tok == token.COMMENT { 113@@ -378,7 +420,7 @@ 114 if p.tok == token.SEMICOLON && p.lit == "\n" { 115 msg += ", found newline" 116 } else { 117- msg += ", found '" + p.tok.String() + "'" 118+ msg += ", found '" + mt.String(p.tok) + "'" 119 if p.tok.IsLiteral() { 120 msg += " " + p.lit 121 } 122@@ -390,7 +432,7 @@ 123 func (p *parser) expect(tok token.Token) token.Pos { 124 pos := p.pos 125 if p.tok != tok { 126- p.errorExpected(pos, "'"+tok.String()+"'") 127+ p.errorExpected(pos, "'"+mt.String(tok)+"'") 128 } 129 p.next() // make progress 130 return pos 131@@ -409,7 +451,7 @@ 132 133 func (p *parser) expectSemi() { 134 // semicolon is optional before a closing ')' or '}' 135- if p.tok != token.RPAREN && p.tok != token.RBRACE { 136+ if p.tok != token.RPAREN && p.tok != token.RBRACE && p.tok != token.RBRACK { // patch: semicolon is optional also before a closing ']' 137 switch p.tok { 138 case token.COMMA: 139 // permit a ',' instead of a ';' but complain 140@@ -454,7 +496,7 @@ 141 case token.BREAK, token.CONST, token.CONTINUE, token.DEFER, 142 token.FALLTHROUGH, token.FOR, token.GO, token.GOTO, 143 token.IF, token.RETURN, token.SELECT, token.SWITCH, 144- token.TYPE, token.VAR: 145+ token.TYPE, token.VAR, mt.FUNCTION: 146 // Return only if parser made some progress since last 147 // sync or if it has not reached 10 sync calls without 148 // progress. Otherwise consume at least one token to 149@@ -489,7 +531,7 @@ 150 func syncDecl(p *parser) { 151 for { 152 switch p.tok { 153- case token.CONST, token.TYPE, token.VAR: 154+ case token.CONST, token.TYPE, token.VAR, token.FUNC, mt.FUNCTION: 155 // see comments in syncStmt 156 if p.pos == p.syncPos && p.syncCnt < 10 { 157 p.syncCnt++ 158@@ -912,12 +954,12 @@ 159 return 160 } 161 162-func (p *parser) parseFuncType() (*ast.FuncType, *ast.Scope) { 163+func (p *parser) parseFuncType(tok token.Token) (*ast.FuncType, *ast.Scope) { 164 if p.trace { 165 defer un(trace(p, "FuncType")) 166 } 167 168- pos := p.expect(token.FUNC) 169+ pos := p.expect(tok) 170 scope := ast.NewScope(p.topScope) // function scope 171 params, results := p.parseSignature(scope) 172 173@@ -1026,8 +1068,8 @@ 174 return p.parseStructType() 175 case token.MUL: 176 return p.parsePointerType() 177- case token.FUNC: 178- typ, _ := p.parseFuncType() 179+ case token.FUNC, mt.LAMBDA: 180+ typ, _ := p.parseFuncType(p.tok) 181 return typ 182 case token.INTERFACE: 183 return p.parseInterfaceType() 184@@ -1041,6 +1083,8 @@ 185 typ := p.parseType() 186 rparen := p.expect(token.RPAREN) 187 return &ast.ParenExpr{Lparen: lparen, X: typ, Rparen: rparen} 188+ case mt.QUOTE, mt.QUASIQUOTE, mt.UNQUOTE: // patch: support quote and friends inside types 189+ return p.parseQuote() 190 } 191 192 // no type found 193@@ -1063,7 +1107,7 @@ 194 defer un(trace(p, "StatementList")) 195 } 196 197- for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF { 198+ for p.tok != mt.TYPECASE && p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF { 199 list = append(list, p.parseStmt()) 200 } 201 202@@ -1103,12 +1147,12 @@ 203 // ---------------------------------------------------------------------------- 204 // Expressions 205 206-func (p *parser) parseFuncTypeOrLit() ast.Expr { 207+func (p *parser) parseFuncTypeOrLit(tok token.Token) ast.Expr { 208 if p.trace { 209 defer un(trace(p, "FuncTypeOrLit")) 210 } 211 212- typ, scope := p.parseFuncType() 213+ typ, scope := p.parseFuncType(tok) 214 if p.tok != token.LBRACE { 215 // function type only 216 return typ 217@@ -1152,8 +1196,20 @@ 218 rparen := p.expect(token.RPAREN) 219 return &ast.ParenExpr{Lparen: lparen, X: x, Rparen: rparen} 220 221- case token.FUNC: 222- return p.parseFuncTypeOrLit() 223+ case token.FUNC, mt.LAMBDA: 224+ // patch: lambda. equivalent to func, useful to resolve ambiguities between closures 225+ // and function/method declarations 226+ return p.parseFuncTypeOrLit(p.tok) 227+ 228+ // patch: quote and friends 229+ // TODO: accept ms.MACRO here and interpret as local macro definition? (i.e. Common Lisp macrolet) 230+ case mt.QUOTE, mt.QUASIQUOTE, mt.UNQUOTE, mt.UNQUOTE_SPLICE: 231+ return p.parseQuote() 232+ 233+ // patch: accept block statements inside expressions. allows to nest macro calls, 234+ // to write { if a { b } else { c } } inside an expression, and many other things 235+ case token.LBRACE: 236+ return p.parseExprBlock() 237 } 238 239 if typ := p.tryIdentOrType(); typ != nil { 240@@ -1432,10 +1488,13 @@ 241 242 // If x is of the form (T), unparen returns unparen(T), otherwise it returns x. 243 func unparen(x ast.Expr) ast.Expr { 244- if p, isParen := x.(*ast.ParenExpr); isParen { 245- x = unparen(p.X) 246+ for { 247+ if p, ok := x.(*ast.ParenExpr); ok { 248+ x = p.X 249+ continue 250+ } 251+ return x 252 } 253- return x 254 } 255 256 // checkExprOrType checks that x is an expression or a type 257@@ -1701,6 +1760,10 @@ 258 // Go spec: The scope of a label is the body of the function 259 // in which it is declared and excludes the body of any nested 260 // function. 261+ if p.labelScope == nil { 262+ p.error(label.Pos(), fmt.Sprintf("syntax error: label outside block: %s", label.Name)) 263+ return p.parseStmt(), false 264+ } 265 stmt := &ast.LabeledStmt{Label: label, Colon: colon, Stmt: p.parseStmt()} 266 p.declare(stmt, nil, p.labelScope, ast.Lbl, label) 267 return stmt, false 268@@ -1884,25 +1947,37 @@ 269 return 270 } 271 272-func (p *parser) parseCaseClause(typeSwitch bool) *ast.CaseClause { 273+func (p *parser) parseCaseClause(typeSwitch bool) ast.Stmt { 274 if p.trace { 275 defer un(trace(p, "CaseClause")) 276 } 277 278 pos := p.pos 279 var list []ast.Expr 280- if p.tok == token.CASE { 281+ if p.tok == mt.TYPECASE { 282+ p.next() 283+ list = p.parseTypeList() 284+ } else if p.tok == token.CASE { 285 p.next() 286 if typeSwitch { 287 list = p.parseTypeList() 288 } else { 289 list = p.parseRhsList() 290 } 291- } else { 292+ } else if p.tok == token.DEFAULT { 293 p.expect(token.DEFAULT) 294+ } else { 295+ switch p.tok { 296+ case token.ILLEGAL, token.EOF, token.COLON, token.SEMICOLON, token.RBRACE, token.RBRACK, token.LPAREN: 297+ p.errorExpected(p.pos, "'case' or 'default'") 298+ default: 299+ // patch: support switch foo { ~,{bar} } 300+ // where bar will expand to case x, y, z: w 301+ return p.parseStmt() 302+ } 303 } 304- 305 colon := p.expect(token.COLON) 306+ 307 p.openScope() 308 body := p.parseStmtList() 309 p.closeScope() 310@@ -1980,7 +2055,8 @@ 311 typeSwitch := p.isTypeSwitchGuard(s2) 312 lbrace := p.expect(token.LBRACE) 313 var list []ast.Stmt 314- for p.tok == token.CASE || p.tok == token.DEFAULT { 315+ // patch: allow ~quote and friends in addition to case: and default: 316+ for p.tok0 != token.EOF && p.tok != token.LPAREN && p.tok != token.RBRACK && p.tok != token.RBRACE { 317 list = append(list, p.parseCaseClause(typeSwitch)) 318 } 319 rbrace := p.expect(token.RBRACE) 320@@ -2159,13 +2235,17 @@ 321 } 322 323 switch p.tok { 324- case token.CONST, token.TYPE, token.VAR: 325+ case token.CONST, token.TYPE, token.VAR, 326+ mt.FUNCTION: // patch: allow function/method declarations inside statements. extremely useful for ~quote and ~quasiquote 327 s = &ast.DeclStmt{Decl: p.parseDecl(syncStmt)} 328 case 329 // tokens that may start an expression 330 token.IDENT, token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operands 331 token.LBRACK, token.STRUCT, token.MAP, token.CHAN, token.INTERFACE, // composite types 332- token.ADD, token.SUB, token.MUL, token.AND, token.XOR, token.ARROW, token.NOT: // unary operators 333+ token.ADD, token.SUB, token.MUL, token.AND, token.XOR, token.ARROW, token.NOT, // unary operators 334+ mt.MACRO, mt.SPLICE, mt.QUOTE, mt.QUASIQUOTE, mt.UNQUOTE, mt.UNQUOTE_SPLICE, // patch: macro, quote and friends 335+ mt.LAMBDA: 336+ 337 s, _ = p.parseSimpleStmt(labelOk) 338 // because of the required look-ahead, labeled statements are 339 // parsed by parseSimpleStmt - don't expect a semicolon after 340@@ -2173,6 +2253,8 @@ 341 if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt { 342 p.expectSemi() 343 } 344+ case token.IMPORT: // patch: allow imports inside statements. useful for ~quote and ~quasiquote 345+ s = &ast.DeclStmt{Decl: p.parseGenDecl(token.IMPORT, p.parseImportSpec)} 346 case token.GO: 347 s = p.parseGoStmt() 348 case token.DEFER: 349@@ -2327,11 +2409,12 @@ 350 // (Global identifiers are resolved in a separate phase after parsing.) 351 spec := &ast.TypeSpec{Doc: doc, Name: ident} 352 p.declare(spec, nil, p.topScope, ast.Typ, ident) 353+ 354 if p.tok == token.ASSIGN { 355 spec.Assign = p.pos 356 p.next() 357 } 358- spec.Type = p.parseType() 359+ spec.Type = p.parseType() 360 p.expectSemi() // call before accessing p.linecomment 361 spec.Comment = p.lineComment 362 363@@ -2369,17 +2452,41 @@ 364 } 365 } 366 367-func (p *parser) parseFuncDecl() *ast.FuncDecl { 368+func (p *parser) parseFuncDecl(tok token.Token) *ast.FuncDecl { 369 if p.trace { 370 defer un(trace(p, "FunctionDecl")) 371 } 372+ decl := p.parseFuncOrMacroDecl(tok) 373+ 374+ // paranoia: empty receiver list is omitted. this should not happen, 375+ // but we use it to distinguish functions from macros, so better safe than sorry. 376+ recv := decl.Recv 377+ if recv != nil && len(recv.List) == 0 { 378+ decl.Recv = nil 379+ } 380+ return decl 381+} 382+ 383+// patch: parse a macro declaration 384+func (p *parser) parseMacroDecl() *ast.FuncDecl { 385+ if p.trace { 386+ defer un(trace(p, "MacroDecl")) 387+ } 388+ decl := p.parseFuncOrMacroDecl(mt.MACRO) 389+ // add zero-length receiver list, to mark decl as a macro 390+ decl.Recv = &ast.FieldList{List: []*ast.Field{}} 391+ return decl 392+} 393+ 394+func (p *parser) parseFuncOrMacroDecl(tok token.Token) *ast.FuncDecl { 395 396 doc := p.leadComment 397- pos := p.expect(token.FUNC) 398+ pos := p.expect(tok) 399 scope := ast.NewScope(p.topScope) // function scope 400 401 var recv *ast.FieldList 402- if p.tok == token.LPAREN { 403+ // patch: macros cannot have a receiver 404+ if tok != mt.MACRO && p.tok == token.LPAREN { 405 recv = p.parseParameters(scope, false) 406 } 407 408@@ -2432,8 +2539,11 @@ 409 case token.TYPE: 410 f = p.parseTypeSpec 411 412- case token.FUNC: 413- return p.parseFuncDecl() 414+ case token.FUNC, mt.FUNCTION: 415+ return p.parseFuncDecl(p.tok) 416+ 417+ case mt.MACRO: // patch: parse a macro declaration 418+ return p.parseMacroDecl() 419 420 default: 421 pos := p.pos 422@@ -2476,8 +2586,11 @@ 423 return nil 424 } 425 426- p.openScope() 427- p.pkgScope = p.topScope 428+ topScope := p.topScope 429+ labelScope := p.labelScope 430+ if topScope == nil { 431+ p.openScope() 432+ } 433 var decls []ast.Decl 434 if p.mode&PackageClauseOnly == 0 { 435 // import decls 436@@ -2492,9 +2605,11 @@ 437 } 438 } 439 } 440- p.closeScope() 441- assert(p.topScope == nil, "unbalanced scopes") 442- assert(p.labelScope == nil, "unbalanced label scopes") 443+ if topScope == nil { 444+ p.closeScope() 445+ } 446+ assert(p.topScope == topScope, "unbalanced scopes") 447+ assert(p.labelScope == labelScope, "unbalanced label scopes") 448 449 // resolve global identifiers within the same file 450 i := 0 451