1""" 2Module for parsing Makefile syntax. 3 4Makefiles use a line-based parsing system. Continuations and substitutions are handled differently based on the 5type of line being parsed: 6 7Lines with makefile syntax condense continuations to a single space, no matter the actual trailing whitespace 8of the first line or the leading whitespace of the continuation. In other situations, trailing whitespace is 9relevant. 10 11Lines with command syntax do not condense continuations: the backslash and newline are part of the command. 12(GNU Make is buggy in this regard, at least on mac). 13 14Lines with an initial tab are commands if they can be (there is a rule or a command immediately preceding). 15Otherwise, they are parsed as makefile syntax. 16 17This file parses into the data structures defined in the parserdata module. Those classes are what actually 18do the dirty work of "executing" the parsed data into a data.Makefile. 19 20Four iterator functions are available: 21* iterdata 22* itermakefilechars 23* itercommandchars 24 25The iterators handle line continuations and comments in different ways, but share a common calling 26convention: 27 28Called with (data, startoffset, tokenlist, finditer) 29 30yield 4-tuples (flatstr, token, tokenoffset, afteroffset) 31flatstr is data, guaranteed to have no tokens (may be '') 32token, tokenoffset, afteroffset *may be None*. That means there is more text 33coming. 34""" 35 36import logging, re, os, sys 37import data, functions, util, parserdata 38 39_log = logging.getLogger('pymake.parser') 40 41class SyntaxError(util.MakeError): 42 pass 43 44_skipws = re.compile('\S') 45class Data(object): 46 """ 47 A single virtual "line", which can be multiple source lines joined with 48 continuations. 49 """ 50 51 __slots__ = ('s', 'lstart', 'lend', 'loc') 52 53 def __init__(self, s, lstart, lend, loc): 54 self.s = s 55 self.lstart = lstart 56 self.lend = lend 57 self.loc = loc 58 59 @staticmethod 60 def fromstring(s, path): 61 return Data(s, 0, len(s), parserdata.Location(path, 1, 0)) 62 63 def getloc(self, offset): 64 assert offset >= self.lstart and offset <= self.lend 65 return self.loc.offset(self.s, self.lstart, offset) 66 67 def skipwhitespace(self, offset): 68 """ 69 Return the offset of the first non-whitespace character in data starting at offset, or None if there are 70 only whitespace characters remaining. 71 """ 72 m = _skipws.search(self.s, offset, self.lend) 73 if m is None: 74 return self.lend 75 76 return m.start(0) 77 78_linere = re.compile(r'\\*\n') 79def enumeratelines(s, filename): 80 """ 81 Enumerate lines in a string as Data objects, joining line 82 continuations. 83 """ 84 85 off = 0 86 lineno = 1 87 curlines = 0 88 for m in _linere.finditer(s): 89 curlines += 1 90 start, end = m.span(0) 91 92 if (start - end) % 2 == 0: 93 # odd number of backslashes is a continuation 94 continue 95 96 yield Data(s, off, end - 1, parserdata.Location(filename, lineno, 0)) 97 98 lineno += curlines 99 curlines = 0 100 off = end 101 102 yield Data(s, off, len(s), parserdata.Location(filename, lineno, 0)) 103 104_alltokens = re.compile(r'''\\*\# | # hash mark preceeded by any number of backslashes 105 := | 106 \+= | 107 \?= | 108 :: | 109 (?:\$(?:$|[\(\{](?:%s)\s+|.)) | # dollar sign followed by EOF, a function keyword with whitespace, or any character 110 :(?![\\/]) | # colon followed by anything except a slash (Windows path detection) 111 [=#{}();,|'"]''' % '|'.join(functions.functionmap.iterkeys()), re.VERBOSE) 112 113def iterdata(d, offset, tokenlist, it): 114 """ 115 Iterate over flat data without line continuations, comments, or any special escaped characters. 116 117 Typically used to parse recursively-expanded variables. 118 """ 119 120 assert len(tokenlist), "Empty tokenlist passed to iterdata is meaningless!" 121 assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) 122 123 if offset == d.lend: 124 return 125 126 s = d.s 127 for m in it: 128 mstart, mend = m.span(0) 129 token = s[mstart:mend] 130 if token in tokenlist or (token[0] == '$' and '$' in tokenlist): 131 yield s[offset:mstart], token, mstart, mend 132 else: 133 yield s[offset:mend], None, None, mend 134 offset = mend 135 136 yield s[offset:d.lend], None, None, None 137 138# multiple backslashes before a newline are unescaped, halving their total number 139_makecontinuations = re.compile(r'(?:\s*|((?:\\\\)+))\\\n\s*') 140def _replacemakecontinuations(m): 141 start, end = m.span(1) 142 if start == -1: 143 return ' ' 144 return ' '.rjust((end - start) / 2 + 1, '\\') 145 146def itermakefilechars(d, offset, tokenlist, it, ignorecomments=False): 147 """ 148 Iterate over data in makefile syntax. Comments are found at unescaped # characters, and escaped newlines 149 are converted to single-space continuations. 150 """ 151 152 assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) 153 154 if offset == d.lend: 155 return 156 157 s = d.s 158 for m in it: 159 mstart, mend = m.span(0) 160 token = s[mstart:mend] 161 162 starttext = _makecontinuations.sub(_replacemakecontinuations, s[offset:mstart]) 163 164 if token[-1] == '#' and not ignorecomments: 165 l = mend - mstart 166 # multiple backslashes before a hash are unescaped, halving their total number 167 if l % 2: 168 # found a comment 169 yield starttext + token[:(l - 1) / 2], None, None, None 170 return 171 else: 172 yield starttext + token[-l / 2:], None, None, mend 173 elif token in tokenlist or (token[0] == '$' and '$' in tokenlist): 174 yield starttext, token, mstart, mend 175 else: 176 yield starttext + token, None, None, mend 177 offset = mend 178 179 yield _makecontinuations.sub(_replacemakecontinuations, s[offset:d.lend]), None, None, None 180 181_findcomment = re.compile(r'\\*\#') 182def flattenmakesyntax(d, offset): 183 """ 184 A shortcut method for flattening line continuations and comments in makefile syntax without 185 looking for other tokens. 186 """ 187 188 assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) 189 if offset == d.lend: 190 return '' 191 192 s = _makecontinuations.sub(_replacemakecontinuations, d.s[offset:d.lend]) 193 194 elements = [] 195 offset = 0 196 for m in _findcomment.finditer(s): 197 mstart, mend = m.span(0) 198 elements.append(s[offset:mstart]) 199 if (mend - mstart) % 2: 200 # even number of backslashes... it's a comment 201 elements.append(''.ljust((mend - mstart - 1) / 2, '\\')) 202 return ''.join(elements) 203 204 # odd number of backslashes 205 elements.append(''.ljust((mend - mstart - 2) / 2, '\\') + '#') 206 offset = mend 207 208 elements.append(s[offset:]) 209 return ''.join(elements) 210 211def itercommandchars(d, offset, tokenlist, it): 212 """ 213 Iterate over command syntax. # comment markers are not special, and escaped newlines are included 214 in the output text. 215 """ 216 217 assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) 218 219 if offset == d.lend: 220 return 221 222 s = d.s 223 for m in it: 224 mstart, mend = m.span(0) 225 token = s[mstart:mend] 226 starttext = s[offset:mstart].replace('\n\t', '\n') 227 228 if token in tokenlist or (token[0] == '$' and '$' in tokenlist): 229 yield starttext, token, mstart, mend 230 else: 231 yield starttext + token, None, None, mend 232 offset = mend 233 234 yield s[offset:d.lend].replace('\n\t', '\n'), None, None, None 235 236_redefines = re.compile('\s*define|\s*endef') 237def iterdefinelines(it, startloc): 238 """ 239 Process the insides of a define. Most characters are included literally. Escaped newlines are treated 240 as they would be in makefile syntax. Internal define/endef pairs are ignored. 241 """ 242 243 results = [] 244 245 definecount = 1 246 for d in it: 247 m = _redefines.match(d.s, d.lstart, d.lend) 248 if m is not None: 249 directive = m.group(0).strip() 250 if directive == 'endef': 251 definecount -= 1 252 if definecount == 0: 253 return _makecontinuations.sub(_replacemakecontinuations, '\n'.join(results)) 254 else: 255 definecount += 1 256 257 results.append(d.s[d.lstart:d.lend]) 258 259 # Falling off the end is an unterminated define! 260 raise SyntaxError("define without matching endef", startloc) 261 262def _ensureend(d, offset, msg): 263 """ 264 Ensure that only whitespace remains in this data. 265 """ 266 267 s = flattenmakesyntax(d, offset) 268 if s != '' and not s.isspace(): 269 raise SyntaxError(msg, d.getloc(offset)) 270 271_eqargstokenlist = ('(', "'", '"') 272 273def ifeq(d, offset): 274 if offset > d.lend - 1: 275 raise SyntaxError("No arguments after conditional", d.getloc(offset)) 276 277 # the variety of formats for this directive is rather maddening 278 token = d.s[offset] 279 if token not in _eqargstokenlist: 280 raise SyntaxError("No arguments after conditional", d.getloc(offset)) 281 282 offset += 1 283 284 if token == '(': 285 arg1, t, offset = parsemakesyntax(d, offset, (',',), itermakefilechars) 286 if t is None: 287 raise SyntaxError("Expected two arguments in conditional", d.getloc(d.lend)) 288 289 arg1.rstrip() 290 291 offset = d.skipwhitespace(offset) 292 arg2, t, offset = parsemakesyntax(d, offset, (')',), itermakefilechars) 293 if t is None: 294 raise SyntaxError("Unexpected text in conditional", d.getloc(offset)) 295 296 _ensureend(d, offset, "Unexpected text after conditional") 297 else: 298 arg1, t, offset = parsemakesyntax(d, offset, (token,), itermakefilechars) 299 if t is None: 300 raise SyntaxError("Unexpected text in conditional", d.getloc(d.lend)) 301 302 offset = d.skipwhitespace(offset) 303 if offset == d.lend: 304 raise SyntaxError("Expected two arguments in conditional", d.getloc(offset)) 305 306 token = d.s[offset] 307 if token not in '\'"': 308 raise SyntaxError("Unexpected text in conditional", d.getloc(offset)) 309 310 arg2, t, offset = parsemakesyntax(d, offset + 1, (token,), itermakefilechars) 311 312 _ensureend(d, offset, "Unexpected text after conditional") 313 314 return parserdata.EqCondition(arg1, arg2) 315 316def ifneq(d, offset): 317 c = ifeq(d, offset) 318 c.expected = False 319 return c 320 321def ifdef(d, offset): 322 e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) 323 e.rstrip() 324 325 return parserdata.IfdefCondition(e) 326 327def ifndef(d, offset): 328 c = ifdef(d, offset) 329 c.expected = False 330 return c 331 332_conditionkeywords = { 333 'ifeq': ifeq, 334 'ifneq': ifneq, 335 'ifdef': ifdef, 336 'ifndef': ifndef 337 } 338 339_conditiontokens = tuple(_conditionkeywords.iterkeys()) 340_conditionre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_conditiontokens)) 341 342_directivestokenlist = _conditiontokens + \ 343 ('else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'includedeps', '-includedeps', 'vpath', 'export', 'unexport') 344 345_directivesre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_directivestokenlist)) 346 347_varsettokens = (':=', '+=', '?=', '=') 348 349def _parsefile(pathname): 350 fd = open(pathname, "rU") 351 stmts = parsestring(fd.read(), pathname) 352 stmts.mtime = os.fstat(fd.fileno()).st_mtime 353 fd.close() 354 return stmts 355 356def _checktime(path, stmts): 357 mtime = os.path.getmtime(path) 358 if mtime != stmts.mtime: 359 _log.debug("Re-parsing makefile '%s': mtimes differ", path) 360 return False 361 362 return True 363 364_parsecache = util.MostUsedCache(50, _parsefile, _checktime) 365 366def parsefile(pathname): 367 """ 368 Parse a filename into a parserdata.StatementList. A cache is used to avoid re-parsing 369 makefiles that have already been parsed and have not changed. 370 """ 371 372 pathname = os.path.realpath(pathname) 373 return _parsecache.get(pathname) 374 375# colon followed by anything except a slash (Windows path detection) 376_depfilesplitter = re.compile(r':(?![\\/])') 377# simple variable references 378_vars = re.compile('\$\((\w+)\)') 379 380def parsedepfile(pathname): 381 """ 382 Parse a filename listing only depencencies into a parserdata.StatementList. 383 Simple variable references are allowed in such files. 384 """ 385 def continuation_iter(lines): 386 current_line = [] 387 for line in lines: 388 line = line.rstrip() 389 if line.endswith("\\"): 390 current_line.append(line.rstrip("\\")) 391 continue 392 if not len(line): 393 continue 394 current_line.append(line) 395 yield ''.join(current_line) 396 current_line = [] 397 if current_line: 398 yield ''.join(current_line) 399 400 def get_expansion(s): 401 if '$' in s: 402 expansion = data.Expansion() 403 # for an input like e.g. "foo $(bar) baz", 404 # _vars.split returns ["foo", "bar", "baz"] 405 # every other element is a variable name. 406 for i, element in enumerate(_vars.split(s)): 407 if i % 2: 408 expansion.appendfunc(functions.VariableRef(None, 409 data.StringExpansion(element, None))) 410 elif element: 411 expansion.appendstr(element) 412 413 return expansion 414 415 return data.StringExpansion(s, None) 416 417 pathname = os.path.realpath(pathname) 418 stmts = parserdata.StatementList() 419 for line in continuation_iter(open(pathname).readlines()): 420 target, deps = _depfilesplitter.split(line, 1) 421 stmts.append(parserdata.Rule(get_expansion(target), 422 get_expansion(deps), False)) 423 return stmts 424 425def parsestring(s, filename): 426 """ 427 Parse a string containing makefile data into a parserdata.StatementList. 428 """ 429 430 currule = False 431 condstack = [parserdata.StatementList()] 432 433 fdlines = enumeratelines(s, filename) 434 for d in fdlines: 435 assert len(condstack) > 0 436 437 offset = d.lstart 438 439 if currule and offset < d.lend and d.s[offset] == '\t': 440 e, token, offset = parsemakesyntax(d, offset + 1, (), itercommandchars) 441 assert token is None 442 assert offset is None 443 condstack[-1].append(parserdata.Command(e)) 444 continue 445 446 # To parse Makefile syntax, we first strip leading whitespace and 447 # look for initial keywords. If there are no keywords, it's either 448 # setting a variable or writing a rule. 449 450 offset = d.skipwhitespace(offset) 451 if offset is None: 452 continue 453 454 m = _directivesre.match(d.s, offset, d.lend) 455 if m is not None: 456 kword = m.group(1) 457 offset = m.end(0) 458 459 if kword == 'endif': 460 _ensureend(d, offset, "Unexpected data after 'endif' directive") 461 if len(condstack) == 1: 462 raise SyntaxError("unmatched 'endif' directive", 463 d.getloc(offset)) 464 465 condstack.pop().endloc = d.getloc(offset) 466 continue 467 468 if kword == 'else': 469 if len(condstack) == 1: 470 raise SyntaxError("unmatched 'else' directive", 471 d.getloc(offset)) 472 473 m = _conditionre.match(d.s, offset, d.lend) 474 if m is None: 475 _ensureend(d, offset, "Unexpected data after 'else' directive.") 476 condstack[-1].addcondition(d.getloc(offset), parserdata.ElseCondition()) 477 else: 478 kword = m.group(1) 479 if kword not in _conditionkeywords: 480 raise SyntaxError("Unexpected condition after 'else' directive.", 481 d.getloc(offset)) 482 483 startoffset = offset 484 offset = d.skipwhitespace(m.end(1)) 485 c = _conditionkeywords[kword](d, offset) 486 condstack[-1].addcondition(d.getloc(startoffset), c) 487 continue 488 489 if kword in _conditionkeywords: 490 c = _conditionkeywords[kword](d, offset) 491 cb = parserdata.ConditionBlock(d.getloc(d.lstart), c) 492 condstack[-1].append(cb) 493 condstack.append(cb) 494 continue 495 496 if kword == 'endef': 497 raise SyntaxError("endef without matching define", d.getloc(offset)) 498 499 if kword == 'define': 500 currule = False 501 vname, t, i = parsemakesyntax(d, offset, (), itermakefilechars) 502 vname.rstrip() 503 504 startloc = d.getloc(d.lstart) 505 value = iterdefinelines(fdlines, startloc) 506 condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=startloc, token='=', targetexp=None)) 507 continue 508 509 if kword in ('include', '-include', 'includedeps', '-includedeps'): 510 if kword.startswith('-'): 511 required = False 512 kword = kword[1:] 513 else: 514 required = True 515 516 deps = kword == 'includedeps' 517 518 currule = False 519 incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) 520 condstack[-1].append(parserdata.Include(incfile, required, deps)) 521 522 continue 523 524 if kword == 'vpath': 525 currule = False 526 e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) 527 condstack[-1].append(parserdata.VPathDirective(e)) 528 continue 529 530 if kword == 'override': 531 currule = False 532 vname, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars) 533 vname.lstrip() 534 vname.rstrip() 535 536 if token is None: 537 raise SyntaxError("Malformed override directive, need =", d.getloc(d.lstart)) 538 539 value = flattenmakesyntax(d, offset).lstrip() 540 541 condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_OVERRIDE)) 542 continue 543 544 if kword == 'export': 545 currule = False 546 e, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars) 547 e.lstrip() 548 e.rstrip() 549 550 if token is None: 551 condstack[-1].append(parserdata.ExportDirective(e, concurrent_set=False)) 552 else: 553 condstack[-1].append(parserdata.ExportDirective(e, concurrent_set=True)) 554 555 value = flattenmakesyntax(d, offset).lstrip() 556 condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None)) 557 558 continue 559 560 if kword == 'unexport': 561 e, token, offset = parsemakesyntax(d, offset, (), itermakefilechars) 562 condstack[-1].append(parserdata.UnexportDirective(e)) 563 continue 564 565 e, token, offset = parsemakesyntax(d, offset, _varsettokens + ('::', ':'), itermakefilechars) 566 if token is None: 567 e.rstrip() 568 e.lstrip() 569 if not e.isempty(): 570 condstack[-1].append(parserdata.EmptyDirective(e)) 571 continue 572 573 # if we encountered real makefile syntax, the current rule is over 574 currule = False 575 576 if token in _varsettokens: 577 e.lstrip() 578 e.rstrip() 579 580 value = flattenmakesyntax(d, offset).lstrip() 581 582 condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None)) 583 else: 584 doublecolon = token == '::' 585 586 # `e` is targets or target patterns, which can end up as 587 # * a rule 588 # * an implicit rule 589 # * a static pattern rule 590 # * a target-specific variable definition 591 # * a pattern-specific variable definition 592 # any of the rules may have order-only prerequisites 593 # delimited by |, and a command delimited by ; 594 targets = e 595 596 e, token, offset = parsemakesyntax(d, offset, 597 _varsettokens + (':', '|', ';'), 598 itermakefilechars) 599 if token in (None, ';'): 600 condstack[-1].append(parserdata.Rule(targets, e, doublecolon)) 601 currule = True 602 603 if token == ';': 604 offset = d.skipwhitespace(offset) 605 e, t, offset = parsemakesyntax(d, offset, (), itercommandchars) 606 condstack[-1].append(parserdata.Command(e)) 607 608 elif token in _varsettokens: 609 e.lstrip() 610 e.rstrip() 611 612 value = flattenmakesyntax(d, offset).lstrip() 613 condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=targets)) 614 elif token == '|': 615 raise SyntaxError('order-only prerequisites not implemented', d.getloc(offset)) 616 else: 617 assert token == ':' 618 # static pattern rule 619 620 pattern = e 621 622 deps, token, offset = parsemakesyntax(d, offset, (';',), itermakefilechars) 623 624 condstack[-1].append(parserdata.StaticPatternRule(targets, pattern, deps, doublecolon)) 625 currule = True 626 627 if token == ';': 628 offset = d.skipwhitespace(offset) 629 e, token, offset = parsemakesyntax(d, offset, (), itercommandchars) 630 condstack[-1].append(parserdata.Command(e)) 631 632 if len(condstack) != 1: 633 raise SyntaxError("Condition never terminated with endif", condstack[-1].loc) 634 635 return condstack[0] 636 637_PARSESTATE_TOPLEVEL = 0 # at the top level 638_PARSESTATE_FUNCTION = 1 # expanding a function call 639_PARSESTATE_VARNAME = 2 # expanding a variable expansion. 640_PARSESTATE_SUBSTFROM = 3 # expanding a variable expansion substitution "from" value 641_PARSESTATE_SUBSTTO = 4 # expanding a variable expansion substitution "to" value 642_PARSESTATE_PARENMATCH = 5 # inside nested parentheses/braces that must be matched 643 644class ParseStackFrame(object): 645 __slots__ = ('parsestate', 'parent', 'expansion', 'tokenlist', 'openbrace', 'closebrace', 'function', 'loc', 'varname', 'substfrom') 646 647 def __init__(self, parsestate, parent, expansion, tokenlist, openbrace, closebrace, function=None, loc=None): 648 self.parsestate = parsestate 649 self.parent = parent 650 self.expansion = expansion 651 self.tokenlist = tokenlist 652 self.openbrace = openbrace 653 self.closebrace = closebrace 654 self.function = function 655 self.loc = loc 656 657 def __str__(self): 658 return "<state=%i expansion=%s tokenlist=%s openbrace=%s closebrace=%s>" % (self.parsestate, self.expansion, self.tokenlist, self.openbrace, self.closebrace) 659 660_matchingbrace = { 661 '(': ')', 662 '{': '}', 663 } 664 665def parsemakesyntax(d, offset, stopon, iterfunc): 666 """ 667 Given Data, parse it into a data.Expansion. 668 669 @param stopon (sequence) 670 Indicate characters where toplevel parsing should stop. 671 672 @param iterfunc (generator function) 673 A function which is used to iterate over d, yielding (char, offset, loc) 674 @see iterdata 675 @see itermakefilechars 676 @see itercommandchars 677 678 @return a tuple (expansion, token, offset). If all the data is consumed, 679 token and offset will be None 680 """ 681 682 assert callable(iterfunc) 683 684 stacktop = ParseStackFrame(_PARSESTATE_TOPLEVEL, None, data.Expansion(loc=d.getloc(d.lstart)), 685 tokenlist=stopon + ('$',), 686 openbrace=None, closebrace=None) 687 688 tokeniterator = _alltokens.finditer(d.s, offset, d.lend) 689 690 di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator) 691 while True: # this is not a for loop because `di` changes during the function 692 assert stacktop is not None 693 try: 694 s, token, tokenoffset, offset = di.next() 695 except StopIteration: 696 break 697 698 stacktop.expansion.appendstr(s) 699 if token is None: 700 continue 701 702 parsestate = stacktop.parsestate 703 704 if token[0] == '$': 705 if tokenoffset + 1 == d.lend: 706 # an unterminated $ expands to nothing 707 break 708 709 loc = d.getloc(tokenoffset) 710 c = token[1] 711 if c == '$': 712 assert len(token) == 2 713 stacktop.expansion.appendstr('$') 714 elif c in ('(', '{'): 715 closebrace = _matchingbrace[c] 716 717 if len(token) > 2: 718 fname = token[2:].rstrip() 719 fn = functions.functionmap[fname](loc) 720 e = data.Expansion() 721 if len(fn) + 1 == fn.maxargs: 722 tokenlist = (c, closebrace, '$') 723 else: 724 tokenlist = (',', c, closebrace, '$') 725 726 stacktop = ParseStackFrame(_PARSESTATE_FUNCTION, stacktop, 727 e, tokenlist, function=fn, 728 openbrace=c, closebrace=closebrace) 729 else: 730 e = data.Expansion() 731 tokenlist = (':', c, closebrace, '$') 732 stacktop = ParseStackFrame(_PARSESTATE_VARNAME, stacktop, 733 e, tokenlist, 734 openbrace=c, closebrace=closebrace, loc=loc) 735 else: 736 assert len(token) == 2 737 e = data.Expansion.fromstring(c, loc) 738 stacktop.expansion.appendfunc(functions.VariableRef(loc, e)) 739 elif token in ('(', '{'): 740 assert token == stacktop.openbrace 741 742 stacktop.expansion.appendstr(token) 743 stacktop = ParseStackFrame(_PARSESTATE_PARENMATCH, stacktop, 744 stacktop.expansion, 745 (token, stacktop.closebrace, '$'), 746 openbrace=token, closebrace=stacktop.closebrace, loc=d.getloc(tokenoffset)) 747 elif parsestate == _PARSESTATE_PARENMATCH: 748 assert token == stacktop.closebrace 749 stacktop.expansion.appendstr(token) 750 stacktop = stacktop.parent 751 elif parsestate == _PARSESTATE_TOPLEVEL: 752 assert stacktop.parent is None 753 return stacktop.expansion.finish(), token, offset 754 elif parsestate == _PARSESTATE_FUNCTION: 755 if token == ',': 756 stacktop.function.append(stacktop.expansion.finish()) 757 758 stacktop.expansion = data.Expansion() 759 if len(stacktop.function) + 1 == stacktop.function.maxargs: 760 tokenlist = (stacktop.openbrace, stacktop.closebrace, '$') 761 stacktop.tokenlist = tokenlist 762 elif token in (')', '}'): 763 fn = stacktop.function 764 fn.append(stacktop.expansion.finish()) 765 fn.setup() 766 767 stacktop = stacktop.parent 768 stacktop.expansion.appendfunc(fn) 769 else: 770 assert False, "Not reached, _PARSESTATE_FUNCTION" 771 elif parsestate == _PARSESTATE_VARNAME: 772 if token == ':': 773 stacktop.varname = stacktop.expansion 774 stacktop.parsestate = _PARSESTATE_SUBSTFROM 775 stacktop.expansion = data.Expansion() 776 stacktop.tokenlist = ('=', stacktop.openbrace, stacktop.closebrace, '$') 777 elif token in (')', '}'): 778 fn = functions.VariableRef(stacktop.loc, stacktop.expansion.finish()) 779 stacktop = stacktop.parent 780 stacktop.expansion.appendfunc(fn) 781 else: 782 assert False, "Not reached, _PARSESTATE_VARNAME" 783 elif parsestate == _PARSESTATE_SUBSTFROM: 784 if token == '=': 785 stacktop.substfrom = stacktop.expansion 786 stacktop.parsestate = _PARSESTATE_SUBSTTO 787 stacktop.expansion = data.Expansion() 788 stacktop.tokenlist = (stacktop.openbrace, stacktop.closebrace, '$') 789 elif token in (')', '}'): 790 # A substitution of the form $(VARNAME:.ee) is probably a mistake, but make 791 # parses it. Issue a warning. Combine the varname and substfrom expansions to 792 # make the compatible varname. See tests/var-substitutions.mk SIMPLE3SUBSTNAME 793 _log.warning("%s: Variable reference looks like substitution without =", stacktop.loc) 794 stacktop.varname.appendstr(':') 795 stacktop.varname.concat(stacktop.expansion) 796 fn = functions.VariableRef(stacktop.loc, stacktop.varname.finish()) 797 stacktop = stacktop.parent 798 stacktop.expansion.appendfunc(fn) 799 else: 800 assert False, "Not reached, _PARSESTATE_SUBSTFROM" 801 elif parsestate == _PARSESTATE_SUBSTTO: 802 assert token in (')','}'), "Not reached, _PARSESTATE_SUBSTTO" 803 804 fn = functions.SubstitutionRef(stacktop.loc, stacktop.varname.finish(), 805 stacktop.substfrom.finish(), stacktop.expansion.finish()) 806 stacktop = stacktop.parent 807 stacktop.expansion.appendfunc(fn) 808 else: 809 assert False, "Unexpected parse state %s" % stacktop.parsestate 810 811 if stacktop.parent is not None and iterfunc == itercommandchars: 812 di = itermakefilechars(d, offset, stacktop.tokenlist, tokeniterator, 813 ignorecomments=True) 814 else: 815 di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator) 816 817 if stacktop.parent is not None: 818 raise SyntaxError("Unterminated function call", d.getloc(offset)) 819 820 assert stacktop.parsestate == _PARSESTATE_TOPLEVEL 821 822 return stacktop.expansion.finish(), None, None 823