1# 2# $Id: em.py 1299 2009-04-02 08:26:37Z vaclavslavik $ $Date: 2003/10/27 $ 3 4""" 5A system for processing Python as markup embedded in text. 6""" 7 8 9__program__ = 'empy' 10__version__ = '3.3' 11__url__ = 'http://www.alcyone.com/software/empy/' 12__author__ = 'Erik Max Francis <max@alcyone.com>' 13__copyright__ = 'Copyright (C) 2002-2003 Erik Max Francis' 14__license__ = 'LGPL' 15 16 17import copy 18import getopt 19import os 20import re 21import string 22import sys 23import types 24 25try: 26 # The equivalent of import cStringIO as StringIO. 27 import cStringIO 28 StringIO = cStringIO 29 del cStringIO 30except ImportError: 31 import StringIO 32 33# For backward compatibility, we can't assume these are defined. 34False, True = 0, 1 35 36# Some basic defaults. 37FAILURE_CODE = 1 38DEFAULT_PREFIX = '@' 39DEFAULT_PSEUDOMODULE_NAME = 'empy' 40DEFAULT_SCRIPT_NAME = '?' 41SIGNIFICATOR_RE_SUFFIX = r"%(\S+)\s*(.*)\s*$" 42SIGNIFICATOR_RE_STRING = DEFAULT_PREFIX + SIGNIFICATOR_RE_SUFFIX 43BANGPATH = '#!' 44DEFAULT_CHUNK_SIZE = 8192 45DEFAULT_ERRORS = 'strict' 46 47# Character information. 48IDENTIFIER_FIRST_CHARS = '_abcdefghijklmnopqrstuvwxyz' \ 49 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 50IDENTIFIER_CHARS = IDENTIFIER_FIRST_CHARS + '0123456789.' 51ENDING_CHARS = {'(': ')', '[': ']', '{': '}'} 52 53# Environment variable names. 54OPTIONS_ENV = 'EMPY_OPTIONS' 55PREFIX_ENV = 'EMPY_PREFIX' 56PSEUDO_ENV = 'EMPY_PSEUDO' 57FLATTEN_ENV = 'EMPY_FLATTEN' 58RAW_ENV = 'EMPY_RAW_ERRORS' 59INTERACTIVE_ENV = 'EMPY_INTERACTIVE' 60BUFFERED_ENV = 'EMPY_BUFFERED_OUTPUT' 61NO_OVERRIDE_ENV = 'EMPY_NO_OVERRIDE' 62UNICODE_ENV = 'EMPY_UNICODE' 63INPUT_ENCODING_ENV = 'EMPY_UNICODE_INPUT_ENCODING' 64OUTPUT_ENCODING_ENV = 'EMPY_UNICODE_OUTPUT_ENCODING' 65INPUT_ERRORS_ENV = 'EMPY_UNICODE_INPUT_ERRORS' 66OUTPUT_ERRORS_ENV = 'EMPY_UNICODE_OUTPUT_ERRORS' 67 68# Interpreter options. 69BANGPATH_OPT = 'processBangpaths' # process bangpaths as comments? 70BUFFERED_OPT = 'bufferedOutput' # fully buffered output? 71RAW_OPT = 'rawErrors' # raw errors? 72EXIT_OPT = 'exitOnError' # exit on error? 73FLATTEN_OPT = 'flatten' # flatten pseudomodule namespace? 74OVERRIDE_OPT = 'override' # override sys.stdout with proxy? 75CALLBACK_OPT = 'noCallbackError' # is no custom callback an error? 76 77# Usage info. 78OPTION_INFO = [ 79("-V --version", "Print version and exit"), 80("-h --help", "Print usage and exit"), 81("-H --extended-help", "Print extended usage and exit"), 82("-k --suppress-errors", "Do not exit on errors; go interactive"), 83("-p --prefix=<char>", "Change prefix to something other than @"), 84(" --no-prefix", "Do not do any markup processing at all"), 85("-m --module=<name>", "Change the internal pseudomodule name"), 86("-f --flatten", "Flatten the members of pseudmodule to start"), 87("-r --raw-errors", "Show raw Python errors"), 88("-i --interactive", "Go into interactive mode after processing"), 89("-n --no-override-stdout", "Do not override sys.stdout with proxy"), 90("-o --output=<filename>", "Specify file for output as write"), 91("-a --append=<filename>", "Specify file for output as append"), 92("-b --buffered-output", "Fully buffer output including open"), 93(" --binary", "Treat the file as a binary"), 94(" --chunk-size=<chunk>", "Use this chunk size for reading binaries"), 95("-P --preprocess=<filename>", "Interpret EmPy file before main processing"), 96("-I --import=<modules>", "Import Python modules before processing"), 97("-D --define=<definition>", "Execute Python assignment statement"), 98("-E --execute=<statement>", "Execute Python statement before processing"), 99("-F --execute-file=<filename>", "Execute Python file before processing"), 100(" --pause-at-end", "Prompt at the ending of processing"), 101(" --relative-path", "Add path of EmPy script to sys.path"), 102(" --no-callback-error", "Custom markup without callback is error"), 103(" --no-bangpath-processing", "Suppress bangpaths as comments"), 104("-u --unicode", "Enable Unicode subsystem (Python 2+ only)"), 105(" --unicode-encoding=<e>", "Set both input and output encodings"), 106(" --unicode-input-encoding=<e>", "Set input encoding"), 107(" --unicode-output-encoding=<e>", "Set output encoding"), 108(" --unicode-errors=<E>", "Set both input and output error handler"), 109(" --unicode-input-errors=<E>", "Set input error handler"), 110(" --unicode-output-errors=<E>", "Set output error handler"), 111] 112 113USAGE_NOTES = """\ 114Notes: Whitespace immediately inside parentheses of @(...) are 115ignored. Whitespace immediately inside braces of @{...} are ignored, 116unless ... spans multiple lines. Use @{ ... }@ to suppress newline 117following expansion. Simple expressions ignore trailing dots; `@x.' 118means `@(x).'. A #! at the start of a file is treated as a @# 119comment.""" 120 121MARKUP_INFO = [ 122("@# ... NL", "Comment; remove everything up to newline"), 123("@? NAME NL", "Set the current context name"), 124("@! INTEGER NL", "Set the current context line number"), 125("@ WHITESPACE", "Remove following whitespace; line continuation"), 126("@\\ ESCAPE_CODE", "A C-style escape sequence"), 127("@@", "Literal @; @ is escaped (duplicated prefix)"), 128("@), @], @}", "Literal close parenthesis, bracket, brace"), 129("@ STRING_LITERAL", "Replace with string literal contents"), 130("@( EXPRESSION )", "Evaluate expression and substitute with str"), 131("@( TEST [? THEN [! ELSE]] )", "If test is true, evaluate then, otherwise else"), 132("@( TRY $ CATCH )", "Expand try expression, or catch if it raises"), 133("@ SIMPLE_EXPRESSION", "Evaluate simple expression and substitute;\n" 134 "e.g., @x, @x.y, @f(a, b), @l[i], etc."), 135("@` EXPRESSION `", "Evaluate expression and substitute with repr"), 136("@: EXPRESSION : [DUMMY] :", "Evaluates to @:...:expansion:"), 137("@{ STATEMENTS }", "Statements are executed for side effects"), 138("@[ CONTROL ]", "Control markups: if E; elif E; for N in E;\n" 139 "while E; try; except E, N; finally; continue;\n" 140 "break; end X"), 141("@%% KEY WHITESPACE VALUE NL", "Significator form of __KEY__ = VALUE"), 142("@< CONTENTS >", "Custom markup; meaning provided by user"), 143] 144 145ESCAPE_INFO = [ 146("@\\0", "NUL, null"), 147("@\\a", "BEL, bell"), 148("@\\b", "BS, backspace"), 149("@\\dDDD", "three-digit decimal code DDD"), 150("@\\e", "ESC, escape"), 151("@\\f", "FF, form feed"), 152("@\\h", "DEL, delete"), 153("@\\n", "LF, linefeed, newline"), 154("@\\N{NAME}", "Unicode character named NAME"), 155("@\\oOOO", "three-digit octal code OOO"), 156("@\\qQQQQ", "four-digit quaternary code QQQQ"), 157("@\\r", "CR, carriage return"), 158("@\\s", "SP, space"), 159("@\\t", "HT, horizontal tab"), 160("@\\uHHHH", "16-bit hexadecimal Unicode HHHH"), 161("@\\UHHHHHHHH", "32-bit hexadecimal Unicode HHHHHHHH"), 162("@\\v", "VT, vertical tab"), 163("@\\xHH", "two-digit hexadecimal code HH"), 164("@\\z", "EOT, end of transmission"), 165] 166 167PSEUDOMODULE_INFO = [ 168("VERSION", "String representing EmPy version"), 169("SIGNIFICATOR_RE_STRING", "Regular expression matching significators"), 170("SIGNIFICATOR_RE_SUFFIX", "The above stub, lacking the prefix"), 171("interpreter", "Currently-executing interpreter instance"), 172("argv", "The EmPy script name and command line arguments"), 173("args", "The command line arguments only"), 174("identify()", "Identify top context as name, line"), 175("setContextName(name)", "Set the name of the current context"), 176("setContextLine(line)", "Set the line number of the current context"), 177("atExit(callable)", "Invoke no-argument function at shutdown"), 178("getGlobals()", "Retrieve this interpreter's globals"), 179("setGlobals(dict)", "Set this interpreter's globals"), 180("updateGlobals(dict)", "Merge dictionary into interpreter's globals"), 181("clearGlobals()", "Start globals over anew"), 182("saveGlobals([deep])", "Save a copy of the globals"), 183("restoreGlobals([pop])", "Restore the most recently saved globals"), 184("defined(name, [loc])", "Find if the name is defined"), 185("evaluate(expression, [loc])", "Evaluate the expression"), 186("serialize(expression, [loc])", "Evaluate and serialize the expression"), 187("execute(statements, [loc])", "Execute the statements"), 188("single(source, [loc])", "Execute the 'single' object"), 189("atomic(name, value, [loc])", "Perform an atomic assignment"), 190("assign(name, value, [loc])", "Perform an arbitrary assignment"), 191("significate(key, [value])", "Significate the given key, value pair"), 192("include(file, [loc])", "Include filename or file-like object"), 193("expand(string, [loc])", "Explicitly expand string and return"), 194("string(data, [name], [loc])", "Process string-like object"), 195("quote(string)", "Quote prefixes in provided string and return"), 196("flatten([keys])", "Flatten module contents into globals namespace"), 197("getPrefix()", "Get current prefix"), 198("setPrefix(char)", "Set new prefix"), 199("stopDiverting()", "Stop diverting; data sent directly to output"), 200("createDiversion(name)", "Create a diversion but do not divert to it"), 201("retrieveDiversion(name)", "Retrieve the actual named diversion object"), 202("startDiversion(name)", "Start diverting to given diversion"), 203("playDiversion(name)", "Recall diversion and then eliminate it"), 204("replayDiversion(name)", "Recall diversion but retain it"), 205("purgeDiversion(name)", "Erase diversion"), 206("playAllDiversions()", "Stop diverting and play all diversions in order"), 207("replayAllDiversions()", "Stop diverting and replay all diversions"), 208("purgeAllDiversions()", "Stop diverting and purge all diversions"), 209("getFilter()", "Get current filter"), 210("resetFilter()", "Reset filter; no filtering"), 211("nullFilter()", "Install null filter"), 212("setFilter(shortcut)", "Install new filter or filter chain"), 213("attachFilter(shortcut)", "Attach single filter to end of current chain"), 214("areHooksEnabled()", "Return whether or not hooks are enabled"), 215("enableHooks()", "Enable hooks (default)"), 216("disableHooks()", "Disable hook invocation"), 217("getHooks()", "Get all the hooks"), 218("clearHooks()", "Clear all hooks"), 219("addHook(hook, [i])", "Register the hook (optionally insert)"), 220("removeHook(hook)", "Remove an already-registered hook from name"), 221("invokeHook(name_, ...)", "Manually invoke hook"), 222("getCallback()", "Get interpreter callback"), 223("registerCallback(callback)", "Register callback with interpreter"), 224("deregisterCallback()", "Deregister callback from interpreter"), 225("invokeCallback(contents)", "Invoke the callback directly"), 226("Interpreter", "The interpreter class"), 227] 228 229ENVIRONMENT_INFO = [ 230(OPTIONS_ENV, "Specified options will be included"), 231(PREFIX_ENV, "Specify the default prefix: -p <value>"), 232(PSEUDO_ENV, "Specify name of pseudomodule: -m <value>"), 233(FLATTEN_ENV, "Flatten empy pseudomodule if defined: -f"), 234(RAW_ENV, "Show raw errors if defined: -r"), 235(INTERACTIVE_ENV, "Enter interactive mode if defined: -i"), 236(BUFFERED_ENV, "Fully buffered output if defined: -b"), 237(NO_OVERRIDE_ENV, "Do not override sys.stdout if defined: -n"), 238(UNICODE_ENV, "Enable Unicode subsystem: -n"), 239(INPUT_ENCODING_ENV, "Unicode input encoding"), 240(OUTPUT_ENCODING_ENV, "Unicode output encoding"), 241(INPUT_ERRORS_ENV, "Unicode input error handler"), 242(OUTPUT_ERRORS_ENV, "Unicode output error handler"), 243] 244 245class Error(Exception): 246 """The base class for all EmPy errors.""" 247 pass 248 249EmpyError = EmPyError = Error # DEPRECATED 250 251class DiversionError(Error): 252 """An error related to diversions.""" 253 pass 254 255class FilterError(Error): 256 """An error related to filters.""" 257 pass 258 259class StackUnderflowError(Error): 260 """A stack underflow.""" 261 pass 262 263class SubsystemError(Error): 264 """An error associated with the Unicode subsystem.""" 265 pass 266 267class FlowError(Error): 268 """An exception related to control flow.""" 269 pass 270 271class ContinueFlow(FlowError): 272 """A continue control flow.""" 273 pass 274 275class BreakFlow(FlowError): 276 """A break control flow.""" 277 pass 278 279class ParseError(Error): 280 """A parse error occurred.""" 281 pass 282 283class TransientParseError(ParseError): 284 """A parse error occurred which may be resolved by feeding more data. 285 Such an error reaching the toplevel is an unexpected EOF error.""" 286 pass 287 288 289class MetaError(Exception): 290 291 """A wrapper around a real Python exception for including a copy of 292 the context.""" 293 294 def __init__(self, contexts, exc): 295 Exception.__init__(self, exc) 296 self.contexts = contexts 297 self.exc = exc 298 299 def __str__(self): 300 backtrace = map(lambda x: str(x), self.contexts) 301 return "%s: %s (%s)" % (self.exc.__class__, self.exc, \ 302 (string.join(backtrace, ', '))) 303 304 305class Subsystem: 306 307 """The subsystem class defers file creation so that it can create 308 Unicode-wrapped files if desired (and possible).""" 309 310 def __init__(self): 311 self.useUnicode = False 312 self.inputEncoding = None 313 self.outputEncoding = None 314 self.errors = None 315 316 def initialize(self, inputEncoding=None, outputEncoding=None, \ 317 inputErrors=None, outputErrors=None): 318 self.useUnicode = True 319 try: 320 unicode 321 import codecs 322 except (NameError, ImportError): 323 raise SubsystemError, "Unicode subsystem unavailable" 324 defaultEncoding = sys.getdefaultencoding() 325 if inputEncoding is None: 326 inputEncoding = defaultEncoding 327 self.inputEncoding = inputEncoding 328 if outputEncoding is None: 329 outputEncoding = defaultEncoding 330 self.outputEncoding = outputEncoding 331 if inputErrors is None: 332 inputErrors = DEFAULT_ERRORS 333 self.inputErrors = inputErrors 334 if outputErrors is None: 335 outputErrors = DEFAULT_ERRORS 336 self.outputErrors = outputErrors 337 338 def assertUnicode(self): 339 if not self.useUnicode: 340 raise SubsystemError, "Unicode subsystem unavailable" 341 342 def open(self, name, mode=None): 343 if self.useUnicode: 344 return self.unicodeOpen(name, mode) 345 else: 346 return self.defaultOpen(name, mode) 347 348 def defaultOpen(self, name, mode=None): 349 if mode is None: 350 mode = 'r' 351 return open(name, mode) 352 353 def unicodeOpen(self, name, mode=None): 354 import codecs 355 if mode is None: 356 mode = 'rb' 357 if mode.find('w') >= 0 or mode.find('a') >= 0: 358 encoding = self.outputEncoding 359 errors = self.outputErrors 360 else: 361 encoding = self.inputEncoding 362 errors = self.inputErrors 363 return codecs.open(name, mode, encoding, errors) 364 365theSubsystem = Subsystem() 366 367 368class Stack: 369 370 """A simple stack that behaves as a sequence (with 0 being the top 371 of the stack, not the bottom).""" 372 373 def __init__(self, seq=None): 374 if seq is None: 375 seq = [] 376 self.data = seq 377 378 def top(self): 379 """Access the top element on the stack.""" 380 try: 381 return self.data[-1] 382 except IndexError: 383 raise StackUnderflowError, "stack is empty for top" 384 385 def pop(self): 386 """Pop the top element off the stack and return it.""" 387 try: 388 return self.data.pop() 389 except IndexError: 390 raise StackUnderflowError, "stack is empty for pop" 391 392 def push(self, object): 393 """Push an element onto the top of the stack.""" 394 self.data.append(object) 395 396 def filter(self, function): 397 """Filter the elements of the stack through the function.""" 398 self.data = filter(function, self.data) 399 400 def purge(self): 401 """Purge the stack.""" 402 self.data = [] 403 404 def clone(self): 405 """Create a duplicate of this stack.""" 406 return self.__class__(self.data[:]) 407 408 def __nonzero__(self): return len(self.data) != 0 409 def __len__(self): return len(self.data) 410 def __getitem__(self, index): return self.data[-(index + 1)] 411 412 def __repr__(self): 413 return '<%s instance at 0x%x [%s]>' % \ 414 (self.__class__, id(self), \ 415 string.join(map(repr, self.data), ', ')) 416 417 418class AbstractFile: 419 420 """An abstracted file that, when buffered, will totally buffer the 421 file, including even the file open.""" 422 423 def __init__(self, filename, mode='w', buffered=False): 424 # The calls below might throw, so start off by marking this 425 # file as "done." This way destruction of a not-completely- 426 # initialized AbstractFile will generate no further errors. 427 self.done = True 428 self.filename = filename 429 self.mode = mode 430 self.buffered = buffered 431 if buffered: 432 self.bufferFile = StringIO.StringIO() 433 else: 434 self.bufferFile = theSubsystem.open(filename, mode) 435 # Okay, we got this far, so the AbstractFile is initialized. 436 # Flag it as "not done." 437 self.done = False 438 439 def __del__(self): 440 self.close() 441 442 def write(self, data): 443 self.bufferFile.write(data) 444 445 def writelines(self, data): 446 self.bufferFile.writelines(data) 447 448 def flush(self): 449 self.bufferFile.flush() 450 451 def close(self): 452 if not self.done: 453 self.commit() 454 self.done = True 455 456 def commit(self): 457 if self.buffered: 458 file = theSubsystem.open(self.filename, self.mode) 459 file.write(self.bufferFile.getvalue()) 460 file.close() 461 else: 462 self.bufferFile.close() 463 464 def abort(self): 465 if self.buffered: 466 self.bufferFile = None 467 else: 468 self.bufferFile.close() 469 self.bufferFile = None 470 self.done = True 471 472 473class Diversion: 474 475 """The representation of an active diversion. Diversions act as 476 (writable) file objects, and then can be recalled either as pure 477 strings or (readable) file objects.""" 478 479 def __init__(self): 480 self.file = StringIO.StringIO() 481 482 # These methods define the writable file-like interface for the 483 # diversion. 484 485 def write(self, data): 486 self.file.write(data) 487 488 def writelines(self, lines): 489 for line in lines: 490 self.write(line) 491 492 def flush(self): 493 self.file.flush() 494 495 def close(self): 496 self.file.close() 497 498 # These methods are specific to diversions. 499 500 def asString(self): 501 """Return the diversion as a string.""" 502 return self.file.getvalue() 503 504 def asFile(self): 505 """Return the diversion as a file.""" 506 return StringIO.StringIO(self.file.getvalue()) 507 508 509class Stream: 510 511 """A wrapper around an (output) file object which supports 512 diversions and filtering.""" 513 514 def __init__(self, file): 515 self.file = file 516 self.currentDiversion = None 517 self.diversions = {} 518 self.filter = file 519 self.done = False 520 521 def write(self, data): 522 if self.currentDiversion is None: 523 self.filter.write(data) 524 else: 525 self.diversions[self.currentDiversion].write(data) 526 527 def writelines(self, lines): 528 for line in lines: 529 self.write(line) 530 531 def flush(self): 532 self.filter.flush() 533 534 def close(self): 535 if not self.done: 536 self.undivertAll(True) 537 self.filter.close() 538 self.done = True 539 540 def shortcut(self, shortcut): 541 """Take a filter shortcut and translate it into a filter, returning 542 it. Sequences don't count here; these should be detected 543 independently.""" 544 if shortcut == 0: 545 return NullFilter() 546 elif type(shortcut) is types.FunctionType or \ 547 type(shortcut) is types.BuiltinFunctionType or \ 548 type(shortcut) is types.BuiltinMethodType or \ 549 type(shortcut) is types.LambdaType: 550 return FunctionFilter(shortcut) 551 elif type(shortcut) is types.StringType: 552 return StringFilter(filter) 553 elif type(shortcut) is types.DictType: 554 raise NotImplementedError, "mapping filters not yet supported" 555 else: 556 # Presume it's a plain old filter. 557 return shortcut 558 559 def last(self): 560 """Find the last filter in the current filter chain, or None if 561 there are no filters installed.""" 562 if self.filter is None: 563 return None 564 thisFilter, lastFilter = self.filter, None 565 while thisFilter is not None and thisFilter is not self.file: 566 lastFilter = thisFilter 567 thisFilter = thisFilter.next() 568 return lastFilter 569 570 def install(self, shortcut=None): 571 """Install a new filter; None means no filter. Handle all the 572 special shortcuts for filters here.""" 573 # Before starting, execute a flush. 574 self.filter.flush() 575 if shortcut is None or shortcut == [] or shortcut == (): 576 # Shortcuts for "no filter." 577 self.filter = self.file 578 else: 579 if type(shortcut) in (types.ListType, types.TupleType): 580 shortcuts = list(shortcut) 581 else: 582 shortcuts = [shortcut] 583 # Run through the shortcut filter names, replacing them with 584 # full-fledged instances of Filter. 585 filters = [] 586 for shortcut in shortcuts: 587 filters.append(self.shortcut(shortcut)) 588 if len(filters) > 1: 589 # If there's more than one filter provided, chain them 590 # together. 591 lastFilter = None 592 for filter in filters: 593 if lastFilter is not None: 594 lastFilter.attach(filter) 595 lastFilter = filter 596 lastFilter.attach(self.file) 597 self.filter = filters[0] 598 else: 599 # If there's only one filter, assume that it's alone or it's 600 # part of a chain that has already been manually chained; 601 # just find the end. 602 filter = filters[0] 603 lastFilter = filter.last() 604 lastFilter.attach(self.file) 605 self.filter = filter 606 607 def attach(self, shortcut): 608 """Attached a solitary filter (no sequences allowed here) at the 609 end of the current filter chain.""" 610 lastFilter = self.last() 611 if lastFilter is None: 612 # Just install it from scratch if there is no active filter. 613 self.install(shortcut) 614 else: 615 # Attach the last filter to this one, and this one to the file. 616 filter = self.shortcut(shortcut) 617 lastFilter.attach(filter) 618 filter.attach(self.file) 619 620 def revert(self): 621 """Reset any current diversions.""" 622 self.currentDiversion = None 623 624 def create(self, name): 625 """Create a diversion if one does not already exist, but do not 626 divert to it yet.""" 627 if name is None: 628 raise DiversionError, "diversion name must be non-None" 629 if not self.diversions.has_key(name): 630 self.diversions[name] = Diversion() 631 632 def retrieve(self, name): 633 """Retrieve the given diversion.""" 634 if name is None: 635 raise DiversionError, "diversion name must be non-None" 636 if self.diversions.has_key(name): 637 return self.diversions[name] 638 else: 639 raise DiversionError, "nonexistent diversion: %s" % name 640 641 def divert(self, name): 642 """Start diverting.""" 643 if name is None: 644 raise DiversionError, "diversion name must be non-None" 645 self.create(name) 646 self.currentDiversion = name 647 648 def undivert(self, name, purgeAfterwards=False): 649 """Undivert a particular diversion.""" 650 if name is None: 651 raise DiversionError, "diversion name must be non-None" 652 if self.diversions.has_key(name): 653 diversion = self.diversions[name] 654 self.filter.write(diversion.asString()) 655 if purgeAfterwards: 656 self.purge(name) 657 else: 658 raise DiversionError, "nonexistent diversion: %s" % name 659 660 def purge(self, name): 661 """Purge the specified diversion.""" 662 if name is None: 663 raise DiversionError, "diversion name must be non-None" 664 if self.diversions.has_key(name): 665 del self.diversions[name] 666 if self.currentDiversion == name: 667 self.currentDiversion = None 668 669 def undivertAll(self, purgeAfterwards=True): 670 """Undivert all pending diversions.""" 671 if self.diversions: 672 self.revert() # revert before undiverting! 673 names = self.diversions.keys() 674 names.sort() 675 for name in names: 676 self.undivert(name) 677 if purgeAfterwards: 678 self.purge(name) 679 680 def purgeAll(self): 681 """Eliminate all existing diversions.""" 682 if self.diversions: 683 self.diversions = {} 684 self.currentDiversion = None 685 686 687class NullFile: 688 689 """A simple class that supports all the file-like object methods 690 but simply does nothing at all.""" 691 692 def __init__(self): pass 693 def write(self, data): pass 694 def writelines(self, lines): pass 695 def flush(self): pass 696 def close(self): pass 697 698 699class UncloseableFile: 700 701 """A simple class which wraps around a delegate file-like object 702 and lets everything through except close calls.""" 703 704 def __init__(self, delegate): 705 self.delegate = delegate 706 707 def write(self, data): 708 self.delegate.write(data) 709 710 def writelines(self, lines): 711 self.delegate.writelines(data) 712 713 def flush(self): 714 self.delegate.flush() 715 716 def close(self): 717 """Eat this one.""" 718 pass 719 720 721class ProxyFile: 722 723 """The proxy file object that is intended to take the place of 724 sys.stdout. The proxy can manage a stack of file objects it is 725 writing to, and an underlying raw file object.""" 726 727 def __init__(self, bottom): 728 self.stack = Stack() 729 self.bottom = bottom 730 731 def current(self): 732 """Get the current stream to write to.""" 733 if self.stack: 734 return self.stack[-1][1] 735 else: 736 return self.bottom 737 738 def push(self, interpreter): 739 self.stack.push((interpreter, interpreter.stream())) 740 741 def pop(self, interpreter): 742 result = self.stack.pop() 743 assert interpreter is result[0] 744 745 def clear(self, interpreter): 746 self.stack.filter(lambda x, i=interpreter: x[0] is not i) 747 748 def write(self, data): 749 self.current().write(data) 750 751 def writelines(self, lines): 752 self.current().writelines(lines) 753 754 def flush(self): 755 self.current().flush() 756 757 def close(self): 758 """Close the current file. If the current file is the bottom, then 759 close it and dispose of it.""" 760 current = self.current() 761 if current is self.bottom: 762 self.bottom = None 763 current.close() 764 765 def _testProxy(self): pass 766 767 768class Filter: 769 770 """An abstract filter.""" 771 772 def __init__(self): 773 if self.__class__ is Filter: 774 raise NotImplementedError 775 self.sink = None 776 777 def next(self): 778 """Return the next filter/file-like object in the sequence, or None.""" 779 return self.sink 780 781 def write(self, data): 782 """The standard write method; this must be overridden in subclasses.""" 783 raise NotImplementedError 784 785 def writelines(self, lines): 786 """Standard writelines wrapper.""" 787 for line in lines: 788 self.write(line) 789 790 def _flush(self): 791 """The _flush method should always flush the sink and should not 792 be overridden.""" 793 self.sink.flush() 794 795 def flush(self): 796 """The flush method can be overridden.""" 797 self._flush() 798 799 def close(self): 800 """Close the filter. Do an explicit flush first, then close the 801 sink.""" 802 self.flush() 803 self.sink.close() 804 805 def attach(self, filter): 806 """Attach a filter to this one.""" 807 if self.sink is not None: 808 # If it's already attached, detach it first. 809 self.detach() 810 self.sink = filter 811 812 def detach(self): 813 """Detach a filter from its sink.""" 814 self.flush() 815 self._flush() # do a guaranteed flush to just to be safe 816 self.sink = None 817 818 def last(self): 819 """Find the last filter in this chain.""" 820 this, last = self, self 821 while this is not None: 822 last = this 823 this = this.next() 824 return last 825 826class NullFilter(Filter): 827 828 """A filter that never sends any output to its sink.""" 829 830 def write(self, data): pass 831 832class FunctionFilter(Filter): 833 834 """A filter that works simply by pumping its input through a 835 function which maps strings into strings.""" 836 837 def __init__(self, function): 838 Filter.__init__(self) 839 self.function = function 840 841 def write(self, data): 842 self.sink.write(self.function(data)) 843 844class StringFilter(Filter): 845 846 """A filter that takes a translation string (256 characters) and 847 filters any incoming data through it.""" 848 849 def __init__(self, table): 850 if not (type(table) == types.StringType and len(table) == 256): 851 raise FilterError, "table must be 256-character string" 852 Filter.__init__(self) 853 self.table = table 854 855 def write(self, data): 856 self.sink.write(string.translate(data, self.table)) 857 858class BufferedFilter(Filter): 859 860 """A buffered filter is one that doesn't modify the source data 861 sent to the sink, but instead holds it for a time. The standard 862 variety only sends the data along when it receives a flush 863 command.""" 864 865 def __init__(self): 866 Filter.__init__(self) 867 self.buffer = '' 868 869 def write(self, data): 870 self.buffer = self.buffer + data 871 872 def flush(self): 873 if self.buffer: 874 self.sink.write(self.buffer) 875 self._flush() 876 877class SizeBufferedFilter(BufferedFilter): 878 879 """A size-buffered filter only in fixed size chunks (excepting the 880 final chunk).""" 881 882 def __init__(self, bufferSize): 883 BufferedFilter.__init__(self) 884 self.bufferSize = bufferSize 885 886 def write(self, data): 887 BufferedFilter.write(self, data) 888 while len(self.buffer) > self.bufferSize: 889 chunk, self.buffer = \ 890 self.buffer[:self.bufferSize], self.buffer[self.bufferSize:] 891 self.sink.write(chunk) 892 893class LineBufferedFilter(BufferedFilter): 894 895 """A line-buffered filter only lets data through when it sees 896 whole lines.""" 897 898 def __init__(self): 899 BufferedFilter.__init__(self) 900 901 def write(self, data): 902 BufferedFilter.write(self, data) 903 chunks = string.split(self.buffer, '\n') 904 for chunk in chunks[:-1]: 905 self.sink.write(chunk + '\n') 906 self.buffer = chunks[-1] 907 908class MaximallyBufferedFilter(BufferedFilter): 909 910 """A maximally-buffered filter only lets its data through on the final 911 close. It ignores flushes.""" 912 913 def __init__(self): 914 BufferedFilter.__init__(self) 915 916 def flush(self): pass 917 918 def close(self): 919 if self.buffer: 920 BufferedFilter.flush(self) 921 self.sink.close() 922 923 924class Context: 925 926 """An interpreter context, which encapsulates a name, an input 927 file object, and a parser object.""" 928 929 DEFAULT_UNIT = 'lines' 930 931 def __init__(self, name, line=0, units=DEFAULT_UNIT): 932 self.name = name 933 self.line = line 934 self.units = units 935 self.pause = False 936 937 def bump(self, quantity=1): 938 if self.pause: 939 self.pause = False 940 else: 941 self.line = self.line + quantity 942 943 def identify(self): 944 return self.name, self.line 945 946 def __str__(self): 947 if self.units == self.DEFAULT_UNIT: 948 return "%s:%s" % (self.name, self.line) 949 else: 950 return "%s:%s[%s]" % (self.name, self.line, self.units) 951 952 953class Hook: 954 955 """The base class for implementing hooks.""" 956 957 def __init__(self): 958 self.interpreter = None 959 960 def register(self, interpreter): 961 self.interpreter = interpreter 962 963 def deregister(self, interpreter): 964 if interpreter is not self.interpreter: 965 raise Error, "hook not associated with this interpreter" 966 self.interpreter = None 967 968 def push(self): 969 self.interpreter.push() 970 971 def pop(self): 972 self.interpreter.pop() 973 974 def null(self): pass 975 976 def atStartup(self): pass 977 def atReady(self): pass 978 def atFinalize(self): pass 979 def atShutdown(self): pass 980 def atParse(self, scanner, locals): pass 981 def atToken(self, token): pass 982 def atHandle(self, meta): pass 983 def atInteract(self): pass 984 985 def beforeInclude(self, name, file, locals): pass 986 def afterInclude(self): pass 987 988 def beforeExpand(self, string, locals): pass 989 def afterExpand(self, result): pass 990 991 def beforeFile(self, name, file, locals): pass 992 def afterFile(self): pass 993 994 def beforeBinary(self, name, file, chunkSize, locals): pass 995 def afterBinary(self): pass 996 997 def beforeString(self, name, string, locals): pass 998 def afterString(self): pass 999 1000 def beforeQuote(self, string): pass 1001 def afterQuote(self, result): pass 1002 1003 def beforeEscape(self, string, more): pass 1004 def afterEscape(self, result): pass 1005 1006 def beforeControl(self, type, rest, locals): pass 1007 def afterControl(self): pass 1008 1009 def beforeSignificate(self, key, value, locals): pass 1010 def afterSignificate(self): pass 1011 1012 def beforeAtomic(self, name, value, locals): pass 1013 def afterAtomic(self): pass 1014 1015 def beforeMulti(self, name, values, locals): pass 1016 def afterMulti(self): pass 1017 1018 def beforeImport(self, name, locals): pass 1019 def afterImport(self): pass 1020 1021 def beforeClause(self, catch, locals): pass 1022 def afterClause(self, exception, variable): pass 1023 1024 def beforeSerialize(self, expression, locals): pass 1025 def afterSerialize(self): pass 1026 1027 def beforeDefined(self, name, locals): pass 1028 def afterDefined(self, result): pass 1029 1030 def beforeLiteral(self, text): pass 1031 def afterLiteral(self): pass 1032 1033 def beforeEvaluate(self, expression, locals): pass 1034 def afterEvaluate(self, result): pass 1035 1036 def beforeExecute(self, statements, locals): pass 1037 def afterExecute(self): pass 1038 1039 def beforeSingle(self, source, locals): pass 1040 def afterSingle(self): pass 1041 1042class VerboseHook(Hook): 1043 1044 """A verbose hook that reports all information received by the 1045 hook interface. This class dynamically scans the Hook base class 1046 to ensure that all hook methods are properly represented.""" 1047 1048 EXEMPT_ATTRIBUTES = ['register', 'deregister', 'push', 'pop'] 1049 1050 def __init__(self, output=sys.stderr): 1051 Hook.__init__(self) 1052 self.output = output 1053 self.indent = 0 1054 1055 class FakeMethod: 1056 """This is a proxy method-like object.""" 1057 def __init__(self, hook, name): 1058 self.hook = hook 1059 self.name = name 1060 1061 def __call__(self, **keywords): 1062 self.hook.output.write("%s%s: %s\n" % \ 1063 (' ' * self.hook.indent, \ 1064 self.name, repr(keywords))) 1065 1066 for attribute in dir(Hook): 1067 if attribute[:1] != '_' and \ 1068 attribute not in self.EXEMPT_ATTRIBUTES: 1069 self.__dict__[attribute] = FakeMethod(self, attribute) 1070 1071 1072class Token: 1073 1074 """An element of expansion.""" 1075 1076 def run(self, interpreter, locals): 1077 raise NotImplementedError 1078 1079 def string(self): 1080 raise NotImplementedError 1081 1082 def __str__(self): return self.string() 1083 1084class NullToken(Token): 1085 """A chunk of data not containing markups.""" 1086 def __init__(self, data): 1087 self.data = data 1088 1089 def run(self, interpreter, locals): 1090 interpreter.write(self.data) 1091 1092 def string(self): 1093 return self.data 1094 1095class ExpansionToken(Token): 1096 """A token that involves an expansion.""" 1097 def __init__(self, prefix, first): 1098 self.prefix = prefix 1099 self.first = first 1100 1101 def scan(self, scanner): 1102 pass 1103 1104 def run(self, interpreter, locals): 1105 pass 1106 1107class WhitespaceToken(ExpansionToken): 1108 """A whitespace markup.""" 1109 def string(self): 1110 return '%s%s' % (self.prefix, self.first) 1111 1112class LiteralToken(ExpansionToken): 1113 """A literal markup.""" 1114 def run(self, interpreter, locals): 1115 interpreter.write(self.first) 1116 1117 def string(self): 1118 return '%s%s' % (self.prefix, self.first) 1119 1120class PrefixToken(ExpansionToken): 1121 """A prefix markup.""" 1122 def run(self, interpreter, locals): 1123 interpreter.write(interpreter.prefix) 1124 1125 def string(self): 1126 return self.prefix * 2 1127 1128class CommentToken(ExpansionToken): 1129 """A comment markup.""" 1130 def scan(self, scanner): 1131 loc = scanner.find('\n') 1132 if loc >= 0: 1133 self.comment = scanner.chop(loc, 1) 1134 else: 1135 raise TransientParseError, "comment expects newline" 1136 1137 def string(self): 1138 return '%s#%s\n' % (self.prefix, self.comment) 1139 1140class ContextNameToken(ExpansionToken): 1141 """A context name change markup.""" 1142 def scan(self, scanner): 1143 loc = scanner.find('\n') 1144 if loc >= 0: 1145 self.name = string.strip(scanner.chop(loc, 1)) 1146 else: 1147 raise TransientParseError, "context name expects newline" 1148 1149 def run(self, interpreter, locals): 1150 context = interpreter.context() 1151 context.name = self.name 1152 1153class ContextLineToken(ExpansionToken): 1154 """A context line change markup.""" 1155 def scan(self, scanner): 1156 loc = scanner.find('\n') 1157 if loc >= 0: 1158 try: 1159 self.line = int(scanner.chop(loc, 1)) 1160 except ValueError: 1161 raise ParseError, "context line requires integer" 1162 else: 1163 raise TransientParseError, "context line expects newline" 1164 1165 def run(self, interpreter, locals): 1166 context = interpreter.context() 1167 context.line = self.line 1168 context.pause = True 1169 1170class EscapeToken(ExpansionToken): 1171 """An escape markup.""" 1172 def scan(self, scanner): 1173 try: 1174 code = scanner.chop(1) 1175 result = None 1176 if code in '()[]{}\'\"\\': # literals 1177 result = code 1178 elif code == '0': # NUL 1179 result = '\x00' 1180 elif code == 'a': # BEL 1181 result = '\x07' 1182 elif code == 'b': # BS 1183 result = '\x08' 1184 elif code == 'd': # decimal code 1185 decimalCode = scanner.chop(3) 1186 result = chr(string.atoi(decimalCode, 10)) 1187 elif code == 'e': # ESC 1188 result = '\x1b' 1189 elif code == 'f': # FF 1190 result = '\x0c' 1191 elif code == 'h': # DEL 1192 result = '\x7f' 1193 elif code == 'n': # LF (newline) 1194 result = '\x0a' 1195 elif code == 'N': # Unicode character name 1196 theSubsystem.assertUnicode() 1197 import unicodedata 1198 if scanner.chop(1) != '{': 1199 raise ParseError, r"Unicode name escape should be \N{...}" 1200 i = scanner.find('}') 1201 name = scanner.chop(i, 1) 1202 try: 1203 result = unicodedata.lookup(name) 1204 except KeyError: 1205 raise SubsystemError, \ 1206 "unknown Unicode character name: %s" % name 1207 elif code == 'o': # octal code 1208 octalCode = scanner.chop(3) 1209 result = chr(string.atoi(octalCode, 8)) 1210 elif code == 'q': # quaternary code 1211 quaternaryCode = scanner.chop(4) 1212 result = chr(string.atoi(quaternaryCode, 4)) 1213 elif code == 'r': # CR 1214 result = '\x0d' 1215 elif code in 's ': # SP 1216 result = ' ' 1217 elif code == 't': # HT 1218 result = '\x09' 1219 elif code in 'u': # Unicode 16-bit hex literal 1220 theSubsystem.assertUnicode() 1221 hexCode = scanner.chop(4) 1222 result = unichr(string.atoi(hexCode, 16)) 1223 elif code in 'U': # Unicode 32-bit hex literal 1224 theSubsystem.assertUnicode() 1225 hexCode = scanner.chop(8) 1226 result = unichr(string.atoi(hexCode, 16)) 1227 elif code == 'v': # VT 1228 result = '\x0b' 1229 elif code == 'x': # hexadecimal code 1230 hexCode = scanner.chop(2) 1231 result = chr(string.atoi(hexCode, 16)) 1232 elif code == 'z': # EOT 1233 result = '\x04' 1234 elif code == '^': # control character 1235 controlCode = string.upper(scanner.chop(1)) 1236 if controlCode >= '@' and controlCode <= '`': 1237 result = chr(ord(controlCode) - ord('@')) 1238 elif controlCode == '?': 1239 result = '\x7f' 1240 else: 1241 raise ParseError, "invalid escape control code" 1242 else: 1243 raise ParseError, "unrecognized escape code" 1244 assert result is not None 1245 self.code = result 1246 except ValueError: 1247 raise ParseError, "invalid numeric escape code" 1248 1249 def run(self, interpreter, locals): 1250 interpreter.write(self.code) 1251 1252 def string(self): 1253 return '%s\\x%02x' % (self.prefix, ord(self.code)) 1254 1255class SignificatorToken(ExpansionToken): 1256 """A significator markup.""" 1257 def scan(self, scanner): 1258 loc = scanner.find('\n') 1259 if loc >= 0: 1260 line = scanner.chop(loc, 1) 1261 if not line: 1262 raise ParseError, "significator must have nonblank key" 1263 if line[0] in ' \t\v\n': 1264 raise ParseError, "no whitespace between % and key" 1265 # Work around a subtle CPython-Jython difference by stripping 1266 # the string before splitting it: 'a '.split(None, 1) has two 1267 # elements in Jython 2.1). 1268 fields = string.split(string.strip(line), None, 1) 1269 if len(fields) == 2 and fields[1] == '': 1270 fields.pop() 1271 self.key = fields[0] 1272 if len(fields) < 2: 1273 fields.append(None) 1274 self.key, self.valueCode = fields 1275 else: 1276 raise TransientParseError, "significator expects newline" 1277 1278 def run(self, interpreter, locals): 1279 value = self.valueCode 1280 if value is not None: 1281 value = interpreter.evaluate(string.strip(value), locals) 1282 interpreter.significate(self.key, value) 1283 1284 def string(self): 1285 if self.valueCode is None: 1286 return '%s%%%s\n' % (self.prefix, self.key) 1287 else: 1288 return '%s%%%s %s\n' % (self.prefix, self.key, self.valueCode) 1289 1290class ExpressionToken(ExpansionToken): 1291 """An expression markup.""" 1292 def scan(self, scanner): 1293 z = scanner.complex('(', ')', 0) 1294 try: 1295 q = scanner.next('$', 0, z, True) 1296 except ParseError: 1297 q = z 1298 try: 1299 i = scanner.next('?', 0, q, True) 1300 try: 1301 j = scanner.next('!', i, q, True) 1302 except ParseError: 1303 try: 1304 j = scanner.next(':', i, q, True) # DEPRECATED 1305 except ParseError: 1306 j = q 1307 except ParseError: 1308 i = j = q 1309 code = scanner.chop(z, 1) 1310 self.testCode = code[:i] 1311 self.thenCode = code[i + 1:j] 1312 self.elseCode = code[j + 1:q] 1313 self.exceptCode = code[q + 1:z] 1314 1315 def run(self, interpreter, locals): 1316 try: 1317 result = interpreter.evaluate(self.testCode, locals) 1318 if self.thenCode: 1319 if result: 1320 result = interpreter.evaluate(self.thenCode, locals) 1321 else: 1322 if self.elseCode: 1323 result = interpreter.evaluate(self.elseCode, locals) 1324 else: 1325 result = None 1326 except SyntaxError: 1327 # Don't catch syntax errors; let them through. 1328 raise 1329 except: 1330 if self.exceptCode: 1331 result = interpreter.evaluate(self.exceptCode, locals) 1332 else: 1333 raise 1334 if result is not None: 1335 interpreter.write(str(result)) 1336 1337 def string(self): 1338 result = self.testCode 1339 if self.thenCode: 1340 result = result + '?' + self.thenCode 1341 if self.elseCode: 1342 result = result + '!' + self.elseCode 1343 if self.exceptCode: 1344 result = result + '$' + self.exceptCode 1345 return '%s(%s)' % (self.prefix, result) 1346 1347class StringLiteralToken(ExpansionToken): 1348 """A string token markup.""" 1349 def scan(self, scanner): 1350 scanner.retreat() 1351 assert scanner[0] == self.first 1352 i = scanner.quote() 1353 self.literal = scanner.chop(i) 1354 1355 def run(self, interpreter, locals): 1356 interpreter.literal(self.literal) 1357 1358 def string(self): 1359 return '%s%s' % (self.prefix, self.literal) 1360 1361class SimpleExpressionToken(ExpansionToken): 1362 """A simple expression markup.""" 1363 def scan(self, scanner): 1364 i = scanner.simple() 1365 self.code = self.first + scanner.chop(i) 1366 1367 def run(self, interpreter, locals): 1368 interpreter.serialize(self.code, locals) 1369 1370 def string(self): 1371 return '%s%s' % (self.prefix, self.code) 1372 1373class ReprToken(ExpansionToken): 1374 """A repr markup.""" 1375 def scan(self, scanner): 1376 i = scanner.next('`', 0) 1377 self.code = scanner.chop(i, 1) 1378 1379 def run(self, interpreter, locals): 1380 interpreter.write(repr(interpreter.evaluate(self.code, locals))) 1381 1382 def string(self): 1383 return '%s`%s`' % (self.prefix, self.code) 1384 1385class InPlaceToken(ExpansionToken): 1386 """An in-place markup.""" 1387 def scan(self, scanner): 1388 i = scanner.next(':', 0) 1389 j = scanner.next(':', i + 1) 1390 self.code = scanner.chop(i, j - i + 1) 1391 1392 def run(self, interpreter, locals): 1393 interpreter.write("%s:%s:" % (interpreter.prefix, self.code)) 1394 try: 1395 interpreter.serialize(self.code, locals) 1396 finally: 1397 interpreter.write(":") 1398 1399 def string(self): 1400 return '%s:%s::' % (self.prefix, self.code) 1401 1402class StatementToken(ExpansionToken): 1403 """A statement markup.""" 1404 def scan(self, scanner): 1405 i = scanner.complex('{', '}', 0) 1406 self.code = scanner.chop(i, 1) 1407 1408 def run(self, interpreter, locals): 1409 interpreter.execute(self.code, locals) 1410 1411 def string(self): 1412 return '%s{%s}' % (self.prefix, self.code) 1413 1414class CustomToken(ExpansionToken): 1415 """A custom markup.""" 1416 def scan(self, scanner): 1417 i = scanner.complex('<', '>', 0) 1418 self.contents = scanner.chop(i, 1) 1419 1420 def run(self, interpreter, locals): 1421 interpreter.invokeCallback(self.contents) 1422 1423 def string(self): 1424 return '%s<%s>' % (self.prefix, self.contents) 1425 1426class ControlToken(ExpansionToken): 1427 1428 """A control token.""" 1429 1430 PRIMARY_TYPES = ['if', 'for', 'while', 'try', 'def'] 1431 SECONDARY_TYPES = ['elif', 'else', 'except', 'finally'] 1432 TERTIARY_TYPES = ['continue', 'break'] 1433 GREEDY_TYPES = ['if', 'elif', 'for', 'while', 'def', 'end'] 1434 END_TYPES = ['end'] 1435 1436 IN_RE = re.compile(r"\bin\b") 1437 1438 def scan(self, scanner): 1439 scanner.acquire() 1440 i = scanner.complex('[', ']', 0) 1441 self.contents = scanner.chop(i, 1) 1442 fields = string.split(string.strip(self.contents), ' ', 1) 1443 if len(fields) > 1: 1444 self.type, self.rest = fields 1445 else: 1446 self.type = fields[0] 1447 self.rest = None 1448 self.subtokens = [] 1449 if self.type in self.GREEDY_TYPES and self.rest is None: 1450 raise ParseError, "control '%s' needs arguments" % self.type 1451 if self.type in self.PRIMARY_TYPES: 1452 self.subscan(scanner, self.type) 1453 self.kind = 'primary' 1454 elif self.type in self.SECONDARY_TYPES: 1455 self.kind = 'secondary' 1456 elif self.type in self.TERTIARY_TYPES: 1457 self.kind = 'tertiary' 1458 elif self.type in self.END_TYPES: 1459 self.kind = 'end' 1460 else: 1461 raise ParseError, "unknown control markup: '%s'" % self.type 1462 scanner.release() 1463 1464 def subscan(self, scanner, primary): 1465 """Do a subscan for contained tokens.""" 1466 while True: 1467 token = scanner.one() 1468 if token is None: 1469 raise TransientParseError, \ 1470 "control '%s' needs more tokens" % primary 1471 if isinstance(token, ControlToken) and \ 1472 token.type in self.END_TYPES: 1473 if token.rest != primary: 1474 raise ParseError, \ 1475 "control must end with 'end %s'" % primary 1476 break 1477 self.subtokens.append(token) 1478 1479 def build(self, allowed=None): 1480 """Process the list of subtokens and divide it into a list of 1481 2-tuples, consisting of the dividing tokens and the list of 1482 subtokens that follow them. If allowed is specified, it will 1483 represent the list of the only secondary markup types which 1484 are allowed.""" 1485 if allowed is None: 1486 allowed = SECONDARY_TYPES 1487 result = [] 1488 latest = [] 1489 result.append((self, latest)) 1490 for subtoken in self.subtokens: 1491 if isinstance(subtoken, ControlToken) and \ 1492 subtoken.kind == 'secondary': 1493 if subtoken.type not in allowed: 1494 raise ParseError, \ 1495 "control unexpected secondary: '%s'" % subtoken.type 1496 latest = [] 1497 result.append((subtoken, latest)) 1498 else: 1499 latest.append(subtoken) 1500 return result 1501 1502 def run(self, interpreter, locals): 1503 interpreter.invoke('beforeControl', type=self.type, rest=self.rest, \ 1504 locals=locals) 1505 if self.type == 'if': 1506 info = self.build(['elif', 'else']) 1507 elseTokens = None 1508 if info[-1][0].type == 'else': 1509 elseTokens = info.pop()[1] 1510 for secondary, subtokens in info: 1511 if secondary.type not in ('if', 'elif'): 1512 raise ParseError, \ 1513 "control 'if' unexpected secondary: '%s'" % secondary.type 1514 if interpreter.evaluate(secondary.rest, locals): 1515 self.subrun(subtokens, interpreter, locals) 1516 break 1517 else: 1518 if elseTokens: 1519 self.subrun(elseTokens, interpreter, locals) 1520 elif self.type == 'for': 1521 sides = self.IN_RE.split(self.rest, 1) 1522 if len(sides) != 2: 1523 raise ParseError, "control expected 'for x in seq'" 1524 iterator, sequenceCode = sides 1525 info = self.build(['else']) 1526 elseTokens = None 1527 if info[-1][0].type == 'else': 1528 elseTokens = info.pop()[1] 1529 if len(info) != 1: 1530 raise ParseError, "control 'for' expects at most one 'else'" 1531 sequence = interpreter.evaluate(sequenceCode, locals) 1532 for element in sequence: 1533 try: 1534 interpreter.assign(iterator, element, locals) 1535 self.subrun(info[0][1], interpreter, locals) 1536 except ContinueFlow: 1537 continue 1538 except BreakFlow: 1539 break 1540 else: 1541 if elseTokens: 1542 self.subrun(elseTokens, interpreter, locals) 1543 elif self.type == 'while': 1544 testCode = self.rest 1545 info = self.build(['else']) 1546 elseTokens = None 1547 if info[-1][0].type == 'else': 1548 elseTokens = info.pop()[1] 1549 if len(info) != 1: 1550 raise ParseError, "control 'while' expects at most one 'else'" 1551 atLeastOnce = False 1552 while True: 1553 try: 1554 if not interpreter.evaluate(testCode, locals): 1555 break 1556 atLeastOnce = True 1557 self.subrun(info[0][1], interpreter, locals) 1558 except ContinueFlow: 1559 continue 1560 except BreakFlow: 1561 break 1562 if not atLeastOnce and elseTokens: 1563 self.subrun(elseTokens, interpreter, locals) 1564 elif self.type == 'try': 1565 info = self.build(['except', 'finally']) 1566 if len(info) == 1: 1567 raise ParseError, "control 'try' needs 'except' or 'finally'" 1568 type = info[-1][0].type 1569 if type == 'except': 1570 for secondary, _tokens in info[1:]: 1571 if secondary.type != 'except': 1572 raise ParseError, \ 1573 "control 'try' cannot have 'except' and 'finally'" 1574 else: 1575 assert type == 'finally' 1576 if len(info) != 2: 1577 raise ParseError, \ 1578 "control 'try' can only have one 'finally'" 1579 if type == 'except': 1580 try: 1581 self.subrun(info[0][1], interpreter, locals) 1582 except FlowError: 1583 raise 1584 except Exception, e: 1585 for secondary, tokens in info[1:]: 1586 exception, variable = interpreter.clause(secondary.rest) 1587 if variable is not None: 1588 interpreter.assign(variable, e) 1589 if isinstance(e, exception): 1590 self.subrun(tokens, interpreter, locals) 1591 break 1592 else: 1593 raise 1594 else: 1595 try: 1596 self.subrun(info[0][1], interpreter, locals) 1597 finally: 1598 self.subrun(info[1][1], interpreter, locals) 1599 elif self.type == 'continue': 1600 raise ContinueFlow, "control 'continue' without 'for', 'while'" 1601 elif self.type == 'break': 1602 raise BreakFlow, "control 'break' without 'for', 'while'" 1603 elif self.type == 'def': 1604 signature = self.rest 1605 definition = self.substring() 1606 code = 'def %s:\n' \ 1607 ' r"""%s"""\n' \ 1608 ' return %s.expand(r"""%s""", locals())\n' % \ 1609 (signature, definition, interpreter.pseudo, definition) 1610 interpreter.execute(code, locals) 1611 elif self.type == 'end': 1612 raise ParseError, "control 'end' requires primary markup" 1613 else: 1614 raise ParseError, \ 1615 "control '%s' cannot be at this level" % self.type 1616 interpreter.invoke('afterControl') 1617 1618 def subrun(self, tokens, interpreter, locals): 1619 """Execute a sequence of tokens.""" 1620 for token in tokens: 1621 token.run(interpreter, locals) 1622 1623 def substring(self): 1624 return string.join(map(str, self.subtokens), '') 1625 1626 def string(self): 1627 if self.kind == 'primary': 1628 return '%s[%s]%s%s[end %s]' % \ 1629 (self.prefix, self.contents, self.substring(), \ 1630 self.prefix, self.type) 1631 else: 1632 return '%s[%s]' % (self.prefix, self.contents) 1633 1634 1635class Scanner: 1636 1637 """A scanner holds a buffer for lookahead parsing and has the 1638 ability to scan for special symbols and indicators in that 1639 buffer.""" 1640 1641 # This is the token mapping table that maps first characters to 1642 # token classes. 1643 TOKEN_MAP = [ 1644 (None, PrefixToken), 1645 (' \t\v\r\n', WhitespaceToken), 1646 (')]}', LiteralToken), 1647 ('\\', EscapeToken), 1648 ('#', CommentToken), 1649 ('?', ContextNameToken), 1650 ('!', ContextLineToken), 1651 ('%', SignificatorToken), 1652 ('(', ExpressionToken), 1653 (IDENTIFIER_FIRST_CHARS, SimpleExpressionToken), 1654 ('\'\"', StringLiteralToken), 1655 ('`', ReprToken), 1656 (':', InPlaceToken), 1657 ('[', ControlToken), 1658 ('{', StatementToken), 1659 ('<', CustomToken), 1660 ] 1661 1662 def __init__(self, prefix, data=''): 1663 self.prefix = prefix 1664 self.pointer = 0 1665 self.buffer = data 1666 self.lock = 0 1667 1668 def __nonzero__(self): return self.pointer < len(self.buffer) 1669 def __len__(self): return len(self.buffer) - self.pointer 1670 def __getitem__(self, index): return self.buffer[self.pointer + index] 1671 1672 def __getslice__(self, start, stop): 1673 if stop > len(self): 1674 stop = len(self) 1675 return self.buffer[self.pointer + start:self.pointer + stop] 1676 1677 def advance(self, count=1): 1678 """Advance the pointer count characters.""" 1679 self.pointer = self.pointer + count 1680 1681 def retreat(self, count=1): 1682 self.pointer = self.pointer - count 1683 if self.pointer < 0: 1684 raise ParseError, "can't retreat back over synced out chars" 1685 1686 def set(self, data): 1687 """Start the scanner digesting a new batch of data; start the pointer 1688 over from scratch.""" 1689 self.pointer = 0 1690 self.buffer = data 1691 1692 def feed(self, data): 1693 """Feed some more data to the scanner.""" 1694 self.buffer = self.buffer + data 1695 1696 def chop(self, count=None, slop=0): 1697 """Chop the first count + slop characters off the front, and return 1698 the first count. If count is not specified, then return 1699 everything.""" 1700 if count is None: 1701 assert slop == 0 1702 count = len(self) 1703 if count > len(self): 1704 raise TransientParseError, "not enough data to read" 1705 result = self[:count] 1706 self.advance(count + slop) 1707 return result 1708 1709 def acquire(self): 1710 """Lock the scanner so it doesn't destroy data on sync.""" 1711 self.lock = self.lock + 1 1712 1713 def release(self): 1714 """Unlock the scanner.""" 1715 self.lock = self.lock - 1 1716 1717 def sync(self): 1718 """Sync up the buffer with the read head.""" 1719 if self.lock == 0 and self.pointer != 0: 1720 self.buffer = self.buffer[self.pointer:] 1721 self.pointer = 0 1722 1723 def unsync(self): 1724 """Undo changes; reset the read head.""" 1725 if self.pointer != 0: 1726 self.lock = 0 1727 self.pointer = 0 1728 1729 def rest(self): 1730 """Get the remainder of the buffer.""" 1731 return self[:] 1732 1733 def read(self, i=0, count=1): 1734 """Read count chars starting from i; raise a transient error if 1735 there aren't enough characters remaining.""" 1736 if len(self) < i + count: 1737 raise TransientParseError, "need more data to read" 1738 else: 1739 return self[i:i + count] 1740 1741 def check(self, i, archetype=None): 1742 """Scan for the next single or triple quote, with the specified 1743 archetype. Return the found quote or None.""" 1744 quote = None 1745 if self[i] in '\'\"': 1746 quote = self[i] 1747 if len(self) - i < 3: 1748 for j in range(i, len(self)): 1749 if self[i] == quote: 1750 return quote 1751 else: 1752 raise TransientParseError, "need to scan for rest of quote" 1753 if self[i + 1] == self[i + 2] == quote: 1754 quote = quote * 3 1755 if quote is not None: 1756 if archetype is None: 1757 return quote 1758 else: 1759 if archetype == quote: 1760 return quote 1761 elif len(archetype) < len(quote) and archetype[0] == quote[0]: 1762 return archetype 1763 else: 1764 return None 1765 else: 1766 return None 1767 1768 def find(self, sub, start=0, end=None): 1769 """Find the next occurrence of the character, or return -1.""" 1770 if end is not None: 1771 return string.find(self.rest(), sub, start, end) 1772 else: 1773 return string.find(self.rest(), sub, start) 1774 1775 def last(self, char, start=0, end=None): 1776 """Find the first character that is _not_ the specified character.""" 1777 if end is None: 1778 end = len(self) 1779 i = start 1780 while i < end: 1781 if self[i] != char: 1782 return i 1783 i = i + 1 1784 else: 1785 raise TransientParseError, "expecting other than %s" % char 1786 1787 def next(self, target, start=0, end=None, mandatory=False): 1788 """Scan for the next occurrence of one of the characters in 1789 the target string; optionally, make the scan mandatory.""" 1790 if mandatory: 1791 assert end is not None 1792 quote = None 1793 if end is None: 1794 end = len(self) 1795 i = start 1796 while i < end: 1797 newQuote = self.check(i, quote) 1798 if newQuote: 1799 if newQuote == quote: 1800 quote = None 1801 else: 1802 quote = newQuote 1803 i = i + len(newQuote) 1804 else: 1805 c = self[i] 1806 if quote: 1807 if c == '\\': 1808 i = i + 1 1809 else: 1810 if c in target: 1811 return i 1812 i = i + 1 1813 else: 1814 if mandatory: 1815 raise ParseError, "expecting %s, not found" % target 1816 else: 1817 raise TransientParseError, "expecting ending character" 1818 1819 def quote(self, start=0, end=None, mandatory=False): 1820 """Scan for the end of the next quote.""" 1821 assert self[start] in '\'\"' 1822 quote = self.check(start) 1823 if end is None: 1824 end = len(self) 1825 i = start + len(quote) 1826 while i < end: 1827 newQuote = self.check(i, quote) 1828 if newQuote: 1829 i = i + len(newQuote) 1830 if newQuote == quote: 1831 return i 1832 else: 1833 c = self[i] 1834 if c == '\\': 1835 i = i + 1 1836 i = i + 1 1837 else: 1838 if mandatory: 1839 raise ParseError, "expecting end of string literal" 1840 else: 1841 raise TransientParseError, "expecting end of string literal" 1842 1843 def nested(self, enter, exit, start=0, end=None): 1844 """Scan from i for an ending sequence, respecting entries and exits 1845 only.""" 1846 depth = 0 1847 if end is None: 1848 end = len(self) 1849 i = start 1850 while i < end: 1851 c = self[i] 1852 if c == enter: 1853 depth = depth + 1 1854 elif c == exit: 1855 depth = depth - 1 1856 if depth < 0: 1857 return i 1858 i = i + 1 1859 else: 1860 raise TransientParseError, "expecting end of complex expression" 1861 1862 def complex(self, enter, exit, start=0, end=None, skip=None): 1863 """Scan from i for an ending sequence, respecting quotes, 1864 entries and exits.""" 1865 quote = None 1866 depth = 0 1867 if end is None: 1868 end = len(self) 1869 last = None 1870 i = start 1871 while i < end: 1872 newQuote = self.check(i, quote) 1873 if newQuote: 1874 if newQuote == quote: 1875 quote = None 1876 else: 1877 quote = newQuote 1878 i = i + len(newQuote) 1879 else: 1880 c = self[i] 1881 if quote: 1882 if c == '\\': 1883 i = i + 1 1884 else: 1885 if skip is None or last != skip: 1886 if c == enter: 1887 depth = depth + 1 1888 elif c == exit: 1889 depth = depth - 1 1890 if depth < 0: 1891 return i 1892 last = c 1893 i = i + 1 1894 else: 1895 raise TransientParseError, "expecting end of complex expression" 1896 1897 def word(self, start=0): 1898 """Scan from i for a simple word.""" 1899 length = len(self) 1900 i = start 1901 while i < length: 1902 if not self[i] in IDENTIFIER_CHARS: 1903 return i 1904 i = i + 1 1905 else: 1906 raise TransientParseError, "expecting end of word" 1907 1908 def phrase(self, start=0): 1909 """Scan from i for a phrase (e.g., 'word', 'f(a, b, c)', 'a[i]', or 1910 combinations like 'x[i](a)'.""" 1911 # Find the word. 1912 i = self.word(start) 1913 while i < len(self) and self[i] in '([{': 1914 enter = self[i] 1915 if enter == '{': 1916 raise ParseError, "curly braces can't open simple expressions" 1917 exit = ENDING_CHARS[enter] 1918 i = self.complex(enter, exit, i + 1) + 1 1919 return i 1920 1921 def simple(self, start=0): 1922 """Scan from i for a simple expression, which consists of one 1923 more phrases separated by dots.""" 1924 i = self.phrase(start) 1925 length = len(self) 1926 while i < length and self[i] == '.': 1927 i = self.phrase(i) 1928 # Make sure we don't end with a trailing dot. 1929 while i > 0 and self[i - 1] == '.': 1930 i = i - 1 1931 return i 1932 1933 def one(self): 1934 """Parse and return one token, or None if the scanner is empty.""" 1935 if not self: 1936 return None 1937 if not self.prefix: 1938 loc = -1 1939 else: 1940 loc = self.find(self.prefix) 1941 if loc < 0: 1942 # If there's no prefix in the buffer, then set the location to 1943 # the end so the whole thing gets processed. 1944 loc = len(self) 1945 if loc == 0: 1946 # If there's a prefix at the beginning of the buffer, process 1947 # an expansion. 1948 prefix = self.chop(1) 1949 assert prefix == self.prefix 1950 first = self.chop(1) 1951 if first == self.prefix: 1952 first = None 1953 for firsts, factory in self.TOKEN_MAP: 1954 if firsts is None: 1955 if first is None: 1956 break 1957 elif first in firsts: 1958 break 1959 else: 1960 raise ParseError, "unknown markup: %s%s" % (self.prefix, first) 1961 token = factory(self.prefix, first) 1962 try: 1963 token.scan(self) 1964 except TransientParseError: 1965 # If a transient parse error occurs, reset the buffer pointer 1966 # so we can (conceivably) try again later. 1967 self.unsync() 1968 raise 1969 else: 1970 # Process everything up to loc as a null token. 1971 data = self.chop(loc) 1972 token = NullToken(data) 1973 self.sync() 1974 return token 1975 1976 1977class Interpreter: 1978 1979 """An interpreter can process chunks of EmPy code.""" 1980 1981 # Constants. 1982 1983 VERSION = __version__ 1984 SIGNIFICATOR_RE_SUFFIX = SIGNIFICATOR_RE_SUFFIX 1985 SIGNIFICATOR_RE_STRING = None 1986 1987 # Types. 1988 1989 Interpreter = None # define this below to prevent a circular reference 1990 Hook = Hook # DEPRECATED 1991 Filter = Filter # DEPRECATED 1992 NullFilter = NullFilter # DEPRECATED 1993 FunctionFilter = FunctionFilter # DEPRECATED 1994 StringFilter = StringFilter # DEPRECATED 1995 BufferedFilter = BufferedFilter # DEPRECATED 1996 SizeBufferedFilter = SizeBufferedFilter # DEPRECATED 1997 LineBufferedFilter = LineBufferedFilter # DEPRECATED 1998 MaximallyBufferedFilter = MaximallyBufferedFilter # DEPRECATED 1999 2000 # Tables. 2001 2002 ESCAPE_CODES = {0x00: '0', 0x07: 'a', 0x08: 'b', 0x1b: 'e', 0x0c: 'f', \ 2003 0x7f: 'h', 0x0a: 'n', 0x0d: 'r', 0x09: 't', 0x0b: 'v', \ 2004 0x04: 'z'} 2005 2006 ASSIGN_TOKEN_RE = re.compile(r"[_a-zA-Z][_a-zA-Z0-9]*|\(|\)|,") 2007 2008 DEFAULT_OPTIONS = {BANGPATH_OPT: True, 2009 BUFFERED_OPT: False, 2010 RAW_OPT: False, 2011 EXIT_OPT: True, 2012 FLATTEN_OPT: False, 2013 OVERRIDE_OPT: True, 2014 CALLBACK_OPT: False} 2015 2016 _wasProxyInstalled = False # was a proxy installed? 2017 2018 # Construction, initialization, destruction. 2019 2020 def __init__(self, output=None, argv=None, prefix=DEFAULT_PREFIX, \ 2021 pseudo=None, options=None, globals=None, hooks=None): 2022 self.interpreter = self # DEPRECATED 2023 # Set up the stream. 2024 if output is None: 2025 output = UncloseableFile(sys.__stdout__) 2026 self.output = output 2027 self.prefix = prefix 2028 if pseudo is None: 2029 pseudo = DEFAULT_PSEUDOMODULE_NAME 2030 self.pseudo = pseudo 2031 if argv is None: 2032 argv = [DEFAULT_SCRIPT_NAME] 2033 self.argv = argv 2034 self.args = argv[1:] 2035 if options is None: 2036 options = {} 2037 self.options = options 2038 # Initialize any hooks. 2039 self.hooksEnabled = None # special sentinel meaning "false until added" 2040 self.hooks = [] 2041 if hooks is None: 2042 hooks = [] 2043 for hook in hooks: 2044 self.register(hook) 2045 # Initialize callback. 2046 self.callback = None 2047 # Finalizers. 2048 self.finals = [] 2049 # The interpreter stacks. 2050 self.contexts = Stack() 2051 self.streams = Stack() 2052 # Now set up the globals. 2053 self.globals = globals 2054 self.fix() 2055 self.history = Stack() 2056 # Install a proxy stdout if one hasn't been already. 2057 self.installProxy() 2058 # Finally, reset the state of all the stacks. 2059 self.reset() 2060 # Okay, now flatten the namespaces if that option has been set. 2061 if self.options.get(FLATTEN_OPT, False): 2062 self.flatten() 2063 # Set up old pseudomodule attributes. 2064 if prefix is None: 2065 self.SIGNIFICATOR_RE_STRING = None 2066 else: 2067 self.SIGNIFICATOR_RE_STRING = prefix + self.SIGNIFICATOR_RE_SUFFIX 2068 self.Interpreter = self.__class__ 2069 # Done. Now declare that we've started up. 2070 self.invoke('atStartup') 2071 2072 def __del__(self): 2073 self.shutdown() 2074 2075 def __repr__(self): 2076 return '<%s pseudomodule/interpreter at 0x%x>' % \ 2077 (self.pseudo, id(self)) 2078 2079 def ready(self): 2080 """Declare the interpreter ready for normal operations.""" 2081 self.invoke('atReady') 2082 2083 def fix(self): 2084 """Reset the globals, stamping in the pseudomodule.""" 2085 if self.globals is None: 2086 self.globals = {} 2087 # Make sure that there is no collision between two interpreters' 2088 # globals. 2089 if self.globals.has_key(self.pseudo): 2090 if self.globals[self.pseudo] is not self: 2091 raise Error, "interpreter globals collision" 2092 self.globals[self.pseudo] = self 2093 2094 def unfix(self): 2095 """Remove the pseudomodule (if present) from the globals.""" 2096 UNWANTED_KEYS = [self.pseudo, '__builtins__'] 2097 for unwantedKey in UNWANTED_KEYS: 2098 if self.globals.has_key(unwantedKey): 2099 del self.globals[unwantedKey] 2100 2101 def update(self, other): 2102 """Update the current globals dictionary with another dictionary.""" 2103 self.globals.update(other) 2104 self.fix() 2105 2106 def clear(self): 2107 """Clear out the globals dictionary with a brand new one.""" 2108 self.globals = {} 2109 self.fix() 2110 2111 def save(self, deep=True): 2112 if deep: 2113 copyMethod = copy.deepcopy 2114 else: 2115 copyMethod = copy.copy 2116 """Save a copy of the current globals on the history stack.""" 2117 self.unfix() 2118 self.history.push(copyMethod(self.globals)) 2119 self.fix() 2120 2121 def restore(self, destructive=True): 2122 """Restore the topmost historic globals.""" 2123 if destructive: 2124 fetchMethod = self.history.pop 2125 else: 2126 fetchMethod = self.history.top 2127 self.unfix() 2128 self.globals = fetchMethod() 2129 self.fix() 2130 2131 def shutdown(self): 2132 """Declare this interpreting session over; close the stream file 2133 object. This method is idempotent.""" 2134 if self.streams is not None: 2135 try: 2136 self.finalize() 2137 self.invoke('atShutdown') 2138 while self.streams: 2139 stream = self.streams.pop() 2140 stream.close() 2141 finally: 2142 self.streams = None 2143 2144 def ok(self): 2145 """Is the interpreter still active?""" 2146 return self.streams is not None 2147 2148 # Writeable file-like methods. 2149 2150 def write(self, data): 2151 self.stream().write(data) 2152 2153 def writelines(self, stuff): 2154 self.stream().writelines(stuff) 2155 2156 def flush(self): 2157 self.stream().flush() 2158 2159 def close(self): 2160 self.shutdown() 2161 2162 # Stack-related activity. 2163 2164 def context(self): 2165 return self.contexts.top() 2166 2167 def stream(self): 2168 return self.streams.top() 2169 2170 def reset(self): 2171 self.contexts.purge() 2172 self.streams.purge() 2173 self.streams.push(Stream(self.output)) 2174 if self.options.get(OVERRIDE_OPT, True): 2175 sys.stdout.clear(self) 2176 2177 def push(self): 2178 if self.options.get(OVERRIDE_OPT, True): 2179 sys.stdout.push(self) 2180 2181 def pop(self): 2182 if self.options.get(OVERRIDE_OPT, True): 2183 sys.stdout.pop(self) 2184 2185 # Higher-level operations. 2186 2187 def include(self, fileOrFilename, locals=None): 2188 """Do an include pass on a file or filename.""" 2189 if type(fileOrFilename) is types.StringType: 2190 # Either it's a string representing a filename ... 2191 filename = fileOrFilename 2192 name = filename 2193 file = theSubsystem.open(filename, 'r') 2194 else: 2195 # ... or a file object. 2196 file = fileOrFilename 2197 name = "<%s>" % str(file.__class__) 2198 self.invoke('beforeInclude', name=name, file=file, locals=locals) 2199 self.file(file, name, locals) 2200 self.invoke('afterInclude') 2201 2202 def expand(self, data, locals=None): 2203 """Do an explicit expansion on a subordinate stream.""" 2204 outFile = StringIO.StringIO() 2205 stream = Stream(outFile) 2206 self.invoke('beforeExpand', string=data, locals=locals) 2207 self.streams.push(stream) 2208 try: 2209 self.string(data, '<expand>', locals) 2210 stream.flush() 2211 expansion = outFile.getvalue() 2212 self.invoke('afterExpand', result=expansion) 2213 return expansion 2214 finally: 2215 self.streams.pop() 2216 2217 def quote(self, data): 2218 """Quote the given string so that if it were expanded it would 2219 evaluate to the original.""" 2220 self.invoke('beforeQuote', string=data) 2221 scanner = Scanner(self.prefix, data) 2222 result = [] 2223 i = 0 2224 try: 2225 j = scanner.next(self.prefix, i) 2226 result.append(data[i:j]) 2227 result.append(self.prefix * 2) 2228 i = j + 1 2229 except TransientParseError: 2230 pass 2231 result.append(data[i:]) 2232 result = string.join(result, '') 2233 self.invoke('afterQuote', result=result) 2234 return result 2235 2236 def escape(self, data, more=''): 2237 """Escape a string so that nonprintable characters are replaced 2238 with compatible EmPy expansions.""" 2239 self.invoke('beforeEscape', string=data, more=more) 2240 result = [] 2241 for char in data: 2242 if char < ' ' or char > '~': 2243 charOrd = ord(char) 2244 if Interpreter.ESCAPE_CODES.has_key(charOrd): 2245 result.append(self.prefix + '\\' + \ 2246 Interpreter.ESCAPE_CODES[charOrd]) 2247 else: 2248 result.append(self.prefix + '\\x%02x' % charOrd) 2249 elif char in more: 2250 result.append(self.prefix + '\\' + char) 2251 else: 2252 result.append(char) 2253 result = string.join(result, '') 2254 self.invoke('afterEscape', result=result) 2255 return result 2256 2257 # Processing. 2258 2259 def wrap(self, callable, args): 2260 """Wrap around an application of a callable and handle errors. 2261 Return whether no error occurred.""" 2262 try: 2263 apply(callable, args) 2264 self.reset() 2265 return True 2266 except KeyboardInterrupt, e: 2267 # Handle keyboard interrupts specially: we should always exit 2268 # from these. 2269 self.fail(e, True) 2270 except Exception, e: 2271 # A standard exception (other than a keyboard interrupt). 2272 self.fail(e) 2273 except: 2274 # If we get here, then either it's an exception not derived from 2275 # Exception or it's a string exception, so get the error type 2276 # from the sys module. 2277 e = sys.exc_type 2278 self.fail(e) 2279 # An error occurred if we leak through to here, so do cleanup. 2280 self.reset() 2281 return False 2282 2283 def interact(self): 2284 """Perform interaction.""" 2285 self.invoke('atInteract') 2286 done = False 2287 while not done: 2288 result = self.wrap(self.file, (sys.stdin, '<interact>')) 2289 if self.options.get(EXIT_OPT, True): 2290 done = True 2291 else: 2292 if result: 2293 done = True 2294 else: 2295 self.reset() 2296 2297 def fail(self, error, fatal=False): 2298 """Handle an actual error that occurred.""" 2299 if self.options.get(BUFFERED_OPT, False): 2300 try: 2301 self.output.abort() 2302 except AttributeError: 2303 # If the output file object doesn't have an abort method, 2304 # something got mismatched, but it's too late to do 2305 # anything about it now anyway, so just ignore it. 2306 pass 2307 meta = self.meta(error) 2308 self.handle(meta) 2309 if self.options.get(RAW_OPT, False): 2310 raise 2311 if fatal or self.options.get(EXIT_OPT, True): 2312 sys.exit(FAILURE_CODE) 2313 2314 def file(self, file, name='<file>', locals=None): 2315 """Parse the entire contents of a file-like object, line by line.""" 2316 context = Context(name) 2317 self.contexts.push(context) 2318 self.invoke('beforeFile', name=name, file=file, locals=locals) 2319 scanner = Scanner(self.prefix) 2320 first = True 2321 done = False 2322 while not done: 2323 self.context().bump() 2324 line = file.readline() 2325 if first: 2326 if self.options.get(BANGPATH_OPT, True) and self.prefix: 2327 # Replace a bangpath at the beginning of the first line 2328 # with an EmPy comment. 2329 if string.find(line, BANGPATH) == 0: 2330 line = self.prefix + '#' + line[2:] 2331 first = False 2332 if line: 2333 scanner.feed(line) 2334 else: 2335 done = True 2336 self.safe(scanner, done, locals) 2337 self.invoke('afterFile') 2338 self.contexts.pop() 2339 2340 def binary(self, file, name='<binary>', chunkSize=0, locals=None): 2341 """Parse the entire contents of a file-like object, in chunks.""" 2342 if chunkSize <= 0: 2343 chunkSize = DEFAULT_CHUNK_SIZE 2344 context = Context(name, units='bytes') 2345 self.contexts.push(context) 2346 self.invoke('beforeBinary', name=name, file=file, \ 2347 chunkSize=chunkSize, locals=locals) 2348 scanner = Scanner(self.prefix) 2349 done = False 2350 while not done: 2351 chunk = file.read(chunkSize) 2352 if chunk: 2353 scanner.feed(chunk) 2354 else: 2355 done = True 2356 self.safe(scanner, done, locals) 2357 self.context().bump(len(chunk)) 2358 self.invoke('afterBinary') 2359 self.contexts.pop() 2360 2361 def string(self, data, name='<string>', locals=None): 2362 """Parse a string.""" 2363 context = Context(name) 2364 self.contexts.push(context) 2365 self.invoke('beforeString', name=name, string=data, locals=locals) 2366 context.bump() 2367 scanner = Scanner(self.prefix, data) 2368 self.safe(scanner, True, locals) 2369 self.invoke('afterString') 2370 self.contexts.pop() 2371 2372 def safe(self, scanner, final=False, locals=None): 2373 """Do a protected parse. Catch transient parse errors; if 2374 final is true, then make a final pass with a terminator, 2375 otherwise ignore the transient parse error (more data is 2376 pending).""" 2377 try: 2378 self.parse(scanner, locals) 2379 except TransientParseError: 2380 if final: 2381 # If the buffer doesn't end with a newline, try tacking on 2382 # a dummy terminator. 2383 buffer = scanner.rest() 2384 if buffer and buffer[-1] != '\n': 2385 scanner.feed(self.prefix + '\n') 2386 # A TransientParseError thrown from here is a real parse 2387 # error. 2388 self.parse(scanner, locals) 2389 2390 def parse(self, scanner, locals=None): 2391 """Parse and run as much from this scanner as possible.""" 2392 self.invoke('atParse', scanner=scanner, locals=locals) 2393 while True: 2394 token = scanner.one() 2395 if token is None: 2396 break 2397 self.invoke('atToken', token=token) 2398 token.run(self, locals) 2399 2400 # Medium-level evaluation and execution. 2401 2402 def tokenize(self, name): 2403 """Take an lvalue string and return a name or a (possibly recursive) 2404 list of names.""" 2405 result = [] 2406 stack = [result] 2407 for garbage in self.ASSIGN_TOKEN_RE.split(name): 2408 garbage = string.strip(garbage) 2409 if garbage: 2410 raise ParseError, "unexpected assignment token: '%s'" % garbage 2411 tokens = self.ASSIGN_TOKEN_RE.findall(name) 2412 # While processing, put a None token at the start of any list in which 2413 # commas actually appear. 2414 for token in tokens: 2415 if token == '(': 2416 stack.append([]) 2417 elif token == ')': 2418 top = stack.pop() 2419 if len(top) == 1: 2420 top = top[0] # no None token means that it's not a 1-tuple 2421 elif top[0] is None: 2422 del top[0] # remove the None token for real tuples 2423 stack[-1].append(top) 2424 elif token == ',': 2425 if len(stack[-1]) == 1: 2426 stack[-1].insert(0, None) 2427 else: 2428 stack[-1].append(token) 2429 # If it's a 1-tuple at the top level, turn it into a real subsequence. 2430 if result and result[0] is None: 2431 result = [result[1:]] 2432 if len(result) == 1: 2433 return result[0] 2434 else: 2435 return result 2436 2437 def significate(self, key, value=None, locals=None): 2438 """Declare a significator.""" 2439 self.invoke('beforeSignificate', key=key, value=value, locals=locals) 2440 name = '__%s__' % key 2441 self.atomic(name, value, locals) 2442 self.invoke('afterSignificate') 2443 2444 def atomic(self, name, value, locals=None): 2445 """Do an atomic assignment.""" 2446 self.invoke('beforeAtomic', name=name, value=value, locals=locals) 2447 if locals is None: 2448 self.globals[name] = value 2449 else: 2450 locals[name] = value 2451 self.invoke('afterAtomic') 2452 2453 def multi(self, names, values, locals=None): 2454 """Do a (potentially recursive) assignment.""" 2455 self.invoke('beforeMulti', names=names, values=values, locals=locals) 2456 # No zip in 1.5, so we have to do it manually. 2457 i = 0 2458 try: 2459 values = tuple(values) 2460 except TypeError: 2461 raise TypeError, "unpack non-sequence" 2462 if len(names) != len(values): 2463 raise ValueError, "unpack tuple of wrong size" 2464 for i in range(len(names)): 2465 name = names[i] 2466 if type(name) is types.StringType: 2467 self.atomic(name, values[i], locals) 2468 else: 2469 self.multi(name, values[i], locals) 2470 self.invoke('afterMulti') 2471 2472 def assign(self, name, value, locals=None): 2473 """Do a potentially complex (including tuple unpacking) assignment.""" 2474 left = self.tokenize(name) 2475 # The return value of tokenize can either be a string or a list of 2476 # (lists of) strings. 2477 if type(left) is types.StringType: 2478 self.atomic(left, value, locals) 2479 else: 2480 self.multi(left, value, locals) 2481 2482 def import_(self, name, locals=None): 2483 """Do an import.""" 2484 self.invoke('beforeImport', name=name, locals=locals) 2485 self.execute('import %s' % name, locals) 2486 self.invoke('afterImport') 2487 2488 def clause(self, catch, locals=None): 2489 """Given the string representation of an except clause, turn it into 2490 a 2-tuple consisting of the class name, and either a variable name 2491 or None.""" 2492 self.invoke('beforeClause', catch=catch, locals=locals) 2493 if catch is None: 2494 exceptionCode, variable = None, None 2495 elif string.find(catch, ',') >= 0: 2496 exceptionCode, variable = string.split(string.strip(catch), ',', 1) 2497 variable = string.strip(variable) 2498 else: 2499 exceptionCode, variable = string.strip(catch), None 2500 if not exceptionCode: 2501 exception = Exception 2502 else: 2503 exception = self.evaluate(exceptionCode, locals) 2504 self.invoke('afterClause', exception=exception, variable=variable) 2505 return exception, variable 2506 2507 def serialize(self, expression, locals=None): 2508 """Do an expansion, involving evaluating an expression, then 2509 converting it to a string and writing that string to the 2510 output if the evaluation is not None.""" 2511 self.invoke('beforeSerialize', expression=expression, locals=locals) 2512 result = self.evaluate(expression, locals) 2513 if result is not None: 2514 self.write(str(result)) 2515 self.invoke('afterSerialize') 2516 2517 def defined(self, name, locals=None): 2518 """Return a Boolean indicating whether or not the name is 2519 defined either in the locals or the globals.""" 2520 self.invoke('beforeDefined', name=name, local=local) 2521 if locals is not None: 2522 if locals.has_key(name): 2523 result = True 2524 else: 2525 result = False 2526 elif self.globals.has_key(name): 2527 result = True 2528 else: 2529 result = False 2530 self.invoke('afterDefined', result=result) 2531 2532 def literal(self, text): 2533 """Process a string literal.""" 2534 self.invoke('beforeLiteral', text=text) 2535 self.serialize(text) 2536 self.invoke('afterLiteral') 2537 2538 # Low-level evaluation and execution. 2539 2540 def evaluate(self, expression, locals=None): 2541 """Evaluate an expression.""" 2542 if expression in ('1', 'True'): return True 2543 if expression in ('0', 'False'): return False 2544 self.push() 2545 try: 2546 self.invoke('beforeEvaluate', \ 2547 expression=expression, locals=locals) 2548 if locals is not None: 2549 result = eval(expression, self.globals, locals) 2550 else: 2551 result = eval(expression, self.globals) 2552 self.invoke('afterEvaluate', result=result) 2553 return result 2554 finally: 2555 self.pop() 2556 2557 def execute(self, statements, locals=None): 2558 """Execute a statement.""" 2559 # If there are any carriage returns (as opposed to linefeeds/newlines) 2560 # in the statements code, then remove them. Even on DOS/Windows 2561 # platforms, 2562 if string.find(statements, '\r') >= 0: 2563 statements = string.replace(statements, '\r', '') 2564 # If there are no newlines in the statements code, then strip any 2565 # leading or trailing whitespace. 2566 if string.find(statements, '\n') < 0: 2567 statements = string.strip(statements) 2568 self.push() 2569 try: 2570 self.invoke('beforeExecute', \ 2571 statements=statements, locals=locals) 2572 if locals is not None: 2573 exec statements in self.globals, locals 2574 else: 2575 exec statements in self.globals 2576 self.invoke('afterExecute') 2577 finally: 2578 self.pop() 2579 2580 def single(self, source, locals=None): 2581 """Execute an expression or statement, just as if it were 2582 entered into the Python interactive interpreter.""" 2583 self.push() 2584 try: 2585 self.invoke('beforeSingle', \ 2586 source=source, locals=locals) 2587 code = compile(source, '<single>', 'single') 2588 if locals is not None: 2589 exec code in self.globals, locals 2590 else: 2591 exec code in self.globals 2592 self.invoke('afterSingle') 2593 finally: 2594 self.pop() 2595 2596 # Hooks. 2597 2598 def register(self, hook, prepend=False): 2599 """Register the provided hook.""" 2600 hook.register(self) 2601 if self.hooksEnabled is None: 2602 # A special optimization so that hooks can be effectively 2603 # disabled until one is added or they are explicitly turned on. 2604 self.hooksEnabled = True 2605 if prepend: 2606 self.hooks.insert(0, hook) 2607 else: 2608 self.hooks.append(hook) 2609 2610 def deregister(self, hook): 2611 """Remove an already registered hook.""" 2612 hook.deregister(self) 2613 self.hooks.remove(hook) 2614 2615 def invoke(self, _name, **keywords): 2616 """Invoke the hook(s) associated with the hook name, should they 2617 exist.""" 2618 if self.hooksEnabled: 2619 for hook in self.hooks: 2620 hook.push() 2621 try: 2622 method = getattr(hook, _name) 2623 apply(method, (), keywords) 2624 finally: 2625 hook.pop() 2626 2627 def finalize(self): 2628 """Execute any remaining final routines.""" 2629 self.push() 2630 self.invoke('atFinalize') 2631 try: 2632 # Pop them off one at a time so they get executed in reverse 2633 # order and we remove them as they're executed in case something 2634 # bad happens. 2635 while self.finals: 2636 final = self.finals.pop() 2637 final() 2638 finally: 2639 self.pop() 2640 2641 # Error handling. 2642 2643 def meta(self, exc=None): 2644 """Construct a MetaError for the interpreter's current state.""" 2645 return MetaError(self.contexts.clone(), exc) 2646 2647 def handle(self, meta): 2648 """Handle a MetaError.""" 2649 first = True 2650 self.invoke('atHandle', meta=meta) 2651 for context in meta.contexts: 2652 if first: 2653 if meta.exc is not None: 2654 desc = "error: %s: %s" % (meta.exc.__class__, meta.exc) 2655 else: 2656 desc = "error" 2657 else: 2658 desc = "from this context" 2659 first = False 2660 sys.stderr.write('%s: %s\n' % (context, desc)) 2661 2662 def installProxy(self): 2663 """Install a proxy if necessary.""" 2664 # Unfortunately, there's no surefire way to make sure that installing 2665 # a sys.stdout proxy is idempotent, what with different interpreters 2666 # running from different modules. The best we can do here is to try 2667 # manipulating the proxy's test function ... 2668 try: 2669 sys.stdout._testProxy() 2670 except AttributeError: 2671 # ... if the current stdout object doesn't have one, then check 2672 # to see if we think _this_ particularly Interpreter class has 2673 # installed it before ... 2674 if Interpreter._wasProxyInstalled: 2675 # ... and if so, we have a proxy problem. 2676 raise Error, "interpreter stdout proxy lost" 2677 else: 2678 # Otherwise, install the proxy and set the flag. 2679 sys.stdout = ProxyFile(sys.stdout) 2680 Interpreter._wasProxyInstalled = True 2681 2682 # 2683 # Pseudomodule routines. 2684 # 2685 2686 # Identification. 2687 2688 def identify(self): 2689 """Identify the topmost context with a 2-tuple of the name and 2690 line number.""" 2691 return self.context().identify() 2692 2693 def atExit(self, callable): 2694 """Register a function to be called at exit.""" 2695 self.finals.append(callable) 2696 2697 # Context manipulation. 2698 2699 def pushContext(self, name='<unnamed>', line=0): 2700 """Create a new context and push it.""" 2701 self.contexts.push(Context(name, line)) 2702 2703 def popContext(self): 2704 """Pop the top context.""" 2705 self.contexts.pop() 2706 2707 def setContextName(self, name): 2708 """Set the name of the topmost context.""" 2709 context = self.context() 2710 context.name = name 2711 2712 def setContextLine(self, line): 2713 """Set the name of the topmost context.""" 2714 context = self.context() 2715 context.line = line 2716 2717 setName = setContextName # DEPRECATED 2718 setLine = setContextLine # DEPRECATED 2719 2720 # Globals manipulation. 2721 2722 def getGlobals(self): 2723 """Retrieve the globals.""" 2724 return self.globals 2725 2726 def setGlobals(self, globals): 2727 """Set the globals to the specified dictionary.""" 2728 self.globals = globals 2729 self.fix() 2730 2731 def updateGlobals(self, otherGlobals): 2732 """Merge another mapping object into this interpreter's globals.""" 2733 self.update(otherGlobals) 2734 2735 def clearGlobals(self): 2736 """Clear out the globals with a brand new dictionary.""" 2737 self.clear() 2738 2739 def saveGlobals(self, deep=True): 2740 """Save a copy of the globals off onto the history stack.""" 2741 self.save(deep) 2742 2743 def restoreGlobals(self, destructive=True): 2744 """Restore the most recently saved copy of the globals.""" 2745 self.restore(destructive) 2746 2747 # Hook support. 2748 2749 def areHooksEnabled(self): 2750 """Return whether or not hooks are presently enabled.""" 2751 if self.hooksEnabled is None: 2752 return True 2753 else: 2754 return self.hooksEnabled 2755 2756 def enableHooks(self): 2757 """Enable hooks.""" 2758 self.hooksEnabled = True 2759 2760 def disableHooks(self): 2761 """Disable hooks.""" 2762 self.hooksEnabled = False 2763 2764 def getHooks(self): 2765 """Get the current hooks.""" 2766 return self.hooks[:] 2767 2768 def clearHooks(self): 2769 """Clear all hooks.""" 2770 self.hooks = [] 2771 2772 def addHook(self, hook, prepend=False): 2773 """Add a new hook; optionally insert it rather than appending it.""" 2774 self.register(hook, prepend) 2775 2776 def removeHook(self, hook): 2777 """Remove a preexisting hook.""" 2778 self.deregister(hook) 2779 2780 def invokeHook(self, _name, **keywords): 2781 """Manually invoke a hook.""" 2782 apply(self.invoke, (_name,), keywords) 2783 2784 # Callbacks. 2785 2786 def getCallback(self): 2787 """Get the callback registered with this interpreter, or None.""" 2788 return self.callback 2789 2790 def registerCallback(self, callback): 2791 """Register a custom markup callback with this interpreter.""" 2792 self.callback = callback 2793 2794 def deregisterCallback(self): 2795 """Remove any previously registered callback with this interpreter.""" 2796 self.callback = None 2797 2798 def invokeCallback(self, contents): 2799 """Invoke the callback.""" 2800 if self.callback is None: 2801 if self.options.get(CALLBACK_OPT, False): 2802 raise Error, "custom markup invoked with no defined callback" 2803 else: 2804 self.callback(contents) 2805 2806 # Pseudomodule manipulation. 2807 2808 def flatten(self, keys=None): 2809 """Flatten the contents of the pseudo-module into the globals 2810 namespace.""" 2811 if keys is None: 2812 keys = self.__dict__.keys() + self.__class__.__dict__.keys() 2813 dict = {} 2814 for key in keys: 2815 # The pseudomodule is really a class instance, so we need to 2816 # fumble use getattr instead of simply fumbling through the 2817 # instance's __dict__. 2818 dict[key] = getattr(self, key) 2819 # Stomp everything into the globals namespace. 2820 self.globals.update(dict) 2821 2822 # Prefix. 2823 2824 def getPrefix(self): 2825 """Get the current prefix.""" 2826 return self.prefix 2827 2828 def setPrefix(self, prefix): 2829 """Set the prefix.""" 2830 self.prefix = prefix 2831 2832 # Diversions. 2833 2834 def stopDiverting(self): 2835 """Stop any diverting.""" 2836 self.stream().revert() 2837 2838 def createDiversion(self, name): 2839 """Create a diversion (but do not divert to it) if it does not 2840 already exist.""" 2841 self.stream().create(name) 2842 2843 def retrieveDiversion(self, name): 2844 """Retrieve the diversion object associated with the name.""" 2845 return self.stream().retrieve(name) 2846 2847 def startDiversion(self, name): 2848 """Start diverting to the given diversion name.""" 2849 self.stream().divert(name) 2850 2851 def playDiversion(self, name): 2852 """Play the given diversion and then purge it.""" 2853 self.stream().undivert(name, True) 2854 2855 def replayDiversion(self, name): 2856 """Replay the diversion without purging it.""" 2857 self.stream().undivert(name, False) 2858 2859 def purgeDiversion(self, name): 2860 """Eliminate the given diversion.""" 2861 self.stream().purge(name) 2862 2863 def playAllDiversions(self): 2864 """Play all existing diversions and then purge them.""" 2865 self.stream().undivertAll(True) 2866 2867 def replayAllDiversions(self): 2868 """Replay all existing diversions without purging them.""" 2869 self.stream().undivertAll(False) 2870 2871 def purgeAllDiversions(self): 2872 """Purge all existing diversions.""" 2873 self.stream().purgeAll() 2874 2875 def getCurrentDiversion(self): 2876 """Get the name of the current diversion.""" 2877 return self.stream().currentDiversion 2878 2879 def getAllDiversions(self): 2880 """Get the names of all existing diversions.""" 2881 names = self.stream().diversions.keys() 2882 names.sort() 2883 return names 2884 2885 # Filter. 2886 2887 def resetFilter(self): 2888 """Reset the filter so that it does no filtering.""" 2889 self.stream().install(None) 2890 2891 def nullFilter(self): 2892 """Install a filter that will consume all text.""" 2893 self.stream().install(0) 2894 2895 def getFilter(self): 2896 """Get the current filter.""" 2897 filter = self.stream().filter 2898 if filter is self.stream().file: 2899 return None 2900 else: 2901 return filter 2902 2903 def setFilter(self, shortcut): 2904 """Set the filter.""" 2905 self.stream().install(shortcut) 2906 2907 def attachFilter(self, shortcut): 2908 """Attach a single filter to the end of the current filter chain.""" 2909 self.stream().attach(shortcut) 2910 2911 2912class Document: 2913 2914 """A representation of an individual EmPy document, as used by a 2915 processor.""" 2916 2917 def __init__(self, ID, filename): 2918 self.ID = ID 2919 self.filename = filename 2920 self.significators = {} 2921 2922 2923class Processor: 2924 2925 """An entity which is capable of processing a hierarchy of EmPy 2926 files and building a dictionary of document objects associated 2927 with them describing their significator contents.""" 2928 2929 DEFAULT_EMPY_EXTENSIONS = ('.em',) 2930 SIGNIFICATOR_RE = re.compile(SIGNIFICATOR_RE_STRING) 2931 2932 def __init__(self, factory=Document): 2933 self.factory = factory 2934 self.documents = {} 2935 2936 def identifier(self, pathname, filename): return filename 2937 2938 def clear(self): 2939 self.documents = {} 2940 2941 def scan(self, basename, extensions=DEFAULT_EMPY_EXTENSIONS): 2942 if type(extensions) is types.StringType: 2943 extensions = (extensions,) 2944 def _noCriteria(x): 2945 return True 2946 def _extensionsCriteria(pathname, extensions=extensions): 2947 if extensions: 2948 for extension in extensions: 2949 if pathname[-len(extension):] == extension: 2950 return True 2951 return False 2952 else: 2953 return True 2954 self.directory(basename, _noCriteria, _extensionsCriteria, None) 2955 self.postprocess() 2956 2957 def postprocess(self): 2958 pass 2959 2960 def directory(self, basename, dirCriteria, fileCriteria, depth=None): 2961 if depth is not None: 2962 if depth <= 0: 2963 return 2964 else: 2965 depth = depth - 1 2966 filenames = os.listdir(basename) 2967 for filename in filenames: 2968 pathname = os.path.join(basename, filename) 2969 if os.path.isdir(pathname): 2970 if dirCriteria(pathname): 2971 self.directory(pathname, dirCriteria, fileCriteria, depth) 2972 elif os.path.isfile(pathname): 2973 if fileCriteria(pathname): 2974 documentID = self.identifier(pathname, filename) 2975 document = self.factory(documentID, pathname) 2976 self.file(document, open(pathname)) 2977 self.documents[documentID] = document 2978 2979 def file(self, document, file): 2980 while True: 2981 line = file.readline() 2982 if not line: 2983 break 2984 self.line(document, line) 2985 2986 def line(self, document, line): 2987 match = self.SIGNIFICATOR_RE.search(line) 2988 if match: 2989 key, valueS = match.groups() 2990 valueS = string.strip(valueS) 2991 if valueS: 2992 value = eval(valueS) 2993 else: 2994 value = None 2995 document.significators[key] = value 2996 2997 2998def expand(_data, _globals=None, \ 2999 _argv=None, _prefix=DEFAULT_PREFIX, _pseudo=None, _options=None, \ 3000 **_locals): 3001 """Do an atomic expansion of the given source data, creating and 3002 shutting down an interpreter dedicated to the task. The sys.stdout 3003 object is saved off and then replaced before this function 3004 returns.""" 3005 if len(_locals) == 0: 3006 # If there were no keyword arguments specified, don't use a locals 3007 # dictionary at all. 3008 _locals = None 3009 output = NullFile() 3010 interpreter = Interpreter(output, argv=_argv, prefix=_prefix, \ 3011 pseudo=_pseudo, options=_options, \ 3012 globals=_globals) 3013 if interpreter.options.get(OVERRIDE_OPT, True): 3014 oldStdout = sys.stdout 3015 try: 3016 result = interpreter.expand(_data, _locals) 3017 finally: 3018 interpreter.shutdown() 3019 if _globals is not None: 3020 interpreter.unfix() # remove pseudomodule to prevent clashes 3021 if interpreter.options.get(OVERRIDE_OPT, True): 3022 sys.stdout = oldStdout 3023 return result 3024 3025def environment(name, default=None): 3026 """Get data from the current environment. If the default is True 3027 or False, then presume that we're only interested in the existence 3028 or non-existence of the environment variable.""" 3029 if os.environ.has_key(name): 3030 # Do the True/False test by value for future compatibility. 3031 if default == False or default == True: 3032 return True 3033 else: 3034 return os.environ[name] 3035 else: 3036 return default 3037 3038def info(table): 3039 DEFAULT_LEFT = 28 3040 maxLeft = 0 3041 maxRight = 0 3042 for left, right in table: 3043 if len(left) > maxLeft: 3044 maxLeft = len(left) 3045 if len(right) > maxRight: 3046 maxRight = len(right) 3047 FORMAT = ' %%-%ds %%s\n' % max(maxLeft, DEFAULT_LEFT) 3048 for left, right in table: 3049 if right.find('\n') >= 0: 3050 for right in right.split('\n'): 3051 sys.stderr.write(FORMAT % (left, right)) 3052 left = '' 3053 else: 3054 sys.stderr.write(FORMAT % (left, right)) 3055 3056def usage(verbose=True): 3057 """Print usage information.""" 3058 programName = sys.argv[0] 3059 def warn(line=''): 3060 sys.stderr.write("%s\n" % line) 3061 warn("""\ 3062Usage: %s [options] [<filename, or '-' for stdin> [<argument>...]] 3063Welcome to EmPy version %s.""" % (programName, __version__)) 3064 warn() 3065 warn("Valid options:") 3066 info(OPTION_INFO) 3067 if verbose: 3068 warn() 3069 warn("The following markups are supported:") 3070 info(MARKUP_INFO) 3071 warn() 3072 warn("Valid escape sequences are:") 3073 info(ESCAPE_INFO) 3074 warn() 3075 warn("The %s pseudomodule contains the following attributes:" % \ 3076 DEFAULT_PSEUDOMODULE_NAME) 3077 info(PSEUDOMODULE_INFO) 3078 warn() 3079 warn("The following environment variables are recognized:") 3080 info(ENVIRONMENT_INFO) 3081 warn() 3082 warn(USAGE_NOTES) 3083 else: 3084 warn() 3085 warn("Type %s -H for more extensive help." % programName) 3086 3087def invoke(args): 3088 """Run a standalone instance of an EmPy interpeter.""" 3089 # Initialize the options. 3090 _output = None 3091 _options = {BUFFERED_OPT: environment(BUFFERED_ENV, False), 3092 RAW_OPT: environment(RAW_ENV, False), 3093 EXIT_OPT: True, 3094 FLATTEN_OPT: environment(FLATTEN_ENV, False), 3095 OVERRIDE_OPT: not environment(NO_OVERRIDE_ENV, False), 3096 CALLBACK_OPT: False} 3097 _preprocessing = [] 3098 _prefix = environment(PREFIX_ENV, DEFAULT_PREFIX) 3099 _pseudo = environment(PSEUDO_ENV, None) 3100 _interactive = environment(INTERACTIVE_ENV, False) 3101 _extraArguments = environment(OPTIONS_ENV) 3102 _binary = -1 # negative for not, 0 for default size, positive for size 3103 _unicode = environment(UNICODE_ENV, False) 3104 _unicodeInputEncoding = environment(INPUT_ENCODING_ENV, None) 3105 _unicodeOutputEncoding = environment(OUTPUT_ENCODING_ENV, None) 3106 _unicodeInputErrors = environment(INPUT_ERRORS_ENV, None) 3107 _unicodeOutputErrors = environment(OUTPUT_ERRORS_ENV, None) 3108 _hooks = [] 3109 _pauseAtEnd = False 3110 _relativePath = False 3111 if _extraArguments is not None: 3112 _extraArguments = string.split(_extraArguments) 3113 args = _extraArguments + args 3114 # Parse the arguments. 3115 pairs, remainder = getopt.getopt(args, 'VhHvkp:m:frino:a:buBP:I:D:E:F:', ['version', 'help', 'extended-help', 'verbose', 'null-hook', 'suppress-errors', 'prefix=', 'no-prefix', 'module=', 'flatten', 'raw-errors', 'interactive', 'no-override-stdout', 'binary', 'chunk-size=', 'output=' 'append=', 'preprocess=', 'import=', 'define=', 'execute=', 'execute-file=', 'buffered-output', 'pause-at-end', 'relative-path', 'no-callback-error', 'no-bangpath-processing', 'unicode', 'unicode-encoding=', 'unicode-input-encoding=', 'unicode-output-encoding=', 'unicode-errors=', 'unicode-input-errors=', 'unicode-output-errors=']) 3116 for option, argument in pairs: 3117 if option in ('-V', '--version'): 3118 sys.stderr.write("%s version %s\n" % (__program__, __version__)) 3119 return 3120 elif option in ('-h', '--help'): 3121 usage(False) 3122 return 3123 elif option in ('-H', '--extended-help'): 3124 usage(True) 3125 return 3126 elif option in ('-v', '--verbose'): 3127 _hooks.append(VerboseHook()) 3128 elif option in ('--null-hook',): 3129 _hooks.append(Hook()) 3130 elif option in ('-k', '--suppress-errors'): 3131 _options[EXIT_OPT] = False 3132 _interactive = True # suppress errors implies interactive mode 3133 elif option in ('-m', '--module'): 3134 _pseudo = argument 3135 elif option in ('-f', '--flatten'): 3136 _options[FLATTEN_OPT] = True 3137 elif option in ('-p', '--prefix'): 3138 _prefix = argument 3139 elif option in ('--no-prefix',): 3140 _prefix = None 3141 elif option in ('-r', '--raw-errors'): 3142 _options[RAW_OPT] = True 3143 elif option in ('-i', '--interactive'): 3144 _interactive = True 3145 elif option in ('-n', '--no-override-stdout'): 3146 _options[OVERRIDE_OPT] = False 3147 elif option in ('-o', '--output'): 3148 _output = argument, 'w', _options[BUFFERED_OPT] 3149 elif option in ('-a', '--append'): 3150 _output = argument, 'a', _options[BUFFERED_OPT] 3151 elif option in ('-b', '--buffered-output'): 3152 _options[BUFFERED_OPT] = True 3153 elif option in ('-B',): # DEPRECATED 3154 _options[BUFFERED_OPT] = True 3155 elif option in ('--binary',): 3156 _binary = 0 3157 elif option in ('--chunk-size',): 3158 _binary = int(argument) 3159 elif option in ('-P', '--preprocess'): 3160 _preprocessing.append(('pre', argument)) 3161 elif option in ('-I', '--import'): 3162 for module in string.split(argument, ','): 3163 module = string.strip(module) 3164 _preprocessing.append(('import', module)) 3165 elif option in ('-D', '--define'): 3166 _preprocessing.append(('define', argument)) 3167 elif option in ('-E', '--execute'): 3168 _preprocessing.append(('exec', argument)) 3169 elif option in ('-F', '--execute-file'): 3170 _preprocessing.append(('file', argument)) 3171 elif option in ('-u', '--unicode'): 3172 _unicode = True 3173 elif option in ('--pause-at-end',): 3174 _pauseAtEnd = True 3175 elif option in ('--relative-path',): 3176 _relativePath = True 3177 elif option in ('--no-callback-error',): 3178 _options[CALLBACK_OPT] = True 3179 elif option in ('--no-bangpath-processing',): 3180 _options[BANGPATH_OPT] = False 3181 elif option in ('--unicode-encoding',): 3182 _unicodeInputEncoding = _unicodeOutputEncoding = argument 3183 elif option in ('--unicode-input-encoding',): 3184 _unicodeInputEncoding = argument 3185 elif option in ('--unicode-output-encoding',): 3186 _unicodeOutputEncoding = argument 3187 elif option in ('--unicode-errors',): 3188 _unicodeInputErrors = _unicodeOutputErrors = argument 3189 elif option in ('--unicode-input-errors',): 3190 _unicodeInputErrors = argument 3191 elif option in ('--unicode-output-errors',): 3192 _unicodeOutputErrors = argument 3193 # Set up the Unicode subsystem if required. 3194 if _unicode or \ 3195 _unicodeInputEncoding or _unicodeOutputEncoding or \ 3196 _unicodeInputErrors or _unicodeOutputErrors: 3197 theSubsystem.initialize(_unicodeInputEncoding, \ 3198 _unicodeOutputEncoding, \ 3199 _unicodeInputErrors, _unicodeOutputErrors) 3200 # Now initialize the output file if something has already been selected. 3201 if _output is not None: 3202 _output = apply(AbstractFile, _output) 3203 # Set up the main filename and the argument. 3204 if not remainder: 3205 remainder.append('-') 3206 filename, arguments = remainder[0], remainder[1:] 3207 # Set up the interpreter. 3208 if _options[BUFFERED_OPT] and _output is None: 3209 raise ValueError, "-b only makes sense with -o or -a arguments" 3210 if _prefix == 'None': 3211 _prefix = None 3212 if _prefix and type(_prefix) is types.StringType and len(_prefix) != 1: 3213 raise Error, "prefix must be single-character string" 3214 interpreter = Interpreter(output=_output, \ 3215 argv=remainder, \ 3216 prefix=_prefix, \ 3217 pseudo=_pseudo, \ 3218 options=_options, \ 3219 hooks=_hooks) 3220 try: 3221 # Execute command-line statements. 3222 i = 0 3223 for which, thing in _preprocessing: 3224 if which == 'pre': 3225 command = interpreter.file 3226 target = theSubsystem.open(thing, 'r') 3227 name = thing 3228 elif which == 'define': 3229 command = interpreter.string 3230 if string.find(thing, '=') >= 0: 3231 target = '%s{%s}' % (_prefix, thing) 3232 else: 3233 target = '%s{%s = None}' % (_prefix, thing) 3234 name = '<define:%d>' % i 3235 elif which == 'exec': 3236 command = interpreter.string 3237 target = '%s{%s}' % (_prefix, thing) 3238 name = '<exec:%d>' % i 3239 elif which == 'file': 3240 command = interpreter.string 3241 name = '<file:%d (%s)>' % (i, thing) 3242 target = '%s{execfile("""%s""")}' % (_prefix, thing) 3243 elif which == 'import': 3244 command = interpreter.string 3245 name = '<import:%d>' % i 3246 target = '%s{import %s}' % (_prefix, thing) 3247 else: 3248 assert 0 3249 interpreter.wrap(command, (target, name)) 3250 i = i + 1 3251 # Now process the primary file. 3252 interpreter.ready() 3253 if filename == '-': 3254 if not _interactive: 3255 name = '<stdin>' 3256 path = '' 3257 file = sys.stdin 3258 else: 3259 name, file = None, None 3260 else: 3261 name = filename 3262 file = theSubsystem.open(filename, 'r') 3263 path = os.path.split(filename)[0] 3264 if _relativePath: 3265 sys.path.insert(0, path) 3266 if file is not None: 3267 if _binary < 0: 3268 interpreter.wrap(interpreter.file, (file, name)) 3269 else: 3270 chunkSize = _binary 3271 interpreter.wrap(interpreter.binary, (file, name, chunkSize)) 3272 # If we're supposed to go interactive afterwards, do it. 3273 if _interactive: 3274 interpreter.interact() 3275 finally: 3276 interpreter.shutdown() 3277 # Finally, if we should pause at the end, do it. 3278 if _pauseAtEnd: 3279 try: 3280 raw_input() 3281 except EOFError: 3282 pass 3283 3284def main(): 3285 invoke(sys.argv[1:]) 3286 3287if __name__ == '__main__': main() 3288