1from __future__ import absolute_import 2 3 4from lark.exceptions import ConfigurationError, assert_config 5 6import sys, os, pickle, hashlib 7from io import open 8import tempfile 9from warnings import warn 10 11from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii, logger, ABC, abstractmethod 12from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files 13from .tree import Tree 14from .common import LexerConf, ParserConf 15 16from .lexer import Lexer, TraditionalLexer, TerminalDef, LexerThread 17from .parse_tree_builder import ParseTreeBuilder 18from .parser_frontends import get_frontend, _get_lexer_callbacks 19from .grammar import Rule 20 21import re 22try: 23 import regex 24except ImportError: 25 regex = None 26 27 28###{standalone 29 30 31class LarkOptions(Serialize): 32 """Specifies the options for Lark 33 34 """ 35 OPTIONS_DOC = """ 36 **=== General Options ===** 37 38 start 39 The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start") 40 debug 41 Display debug information and extra warnings. Use only when debugging (default: False) 42 When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed. 43 transformer 44 Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster) 45 propagate_positions 46 Propagates (line, column, end_line, end_column) attributes into all tree branches. 47 Accepts ``False``, ``True``, or a callable, which will filter which nodes to ignore when propagating. 48 maybe_placeholders 49 When ``True``, the ``[]`` operator returns ``None`` when not matched. 50 51 When ``False``, ``[]`` behaves like the ``?`` operator, and returns no value at all. 52 (default= ``False``. Recommended to set to ``True``) 53 cache 54 Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now. 55 56 - When ``False``, does nothing (default) 57 - When ``True``, caches to a temporary file in the local directory 58 - When given a string, caches to the path pointed by the string 59 regex 60 When True, uses the ``regex`` module instead of the stdlib ``re``. 61 g_regex_flags 62 Flags that are applied to all terminals (both regex and strings) 63 keep_all_tokens 64 Prevent the tree builder from automagically removing "punctuation" tokens (default: False) 65 tree_class 66 Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``. 67 68 **=== Algorithm Options ===** 69 70 parser 71 Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley"). 72 (there is also a "cyk" option for legacy) 73 lexer 74 Decides whether or not to use a lexer stage 75 76 - "auto" (default): Choose for me based on the parser 77 - "standard": Use a standard lexer 78 - "contextual": Stronger lexer (only works with parser="lalr") 79 - "dynamic": Flexible and powerful (only with parser="earley") 80 - "dynamic_complete": Same as dynamic, but tries *every* variation of tokenizing possible. 81 ambiguity 82 Decides how to handle ambiguity in the parse. Only relevant if parser="earley" 83 84 - "resolve": The parser will automatically choose the simplest derivation 85 (it chooses consistently: greedy for tokens, non-greedy for rules) 86 - "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest). 87 - "forest": The parser will return the root of the shared packed parse forest. 88 89 **=== Misc. / Domain Specific Options ===** 90 91 postlex 92 Lexer post-processing (Default: None) Only works with the standard and contextual lexers. 93 priority 94 How priorities should be evaluated - auto, none, normal, invert (Default: auto) 95 lexer_callbacks 96 Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution. 97 use_bytes 98 Accept an input of type ``bytes`` instead of ``str`` (Python 3 only). 99 edit_terminals 100 A callback for editing the terminals before parse. 101 import_paths 102 A List of either paths or loader functions to specify from where grammars are imported 103 source_path 104 Override the source of from where the grammar was loaded. Useful for relative imports and unconventional grammar loading 105 **=== End of Options ===** 106 """ 107 if __doc__: 108 __doc__ += OPTIONS_DOC 109 110 111 # Adding a new option needs to be done in multiple places: 112 # - In the dictionary below. This is the primary truth of which options `Lark.__init__` accepts 113 # - In the docstring above. It is used both for the docstring of `LarkOptions` and `Lark`, and in readthedocs 114 # - In `lark-stubs/lark.pyi`: 115 # - As attribute to `LarkOptions` 116 # - As parameter to `Lark.__init__` 117 # - Potentially in `_LOAD_ALLOWED_OPTIONS` below this class, when the option doesn't change how the grammar is loaded 118 # - Potentially in `lark.tools.__init__`, if it makes sense, and it can easily be passed as a cmd argument 119 _defaults = { 120 'debug': False, 121 'keep_all_tokens': False, 122 'tree_class': None, 123 'cache': False, 124 'postlex': None, 125 'parser': 'earley', 126 'lexer': 'auto', 127 'transformer': None, 128 'start': 'start', 129 'priority': 'auto', 130 'ambiguity': 'auto', 131 'regex': False, 132 'propagate_positions': False, 133 'lexer_callbacks': {}, 134 'maybe_placeholders': False, 135 'edit_terminals': None, 136 'g_regex_flags': 0, 137 'use_bytes': False, 138 'import_paths': [], 139 'source_path': None, 140 } 141 142 def __init__(self, options_dict): 143 o = dict(options_dict) 144 145 options = {} 146 for name, default in self._defaults.items(): 147 if name in o: 148 value = o.pop(name) 149 if isinstance(default, bool) and name not in ('cache', 'use_bytes', 'propagate_positions'): 150 value = bool(value) 151 else: 152 value = default 153 154 options[name] = value 155 156 if isinstance(options['start'], STRING_TYPE): 157 options['start'] = [options['start']] 158 159 self.__dict__['options'] = options 160 161 162 assert_config(self.parser, ('earley', 'lalr', 'cyk', None)) 163 164 if self.parser == 'earley' and self.transformer: 165 raise ConfigurationError('Cannot specify an embedded transformer when using the Earley algorithm. ' 166 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)') 167 168 if o: 169 raise ConfigurationError("Unknown options: %s" % o.keys()) 170 171 def __getattr__(self, name): 172 try: 173 return self.__dict__['options'][name] 174 except KeyError as e: 175 raise AttributeError(e) 176 177 def __setattr__(self, name, value): 178 assert_config(name, self.options.keys(), "%r isn't a valid option. Expected one of: %s") 179 self.options[name] = value 180 181 def serialize(self, memo): 182 return self.options 183 184 @classmethod 185 def deserialize(cls, data, memo): 186 return cls(data) 187 188 189# Options that can be passed to the Lark parser, even when it was loaded from cache/standalone. 190# These option are only used outside of `load_grammar`. 191_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'} 192 193_VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None) 194_VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest') 195 196 197class PostLex(ABC): 198 @abstractmethod 199 def process(self, stream): 200 return stream 201 202 always_accept = () 203 204 205class Lark(Serialize): 206 """Main interface for the library. 207 208 It's mostly a thin wrapper for the many different parsers, and for the tree constructor. 209 210 Parameters: 211 grammar: a string or file-object containing the grammar spec (using Lark's ebnf syntax) 212 options: a dictionary controlling various aspects of Lark. 213 214 Example: 215 >>> Lark(r'''start: "foo" ''') 216 Lark(...) 217 """ 218 def __init__(self, grammar, **options): 219 self.options = LarkOptions(options) 220 221 # Set regex or re module 222 use_regex = self.options.regex 223 if use_regex: 224 if regex: 225 re_module = regex 226 else: 227 raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.') 228 else: 229 re_module = re 230 231 # Some, but not all file-like objects have a 'name' attribute 232 if self.options.source_path is None: 233 try: 234 self.source_path = grammar.name 235 except AttributeError: 236 self.source_path = '<string>' 237 else: 238 self.source_path = self.options.source_path 239 240 # Drain file-like objects to get their contents 241 try: 242 read = grammar.read 243 except AttributeError: 244 pass 245 else: 246 grammar = read() 247 248 cache_fn = None 249 cache_md5 = None 250 if isinstance(grammar, STRING_TYPE): 251 self.source_grammar = grammar 252 if self.options.use_bytes: 253 if not isascii(grammar): 254 raise ConfigurationError("Grammar must be ascii only, when use_bytes=True") 255 if sys.version_info[0] == 2 and self.options.use_bytes != 'force': 256 raise ConfigurationError("`use_bytes=True` may have issues on python2." 257 "Use `use_bytes='force'` to use it at your own risk.") 258 259 if self.options.cache: 260 if self.options.parser != 'lalr': 261 raise ConfigurationError("cache only works with parser='lalr' for now") 262 263 unhashable = ('transformer', 'postlex', 'lexer_callbacks', 'edit_terminals') 264 options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable) 265 from . import __version__ 266 s = grammar + options_str + __version__ + str(sys.version_info[:2]) 267 cache_md5 = hashlib.md5(s.encode('utf8')).hexdigest() 268 269 if isinstance(self.options.cache, STRING_TYPE): 270 cache_fn = self.options.cache 271 else: 272 if self.options.cache is not True: 273 raise ConfigurationError("cache argument must be bool or str") 274 # Python2.7 doesn't support * syntax in tuples 275 cache_fn = tempfile.gettempdir() + '/.lark_cache_%s_%s_%s.tmp' % ((cache_md5,) + sys.version_info[:2]) 276 277 if FS.exists(cache_fn): 278 logger.debug('Loading grammar from cache: %s', cache_fn) 279 # Remove options that aren't relevant for loading from cache 280 for name in (set(options) - _LOAD_ALLOWED_OPTIONS): 281 del options[name] 282 with FS.open(cache_fn, 'rb') as f: 283 old_options = self.options 284 try: 285 file_md5 = f.readline().rstrip(b'\n') 286 cached_used_files = pickle.load(f) 287 if file_md5 == cache_md5.encode('utf8') and verify_used_files(cached_used_files): 288 cached_parser_data = pickle.load(f) 289 self._load(cached_parser_data, **options) 290 return 291 except Exception: # We should probably narrow done which errors we catch here. 292 logger.exception("Failed to load Lark from cache: %r. We will try to carry on." % cache_fn) 293 294 # In theory, the Lark instance might have been messed up by the call to `_load`. 295 # In practice the only relevant thing that might have been overriden should be `options` 296 self.options = old_options 297 298 299 # Parse the grammar file and compose the grammars 300 self.grammar, used_files = load_grammar(grammar, self.source_path, self.options.import_paths, self.options.keep_all_tokens) 301 else: 302 assert isinstance(grammar, Grammar) 303 self.grammar = grammar 304 305 306 if self.options.lexer == 'auto': 307 if self.options.parser == 'lalr': 308 self.options.lexer = 'contextual' 309 elif self.options.parser == 'earley': 310 if self.options.postlex is not None: 311 logger.info("postlex can't be used with the dynamic lexer, so we use standard instead. " 312 "Consider using lalr with contextual instead of earley") 313 self.options.lexer = 'standard' 314 else: 315 self.options.lexer = 'dynamic' 316 elif self.options.parser == 'cyk': 317 self.options.lexer = 'standard' 318 else: 319 assert False, self.options.parser 320 lexer = self.options.lexer 321 if isinstance(lexer, type): 322 assert issubclass(lexer, Lexer) # XXX Is this really important? Maybe just ensure interface compliance 323 else: 324 assert_config(lexer, ('standard', 'contextual', 'dynamic', 'dynamic_complete')) 325 if self.options.postlex is not None and 'dynamic' in lexer: 326 raise ConfigurationError("Can't use postlex with a dynamic lexer. Use standard or contextual instead") 327 328 if self.options.ambiguity == 'auto': 329 if self.options.parser == 'earley': 330 self.options.ambiguity = 'resolve' 331 else: 332 assert_config(self.options.parser, ('earley', 'cyk'), "%r doesn't support disambiguation. Use one of these parsers instead: %s") 333 334 if self.options.priority == 'auto': 335 self.options.priority = 'normal' 336 337 if self.options.priority not in _VALID_PRIORITY_OPTIONS: 338 raise ConfigurationError("invalid priority option: %r. Must be one of %r" % (self.options.priority, _VALID_PRIORITY_OPTIONS)) 339 assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"' 340 if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS: 341 raise ConfigurationError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS)) 342 343 if self.options.parser is None: 344 terminals_to_keep = '*' 345 elif self.options.postlex is not None: 346 terminals_to_keep = set(self.options.postlex.always_accept) 347 else: 348 terminals_to_keep = set() 349 350 # Compile the EBNF grammar into BNF 351 self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start, terminals_to_keep) 352 353 if self.options.edit_terminals: 354 for t in self.terminals: 355 self.options.edit_terminals(t) 356 357 self._terminals_dict = {t.name: t for t in self.terminals} 358 359 # If the user asked to invert the priorities, negate them all here. 360 # This replaces the old 'resolve__antiscore_sum' option. 361 if self.options.priority == 'invert': 362 for rule in self.rules: 363 if rule.options.priority is not None: 364 rule.options.priority = -rule.options.priority 365 # Else, if the user asked to disable priorities, strip them from the 366 # rules. This allows the Earley parsers to skip an extra forest walk 367 # for improved performance, if you don't need them (or didn't specify any). 368 elif self.options.priority is None: 369 for rule in self.rules: 370 if rule.options.priority is not None: 371 rule.options.priority = None 372 373 # TODO Deprecate lexer_callbacks? 374 self.lexer_conf = LexerConf( 375 self.terminals, re_module, self.ignore_tokens, self.options.postlex, 376 self.options.lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes 377 ) 378 379 if self.options.parser: 380 self.parser = self._build_parser() 381 elif lexer: 382 self.lexer = self._build_lexer() 383 384 if cache_fn: 385 logger.debug('Saving grammar to cache: %s', cache_fn) 386 with FS.open(cache_fn, 'wb') as f: 387 f.write(cache_md5.encode('utf8') + b'\n') 388 pickle.dump(used_files, f) 389 self.save(f) 390 391 if __doc__: 392 __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC 393 394 __serialize_fields__ = 'parser', 'rules', 'options' 395 396 def _build_lexer(self, dont_ignore=False): 397 lexer_conf = self.lexer_conf 398 if dont_ignore: 399 from copy import copy 400 lexer_conf = copy(lexer_conf) 401 lexer_conf.ignore = () 402 return TraditionalLexer(lexer_conf) 403 404 def _prepare_callbacks(self): 405 self._callbacks = {} 406 # we don't need these callbacks if we aren't building a tree 407 if self.options.ambiguity != 'forest': 408 self._parse_tree_builder = ParseTreeBuilder( 409 self.rules, 410 self.options.tree_class or Tree, 411 self.options.propagate_positions, 412 self.options.parser != 'lalr' and self.options.ambiguity == 'explicit', 413 self.options.maybe_placeholders 414 ) 415 self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer) 416 self._callbacks.update(_get_lexer_callbacks(self.options.transformer, self.terminals)) 417 418 def _build_parser(self): 419 self._prepare_callbacks() 420 parser_class = get_frontend(self.options.parser, self.options.lexer) 421 parser_conf = ParserConf(self.rules, self._callbacks, self.options.start) 422 return parser_class(self.lexer_conf, parser_conf, options=self.options) 423 424 def save(self, f): 425 """Saves the instance into the given file object 426 427 Useful for caching and multiprocessing. 428 """ 429 data, m = self.memo_serialize([TerminalDef, Rule]) 430 pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL) 431 432 @classmethod 433 def load(cls, f): 434 """Loads an instance from the given file object 435 436 Useful for caching and multiprocessing. 437 """ 438 inst = cls.__new__(cls) 439 return inst._load(f) 440 441 def _deserialize_lexer_conf(self, data, memo, options): 442 lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo) 443 lexer_conf.callbacks = options.lexer_callbacks or {} 444 lexer_conf.re_module = regex if options.regex else re 445 lexer_conf.use_bytes = options.use_bytes 446 lexer_conf.g_regex_flags = options.g_regex_flags 447 lexer_conf.skip_validation = True 448 lexer_conf.postlex = options.postlex 449 return lexer_conf 450 451 def _load(self, f, **kwargs): 452 if isinstance(f, dict): 453 d = f 454 else: 455 d = pickle.load(f) 456 memo_json = d['memo'] 457 data = d['data'] 458 459 assert memo_json 460 memo = SerializeMemoizer.deserialize(memo_json, {'Rule': Rule, 'TerminalDef': TerminalDef}, {}) 461 options = dict(data['options']) 462 if (set(kwargs) - _LOAD_ALLOWED_OPTIONS) & set(LarkOptions._defaults): 463 raise ConfigurationError("Some options are not allowed when loading a Parser: {}" 464 .format(set(kwargs) - _LOAD_ALLOWED_OPTIONS)) 465 options.update(kwargs) 466 self.options = LarkOptions.deserialize(options, memo) 467 self.rules = [Rule.deserialize(r, memo) for r in data['rules']] 468 self.source_path = '<deserialized>' 469 parser_class = get_frontend(self.options.parser, self.options.lexer) 470 self.lexer_conf = self._deserialize_lexer_conf(data['parser'], memo, self.options) 471 self.terminals = self.lexer_conf.terminals 472 self._prepare_callbacks() 473 self._terminals_dict = {t.name: t for t in self.terminals} 474 self.parser = parser_class.deserialize( 475 data['parser'], 476 memo, 477 self.lexer_conf, 478 self._callbacks, 479 self.options, # Not all, but multiple attributes are used 480 ) 481 return self 482 483 @classmethod 484 def _load_from_dict(cls, data, memo, **kwargs): 485 inst = cls.__new__(cls) 486 return inst._load({'data': data, 'memo': memo}, **kwargs) 487 488 @classmethod 489 def open(cls, grammar_filename, rel_to=None, **options): 490 """Create an instance of Lark with the grammar given by its filename 491 492 If ``rel_to`` is provided, the function will find the grammar filename in relation to it. 493 494 Example: 495 496 >>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr") 497 Lark(...) 498 499 """ 500 if rel_to: 501 basepath = os.path.dirname(rel_to) 502 grammar_filename = os.path.join(basepath, grammar_filename) 503 with open(grammar_filename, encoding='utf8') as f: 504 return cls(f, **options) 505 506 @classmethod 507 def open_from_package(cls, package, grammar_path, search_paths=("",), **options): 508 """Create an instance of Lark with the grammar loaded from within the package `package`. 509 This allows grammar loading from zipapps. 510 511 Imports in the grammar will use the `package` and `search_paths` provided, through `FromPackageLoader` 512 513 Example: 514 515 Lark.open_from_package(__name__, "example.lark", ("grammars",), parser=...) 516 """ 517 package_loader = FromPackageLoader(package, search_paths) 518 full_path, text = package_loader(None, grammar_path) 519 options.setdefault('source_path', full_path) 520 options.setdefault('import_paths', []) 521 options['import_paths'].append(package_loader) 522 return cls(text, **options) 523 524 def __repr__(self): 525 return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source_path, self.options.parser, self.options.lexer) 526 527 528 def lex(self, text, dont_ignore=False): 529 """Only lex (and postlex) the text, without parsing it. Only relevant when lexer='standard' 530 531 When dont_ignore=True, the lexer will return all tokens, even those marked for %ignore. 532 533 :raises UnexpectedCharacters: In case the lexer cannot find a suitable match. 534 """ 535 if not hasattr(self, 'lexer') or dont_ignore: 536 lexer = self._build_lexer(dont_ignore) 537 else: 538 lexer = self.lexer 539 lexer_thread = LexerThread(lexer, text) 540 stream = lexer_thread.lex(None) 541 if self.options.postlex: 542 return self.options.postlex.process(stream) 543 return stream 544 545 def get_terminal(self, name): 546 """Get information about a terminal""" 547 return self._terminals_dict[name] 548 549 def parse_interactive(self, text=None, start=None): 550 """Start an interactive parsing session. 551 552 Parameters: 553 text (str, optional): Text to be parsed. Required for ``resume_parse()``. 554 start (str, optional): Start symbol 555 556 Returns: 557 A new InteractiveParser instance. 558 559 See Also: ``Lark.parse()`` 560 """ 561 return self.parser.parse_interactive(text, start=start) 562 563 def parse(self, text, start=None, on_error=None): 564 """Parse the given text, according to the options provided. 565 566 Parameters: 567 text (str): Text to be parsed. 568 start (str, optional): Required if Lark was given multiple possible start symbols (using the start option). 569 on_error (function, optional): if provided, will be called on UnexpectedToken error. Return true to resume parsing. 570 LALR only. See examples/advanced/error_handling.py for an example of how to use on_error. 571 572 Returns: 573 If a transformer is supplied to ``__init__``, returns whatever is the 574 result of the transformation. Otherwise, returns a Tree instance. 575 576 :raises UnexpectedInput: On a parse error, one of these sub-exceptions will rise: 577 ``UnexpectedCharacters``, ``UnexpectedToken``, or ``UnexpectedEOF``. 578 For convenience, these sub-exceptions also inherit from ``ParserError`` and ``LexerError``. 579 580 """ 581 return self.parser.parse(text, start=start, on_error=on_error) 582 583 @property 584 def source(self): 585 warn("Attribute Lark.source was renamed to Lark.source_path", DeprecationWarning) 586 return self.source_path 587 588 @source.setter 589 def source(self, value): 590 self.source_path = value 591 592 @property 593 def grammar_source(self): 594 warn("Attribute Lark.grammar_source was renamed to Lark.source_grammar", DeprecationWarning) 595 return self.source_grammar 596 597 @grammar_source.setter 598 def grammar_source(self, value): 599 self.source_grammar = value 600 601 602###} 603