1from __future__ import absolute_import
2
3
4from lark.exceptions import ConfigurationError, assert_config
5
6import sys, os, pickle, hashlib
7from io import open
8import tempfile
9from warnings import warn
10
11from .utils import STRING_TYPE, Serialize, SerializeMemoizer, FS, isascii, logger, ABC, abstractmethod
12from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files
13from .tree import Tree
14from .common import LexerConf, ParserConf
15
16from .lexer import Lexer, TraditionalLexer, TerminalDef, LexerThread
17from .parse_tree_builder import ParseTreeBuilder
18from .parser_frontends import get_frontend, _get_lexer_callbacks
19from .grammar import Rule
20
21import re
22try:
23    import regex
24except ImportError:
25    regex = None
26
27
28###{standalone
29
30
31class LarkOptions(Serialize):
32    """Specifies the options for Lark
33
34    """
35    OPTIONS_DOC = """
36    **===  General Options  ===**
37
38    start
39            The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start")
40    debug
41            Display debug information and extra warnings. Use only when debugging (default: False)
42            When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed.
43    transformer
44            Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster)
45    propagate_positions
46            Propagates (line, column, end_line, end_column) attributes into all tree branches.
47            Accepts ``False``, ``True``, or a callable, which will filter which nodes to ignore when propagating.
48    maybe_placeholders
49            When ``True``, the ``[]`` operator returns ``None`` when not matched.
50
51            When ``False``,  ``[]`` behaves like the ``?`` operator, and returns no value at all.
52            (default= ``False``. Recommended to set to ``True``)
53    cache
54            Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now.
55
56            - When ``False``, does nothing (default)
57            - When ``True``, caches to a temporary file in the local directory
58            - When given a string, caches to the path pointed by the string
59    regex
60            When True, uses the ``regex`` module instead of the stdlib ``re``.
61    g_regex_flags
62            Flags that are applied to all terminals (both regex and strings)
63    keep_all_tokens
64            Prevent the tree builder from automagically removing "punctuation" tokens (default: False)
65    tree_class
66            Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``.
67
68    **=== Algorithm Options ===**
69
70    parser
71            Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley").
72            (there is also a "cyk" option for legacy)
73    lexer
74            Decides whether or not to use a lexer stage
75
76            - "auto" (default): Choose for me based on the parser
77            - "standard": Use a standard lexer
78            - "contextual": Stronger lexer (only works with parser="lalr")
79            - "dynamic": Flexible and powerful (only with parser="earley")
80            - "dynamic_complete": Same as dynamic, but tries *every* variation of tokenizing possible.
81    ambiguity
82            Decides how to handle ambiguity in the parse. Only relevant if parser="earley"
83
84            - "resolve": The parser will automatically choose the simplest derivation
85              (it chooses consistently: greedy for tokens, non-greedy for rules)
86            - "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
87            - "forest": The parser will return the root of the shared packed parse forest.
88
89    **=== Misc. / Domain Specific Options ===**
90
91    postlex
92            Lexer post-processing (Default: None) Only works with the standard and contextual lexers.
93    priority
94            How priorities should be evaluated - auto, none, normal, invert (Default: auto)
95    lexer_callbacks
96            Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
97    use_bytes
98            Accept an input of type ``bytes`` instead of ``str`` (Python 3 only).
99    edit_terminals
100            A callback for editing the terminals before parse.
101    import_paths
102            A List of either paths or loader functions to specify from where grammars are imported
103    source_path
104            Override the source of from where the grammar was loaded. Useful for relative imports and unconventional grammar loading
105    **=== End of Options ===**
106    """
107    if __doc__:
108        __doc__ += OPTIONS_DOC
109
110
111    # Adding a new option needs to be done in multiple places:
112    # - In the dictionary below. This is the primary truth of which options `Lark.__init__` accepts
113    # - In the docstring above. It is used both for the docstring of `LarkOptions` and `Lark`, and in readthedocs
114    # - In `lark-stubs/lark.pyi`:
115    #   - As attribute to `LarkOptions`
116    #   - As parameter to `Lark.__init__`
117    # - Potentially in `_LOAD_ALLOWED_OPTIONS` below this class, when the option doesn't change how the grammar is loaded
118    # - Potentially in `lark.tools.__init__`, if it makes sense, and it can easily be passed as a cmd argument
119    _defaults = {
120        'debug': False,
121        'keep_all_tokens': False,
122        'tree_class': None,
123        'cache': False,
124        'postlex': None,
125        'parser': 'earley',
126        'lexer': 'auto',
127        'transformer': None,
128        'start': 'start',
129        'priority': 'auto',
130        'ambiguity': 'auto',
131        'regex': False,
132        'propagate_positions': False,
133        'lexer_callbacks': {},
134        'maybe_placeholders': False,
135        'edit_terminals': None,
136        'g_regex_flags': 0,
137        'use_bytes': False,
138        'import_paths': [],
139        'source_path': None,
140    }
141
142    def __init__(self, options_dict):
143        o = dict(options_dict)
144
145        options = {}
146        for name, default in self._defaults.items():
147            if name in o:
148                value = o.pop(name)
149                if isinstance(default, bool) and name not in ('cache', 'use_bytes', 'propagate_positions'):
150                    value = bool(value)
151            else:
152                value = default
153
154            options[name] = value
155
156        if isinstance(options['start'], STRING_TYPE):
157            options['start'] = [options['start']]
158
159        self.__dict__['options'] = options
160
161
162        assert_config(self.parser, ('earley', 'lalr', 'cyk', None))
163
164        if self.parser == 'earley' and self.transformer:
165            raise ConfigurationError('Cannot specify an embedded transformer when using the Earley algorithm. '
166                             'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)')
167
168        if o:
169            raise ConfigurationError("Unknown options: %s" % o.keys())
170
171    def __getattr__(self, name):
172        try:
173            return self.__dict__['options'][name]
174        except KeyError as e:
175            raise AttributeError(e)
176
177    def __setattr__(self, name, value):
178        assert_config(name, self.options.keys(), "%r isn't a valid option. Expected one of: %s")
179        self.options[name] = value
180
181    def serialize(self, memo):
182        return self.options
183
184    @classmethod
185    def deserialize(cls, data, memo):
186        return cls(data)
187
188
189# Options that can be passed to the Lark parser, even when it was loaded from cache/standalone.
190# These option are only used outside of `load_grammar`.
191_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'}
192
193_VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None)
194_VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest')
195
196
197class PostLex(ABC):
198    @abstractmethod
199    def process(self, stream):
200        return stream
201
202    always_accept = ()
203
204
205class Lark(Serialize):
206    """Main interface for the library.
207
208    It's mostly a thin wrapper for the many different parsers, and for the tree constructor.
209
210    Parameters:
211        grammar: a string or file-object containing the grammar spec (using Lark's ebnf syntax)
212        options: a dictionary controlling various aspects of Lark.
213
214    Example:
215        >>> Lark(r'''start: "foo" ''')
216        Lark(...)
217    """
218    def __init__(self, grammar, **options):
219        self.options = LarkOptions(options)
220
221        # Set regex or re module
222        use_regex = self.options.regex
223        if use_regex:
224            if regex:
225                re_module = regex
226            else:
227                raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.')
228        else:
229            re_module = re
230
231        # Some, but not all file-like objects have a 'name' attribute
232        if self.options.source_path is None:
233            try:
234                self.source_path = grammar.name
235            except AttributeError:
236                self.source_path = '<string>'
237        else:
238            self.source_path = self.options.source_path
239
240        # Drain file-like objects to get their contents
241        try:
242            read = grammar.read
243        except AttributeError:
244            pass
245        else:
246            grammar = read()
247
248        cache_fn = None
249        cache_md5 = None
250        if isinstance(grammar, STRING_TYPE):
251            self.source_grammar = grammar
252            if self.options.use_bytes:
253                if not isascii(grammar):
254                    raise ConfigurationError("Grammar must be ascii only, when use_bytes=True")
255                if sys.version_info[0] == 2 and self.options.use_bytes != 'force':
256                    raise ConfigurationError("`use_bytes=True` may have issues on python2."
257                                              "Use `use_bytes='force'` to use it at your own risk.")
258
259            if self.options.cache:
260                if self.options.parser != 'lalr':
261                    raise ConfigurationError("cache only works with parser='lalr' for now")
262
263                unhashable = ('transformer', 'postlex', 'lexer_callbacks', 'edit_terminals')
264                options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable)
265                from . import __version__
266                s = grammar + options_str + __version__ + str(sys.version_info[:2])
267                cache_md5 = hashlib.md5(s.encode('utf8')).hexdigest()
268
269                if isinstance(self.options.cache, STRING_TYPE):
270                    cache_fn = self.options.cache
271                else:
272                    if self.options.cache is not True:
273                        raise ConfigurationError("cache argument must be bool or str")
274                    # Python2.7 doesn't support * syntax in tuples
275                    cache_fn = tempfile.gettempdir() + '/.lark_cache_%s_%s_%s.tmp' % ((cache_md5,) + sys.version_info[:2])
276
277                if FS.exists(cache_fn):
278                    logger.debug('Loading grammar from cache: %s', cache_fn)
279                    # Remove options that aren't relevant for loading from cache
280                    for name in (set(options) - _LOAD_ALLOWED_OPTIONS):
281                        del options[name]
282                    with FS.open(cache_fn, 'rb') as f:
283                        old_options = self.options
284                        try:
285                            file_md5 = f.readline().rstrip(b'\n')
286                            cached_used_files = pickle.load(f)
287                            if file_md5 == cache_md5.encode('utf8') and verify_used_files(cached_used_files):
288                                cached_parser_data = pickle.load(f)
289                                self._load(cached_parser_data, **options)
290                                return
291                        except Exception: # We should probably narrow done which errors we catch here.
292                            logger.exception("Failed to load Lark from cache: %r. We will try to carry on." % cache_fn)
293
294                            # In theory, the Lark instance might have been messed up by the call to `_load`.
295                            # In practice the only relevant thing that might have been overriden should be `options`
296                            self.options = old_options
297
298
299            # Parse the grammar file and compose the grammars
300            self.grammar, used_files = load_grammar(grammar, self.source_path, self.options.import_paths, self.options.keep_all_tokens)
301        else:
302            assert isinstance(grammar, Grammar)
303            self.grammar = grammar
304
305
306        if self.options.lexer == 'auto':
307            if self.options.parser == 'lalr':
308                self.options.lexer = 'contextual'
309            elif self.options.parser == 'earley':
310                if self.options.postlex is not None:
311                    logger.info("postlex can't be used with the dynamic lexer, so we use standard instead. "
312                                "Consider using lalr with contextual instead of earley")
313                    self.options.lexer = 'standard'
314                else:
315                    self.options.lexer = 'dynamic'
316            elif self.options.parser == 'cyk':
317                self.options.lexer = 'standard'
318            else:
319                assert False, self.options.parser
320        lexer = self.options.lexer
321        if isinstance(lexer, type):
322            assert issubclass(lexer, Lexer)     # XXX Is this really important? Maybe just ensure interface compliance
323        else:
324            assert_config(lexer, ('standard', 'contextual', 'dynamic', 'dynamic_complete'))
325            if self.options.postlex is not None and 'dynamic' in lexer:
326                raise ConfigurationError("Can't use postlex with a dynamic lexer. Use standard or contextual instead")
327
328        if self.options.ambiguity == 'auto':
329            if self.options.parser == 'earley':
330                self.options.ambiguity = 'resolve'
331        else:
332            assert_config(self.options.parser, ('earley', 'cyk'), "%r doesn't support disambiguation. Use one of these parsers instead: %s")
333
334        if self.options.priority == 'auto':
335            self.options.priority = 'normal'
336
337        if self.options.priority not in _VALID_PRIORITY_OPTIONS:
338            raise ConfigurationError("invalid priority option: %r. Must be one of %r" % (self.options.priority, _VALID_PRIORITY_OPTIONS))
339        assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"'
340        if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS:
341            raise ConfigurationError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS))
342
343        if self.options.parser is None:
344            terminals_to_keep = '*'
345        elif self.options.postlex is not None:
346            terminals_to_keep = set(self.options.postlex.always_accept)
347        else:
348            terminals_to_keep = set()
349
350        # Compile the EBNF grammar into BNF
351        self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start, terminals_to_keep)
352
353        if self.options.edit_terminals:
354            for t in self.terminals:
355                self.options.edit_terminals(t)
356
357        self._terminals_dict = {t.name: t for t in self.terminals}
358
359        # If the user asked to invert the priorities, negate them all here.
360        # This replaces the old 'resolve__antiscore_sum' option.
361        if self.options.priority == 'invert':
362            for rule in self.rules:
363                if rule.options.priority is not None:
364                    rule.options.priority = -rule.options.priority
365        # Else, if the user asked to disable priorities, strip them from the
366        # rules. This allows the Earley parsers to skip an extra forest walk
367        # for improved performance, if you don't need them (or didn't specify any).
368        elif self.options.priority is None:
369            for rule in self.rules:
370                if rule.options.priority is not None:
371                    rule.options.priority = None
372
373        # TODO Deprecate lexer_callbacks?
374        self.lexer_conf = LexerConf(
375                self.terminals, re_module, self.ignore_tokens, self.options.postlex,
376                self.options.lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes
377            )
378
379        if self.options.parser:
380            self.parser = self._build_parser()
381        elif lexer:
382            self.lexer = self._build_lexer()
383
384        if cache_fn:
385            logger.debug('Saving grammar to cache: %s', cache_fn)
386            with FS.open(cache_fn, 'wb') as f:
387                f.write(cache_md5.encode('utf8') + b'\n')
388                pickle.dump(used_files, f)
389                self.save(f)
390
391    if __doc__:
392        __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC
393
394    __serialize_fields__ = 'parser', 'rules', 'options'
395
396    def _build_lexer(self, dont_ignore=False):
397        lexer_conf = self.lexer_conf
398        if dont_ignore:
399            from copy import copy
400            lexer_conf = copy(lexer_conf)
401            lexer_conf.ignore = ()
402        return TraditionalLexer(lexer_conf)
403
404    def _prepare_callbacks(self):
405        self._callbacks = {}
406        # we don't need these callbacks if we aren't building a tree
407        if self.options.ambiguity != 'forest':
408            self._parse_tree_builder = ParseTreeBuilder(
409                    self.rules,
410                    self.options.tree_class or Tree,
411                    self.options.propagate_positions,
412                    self.options.parser != 'lalr' and self.options.ambiguity == 'explicit',
413                    self.options.maybe_placeholders
414                )
415            self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
416        self._callbacks.update(_get_lexer_callbacks(self.options.transformer, self.terminals))
417
418    def _build_parser(self):
419        self._prepare_callbacks()
420        parser_class = get_frontend(self.options.parser, self.options.lexer)
421        parser_conf = ParserConf(self.rules, self._callbacks, self.options.start)
422        return parser_class(self.lexer_conf, parser_conf, options=self.options)
423
424    def save(self, f):
425        """Saves the instance into the given file object
426
427        Useful for caching and multiprocessing.
428        """
429        data, m = self.memo_serialize([TerminalDef, Rule])
430        pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL)
431
432    @classmethod
433    def load(cls, f):
434        """Loads an instance from the given file object
435
436        Useful for caching and multiprocessing.
437        """
438        inst = cls.__new__(cls)
439        return inst._load(f)
440
441    def _deserialize_lexer_conf(self, data, memo, options):
442        lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo)
443        lexer_conf.callbacks = options.lexer_callbacks or {}
444        lexer_conf.re_module = regex if options.regex else re
445        lexer_conf.use_bytes = options.use_bytes
446        lexer_conf.g_regex_flags = options.g_regex_flags
447        lexer_conf.skip_validation = True
448        lexer_conf.postlex = options.postlex
449        return lexer_conf
450
451    def _load(self, f, **kwargs):
452        if isinstance(f, dict):
453            d = f
454        else:
455            d = pickle.load(f)
456        memo_json = d['memo']
457        data = d['data']
458
459        assert memo_json
460        memo = SerializeMemoizer.deserialize(memo_json, {'Rule': Rule, 'TerminalDef': TerminalDef}, {})
461        options = dict(data['options'])
462        if (set(kwargs) - _LOAD_ALLOWED_OPTIONS) & set(LarkOptions._defaults):
463            raise ConfigurationError("Some options are not allowed when loading a Parser: {}"
464                             .format(set(kwargs) - _LOAD_ALLOWED_OPTIONS))
465        options.update(kwargs)
466        self.options = LarkOptions.deserialize(options, memo)
467        self.rules = [Rule.deserialize(r, memo) for r in data['rules']]
468        self.source_path = '<deserialized>'
469        parser_class = get_frontend(self.options.parser, self.options.lexer)
470        self.lexer_conf = self._deserialize_lexer_conf(data['parser'], memo, self.options)
471        self.terminals = self.lexer_conf.terminals
472        self._prepare_callbacks()
473        self._terminals_dict = {t.name: t for t in self.terminals}
474        self.parser = parser_class.deserialize(
475            data['parser'],
476            memo,
477            self.lexer_conf,
478            self._callbacks,
479            self.options,  # Not all, but multiple attributes are used
480        )
481        return self
482
483    @classmethod
484    def _load_from_dict(cls, data, memo, **kwargs):
485        inst = cls.__new__(cls)
486        return inst._load({'data': data, 'memo': memo}, **kwargs)
487
488    @classmethod
489    def open(cls, grammar_filename, rel_to=None, **options):
490        """Create an instance of Lark with the grammar given by its filename
491
492        If ``rel_to`` is provided, the function will find the grammar filename in relation to it.
493
494        Example:
495
496            >>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr")
497            Lark(...)
498
499        """
500        if rel_to:
501            basepath = os.path.dirname(rel_to)
502            grammar_filename = os.path.join(basepath, grammar_filename)
503        with open(grammar_filename, encoding='utf8') as f:
504            return cls(f, **options)
505
506    @classmethod
507    def open_from_package(cls, package, grammar_path, search_paths=("",), **options):
508        """Create an instance of Lark with the grammar loaded from within the package `package`.
509        This allows grammar loading from zipapps.
510
511        Imports in the grammar will use the `package` and `search_paths` provided, through `FromPackageLoader`
512
513        Example:
514
515            Lark.open_from_package(__name__, "example.lark", ("grammars",), parser=...)
516        """
517        package_loader = FromPackageLoader(package, search_paths)
518        full_path, text = package_loader(None, grammar_path)
519        options.setdefault('source_path', full_path)
520        options.setdefault('import_paths', [])
521        options['import_paths'].append(package_loader)
522        return cls(text, **options)
523
524    def __repr__(self):
525        return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source_path, self.options.parser, self.options.lexer)
526
527
528    def lex(self, text, dont_ignore=False):
529        """Only lex (and postlex) the text, without parsing it. Only relevant when lexer='standard'
530
531        When dont_ignore=True, the lexer will return all tokens, even those marked for %ignore.
532
533        :raises UnexpectedCharacters: In case the lexer cannot find a suitable match.
534        """
535        if not hasattr(self, 'lexer') or dont_ignore:
536            lexer = self._build_lexer(dont_ignore)
537        else:
538            lexer = self.lexer
539        lexer_thread = LexerThread(lexer, text)
540        stream = lexer_thread.lex(None)
541        if self.options.postlex:
542            return self.options.postlex.process(stream)
543        return stream
544
545    def get_terminal(self, name):
546        """Get information about a terminal"""
547        return self._terminals_dict[name]
548
549    def parse_interactive(self, text=None, start=None):
550        """Start an interactive parsing session.
551
552        Parameters:
553            text (str, optional): Text to be parsed. Required for ``resume_parse()``.
554            start (str, optional): Start symbol
555
556        Returns:
557            A new InteractiveParser instance.
558
559        See Also: ``Lark.parse()``
560        """
561        return self.parser.parse_interactive(text, start=start)
562
563    def parse(self, text, start=None, on_error=None):
564        """Parse the given text, according to the options provided.
565
566        Parameters:
567            text (str): Text to be parsed.
568            start (str, optional): Required if Lark was given multiple possible start symbols (using the start option).
569            on_error (function, optional): if provided, will be called on UnexpectedToken error. Return true to resume parsing.
570                LALR only. See examples/advanced/error_handling.py for an example of how to use on_error.
571
572        Returns:
573            If a transformer is supplied to ``__init__``, returns whatever is the
574            result of the transformation. Otherwise, returns a Tree instance.
575
576        :raises UnexpectedInput: On a parse error, one of these sub-exceptions will rise:
577                ``UnexpectedCharacters``, ``UnexpectedToken``, or ``UnexpectedEOF``.
578                For convenience, these sub-exceptions also inherit from ``ParserError`` and ``LexerError``.
579
580        """
581        return self.parser.parse(text, start=start, on_error=on_error)
582
583    @property
584    def source(self):
585        warn("Attribute Lark.source was renamed to Lark.source_path", DeprecationWarning)
586        return self.source_path
587
588    @source.setter
589    def source(self, value):
590        self.source_path = value
591
592    @property
593    def grammar_source(self):
594        warn("Attribute Lark.grammar_source was renamed to Lark.source_grammar", DeprecationWarning)
595        return self.source_grammar
596
597    @grammar_source.setter
598    def grammar_source(self, value):
599        self.source_grammar = value
600
601
602###}
603