1from .exceptions import ConfigurationError, GrammarError, assert_config
2from .utils import get_regexp_width, Serialize
3from .parsers.grammar_analysis import GrammarAnalyzer
4from .lexer import LexerThread, TraditionalLexer, ContextualLexer, Lexer, Token, TerminalDef
5from .parsers import earley, xearley, cyk
6from .parsers.lalr_parser import LALR_Parser
7from .tree import Tree
8from .common import LexerConf, ParserConf
9try:
10    import regex
11except ImportError:
12    regex = None
13import re
14
15###{standalone
16
17def _wrap_lexer(lexer_class):
18    future_interface = getattr(lexer_class, '__future_interface__', False)
19    if future_interface:
20        return lexer_class
21    else:
22        class CustomLexerWrapper(Lexer):
23            def __init__(self, lexer_conf):
24                self.lexer = lexer_class(lexer_conf)
25            def lex(self, lexer_state, parser_state):
26                return self.lexer.lex(lexer_state.text)
27        return CustomLexerWrapper
28
29
30class MakeParsingFrontend:
31    def __init__(self, parser_type, lexer_type):
32        self.parser_type = parser_type
33        self.lexer_type = lexer_type
34
35    def __call__(self, lexer_conf, parser_conf, options):
36        assert isinstance(lexer_conf, LexerConf)
37        assert isinstance(parser_conf, ParserConf)
38        parser_conf.parser_type = self.parser_type
39        lexer_conf.lexer_type = self.lexer_type
40        return ParsingFrontend(lexer_conf, parser_conf, options)
41
42    def deserialize(self, data, memo, lexer_conf, callbacks, options):
43        parser_conf = ParserConf.deserialize(data['parser_conf'], memo)
44        parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug)
45        parser_conf.callbacks = callbacks
46        return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser)
47
48
49
50
51class ParsingFrontend(Serialize):
52    __serialize_fields__ = 'lexer_conf', 'parser_conf', 'parser', 'options'
53
54    def __init__(self, lexer_conf, parser_conf, options, parser=None):
55        self.parser_conf = parser_conf
56        self.lexer_conf = lexer_conf
57        self.options = options
58
59        # Set-up parser
60        if parser:  # From cache
61            self.parser = parser
62        else:
63            create_parser = {
64                'lalr': create_lalr_parser,
65                'earley': create_earley_parser,
66                'cyk': CYK_FrontEnd,
67            }[parser_conf.parser_type]
68            self.parser = create_parser(lexer_conf, parser_conf, options)
69
70        # Set-up lexer
71        lexer_type = lexer_conf.lexer_type
72        self.skip_lexer = False
73        if lexer_type in ('dynamic', 'dynamic_complete'):
74            assert lexer_conf.postlex is None
75            self.skip_lexer = True
76            return
77
78        try:
79            create_lexer = {
80                'standard': create_traditional_lexer,
81                'contextual': create_contextual_lexer,
82            }[lexer_type]
83        except KeyError:
84            assert issubclass(lexer_type, Lexer), lexer_type
85            self.lexer = _wrap_lexer(lexer_type)(lexer_conf)
86        else:
87            self.lexer = create_lexer(lexer_conf, self.parser, lexer_conf.postlex)
88
89        if lexer_conf.postlex:
90            self.lexer = PostLexConnector(self.lexer, lexer_conf.postlex)
91
92    def _verify_start(self, start=None):
93        if start is None:
94            start_decls = self.parser_conf.start
95            if len(start_decls) > 1:
96                raise ConfigurationError("Lark initialized with more than 1 possible start rule. Must specify which start rule to parse", start_decls)
97            start ,= start_decls
98        elif start not in self.parser_conf.start:
99            raise ConfigurationError("Unknown start rule %s. Must be one of %r" % (start, self.parser_conf.start))
100        return start
101
102    def parse(self, text, start=None, on_error=None):
103        chosen_start = self._verify_start(start)
104        stream = text if self.skip_lexer else LexerThread(self.lexer, text)
105        kw = {} if on_error is None else {'on_error': on_error}
106        return self.parser.parse(stream, chosen_start, **kw)
107
108    def parse_interactive(self, text=None, start=None):
109        chosen_start = self._verify_start(start)
110        if self.parser_conf.parser_type != 'lalr':
111            raise ConfigurationError("parse_interactive() currently only works with parser='lalr' ")
112        stream = text if self.skip_lexer else LexerThread(self.lexer, text)
113        return self.parser.parse_interactive(stream, chosen_start)
114
115
116def get_frontend(parser, lexer):
117    assert_config(parser, ('lalr', 'earley', 'cyk'))
118    if not isinstance(lexer, type):     # not custom lexer?
119        expected = {
120            'lalr': ('standard', 'contextual'),
121            'earley': ('standard', 'dynamic', 'dynamic_complete'),
122            'cyk': ('standard', ),
123         }[parser]
124        assert_config(lexer, expected, 'Parser %r does not support lexer %%r, expected one of %%s' % parser)
125
126    return MakeParsingFrontend(parser, lexer)
127
128
129def _get_lexer_callbacks(transformer, terminals):
130    result = {}
131    for terminal in terminals:
132        callback = getattr(transformer, terminal.name, None)
133        if callback is not None:
134            result[terminal.name] = callback
135    return result
136
137class PostLexConnector:
138    def __init__(self, lexer, postlexer):
139        self.lexer = lexer
140        self.postlexer = postlexer
141
142    def make_lexer_state(self, text):
143        return self.lexer.make_lexer_state(text)
144
145    def lex(self, lexer_state, parser_state):
146        i = self.lexer.lex(lexer_state, parser_state)
147        return self.postlexer.process(i)
148
149
150
151def create_traditional_lexer(lexer_conf, parser, postlex):
152    return TraditionalLexer(lexer_conf)
153
154def create_contextual_lexer(lexer_conf, parser, postlex):
155    states = {idx:list(t.keys()) for idx, t in parser._parse_table.states.items()}
156    always_accept = postlex.always_accept if postlex else ()
157    return ContextualLexer(lexer_conf, states, always_accept=always_accept)
158
159def create_lalr_parser(lexer_conf, parser_conf, options=None):
160    debug = options.debug if options else False
161    return LALR_Parser(parser_conf, debug=debug)
162
163
164create_earley_parser = NotImplemented
165CYK_FrontEnd = NotImplemented
166###}
167
168class EarleyRegexpMatcher:
169    def __init__(self, lexer_conf):
170        self.regexps = {}
171        for t in lexer_conf.terminals:
172            if t.priority != 1:
173                raise GrammarError("Dynamic Earley doesn't support weights on terminals", t, t.priority)
174            regexp = t.pattern.to_regexp()
175            try:
176                width = get_regexp_width(regexp)[0]
177            except ValueError:
178                raise GrammarError("Bad regexp in token %s: %s" % (t.name, regexp))
179            else:
180                if width == 0:
181                    raise GrammarError("Dynamic Earley doesn't allow zero-width regexps", t)
182            if lexer_conf.use_bytes:
183                regexp = regexp.encode('utf-8')
184
185            self.regexps[t.name] = lexer_conf.re_module.compile(regexp, lexer_conf.g_regex_flags)
186
187    def match(self, term, text, index=0):
188        return self.regexps[term.name].match(text, index)
189
190
191def create_earley_parser__dynamic(lexer_conf, parser_conf, options=None, **kw):
192        earley_matcher = EarleyRegexpMatcher(lexer_conf)
193        return xearley.Parser(parser_conf, earley_matcher.match, ignore=lexer_conf.ignore, **kw)
194
195def _match_earley_basic(term, token):
196    return term.name == token.type
197
198def create_earley_parser__basic(lexer_conf, parser_conf, options, **kw):
199    return earley.Parser(parser_conf, _match_earley_basic, **kw)
200
201def create_earley_parser(lexer_conf, parser_conf, options):
202    resolve_ambiguity = options.ambiguity == 'resolve'
203    debug = options.debug if options else False
204    tree_class = options.tree_class or Tree if options.ambiguity != 'forest' else None
205
206    extra = {}
207    if lexer_conf.lexer_type == 'dynamic':
208        f = create_earley_parser__dynamic
209    elif lexer_conf.lexer_type == 'dynamic_complete':
210        extra['complete_lex'] =True
211        f = create_earley_parser__dynamic
212    else:
213        f = create_earley_parser__basic
214
215    return f(lexer_conf, parser_conf, options, resolve_ambiguity=resolve_ambiguity, debug=debug, tree_class=tree_class, **extra)
216
217
218
219class CYK_FrontEnd:
220    def __init__(self, lexer_conf, parser_conf, options=None):
221        self._analysis = GrammarAnalyzer(parser_conf)
222        self.parser = cyk.Parser(parser_conf.rules)
223
224        self.callbacks = parser_conf.callbacks
225
226    def parse(self, lexer_thread, start):
227        tokens = list(lexer_thread.lex(None))
228        tree = self.parser.parse(tokens, start)
229        return self._transform(tree)
230
231    def _transform(self, tree):
232        subtrees = list(tree.iter_subtrees())
233        for subtree in subtrees:
234            subtree.children = [self._apply_callback(c) if isinstance(c, Tree) else c for c in subtree.children]
235
236        return self._apply_callback(tree)
237
238    def _apply_callback(self, tree):
239        return self.callbacks[tree.rule](tree.children)
240