1from .exceptions import ConfigurationError, GrammarError, assert_config 2from .utils import get_regexp_width, Serialize 3from .parsers.grammar_analysis import GrammarAnalyzer 4from .lexer import LexerThread, TraditionalLexer, ContextualLexer, Lexer, Token, TerminalDef 5from .parsers import earley, xearley, cyk 6from .parsers.lalr_parser import LALR_Parser 7from .tree import Tree 8from .common import LexerConf, ParserConf 9try: 10 import regex 11except ImportError: 12 regex = None 13import re 14 15###{standalone 16 17def _wrap_lexer(lexer_class): 18 future_interface = getattr(lexer_class, '__future_interface__', False) 19 if future_interface: 20 return lexer_class 21 else: 22 class CustomLexerWrapper(Lexer): 23 def __init__(self, lexer_conf): 24 self.lexer = lexer_class(lexer_conf) 25 def lex(self, lexer_state, parser_state): 26 return self.lexer.lex(lexer_state.text) 27 return CustomLexerWrapper 28 29 30class MakeParsingFrontend: 31 def __init__(self, parser_type, lexer_type): 32 self.parser_type = parser_type 33 self.lexer_type = lexer_type 34 35 def __call__(self, lexer_conf, parser_conf, options): 36 assert isinstance(lexer_conf, LexerConf) 37 assert isinstance(parser_conf, ParserConf) 38 parser_conf.parser_type = self.parser_type 39 lexer_conf.lexer_type = self.lexer_type 40 return ParsingFrontend(lexer_conf, parser_conf, options) 41 42 def deserialize(self, data, memo, lexer_conf, callbacks, options): 43 parser_conf = ParserConf.deserialize(data['parser_conf'], memo) 44 parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug) 45 parser_conf.callbacks = callbacks 46 return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser) 47 48 49 50 51class ParsingFrontend(Serialize): 52 __serialize_fields__ = 'lexer_conf', 'parser_conf', 'parser', 'options' 53 54 def __init__(self, lexer_conf, parser_conf, options, parser=None): 55 self.parser_conf = parser_conf 56 self.lexer_conf = lexer_conf 57 self.options = options 58 59 # Set-up parser 60 if parser: # From cache 61 self.parser = parser 62 else: 63 create_parser = { 64 'lalr': create_lalr_parser, 65 'earley': create_earley_parser, 66 'cyk': CYK_FrontEnd, 67 }[parser_conf.parser_type] 68 self.parser = create_parser(lexer_conf, parser_conf, options) 69 70 # Set-up lexer 71 lexer_type = lexer_conf.lexer_type 72 self.skip_lexer = False 73 if lexer_type in ('dynamic', 'dynamic_complete'): 74 assert lexer_conf.postlex is None 75 self.skip_lexer = True 76 return 77 78 try: 79 create_lexer = { 80 'standard': create_traditional_lexer, 81 'contextual': create_contextual_lexer, 82 }[lexer_type] 83 except KeyError: 84 assert issubclass(lexer_type, Lexer), lexer_type 85 self.lexer = _wrap_lexer(lexer_type)(lexer_conf) 86 else: 87 self.lexer = create_lexer(lexer_conf, self.parser, lexer_conf.postlex) 88 89 if lexer_conf.postlex: 90 self.lexer = PostLexConnector(self.lexer, lexer_conf.postlex) 91 92 def _verify_start(self, start=None): 93 if start is None: 94 start_decls = self.parser_conf.start 95 if len(start_decls) > 1: 96 raise ConfigurationError("Lark initialized with more than 1 possible start rule. Must specify which start rule to parse", start_decls) 97 start ,= start_decls 98 elif start not in self.parser_conf.start: 99 raise ConfigurationError("Unknown start rule %s. Must be one of %r" % (start, self.parser_conf.start)) 100 return start 101 102 def parse(self, text, start=None, on_error=None): 103 chosen_start = self._verify_start(start) 104 stream = text if self.skip_lexer else LexerThread(self.lexer, text) 105 kw = {} if on_error is None else {'on_error': on_error} 106 return self.parser.parse(stream, chosen_start, **kw) 107 108 def parse_interactive(self, text=None, start=None): 109 chosen_start = self._verify_start(start) 110 if self.parser_conf.parser_type != 'lalr': 111 raise ConfigurationError("parse_interactive() currently only works with parser='lalr' ") 112 stream = text if self.skip_lexer else LexerThread(self.lexer, text) 113 return self.parser.parse_interactive(stream, chosen_start) 114 115 116def get_frontend(parser, lexer): 117 assert_config(parser, ('lalr', 'earley', 'cyk')) 118 if not isinstance(lexer, type): # not custom lexer? 119 expected = { 120 'lalr': ('standard', 'contextual'), 121 'earley': ('standard', 'dynamic', 'dynamic_complete'), 122 'cyk': ('standard', ), 123 }[parser] 124 assert_config(lexer, expected, 'Parser %r does not support lexer %%r, expected one of %%s' % parser) 125 126 return MakeParsingFrontend(parser, lexer) 127 128 129def _get_lexer_callbacks(transformer, terminals): 130 result = {} 131 for terminal in terminals: 132 callback = getattr(transformer, terminal.name, None) 133 if callback is not None: 134 result[terminal.name] = callback 135 return result 136 137class PostLexConnector: 138 def __init__(self, lexer, postlexer): 139 self.lexer = lexer 140 self.postlexer = postlexer 141 142 def make_lexer_state(self, text): 143 return self.lexer.make_lexer_state(text) 144 145 def lex(self, lexer_state, parser_state): 146 i = self.lexer.lex(lexer_state, parser_state) 147 return self.postlexer.process(i) 148 149 150 151def create_traditional_lexer(lexer_conf, parser, postlex): 152 return TraditionalLexer(lexer_conf) 153 154def create_contextual_lexer(lexer_conf, parser, postlex): 155 states = {idx:list(t.keys()) for idx, t in parser._parse_table.states.items()} 156 always_accept = postlex.always_accept if postlex else () 157 return ContextualLexer(lexer_conf, states, always_accept=always_accept) 158 159def create_lalr_parser(lexer_conf, parser_conf, options=None): 160 debug = options.debug if options else False 161 return LALR_Parser(parser_conf, debug=debug) 162 163 164create_earley_parser = NotImplemented 165CYK_FrontEnd = NotImplemented 166###} 167 168class EarleyRegexpMatcher: 169 def __init__(self, lexer_conf): 170 self.regexps = {} 171 for t in lexer_conf.terminals: 172 if t.priority != 1: 173 raise GrammarError("Dynamic Earley doesn't support weights on terminals", t, t.priority) 174 regexp = t.pattern.to_regexp() 175 try: 176 width = get_regexp_width(regexp)[0] 177 except ValueError: 178 raise GrammarError("Bad regexp in token %s: %s" % (t.name, regexp)) 179 else: 180 if width == 0: 181 raise GrammarError("Dynamic Earley doesn't allow zero-width regexps", t) 182 if lexer_conf.use_bytes: 183 regexp = regexp.encode('utf-8') 184 185 self.regexps[t.name] = lexer_conf.re_module.compile(regexp, lexer_conf.g_regex_flags) 186 187 def match(self, term, text, index=0): 188 return self.regexps[term.name].match(text, index) 189 190 191def create_earley_parser__dynamic(lexer_conf, parser_conf, options=None, **kw): 192 earley_matcher = EarleyRegexpMatcher(lexer_conf) 193 return xearley.Parser(parser_conf, earley_matcher.match, ignore=lexer_conf.ignore, **kw) 194 195def _match_earley_basic(term, token): 196 return term.name == token.type 197 198def create_earley_parser__basic(lexer_conf, parser_conf, options, **kw): 199 return earley.Parser(parser_conf, _match_earley_basic, **kw) 200 201def create_earley_parser(lexer_conf, parser_conf, options): 202 resolve_ambiguity = options.ambiguity == 'resolve' 203 debug = options.debug if options else False 204 tree_class = options.tree_class or Tree if options.ambiguity != 'forest' else None 205 206 extra = {} 207 if lexer_conf.lexer_type == 'dynamic': 208 f = create_earley_parser__dynamic 209 elif lexer_conf.lexer_type == 'dynamic_complete': 210 extra['complete_lex'] =True 211 f = create_earley_parser__dynamic 212 else: 213 f = create_earley_parser__basic 214 215 return f(lexer_conf, parser_conf, options, resolve_ambiguity=resolve_ambiguity, debug=debug, tree_class=tree_class, **extra) 216 217 218 219class CYK_FrontEnd: 220 def __init__(self, lexer_conf, parser_conf, options=None): 221 self._analysis = GrammarAnalyzer(parser_conf) 222 self.parser = cyk.Parser(parser_conf.rules) 223 224 self.callbacks = parser_conf.callbacks 225 226 def parse(self, lexer_thread, start): 227 tokens = list(lexer_thread.lex(None)) 228 tree = self.parser.parse(tokens, start) 229 return self._transform(tree) 230 231 def _transform(self, tree): 232 subtrees = list(tree.iter_subtrees()) 233 for subtree in subtrees: 234 subtree.children = [self._apply_callback(c) if isinstance(c, Tree) else c for c in subtree.children] 235 236 return self._apply_callback(tree) 237 238 def _apply_callback(self, tree): 239 return self.callbacks[tree.rule](tree.children) 240