1# 2# TreeFragments - parsing of strings to trees 3# 4 5""" 6Support for parsing strings into code trees. 7""" 8 9from __future__ import absolute_import 10 11import re 12from io import StringIO 13 14from .Scanning import PyrexScanner, StringSourceDescriptor 15from .Symtab import ModuleScope 16from . import PyrexTypes 17from .Visitor import VisitorTransform 18from .Nodes import Node, StatListNode 19from .ExprNodes import NameNode 20from .StringEncoding import _unicode 21from . import Parsing 22from . import Main 23from . import UtilNodes 24 25 26class StringParseContext(Main.Context): 27 def __init__(self, name, include_directories=None, compiler_directives=None, cpp=False): 28 if include_directories is None: 29 include_directories = [] 30 if compiler_directives is None: 31 compiler_directives = {} 32 # TODO: see if "language_level=3" also works for our internal code here. 33 Main.Context.__init__(self, include_directories, compiler_directives, cpp=cpp, language_level=2) 34 self.module_name = name 35 36 def find_module(self, module_name, relative_to=None, pos=None, need_pxd=1, absolute_fallback=True): 37 if module_name not in (self.module_name, 'cython'): 38 raise AssertionError("Not yet supporting any cimports/includes from string code snippets") 39 return ModuleScope(module_name, parent_module=None, context=self) 40 41 42def parse_from_strings(name, code, pxds=None, level=None, initial_pos=None, 43 context=None, allow_struct_enum_decorator=False): 44 """ 45 Utility method to parse a (unicode) string of code. This is mostly 46 used for internal Cython compiler purposes (creating code snippets 47 that transforms should emit, as well as unit testing). 48 49 code - a unicode string containing Cython (module-level) code 50 name - a descriptive name for the code source (to use in error messages etc.) 51 52 RETURNS 53 54 The tree, i.e. a ModuleNode. The ModuleNode's scope attribute is 55 set to the scope used when parsing. 56 """ 57 if context is None: 58 context = StringParseContext(name) 59 # Since source files carry an encoding, it makes sense in this context 60 # to use a unicode string so that code fragments don't have to bother 61 # with encoding. This means that test code passed in should not have an 62 # encoding header. 63 assert isinstance(code, _unicode), "unicode code snippets only please" 64 encoding = "UTF-8" 65 66 module_name = name 67 if initial_pos is None: 68 initial_pos = (name, 1, 0) 69 code_source = StringSourceDescriptor(name, code) 70 71 scope = context.find_module(module_name, pos=initial_pos, need_pxd=False) 72 73 buf = StringIO(code) 74 75 scanner = PyrexScanner(buf, code_source, source_encoding = encoding, 76 scope = scope, context = context, initial_pos = initial_pos) 77 ctx = Parsing.Ctx(allow_struct_enum_decorator=allow_struct_enum_decorator) 78 79 if level is None: 80 tree = Parsing.p_module(scanner, 0, module_name, ctx=ctx) 81 tree.scope = scope 82 tree.is_pxd = False 83 else: 84 tree = Parsing.p_code(scanner, level=level, ctx=ctx) 85 86 tree.scope = scope 87 return tree 88 89 90class TreeCopier(VisitorTransform): 91 def visit_Node(self, node): 92 if node is None: 93 return node 94 else: 95 c = node.clone_node() 96 self.visitchildren(c) 97 return c 98 99 100class ApplyPositionAndCopy(TreeCopier): 101 def __init__(self, pos): 102 super(ApplyPositionAndCopy, self).__init__() 103 self.pos = pos 104 105 def visit_Node(self, node): 106 copy = super(ApplyPositionAndCopy, self).visit_Node(node) 107 copy.pos = self.pos 108 return copy 109 110 111class TemplateTransform(VisitorTransform): 112 """ 113 Makes a copy of a template tree while doing substitutions. 114 115 A dictionary "substitutions" should be passed in when calling 116 the transform; mapping names to replacement nodes. Then replacement 117 happens like this: 118 - If an ExprStatNode contains a single NameNode, whose name is 119 a key in the substitutions dictionary, the ExprStatNode is 120 replaced with a copy of the tree given in the dictionary. 121 It is the responsibility of the caller that the replacement 122 node is a valid statement. 123 - If a single NameNode is otherwise encountered, it is replaced 124 if its name is listed in the substitutions dictionary in the 125 same way. It is the responsibility of the caller to make sure 126 that the replacement nodes is a valid expression. 127 128 Also a list "temps" should be passed. Any names listed will 129 be transformed into anonymous, temporary names. 130 131 Currently supported for tempnames is: 132 NameNode 133 (various function and class definition nodes etc. should be added to this) 134 135 Each replacement node gets the position of the substituted node 136 recursively applied to every member node. 137 """ 138 139 temp_name_counter = 0 140 141 def __call__(self, node, substitutions, temps, pos): 142 self.substitutions = substitutions 143 self.pos = pos 144 tempmap = {} 145 temphandles = [] 146 for temp in temps: 147 TemplateTransform.temp_name_counter += 1 148 handle = UtilNodes.TempHandle(PyrexTypes.py_object_type) 149 tempmap[temp] = handle 150 temphandles.append(handle) 151 self.tempmap = tempmap 152 result = super(TemplateTransform, self).__call__(node) 153 if temps: 154 result = UtilNodes.TempsBlockNode(self.get_pos(node), 155 temps=temphandles, 156 body=result) 157 return result 158 159 def get_pos(self, node): 160 if self.pos: 161 return self.pos 162 else: 163 return node.pos 164 165 def visit_Node(self, node): 166 if node is None: 167 return None 168 else: 169 c = node.clone_node() 170 if self.pos is not None: 171 c.pos = self.pos 172 self.visitchildren(c) 173 return c 174 175 def try_substitution(self, node, key): 176 sub = self.substitutions.get(key) 177 if sub is not None: 178 pos = self.pos 179 if pos is None: pos = node.pos 180 return ApplyPositionAndCopy(pos)(sub) 181 else: 182 return self.visit_Node(node) # make copy as usual 183 184 def visit_NameNode(self, node): 185 temphandle = self.tempmap.get(node.name) 186 if temphandle: 187 # Replace name with temporary 188 return temphandle.ref(self.get_pos(node)) 189 else: 190 return self.try_substitution(node, node.name) 191 192 def visit_ExprStatNode(self, node): 193 # If an expression-as-statement consists of only a replaceable 194 # NameNode, we replace the entire statement, not only the NameNode 195 if isinstance(node.expr, NameNode): 196 return self.try_substitution(node, node.expr.name) 197 else: 198 return self.visit_Node(node) 199 200 201def copy_code_tree(node): 202 return TreeCopier()(node) 203 204 205_match_indent = re.compile(u"^ *").match 206 207 208def strip_common_indent(lines): 209 """Strips empty lines and common indentation from the list of strings given in lines""" 210 # TODO: Facilitate textwrap.indent instead 211 lines = [x for x in lines if x.strip() != u""] 212 if lines: 213 minindent = min([len(_match_indent(x).group(0)) for x in lines]) 214 lines = [x[minindent:] for x in lines] 215 return lines 216 217 218class TreeFragment(object): 219 def __init__(self, code, name=None, pxds=None, temps=None, pipeline=None, level=None, initial_pos=None): 220 if pxds is None: 221 pxds = {} 222 if temps is None: 223 temps = [] 224 if pipeline is None: 225 pipeline = [] 226 if not name: 227 name = "(tree fragment)" 228 229 if isinstance(code, _unicode): 230 def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n"))) 231 232 fmt_code = fmt(code) 233 fmt_pxds = {} 234 for key, value in pxds.items(): 235 fmt_pxds[key] = fmt(value) 236 mod = t = parse_from_strings(name, fmt_code, fmt_pxds, level=level, initial_pos=initial_pos) 237 if level is None: 238 t = t.body # Make sure a StatListNode is at the top 239 if not isinstance(t, StatListNode): 240 t = StatListNode(pos=mod.pos, stats=[t]) 241 for transform in pipeline: 242 if transform is None: 243 continue 244 t = transform(t) 245 self.root = t 246 elif isinstance(code, Node): 247 if pxds: 248 raise NotImplementedError() 249 self.root = code 250 else: 251 raise ValueError("Unrecognized code format (accepts unicode and Node)") 252 self.temps = temps 253 254 def copy(self): 255 return copy_code_tree(self.root) 256 257 def substitute(self, nodes=None, temps=None, pos = None): 258 if nodes is None: 259 nodes = {} 260 if temps is None: 261 temps = [] 262 return TemplateTransform()(self.root, 263 substitutions = nodes, 264 temps = self.temps + temps, pos = pos) 265 266 267class SetPosTransform(VisitorTransform): 268 def __init__(self, pos): 269 super(SetPosTransform, self).__init__() 270 self.pos = pos 271 272 def visit_Node(self, node): 273 node.pos = self.pos 274 self.visitchildren(node) 275 return node 276