1# -*- coding: utf-8 -*- 2"""Implements the xonsh executer.""" 3import sys 4import types 5import inspect 6import builtins 7import collections.abc as cabc 8 9from xonsh.ast import CtxAwareTransformer 10from xonsh.parser import Parser 11from xonsh.tools import ( 12 subproc_toks, 13 find_next_break, 14 get_logical_line, 15 replace_logical_line, 16 balanced_parens, 17) 18from xonsh.built_ins import load_builtins, unload_builtins 19 20 21class Execer(object): 22 """Executes xonsh code in a context.""" 23 24 def __init__( 25 self, 26 filename="<xonsh-code>", 27 debug_level=0, 28 parser_args=None, 29 unload=True, 30 xonsh_ctx=None, 31 scriptcache=True, 32 cacheall=False, 33 ): 34 """Parameters 35 ---------- 36 filename : str, optional 37 File we are to execute. 38 debug_level : int, optional 39 Debugging level to use in lexing and parsing. 40 parser_args : dict, optional 41 Arguments to pass down to the parser. 42 unload : bool, optional 43 Whether or not to unload xonsh builtins upon deletion. 44 xonsh_ctx : dict or None, optional 45 Xonsh xontext to load as builtins.__xonsh_ctx__ 46 scriptcache : bool, optional 47 Whether or not to use a precompiled bytecode cache when execing 48 code, default: True. 49 cacheall : bool, optional 50 Whether or not to cache all xonsh code, and not just files. If this 51 is set to true, it will cache command line input too, default: False. 52 """ 53 parser_args = parser_args or {} 54 self.parser = Parser(**parser_args) 55 self.filename = filename 56 self.debug_level = debug_level 57 self.unload = unload 58 self.scriptcache = scriptcache 59 self.cacheall = cacheall 60 self.ctxtransformer = CtxAwareTransformer(self.parser) 61 load_builtins(execer=self, ctx=xonsh_ctx) 62 63 def __del__(self): 64 if self.unload: 65 unload_builtins() 66 67 def parse(self, input, ctx, mode="exec", filename=None, transform=True): 68 """Parses xonsh code in a context-aware fashion. For context-free 69 parsing, please use the Parser class directly or pass in 70 transform=False. 71 """ 72 if filename is None: 73 filename = self.filename 74 if not transform: 75 return self.parser.parse( 76 input, filename=filename, mode=mode, debug_level=(self.debug_level > 2) 77 ) 78 79 # Parsing actually happens in a couple of phases. The first is a 80 # shortcut for a context-free parser. Normally, all subprocess 81 # lines should be wrapped in $(), to indicate that they are a 82 # subproc. But that would be super annoying. Unfortunately, Python 83 # mode - after indentation - is whitespace agnostic while, using 84 # the Python token, subproc mode is whitespace aware. That is to say, 85 # in Python mode "ls -l", "ls-l", and "ls - l" all parse to the 86 # same AST because whitespace doesn't matter to the minus binary op. 87 # However, these phases all have very different meaning in subproc 88 # mode. The 'right' way to deal with this is to make the entire 89 # grammar whitespace aware, and then ignore all of the whitespace 90 # tokens for all of the Python rules. The lazy way implemented here 91 # is to parse a line a second time with a $() wrapper if it fails 92 # the first time. This is a context-free phase. 93 tree, input = self._parse_ctx_free(input, mode=mode, filename=filename) 94 if tree is None: 95 return None 96 97 # Now we need to perform context-aware AST transformation. This is 98 # because the "ls -l" is valid Python. The only way that we know 99 # it is not actually Python is by checking to see if the first token 100 # (ls) is part of the execution context. If it isn't, then we will 101 # assume that this line is supposed to be a subprocess line, assuming 102 # it also is valid as a subprocess line. 103 if ctx is None: 104 ctx = set() 105 elif isinstance(ctx, cabc.Mapping): 106 ctx = set(ctx.keys()) 107 tree = self.ctxtransformer.ctxvisit( 108 tree, input, ctx, mode=mode, debug_level=self.debug_level 109 ) 110 return tree 111 112 def compile( 113 self, 114 input, 115 mode="exec", 116 glbs=None, 117 locs=None, 118 stacklevel=2, 119 filename=None, 120 transform=True, 121 ): 122 """Compiles xonsh code into a Python code object, which may then 123 be execed or evaled. 124 """ 125 if filename is None: 126 filename = self.filename 127 if glbs is None or locs is None: 128 frame = inspect.stack()[stacklevel][0] 129 glbs = frame.f_globals if glbs is None else glbs 130 locs = frame.f_locals if locs is None else locs 131 ctx = set(dir(builtins)) | set(glbs.keys()) | set(locs.keys()) 132 tree = self.parse(input, ctx, mode=mode, filename=filename, transform=transform) 133 if tree is None: 134 return None # handles comment only input 135 code = compile(tree, filename, mode) 136 return code 137 138 def eval( 139 self, input, glbs=None, locs=None, stacklevel=2, filename=None, transform=True 140 ): 141 """Evaluates (and returns) xonsh code.""" 142 if isinstance(input, types.CodeType): 143 code = input 144 else: 145 if filename is None: 146 filename = self.filename 147 code = self.compile( 148 input=input, 149 glbs=glbs, 150 locs=locs, 151 mode="eval", 152 stacklevel=stacklevel, 153 filename=filename, 154 transform=transform, 155 ) 156 if code is None: 157 return None # handles comment only input 158 return eval(code, glbs, locs) 159 160 def exec( 161 self, 162 input, 163 mode="exec", 164 glbs=None, 165 locs=None, 166 stacklevel=2, 167 filename=None, 168 transform=True, 169 ): 170 """Execute xonsh code.""" 171 if isinstance(input, types.CodeType): 172 code = input 173 else: 174 if filename is None: 175 filename = self.filename 176 code = self.compile( 177 input=input, 178 glbs=glbs, 179 locs=locs, 180 mode=mode, 181 stacklevel=stacklevel, 182 filename=filename, 183 transform=transform, 184 ) 185 if code is None: 186 return None # handles comment only input 187 return exec(code, glbs, locs) 188 189 def _print_debug_wrapping( 190 self, line, sbpline, last_error_line, last_error_col, maxcol=None 191 ): 192 """print some debugging info if asked for.""" 193 if self.debug_level > 1: 194 msg = "{0}:{1}:{2}{3} - {4}\n" "{0}:{1}:{2}{3} + {5}" 195 mstr = "" if maxcol is None else ":" + str(maxcol) 196 msg = msg.format( 197 self.filename, last_error_line, last_error_col, mstr, line, sbpline 198 ) 199 print(msg, file=sys.stderr) 200 201 def _parse_ctx_free(self, input, mode="exec", filename=None, logical_input=False): 202 last_error_line = last_error_col = -1 203 parsed = False 204 original_error = None 205 greedy = False 206 if filename is None: 207 filename = self.filename 208 while not parsed: 209 try: 210 tree = self.parser.parse( 211 input, 212 filename=filename, 213 mode=mode, 214 debug_level=(self.debug_level > 2), 215 ) 216 parsed = True 217 except IndentationError as e: 218 if original_error is None: 219 raise e 220 else: 221 raise original_error 222 except SyntaxError as e: 223 if original_error is None: 224 original_error = e 225 if (e.loc is None) or ( 226 last_error_line == e.loc.lineno 227 and last_error_col in (e.loc.column + 1, e.loc.column) 228 ): 229 raise original_error from None 230 elif last_error_line != e.loc.lineno: 231 original_error = e 232 last_error_col = e.loc.column 233 last_error_line = e.loc.lineno 234 idx = last_error_line - 1 235 lines = input.splitlines() 236 line, nlogical, idx = get_logical_line(lines, idx) 237 if nlogical > 1 and not logical_input: 238 _, sbpline = self._parse_ctx_free( 239 line, mode=mode, filename=filename, logical_input=True 240 ) 241 self._print_debug_wrapping( 242 line, sbpline, last_error_line, last_error_col, maxcol=None 243 ) 244 replace_logical_line(lines, sbpline, idx, nlogical) 245 last_error_col += 3 246 input = "\n".join(lines) 247 continue 248 if input.endswith("\n"): 249 lines.append("") 250 if len(line.strip()) == 0: 251 # whitespace only lines are not valid syntax in Python's 252 # interactive mode='single', who knew?! Just ignore them. 253 # this might cause actual syntax errors to have bad line 254 # numbers reported, but should only affect interactive mode 255 del lines[idx] 256 last_error_line = last_error_col = -1 257 input = "\n".join(lines) 258 continue 259 260 if last_error_line > 1 and lines[idx - 1].rstrip()[-1:] == ":": 261 # catch non-indented blocks and raise error. 262 prev_indent = len(lines[idx - 1]) - len(lines[idx - 1].lstrip()) 263 curr_indent = len(lines[idx]) - len(lines[idx].lstrip()) 264 if prev_indent == curr_indent: 265 raise original_error 266 lexer = self.parser.lexer 267 maxcol = ( 268 None 269 if greedy 270 else find_next_break(line, mincol=last_error_col, lexer=lexer) 271 ) 272 if not greedy and maxcol in (e.loc.column + 1, e.loc.column): 273 # go greedy the first time if the syntax error was because 274 # we hit an end token out of place. This usually indicates 275 # a subshell or maybe a macro. 276 if not balanced_parens(line, maxcol=maxcol): 277 greedy = True 278 maxcol = None 279 sbpline = subproc_toks( 280 line, returnline=True, greedy=greedy, maxcol=maxcol, lexer=lexer 281 ) 282 if sbpline is None: 283 # subprocess line had no valid tokens, 284 if len(line.partition("#")[0].strip()) == 0: 285 # likely because it only contained a comment. 286 del lines[idx] 287 last_error_line = last_error_col = -1 288 input = "\n".join(lines) 289 continue 290 elif not greedy: 291 greedy = True 292 continue 293 else: 294 # or for some other syntax error 295 raise original_error 296 elif sbpline[last_error_col:].startswith( 297 "![![" 298 ) or sbpline.lstrip().startswith("![!["): 299 # if we have already wrapped this in subproc tokens 300 # and it still doesn't work, adding more won't help 301 # anything 302 if not greedy: 303 greedy = True 304 continue 305 else: 306 raise original_error 307 # replace the line 308 self._print_debug_wrapping( 309 line, sbpline, last_error_line, last_error_col, maxcol=maxcol 310 ) 311 replace_logical_line(lines, sbpline, idx, nlogical) 312 last_error_col += 3 313 input = "\n".join(lines) 314 return tree, input 315