1# -*- coding: utf-8 -*- 2"""The xonsh built-ins. 3 4Note that this module is named 'built_ins' so as not to be confused with the 5special Python builtins module. 6""" 7import io 8import os 9import re 10import sys 11import types 12import shlex 13import signal 14import atexit 15import pathlib 16import inspect 17import builtins 18import itertools 19import subprocess 20import contextlib 21import collections.abc as cabc 22 23from xonsh.ast import AST 24from xonsh.lazyasd import LazyObject, lazyobject 25from xonsh.inspectors import Inspector 26from xonsh.aliases import Aliases, make_default_aliases 27from xonsh.environ import Env, default_env, locate_binary 28from xonsh.jobs import add_job 29from xonsh.platform import ON_POSIX, ON_WINDOWS 30from xonsh.proc import ( 31 PopenThread, 32 ProcProxyThread, 33 ProcProxy, 34 ConsoleParallelReader, 35 pause_call_resume, 36 CommandPipeline, 37 HiddenCommandPipeline, 38 STDOUT_CAPTURE_KINDS, 39) 40from xonsh.tools import ( 41 suggest_commands, 42 expand_path, 43 globpath, 44 XonshError, 45 XonshCalledProcessError, 46) 47from xonsh.lazyimps import pty, termios 48from xonsh.commands_cache import CommandsCache 49from xonsh.events import events 50 51import xonsh.completers.init 52 53BUILTINS_LOADED = False 54INSPECTOR = LazyObject(Inspector, globals(), "INSPECTOR") 55 56 57@lazyobject 58def AT_EXIT_SIGNALS(): 59 sigs = ( 60 signal.SIGABRT, 61 signal.SIGFPE, 62 signal.SIGILL, 63 signal.SIGSEGV, 64 signal.SIGTERM, 65 ) 66 if ON_POSIX: 67 sigs += (signal.SIGTSTP, signal.SIGQUIT, signal.SIGHUP) 68 return sigs 69 70 71def resetting_signal_handle(sig, f): 72 """Sets a new signal handle that will automatically restore the old value 73 once the new handle is finished. 74 """ 75 oldh = signal.getsignal(sig) 76 77 def newh(s=None, frame=None): 78 f(s, frame) 79 signal.signal(sig, oldh) 80 if sig != 0: 81 sys.exit(sig) 82 83 signal.signal(sig, newh) 84 85 86def helper(x, name=""): 87 """Prints help about, and then returns that variable.""" 88 INSPECTOR.pinfo(x, oname=name, detail_level=0) 89 return x 90 91 92def superhelper(x, name=""): 93 """Prints help about, and then returns that variable.""" 94 INSPECTOR.pinfo(x, oname=name, detail_level=1) 95 return x 96 97 98def reglob(path, parts=None, i=None): 99 """Regular expression-based globbing.""" 100 if parts is None: 101 path = os.path.normpath(path) 102 drive, tail = os.path.splitdrive(path) 103 parts = tail.split(os.sep) 104 d = os.sep if os.path.isabs(path) else "." 105 d = os.path.join(drive, d) 106 return reglob(d, parts, i=0) 107 base = subdir = path 108 if i == 0: 109 if not os.path.isabs(base): 110 base = "" 111 elif len(parts) > 1: 112 i += 1 113 regex = os.path.join(base, parts[i]) 114 if ON_WINDOWS: 115 # currently unable to access regex backslash sequences 116 # on Windows due to paths using \. 117 regex = regex.replace("\\", "\\\\") 118 regex = re.compile(regex) 119 files = os.listdir(subdir) 120 files.sort() 121 paths = [] 122 i1 = i + 1 123 if i1 == len(parts): 124 for f in files: 125 p = os.path.join(base, f) 126 if regex.fullmatch(p) is not None: 127 paths.append(p) 128 else: 129 for f in files: 130 p = os.path.join(base, f) 131 if regex.fullmatch(p) is None or not os.path.isdir(p): 132 continue 133 paths += reglob(p, parts=parts, i=i1) 134 return paths 135 136 137def path_literal(s): 138 s = expand_path(s) 139 return pathlib.Path(s) 140 141 142def regexsearch(s): 143 s = expand_path(s) 144 return reglob(s) 145 146 147def globsearch(s): 148 csc = builtins.__xonsh_env__.get("CASE_SENSITIVE_COMPLETIONS") 149 glob_sorted = builtins.__xonsh_env__.get("GLOB_SORTED") 150 dotglob = builtins.__xonsh_env__.get("DOTGLOB") 151 return globpath( 152 s, 153 ignore_case=(not csc), 154 return_empty=True, 155 sort_result=glob_sorted, 156 include_dotfiles=dotglob, 157 ) 158 159 160def pathsearch(func, s, pymode=False, pathobj=False): 161 """ 162 Takes a string and returns a list of file paths that match (regex, glob, 163 or arbitrary search function). If pathobj=True, the return is a list of 164 pathlib.Path objects instead of strings. 165 """ 166 if not callable(func) or len(inspect.signature(func).parameters) != 1: 167 error = "%r is not a known path search function" 168 raise XonshError(error % func) 169 o = func(s) 170 if pathobj and pymode: 171 o = list(map(pathlib.Path, o)) 172 no_match = [] if pymode else [s] 173 return o if len(o) != 0 else no_match 174 175 176RE_SHEBANG = LazyObject(lambda: re.compile(r"#![ \t]*(.+?)$"), globals(), "RE_SHEBANG") 177 178 179def _is_binary(fname, limit=80): 180 with open(fname, "rb") as f: 181 for i in range(limit): 182 char = f.read(1) 183 if char == b"\0": 184 return True 185 if char == b"\n": 186 return False 187 if char == b"": 188 return False 189 return False 190 191 192def _un_shebang(x): 193 if x == "/usr/bin/env": 194 return [] 195 elif any(x.startswith(i) for i in ["/usr/bin", "/usr/local/bin", "/bin"]): 196 x = os.path.basename(x) 197 elif x.endswith("python") or x.endswith("python.exe"): 198 x = "python" 199 if x == "xonsh": 200 return ["python", "-m", "xonsh.main"] 201 return [x] 202 203 204def get_script_subproc_command(fname, args): 205 """Given the name of a script outside the path, returns a list representing 206 an appropriate subprocess command to execute the script. Raises 207 PermissionError if the script is not executable. 208 """ 209 # make sure file is executable 210 if not os.access(fname, os.X_OK): 211 raise PermissionError 212 if ON_POSIX and not os.access(fname, os.R_OK): 213 # on some systems, some important programs (e.g. sudo) will have 214 # execute permissions but not read/write permissions. This enables 215 # things with the SUID set to be run. Needs to come before _is_binary() 216 # is called, because that function tries to read the file. 217 return [fname] + args 218 elif _is_binary(fname): 219 # if the file is a binary, we should call it directly 220 return [fname] + args 221 if ON_WINDOWS: 222 # Windows can execute various filetypes directly 223 # as given in PATHEXT 224 _, ext = os.path.splitext(fname) 225 if ext.upper() in builtins.__xonsh_env__.get("PATHEXT"): 226 return [fname] + args 227 # find interpreter 228 with open(fname, "rb") as f: 229 first_line = f.readline().decode().strip() 230 m = RE_SHEBANG.match(first_line) 231 # xonsh is the default interpreter 232 if m is None: 233 interp = ["xonsh"] 234 else: 235 interp = m.group(1).strip() 236 if len(interp) > 0: 237 interp = shlex.split(interp) 238 else: 239 interp = ["xonsh"] 240 if ON_WINDOWS: 241 o = [] 242 for i in interp: 243 o.extend(_un_shebang(i)) 244 interp = o 245 return interp + [fname] + args 246 247 248@lazyobject 249def _REDIR_REGEX(): 250 name = "(o(?:ut)?|e(?:rr)?|a(?:ll)?|&?\d?)" 251 return re.compile("{r}(>?>|<){r}$".format(r=name)) 252 253 254_MODES = LazyObject(lambda: {">>": "a", ">": "w", "<": "r"}, globals(), "_MODES") 255_WRITE_MODES = LazyObject(lambda: frozenset({"w", "a"}), globals(), "_WRITE_MODES") 256_REDIR_ALL = LazyObject(lambda: frozenset({"&", "a", "all"}), globals(), "_REDIR_ALL") 257_REDIR_ERR = LazyObject(lambda: frozenset({"2", "e", "err"}), globals(), "_REDIR_ERR") 258_REDIR_OUT = LazyObject( 259 lambda: frozenset({"", "1", "o", "out"}), globals(), "_REDIR_OUT" 260) 261_E2O_MAP = LazyObject( 262 lambda: frozenset( 263 {"{}>{}".format(e, o) for e in _REDIR_ERR for o in _REDIR_OUT if o != ""} 264 ), 265 globals(), 266 "_E2O_MAP", 267) 268_O2E_MAP = LazyObject( 269 lambda: frozenset( 270 {"{}>{}".format(o, e) for e in _REDIR_ERR for o in _REDIR_OUT if o != ""} 271 ), 272 globals(), 273 "_O2E_MAP", 274) 275 276 277def _is_redirect(x): 278 return isinstance(x, str) and _REDIR_REGEX.match(x) 279 280 281def safe_open(fname, mode, buffering=-1): 282 """Safely attempts to open a file in for xonsh subprocs.""" 283 # file descriptors 284 try: 285 return io.open(fname, mode, buffering=buffering) 286 except PermissionError: 287 raise XonshError("xonsh: {0}: permission denied".format(fname)) 288 except FileNotFoundError: 289 raise XonshError("xonsh: {0}: no such file or directory".format(fname)) 290 except Exception: 291 raise XonshError("xonsh: {0}: unable to open file".format(fname)) 292 293 294def safe_close(x): 295 """Safely attempts to close an object.""" 296 if not isinstance(x, io.IOBase): 297 return 298 if x.closed: 299 return 300 try: 301 x.close() 302 except Exception: 303 pass 304 305 306def _parse_redirects(r, loc=None): 307 """returns origin, mode, destination tuple""" 308 orig, mode, dest = _REDIR_REGEX.match(r).groups() 309 # redirect to fd 310 if dest.startswith("&"): 311 try: 312 dest = int(dest[1:]) 313 if loc is None: 314 loc, dest = dest, "" # NOQA 315 else: 316 e = "Unrecognized redirection command: {}".format(r) 317 raise XonshError(e) 318 except (ValueError, XonshError): 319 raise 320 except Exception: 321 pass 322 mode = _MODES.get(mode, None) 323 if mode == "r" and (len(orig) > 0 or len(dest) > 0): 324 raise XonshError("Unrecognized redirection command: {}".format(r)) 325 elif mode in _WRITE_MODES and len(dest) > 0: 326 raise XonshError("Unrecognized redirection command: {}".format(r)) 327 return orig, mode, dest 328 329 330def _redirect_streams(r, loc=None): 331 """Returns stdin, stdout, stderr tuple of redirections.""" 332 stdin = stdout = stderr = None 333 no_ampersand = r.replace("&", "") 334 # special case of redirecting stderr to stdout 335 if no_ampersand in _E2O_MAP: 336 stderr = subprocess.STDOUT 337 return stdin, stdout, stderr 338 elif no_ampersand in _O2E_MAP: 339 stdout = 2 # using 2 as a flag, rather than using a file object 340 return stdin, stdout, stderr 341 # get streams 342 orig, mode, dest = _parse_redirects(r) 343 if mode == "r": 344 stdin = safe_open(loc, mode) 345 elif mode in _WRITE_MODES: 346 if orig in _REDIR_ALL: 347 stdout = stderr = safe_open(loc, mode) 348 elif orig in _REDIR_OUT: 349 stdout = safe_open(loc, mode) 350 elif orig in _REDIR_ERR: 351 stderr = safe_open(loc, mode) 352 else: 353 raise XonshError("Unrecognized redirection command: {}".format(r)) 354 else: 355 raise XonshError("Unrecognized redirection command: {}".format(r)) 356 return stdin, stdout, stderr 357 358 359def default_signal_pauser(n, f): 360 """Pauses a signal, as needed.""" 361 signal.pause() 362 363 364def no_pg_xonsh_preexec_fn(): 365 """Default subprocess preexec function for when there is no existing 366 pipeline group. 367 """ 368 os.setpgrp() 369 signal.signal(signal.SIGTSTP, default_signal_pauser) 370 371 372class SubprocSpec: 373 """A container for specifying how a subprocess command should be 374 executed. 375 """ 376 377 kwnames = ("stdin", "stdout", "stderr", "universal_newlines") 378 379 def __init__( 380 self, 381 cmd, 382 *, 383 cls=subprocess.Popen, 384 stdin=None, 385 stdout=None, 386 stderr=None, 387 universal_newlines=False, 388 captured=False 389 ): 390 """ 391 Parameters 392 ---------- 393 cmd : list of str 394 Command to be run. 395 cls : Popen-like 396 Class to run the subprocess with. 397 stdin : file-like 398 Popen file descriptor or flag for stdin. 399 stdout : file-like 400 Popen file descriptor or flag for stdout. 401 stderr : file-like 402 Popen file descriptor or flag for stderr. 403 universal_newlines : bool 404 Whether or not to use universal newlines. 405 captured : bool or str, optional 406 The flag for if the subprocess is captured, may be one of: 407 False for $[], 'stdout' for $(), 'hiddenobject' for ![], or 408 'object' for !(). 409 410 Attributes 411 ---------- 412 args : list of str 413 Arguments as originally supplied. 414 alias : list of str, callable, or None 415 The alias that was resolved for this command, if any. 416 binary_loc : str or None 417 Path to binary to execute. 418 is_proxy : bool 419 Whether or not the subprocess is or should be run as a proxy. 420 background : bool 421 Whether or not the subprocess should be started in the background. 422 threadable : bool 423 Whether or not the subprocess is able to be run in a background 424 thread, rather than the main thread. 425 last_in_pipeline : bool 426 Whether the subprocess is the last in the execution pipeline. 427 captured_stdout : file-like 428 Handle to captured stdin 429 captured_stderr : file-like 430 Handle to captured stderr 431 stack : list of FrameInfo namedtuples or None 432 The stack of the call-site of alias, if the alias requires it. 433 None otherwise. 434 """ 435 self._stdin = self._stdout = self._stderr = None 436 # args 437 self.cmd = list(cmd) 438 self.cls = cls 439 self.stdin = stdin 440 self.stdout = stdout 441 self.stderr = stderr 442 self.universal_newlines = universal_newlines 443 self.captured = captured 444 # pure attrs 445 self.args = list(cmd) 446 self.alias = None 447 self.binary_loc = None 448 self.is_proxy = False 449 self.background = False 450 self.threadable = True 451 self.last_in_pipeline = False 452 self.captured_stdout = None 453 self.captured_stderr = None 454 self.stack = None 455 456 def __str__(self): 457 s = self.__class__.__name__ + "(" + str(self.cmd) + ", " 458 s += self.cls.__name__ + ", " 459 kws = [n + "=" + str(getattr(self, n)) for n in self.kwnames] 460 s += ", ".join(kws) + ")" 461 return s 462 463 def __repr__(self): 464 s = self.__class__.__name__ + "(" + repr(self.cmd) + ", " 465 s += self.cls.__name__ + ", " 466 kws = [n + "=" + repr(getattr(self, n)) for n in self.kwnames] 467 s += ", ".join(kws) + ")" 468 return s 469 470 # 471 # Properties 472 # 473 474 @property 475 def stdin(self): 476 return self._stdin 477 478 @stdin.setter 479 def stdin(self, value): 480 if self._stdin is None: 481 self._stdin = value 482 elif value is None: 483 pass 484 else: 485 safe_close(value) 486 msg = "Multiple inputs for stdin for {0!r}" 487 msg = msg.format(" ".join(self.args)) 488 raise XonshError(msg) 489 490 @property 491 def stdout(self): 492 return self._stdout 493 494 @stdout.setter 495 def stdout(self, value): 496 if self._stdout is None: 497 self._stdout = value 498 elif value is None: 499 pass 500 else: 501 safe_close(value) 502 msg = "Multiple redirections for stdout for {0!r}" 503 msg = msg.format(" ".join(self.args)) 504 raise XonshError(msg) 505 506 @property 507 def stderr(self): 508 return self._stderr 509 510 @stderr.setter 511 def stderr(self, value): 512 if self._stderr is None: 513 self._stderr = value 514 elif value is None: 515 pass 516 else: 517 safe_close(value) 518 msg = "Multiple redirections for stderr for {0!r}" 519 msg = msg.format(" ".join(self.args)) 520 raise XonshError(msg) 521 522 # 523 # Execution methods 524 # 525 526 def run(self, *, pipeline_group=None): 527 """Launches the subprocess and returns the object.""" 528 kwargs = {n: getattr(self, n) for n in self.kwnames} 529 self.prep_env(kwargs) 530 self.prep_preexec_fn(kwargs, pipeline_group=pipeline_group) 531 if callable(self.alias): 532 if "preexec_fn" in kwargs: 533 kwargs.pop("preexec_fn") 534 p = self.cls(self.alias, self.cmd, **kwargs) 535 else: 536 self._fix_null_cmd_bytes() 537 p = self._run_binary(kwargs) 538 p.spec = self 539 p.last_in_pipeline = self.last_in_pipeline 540 p.captured_stdout = self.captured_stdout 541 p.captured_stderr = self.captured_stderr 542 return p 543 544 def _run_binary(self, kwargs): 545 try: 546 bufsize = 1 547 p = self.cls(self.cmd, bufsize=bufsize, **kwargs) 548 except PermissionError: 549 e = "xonsh: subprocess mode: permission denied: {0}" 550 raise XonshError(e.format(self.cmd[0])) 551 except FileNotFoundError: 552 cmd0 = self.cmd[0] 553 e = "xonsh: subprocess mode: command not found: {0}".format(cmd0) 554 env = builtins.__xonsh_env__ 555 sug = suggest_commands(cmd0, env, builtins.aliases) 556 if len(sug.strip()) > 0: 557 e += "\n" + suggest_commands(cmd0, env, builtins.aliases) 558 raise XonshError(e) 559 return p 560 561 def prep_env(self, kwargs): 562 """Prepares the environment to use in the subprocess.""" 563 denv = builtins.__xonsh_env__.detype() 564 if ON_WINDOWS: 565 # Over write prompt variable as xonsh's $PROMPT does 566 # not make much sense for other subprocs 567 denv["PROMPT"] = "$P$G" 568 kwargs["env"] = denv 569 570 def prep_preexec_fn(self, kwargs, pipeline_group=None): 571 """Prepares the 'preexec_fn' keyword argument""" 572 if not ON_POSIX: 573 return 574 if not builtins.__xonsh_env__.get("XONSH_INTERACTIVE"): 575 return 576 if pipeline_group is None: 577 xonsh_preexec_fn = no_pg_xonsh_preexec_fn 578 else: 579 580 def xonsh_preexec_fn(): 581 """Preexec function bound to a pipeline group.""" 582 os.setpgid(0, pipeline_group) 583 signal.signal(signal.SIGTSTP, default_signal_pauser) 584 585 kwargs["preexec_fn"] = xonsh_preexec_fn 586 587 def _fix_null_cmd_bytes(self): 588 # Popen does not accept null bytes in its input commands. 589 # That doesn't stop some subprocesses from using them. Here we 590 # escape them just in case. 591 cmd = self.cmd 592 for i in range(len(cmd)): 593 cmd[i] = cmd[i].replace("\0", "\\0") 594 595 # 596 # Building methods 597 # 598 599 @classmethod 600 def build(kls, cmd, *, cls=subprocess.Popen, **kwargs): 601 """Creates an instance of the subprocess command, with any 602 modifications and adjustments based on the actual cmd that 603 was received. 604 """ 605 # modifications that do not alter cmds may come before creating instance 606 spec = kls(cmd, cls=cls, **kwargs) 607 # modifications that alter cmds must come after creating instance 608 # perform initial redirects 609 spec.redirect_leading() 610 spec.redirect_trailing() 611 # apply aliases 612 spec.resolve_alias() 613 spec.resolve_binary_loc() 614 spec.resolve_auto_cd() 615 spec.resolve_executable_commands() 616 spec.resolve_alias_cls() 617 spec.resolve_stack() 618 return spec 619 620 def redirect_leading(self): 621 """Manage leading redirects such as with '< input.txt COMMAND'. """ 622 while len(self.cmd) >= 3 and self.cmd[0] == "<": 623 self.stdin = safe_open(self.cmd[1], "r") 624 self.cmd = self.cmd[2:] 625 626 def redirect_trailing(self): 627 """Manages trailing redirects.""" 628 while True: 629 cmd = self.cmd 630 if len(cmd) >= 3 and _is_redirect(cmd[-2]): 631 streams = _redirect_streams(cmd[-2], cmd[-1]) 632 self.stdin, self.stdout, self.stderr = streams 633 self.cmd = cmd[:-2] 634 elif len(cmd) >= 2 and _is_redirect(cmd[-1]): 635 streams = _redirect_streams(cmd[-1]) 636 self.stdin, self.stdout, self.stderr = streams 637 self.cmd = cmd[:-1] 638 else: 639 break 640 641 def resolve_alias(self): 642 """Sets alias in command, if applicable.""" 643 cmd0 = self.cmd[0] 644 if callable(cmd0): 645 alias = cmd0 646 else: 647 alias = builtins.aliases.get(cmd0, None) 648 self.alias = alias 649 650 def resolve_binary_loc(self): 651 """Sets the binary location""" 652 alias = self.alias 653 if alias is None: 654 binary_loc = locate_binary(self.cmd[0]) 655 elif callable(alias): 656 binary_loc = None 657 else: 658 binary_loc = locate_binary(alias[0]) 659 self.binary_loc = binary_loc 660 661 def resolve_auto_cd(self): 662 """Implements AUTO_CD functionality.""" 663 if not ( 664 self.alias is None 665 and self.binary_loc is None 666 and len(self.cmd) == 1 667 and builtins.__xonsh_env__.get("AUTO_CD") 668 and os.path.isdir(self.cmd[0]) 669 ): 670 return 671 self.cmd.insert(0, "cd") 672 self.alias = builtins.aliases.get("cd", None) 673 674 def resolve_executable_commands(self): 675 """Resolve command executables, if applicable.""" 676 alias = self.alias 677 if alias is None: 678 pass 679 elif callable(alias): 680 self.cmd.pop(0) 681 return 682 else: 683 self.cmd = alias + self.cmd[1:] 684 # resolve any redirects the aliases may have applied 685 self.redirect_leading() 686 self.redirect_trailing() 687 if self.binary_loc is None: 688 return 689 try: 690 self.cmd = get_script_subproc_command(self.binary_loc, self.cmd[1:]) 691 except PermissionError: 692 e = "xonsh: subprocess mode: permission denied: {0}" 693 raise XonshError(e.format(self.cmd[0])) 694 695 def resolve_alias_cls(self): 696 """Determine which proxy class to run an alias with.""" 697 alias = self.alias 698 if not callable(alias): 699 return 700 self.is_proxy = True 701 thable = getattr(alias, "__xonsh_threadable__", True) 702 cls = ProcProxyThread if thable else ProcProxy 703 self.cls = cls 704 self.threadable = thable 705 # also check capturability, while we are here 706 cpable = getattr(alias, "__xonsh_capturable__", self.captured) 707 self.captured = cpable 708 709 def resolve_stack(self): 710 """Computes the stack for a callable alias's call-site, if needed.""" 711 if not callable(self.alias): 712 return 713 # check that we actual need the stack 714 sig = inspect.signature(self.alias) 715 if len(sig.parameters) <= 5 and "stack" not in sig.parameters: 716 return 717 # compute the stack, and filter out these build methods 718 # run_subproc() is the 4th command in the stack 719 # we want to filter out one up, e.g. subproc_captured_hiddenobject() 720 # after that the stack from the call site starts. 721 stack = inspect.stack(context=0) 722 assert stack[3][3] == "run_subproc", "xonsh stack has changed!" 723 del stack[:5] 724 self.stack = stack 725 726 727def _safe_pipe_properties(fd, use_tty=False): 728 """Makes sure that a pipe file descriptor properties are sane.""" 729 if not use_tty: 730 return 731 # due to some weird, long standing issue in Python, PTYs come out 732 # replacing newline \n with \r\n. This causes issues for raw unix 733 # protocols, like git and ssh, which expect unix line endings. 734 # see https://mail.python.org/pipermail/python-list/2013-June/650460.html 735 # for more details and the following solution. 736 props = termios.tcgetattr(fd) 737 props[1] = props[1] & (~termios.ONLCR) | termios.ONLRET 738 termios.tcsetattr(fd, termios.TCSANOW, props) 739 740 741def _update_last_spec(last): 742 captured = last.captured 743 last.last_in_pipeline = True 744 if not captured: 745 return 746 callable_alias = callable(last.alias) 747 if callable_alias: 748 pass 749 else: 750 cmds_cache = builtins.__xonsh_commands_cache__ 751 thable = cmds_cache.predict_threadable( 752 last.args 753 ) and cmds_cache.predict_threadable(last.cmd) 754 if captured and thable: 755 last.cls = PopenThread 756 elif not thable: 757 # foreground processes should use Popen 758 last.threadable = False 759 if captured == "object" or captured == "hiddenobject": 760 # CommandPipeline objects should not pipe stdout, stderr 761 return 762 # cannot used PTY pipes for aliases, for some dark reason, 763 # and must use normal pipes instead. 764 use_tty = ON_POSIX and not callable_alias 765 # Do not set standard in! Popen is not a fan of redirections here 766 # set standard out 767 if last.stdout is not None: 768 last.universal_newlines = True 769 elif captured in STDOUT_CAPTURE_KINDS: 770 last.universal_newlines = False 771 r, w = os.pipe() 772 last.stdout = safe_open(w, "wb") 773 last.captured_stdout = safe_open(r, "rb") 774 elif builtins.__xonsh_stdout_uncaptured__ is not None: 775 last.universal_newlines = True 776 last.stdout = builtins.__xonsh_stdout_uncaptured__ 777 last.captured_stdout = last.stdout 778 elif ON_WINDOWS and not callable_alias: 779 last.universal_newlines = True 780 last.stdout = None # must truly stream on windows 781 last.captured_stdout = ConsoleParallelReader(1) 782 else: 783 last.universal_newlines = True 784 r, w = pty.openpty() if use_tty else os.pipe() 785 _safe_pipe_properties(w, use_tty=use_tty) 786 last.stdout = safe_open(w, "w") 787 _safe_pipe_properties(r, use_tty=use_tty) 788 last.captured_stdout = safe_open(r, "r") 789 # set standard error 790 if last.stderr is not None: 791 pass 792 elif captured == "object": 793 r, w = os.pipe() 794 last.stderr = safe_open(w, "w") 795 last.captured_stderr = safe_open(r, "r") 796 elif builtins.__xonsh_stderr_uncaptured__ is not None: 797 last.stderr = builtins.__xonsh_stderr_uncaptured__ 798 last.captured_stderr = last.stderr 799 elif ON_WINDOWS and not callable_alias: 800 last.universal_newlines = True 801 last.stderr = None # must truly stream on windows 802 else: 803 r, w = pty.openpty() if use_tty else os.pipe() 804 _safe_pipe_properties(w, use_tty=use_tty) 805 last.stderr = safe_open(w, "w") 806 _safe_pipe_properties(r, use_tty=use_tty) 807 last.captured_stderr = safe_open(r, "r") 808 # redirect stdout to stderr, if we should 809 if isinstance(last.stdout, int) and last.stdout == 2: 810 # need to use private interface to avoid duplication. 811 last._stdout = last.stderr 812 # redirect stderr to stdout, if we should 813 if callable_alias and last.stderr == subprocess.STDOUT: 814 last._stderr = last.stdout 815 last.captured_stderr = last.captured_stdout 816 817 818def cmds_to_specs(cmds, captured=False): 819 """Converts a list of cmds to a list of SubprocSpec objects that are 820 ready to be executed. 821 """ 822 # first build the subprocs independently and separate from the redirects 823 specs = [] 824 redirects = [] 825 for cmd in cmds: 826 if isinstance(cmd, str): 827 redirects.append(cmd) 828 else: 829 if cmd[-1] == "&": 830 cmd = cmd[:-1] 831 redirects.append("&") 832 spec = SubprocSpec.build(cmd, captured=captured) 833 specs.append(spec) 834 # now modify the subprocs based on the redirects. 835 for i, redirect in enumerate(redirects): 836 if redirect == "|": 837 # these should remain integer file descriptors, and not Python 838 # file objects since they connect processes. 839 r, w = os.pipe() 840 specs[i].stdout = w 841 specs[i + 1].stdin = r 842 elif redirect == "&" and i == len(redirects) - 1: 843 specs[-1].background = True 844 else: 845 raise XonshError("unrecognized redirect {0!r}".format(redirect)) 846 # Apply boundary conditions 847 _update_last_spec(specs[-1]) 848 return specs 849 850 851def _should_set_title(captured=False): 852 env = builtins.__xonsh_env__ 853 return ( 854 env.get("XONSH_INTERACTIVE") 855 and not env.get("XONSH_STORE_STDOUT") 856 and captured not in STDOUT_CAPTURE_KINDS 857 and hasattr(builtins, "__xonsh_shell__") 858 ) 859 860 861def run_subproc(cmds, captured=False): 862 """Runs a subprocess, in its many forms. This takes a list of 'commands,' 863 which may be a list of command line arguments or a string, representing 864 a special connecting character. For example:: 865 866 $ ls | grep wakka 867 868 is represented by the following cmds:: 869 870 [['ls'], '|', ['grep', 'wakka']] 871 872 Lastly, the captured argument affects only the last real command. 873 """ 874 specs = cmds_to_specs(cmds, captured=captured) 875 captured = specs[-1].captured 876 if captured == "hiddenobject": 877 command = HiddenCommandPipeline(specs) 878 else: 879 command = CommandPipeline(specs) 880 proc = command.proc 881 background = command.spec.background 882 if not all(x.is_proxy for x in specs): 883 add_job( 884 { 885 "cmds": cmds, 886 "pids": [i.pid for i in command.procs], 887 "obj": proc, 888 "bg": background, 889 "pipeline": command, 890 "pgrp": command.term_pgid, 891 } 892 ) 893 if _should_set_title(captured=captured): 894 # set title here to get currently executing command 895 pause_call_resume(proc, builtins.__xonsh_shell__.settitle) 896 # create command or return if backgrounding. 897 if background: 898 return 899 # now figure out what we should return. 900 if captured == "stdout": 901 command.end() 902 return command.output 903 elif captured == "object": 904 return command 905 elif captured == "hiddenobject": 906 command.end() 907 return command 908 else: 909 command.end() 910 return 911 912 913def subproc_captured_stdout(*cmds): 914 """Runs a subprocess, capturing the output. Returns the stdout 915 that was produced as a str. 916 """ 917 return run_subproc(cmds, captured="stdout") 918 919 920def subproc_captured_inject(*cmds): 921 """Runs a subprocess, capturing the output. Returns a list of 922 whitespace-separated strings of the stdout that was produced. 923 The string is split using xonsh's lexer, rather than Python's str.split() 924 or shlex.split(). 925 """ 926 s = run_subproc(cmds, captured="stdout") 927 toks = builtins.__xonsh_execer__.parser.lexer.split(s.strip()) 928 return toks 929 930 931def subproc_captured_object(*cmds): 932 """ 933 Runs a subprocess, capturing the output. Returns an instance of 934 CommandPipeline representing the completed command. 935 """ 936 return run_subproc(cmds, captured="object") 937 938 939def subproc_captured_hiddenobject(*cmds): 940 """Runs a subprocess, capturing the output. Returns an instance of 941 HiddenCommandPipeline representing the completed command. 942 """ 943 return run_subproc(cmds, captured="hiddenobject") 944 945 946def subproc_uncaptured(*cmds): 947 """Runs a subprocess, without capturing the output. Returns the stdout 948 that was produced as a str. 949 """ 950 return run_subproc(cmds, captured=False) 951 952 953def ensure_list_of_strs(x): 954 """Ensures that x is a list of strings.""" 955 if isinstance(x, str): 956 rtn = [x] 957 elif isinstance(x, cabc.Sequence): 958 rtn = [i if isinstance(i, str) else str(i) for i in x] 959 else: 960 rtn = [str(x)] 961 return rtn 962 963 964def list_of_strs_or_callables(x): 965 """Ensures that x is a list of strings or functions""" 966 if isinstance(x, str) or callable(x): 967 rtn = [x] 968 elif isinstance(x, cabc.Iterable): 969 rtn = [i if isinstance(i, str) or callable(i) else str(i) for i in x] 970 else: 971 rtn = [str(x)] 972 return rtn 973 974 975def list_of_list_of_strs_outer_product(x): 976 """Takes an outer product of a list of strings""" 977 lolos = map(ensure_list_of_strs, x) 978 rtn = [] 979 for los in itertools.product(*lolos): 980 s = "".join(los) 981 if "*" in s: 982 rtn.extend(builtins.__xonsh_glob__(s)) 983 else: 984 rtn.append(builtins.__xonsh_expand_path__(s)) 985 return rtn 986 987 988@lazyobject 989def MACRO_FLAG_KINDS(): 990 return { 991 "s": str, 992 "str": str, 993 "string": str, 994 "a": AST, 995 "ast": AST, 996 "c": types.CodeType, 997 "code": types.CodeType, 998 "compile": types.CodeType, 999 "v": eval, 1000 "eval": eval, 1001 "x": exec, 1002 "exec": exec, 1003 "t": type, 1004 "type": type, 1005 } 1006 1007 1008def _convert_kind_flag(x): 1009 """Puts a kind flag (string) a canonical form.""" 1010 x = x.lower() 1011 kind = MACRO_FLAG_KINDS.get(x, None) 1012 if kind is None: 1013 raise TypeError("{0!r} not a recognized macro type.".format(x)) 1014 return kind 1015 1016 1017def convert_macro_arg(raw_arg, kind, glbs, locs, *, name="<arg>", macroname="<macro>"): 1018 """Converts a string macro argument based on the requested kind. 1019 1020 Parameters 1021 ---------- 1022 raw_arg : str 1023 The str representation of the macro argument. 1024 kind : object 1025 A flag or type representing how to convert the argument. 1026 glbs : Mapping 1027 The globals from the call site. 1028 locs : Mapping or None 1029 The locals from the call site. 1030 name : str, optional 1031 The macro argument name. 1032 macroname : str, optional 1033 The name of the macro itself. 1034 1035 Returns 1036 ------- 1037 The converted argument. 1038 """ 1039 # munge kind and mode to start 1040 mode = None 1041 if isinstance(kind, cabc.Sequence) and not isinstance(kind, str): 1042 # have (kind, mode) tuple 1043 kind, mode = kind 1044 if isinstance(kind, str): 1045 kind = _convert_kind_flag(kind) 1046 if kind is str or kind is None: 1047 return raw_arg # short circuit since there is nothing else to do 1048 # select from kind and convert 1049 execer = builtins.__xonsh_execer__ 1050 filename = macroname + "(" + name + ")" 1051 if kind is AST: 1052 ctx = set(dir(builtins)) | set(glbs.keys()) 1053 if locs is not None: 1054 ctx |= set(locs.keys()) 1055 mode = mode or "eval" 1056 arg = execer.parse(raw_arg, ctx, mode=mode, filename=filename) 1057 elif kind is types.CodeType or kind is compile: # NOQA 1058 mode = mode or "eval" 1059 arg = execer.compile( 1060 raw_arg, mode=mode, glbs=glbs, locs=locs, filename=filename 1061 ) 1062 elif kind is eval: 1063 arg = execer.eval(raw_arg, glbs=glbs, locs=locs, filename=filename) 1064 elif kind is exec: 1065 mode = mode or "exec" 1066 if not raw_arg.endswith("\n"): 1067 raw_arg += "\n" 1068 arg = execer.exec(raw_arg, mode=mode, glbs=glbs, locs=locs, filename=filename) 1069 elif kind is type: 1070 arg = type(execer.eval(raw_arg, glbs=glbs, locs=locs, filename=filename)) 1071 else: 1072 msg = "kind={0!r} and mode={1!r} was not recognized for macro " "argument {2!r}" 1073 raise TypeError(msg.format(kind, mode, name)) 1074 return arg 1075 1076 1077@contextlib.contextmanager 1078def in_macro_call(f, glbs, locs): 1079 """Attaches macro globals and locals temporarily to function as a 1080 context manager. 1081 1082 Parameters 1083 ---------- 1084 f : callable object 1085 The function that is called as ``f(*args)``. 1086 glbs : Mapping 1087 The globals from the call site. 1088 locs : Mapping or None 1089 The locals from the call site. 1090 """ 1091 prev_glbs = getattr(f, "macro_globals", None) 1092 prev_locs = getattr(f, "macro_locals", None) 1093 f.macro_globals = glbs 1094 f.macro_locals = locs 1095 yield 1096 if prev_glbs is None: 1097 del f.macro_globals 1098 else: 1099 f.macro_globals = prev_glbs 1100 if prev_locs is None: 1101 del f.macro_locals 1102 else: 1103 f.macro_locals = prev_locs 1104 1105 1106def call_macro(f, raw_args, glbs, locs): 1107 """Calls a function as a macro, returning its result. 1108 1109 Parameters 1110 ---------- 1111 f : callable object 1112 The function that is called as ``f(*args)``. 1113 raw_args : tuple of str 1114 The str representation of arguments of that were passed into the 1115 macro. These strings will be parsed, compiled, evaled, or left as 1116 a string depending on the annotations of f. 1117 glbs : Mapping 1118 The globals from the call site. 1119 locs : Mapping or None 1120 The locals from the call site. 1121 """ 1122 sig = inspect.signature(f) 1123 empty = inspect.Parameter.empty 1124 macroname = f.__name__ 1125 i = 0 1126 args = [] 1127 for (key, param), raw_arg in zip(sig.parameters.items(), raw_args): 1128 i += 1 1129 if raw_arg == "*": 1130 break 1131 kind = param.annotation 1132 if kind is empty or kind is None: 1133 kind = str 1134 arg = convert_macro_arg( 1135 raw_arg, kind, glbs, locs, name=key, macroname=macroname 1136 ) 1137 args.append(arg) 1138 reg_args, kwargs = _eval_regular_args(raw_args[i:], glbs, locs) 1139 args += reg_args 1140 with in_macro_call(f, glbs, locs): 1141 rtn = f(*args, **kwargs) 1142 return rtn 1143 1144 1145@lazyobject 1146def KWARG_RE(): 1147 return re.compile("([A-Za-z_]\w*=|\*\*)") 1148 1149 1150def _starts_as_arg(s): 1151 """Tests if a string starts as a non-kwarg string would.""" 1152 return KWARG_RE.match(s) is None 1153 1154 1155def _eval_regular_args(raw_args, glbs, locs): 1156 if not raw_args: 1157 return [], {} 1158 arglist = list(itertools.takewhile(_starts_as_arg, raw_args)) 1159 kwarglist = raw_args[len(arglist) :] 1160 execer = builtins.__xonsh_execer__ 1161 if not arglist: 1162 args = arglist 1163 kwargstr = "dict({})".format(", ".join(kwarglist)) 1164 kwargs = execer.eval(kwargstr, glbs=glbs, locs=locs) 1165 elif not kwarglist: 1166 argstr = "({},)".format(", ".join(arglist)) 1167 args = execer.eval(argstr, glbs=glbs, locs=locs) 1168 kwargs = {} 1169 else: 1170 argstr = "({},)".format(", ".join(arglist)) 1171 kwargstr = "dict({})".format(", ".join(kwarglist)) 1172 both = "({}, {})".format(argstr, kwargstr) 1173 args, kwargs = execer.eval(both, glbs=glbs, locs=locs) 1174 return args, kwargs 1175 1176 1177def enter_macro(obj, raw_block, glbs, locs): 1178 """Prepares to enter a context manager macro by attaching the contents 1179 of the macro block, globals, and locals to the object. These modifications 1180 are made in-place and the original object is returned. 1181 1182 1183 Parameters 1184 ---------- 1185 obj : context manager 1186 The object that is about to be entered via a with-statement. 1187 raw_block : str 1188 The str of the block that is the context body. 1189 This string will be parsed, compiled, evaled, or left as 1190 a string depending on the return annotation of obj.__enter__. 1191 glbs : Mapping 1192 The globals from the context site. 1193 locs : Mapping or None 1194 The locals from the context site. 1195 1196 Returns 1197 ------- 1198 obj : context manager 1199 The same context manager but with the new macro information applied. 1200 """ 1201 # recurse down sequences 1202 if isinstance(obj, cabc.Sequence): 1203 for x in obj: 1204 enter_macro(x, raw_block, glbs, locs) 1205 return obj 1206 # convert block as needed 1207 kind = getattr(obj, "__xonsh_block__", str) 1208 macroname = getattr(obj, "__name__", "<context>") 1209 block = convert_macro_arg( 1210 raw_block, kind, glbs, locs, name="<with!>", macroname=macroname 1211 ) 1212 # attach attrs 1213 obj.macro_globals = glbs 1214 obj.macro_locals = locs 1215 obj.macro_block = block 1216 return obj 1217 1218 1219def load_builtins(execer=None, ctx=None): 1220 """Loads the xonsh builtins into the Python builtins. Sets the 1221 BUILTINS_LOADED variable to True. 1222 """ 1223 global BUILTINS_LOADED 1224 # private built-ins 1225 builtins.__xonsh_config__ = {} 1226 builtins.__xonsh_env__ = Env(default_env()) 1227 builtins.__xonsh_help__ = helper 1228 builtins.__xonsh_superhelp__ = superhelper 1229 builtins.__xonsh_pathsearch__ = pathsearch 1230 builtins.__xonsh_globsearch__ = globsearch 1231 builtins.__xonsh_regexsearch__ = regexsearch 1232 builtins.__xonsh_glob__ = globpath 1233 builtins.__xonsh_expand_path__ = expand_path 1234 builtins.__xonsh_exit__ = False 1235 builtins.__xonsh_stdout_uncaptured__ = None 1236 builtins.__xonsh_stderr_uncaptured__ = None 1237 if hasattr(builtins, "exit"): 1238 builtins.__xonsh_pyexit__ = builtins.exit 1239 del builtins.exit 1240 if hasattr(builtins, "quit"): 1241 builtins.__xonsh_pyquit__ = builtins.quit 1242 del builtins.quit 1243 builtins.__xonsh_subproc_captured_stdout__ = subproc_captured_stdout 1244 builtins.__xonsh_subproc_captured_inject__ = subproc_captured_inject 1245 builtins.__xonsh_subproc_captured_object__ = subproc_captured_object 1246 builtins.__xonsh_subproc_captured_hiddenobject__ = subproc_captured_hiddenobject 1247 builtins.__xonsh_subproc_uncaptured__ = subproc_uncaptured 1248 builtins.__xonsh_execer__ = execer 1249 builtins.__xonsh_commands_cache__ = CommandsCache() 1250 builtins.__xonsh_all_jobs__ = {} 1251 builtins.__xonsh_ensure_list_of_strs__ = ensure_list_of_strs 1252 builtins.__xonsh_list_of_strs_or_callables__ = list_of_strs_or_callables 1253 builtins.__xonsh_list_of_list_of_strs_outer_product__ = ( 1254 list_of_list_of_strs_outer_product 1255 ) 1256 builtins.__xonsh_completers__ = xonsh.completers.init.default_completers() 1257 builtins.__xonsh_call_macro__ = call_macro 1258 builtins.__xonsh_enter_macro__ = enter_macro 1259 builtins.__xonsh_path_literal__ = path_literal 1260 # public built-ins 1261 builtins.XonshError = XonshError 1262 builtins.XonshCalledProcessError = XonshCalledProcessError 1263 builtins.evalx = None if execer is None else execer.eval 1264 builtins.execx = None if execer is None else execer.exec 1265 builtins.compilex = None if execer is None else execer.compile 1266 builtins.events = events 1267 1268 # sneak the path search functions into the aliases 1269 # Need this inline/lazy import here since we use locate_binary that 1270 # relies on __xonsh_env__ in default aliases 1271 builtins.default_aliases = builtins.aliases = Aliases(make_default_aliases()) 1272 builtins.__xonsh_history__ = None 1273 atexit.register(_lastflush) 1274 for sig in AT_EXIT_SIGNALS: 1275 resetting_signal_handle(sig, _lastflush) 1276 BUILTINS_LOADED = True 1277 1278 1279def _lastflush(s=None, f=None): 1280 if hasattr(builtins, "__xonsh_history__"): 1281 if builtins.__xonsh_history__ is not None: 1282 builtins.__xonsh_history__.flush(at_exit=True) 1283 1284 1285def unload_builtins(): 1286 """Removes the xonsh builtins from the Python builtins, if the 1287 BUILTINS_LOADED is True, sets BUILTINS_LOADED to False, and returns. 1288 """ 1289 global BUILTINS_LOADED 1290 env = getattr(builtins, "__xonsh_env__", None) 1291 if isinstance(env, Env): 1292 env.undo_replace_env() 1293 if hasattr(builtins, "__xonsh_pyexit__"): 1294 builtins.exit = builtins.__xonsh_pyexit__ 1295 if hasattr(builtins, "__xonsh_pyquit__"): 1296 builtins.quit = builtins.__xonsh_pyquit__ 1297 if not BUILTINS_LOADED: 1298 return 1299 names = [ 1300 "__xonsh_config__", 1301 "__xonsh_env__", 1302 "__xonsh_ctx__", 1303 "__xonsh_help__", 1304 "__xonsh_superhelp__", 1305 "__xonsh_pathsearch__", 1306 "__xonsh_globsearch__", 1307 "__xonsh_regexsearch__", 1308 "__xonsh_glob__", 1309 "__xonsh_expand_path__", 1310 "__xonsh_exit__", 1311 "__xonsh_stdout_uncaptured__", 1312 "__xonsh_stderr_uncaptured__", 1313 "__xonsh_pyexit__", 1314 "__xonsh_pyquit__", 1315 "__xonsh_subproc_captured_stdout__", 1316 "__xonsh_subproc_captured_inject__", 1317 "__xonsh_subproc_captured_object__", 1318 "__xonsh_subproc_captured_hiddenobject__", 1319 "__xonsh_subproc_uncaptured__", 1320 "__xonsh_execer__", 1321 "__xonsh_commands_cache__", 1322 "__xonsh_completers__", 1323 "__xonsh_call_macro__", 1324 "__xonsh_enter_macro__", 1325 "__xonsh_path_literal__", 1326 "XonshError", 1327 "XonshCalledProcessError", 1328 "evalx", 1329 "execx", 1330 "compilex", 1331 "default_aliases", 1332 "__xonsh_all_jobs__", 1333 "__xonsh_ensure_list_of_strs__", 1334 "__xonsh_list_of_strs_or_callables__", 1335 "__xonsh_list_of_list_of_strs_outer_product__", 1336 "__xonsh_history__", 1337 ] 1338 for name in names: 1339 if hasattr(builtins, name): 1340 delattr(builtins, name) 1341 BUILTINS_LOADED = False 1342 1343 1344@contextlib.contextmanager 1345def xonsh_builtins(execer=None): 1346 """A context manager for using the xonsh builtins only in a limited 1347 scope. Likely useful in testing. 1348 """ 1349 load_builtins(execer=execer) 1350 yield 1351 unload_builtins() 1352