106f32e7eSjoergfrom __future__ import absolute_import 206f32e7eSjoergimport errno 306f32e7eSjoergimport io 406f32e7eSjoergimport itertools 506f32e7eSjoergimport getopt 606f32e7eSjoergimport os, signal, subprocess, sys 706f32e7eSjoergimport re 806f32e7eSjoergimport stat 906f32e7eSjoergimport platform 1006f32e7eSjoergimport shutil 1106f32e7eSjoergimport tempfile 1206f32e7eSjoergimport threading 1306f32e7eSjoerg 1406f32e7eSjoergimport io 1506f32e7eSjoergtry: 1606f32e7eSjoerg from StringIO import StringIO 1706f32e7eSjoergexcept ImportError: 1806f32e7eSjoerg from io import StringIO 1906f32e7eSjoerg 20*da58b97aSjoergfrom lit.ShCommands import GlobItem, Command 2106f32e7eSjoergimport lit.ShUtil as ShUtil 2206f32e7eSjoergimport lit.Test as Test 2306f32e7eSjoergimport lit.util 2406f32e7eSjoergfrom lit.util import to_bytes, to_string, to_unicode 2506f32e7eSjoergfrom lit.BooleanExpression import BooleanExpression 2606f32e7eSjoerg 2706f32e7eSjoergclass InternalShellError(Exception): 2806f32e7eSjoerg def __init__(self, command, message): 2906f32e7eSjoerg self.command = command 3006f32e7eSjoerg self.message = message 3106f32e7eSjoerg 3206f32e7eSjoergkIsWindows = platform.system() == 'Windows' 3306f32e7eSjoerg 3406f32e7eSjoerg# Don't use close_fds on Windows. 3506f32e7eSjoergkUseCloseFDs = not kIsWindows 3606f32e7eSjoerg 3706f32e7eSjoerg# Use temporary files to replace /dev/null on Windows. 3806f32e7eSjoergkAvoidDevNull = kIsWindows 3906f32e7eSjoergkDevNull = "/dev/null" 4006f32e7eSjoerg 4106f32e7eSjoerg# A regex that matches %dbg(ARG), which lit inserts at the beginning of each 4206f32e7eSjoerg# run command pipeline such that ARG specifies the pipeline's source line 4306f32e7eSjoerg# number. lit later expands each %dbg(ARG) to a command that behaves as a null 4406f32e7eSjoerg# command in the target shell so that the line number is seen in lit's verbose 4506f32e7eSjoerg# mode. 4606f32e7eSjoerg# 4706f32e7eSjoerg# This regex captures ARG. ARG must not contain a right parenthesis, which 4806f32e7eSjoerg# terminates %dbg. ARG must not contain quotes, in which ARG might be enclosed 4906f32e7eSjoerg# during expansion. 5006f32e7eSjoergkPdbgRegex = '%dbg\\(([^)\'"]*)\\)' 5106f32e7eSjoerg 5206f32e7eSjoergclass ShellEnvironment(object): 5306f32e7eSjoerg 5406f32e7eSjoerg """Mutable shell environment containing things like CWD and env vars. 5506f32e7eSjoerg 5606f32e7eSjoerg Environment variables are not implemented, but cwd tracking is. 5706f32e7eSjoerg """ 5806f32e7eSjoerg 5906f32e7eSjoerg def __init__(self, cwd, env): 6006f32e7eSjoerg self.cwd = cwd 6106f32e7eSjoerg self.env = dict(env) 6206f32e7eSjoerg 6306f32e7eSjoergclass TimeoutHelper(object): 6406f32e7eSjoerg """ 6506f32e7eSjoerg Object used to helper manage enforcing a timeout in 6606f32e7eSjoerg _executeShCmd(). It is passed through recursive calls 6706f32e7eSjoerg to collect processes that have been executed so that when 6806f32e7eSjoerg the timeout happens they can be killed. 6906f32e7eSjoerg """ 7006f32e7eSjoerg def __init__(self, timeout): 7106f32e7eSjoerg self.timeout = timeout 7206f32e7eSjoerg self._procs = [] 7306f32e7eSjoerg self._timeoutReached = False 7406f32e7eSjoerg self._doneKillPass = False 7506f32e7eSjoerg # This lock will be used to protect concurrent access 7606f32e7eSjoerg # to _procs and _doneKillPass 7706f32e7eSjoerg self._lock = None 7806f32e7eSjoerg self._timer = None 7906f32e7eSjoerg 8006f32e7eSjoerg def cancel(self): 8106f32e7eSjoerg if not self.active(): 8206f32e7eSjoerg return 8306f32e7eSjoerg self._timer.cancel() 8406f32e7eSjoerg 8506f32e7eSjoerg def active(self): 8606f32e7eSjoerg return self.timeout > 0 8706f32e7eSjoerg 8806f32e7eSjoerg def addProcess(self, proc): 8906f32e7eSjoerg if not self.active(): 9006f32e7eSjoerg return 9106f32e7eSjoerg needToRunKill = False 9206f32e7eSjoerg with self._lock: 9306f32e7eSjoerg self._procs.append(proc) 9406f32e7eSjoerg # Avoid re-entering the lock by finding out if kill needs to be run 9506f32e7eSjoerg # again here but call it if necessary once we have left the lock. 9606f32e7eSjoerg # We could use a reentrant lock here instead but this code seems 9706f32e7eSjoerg # clearer to me. 9806f32e7eSjoerg needToRunKill = self._doneKillPass 9906f32e7eSjoerg 10006f32e7eSjoerg # The initial call to _kill() from the timer thread already happened so 10106f32e7eSjoerg # we need to call it again from this thread, otherwise this process 10206f32e7eSjoerg # will be left to run even though the timeout was already hit 10306f32e7eSjoerg if needToRunKill: 10406f32e7eSjoerg assert self.timeoutReached() 10506f32e7eSjoerg self._kill() 10606f32e7eSjoerg 10706f32e7eSjoerg def startTimer(self): 10806f32e7eSjoerg if not self.active(): 10906f32e7eSjoerg return 11006f32e7eSjoerg 11106f32e7eSjoerg # Do some late initialisation that's only needed 11206f32e7eSjoerg # if there is a timeout set 11306f32e7eSjoerg self._lock = threading.Lock() 11406f32e7eSjoerg self._timer = threading.Timer(self.timeout, self._handleTimeoutReached) 11506f32e7eSjoerg self._timer.start() 11606f32e7eSjoerg 11706f32e7eSjoerg def _handleTimeoutReached(self): 11806f32e7eSjoerg self._timeoutReached = True 11906f32e7eSjoerg self._kill() 12006f32e7eSjoerg 12106f32e7eSjoerg def timeoutReached(self): 12206f32e7eSjoerg return self._timeoutReached 12306f32e7eSjoerg 12406f32e7eSjoerg def _kill(self): 12506f32e7eSjoerg """ 12606f32e7eSjoerg This method may be called multiple times as we might get unlucky 12706f32e7eSjoerg and be in the middle of creating a new process in _executeShCmd() 12806f32e7eSjoerg which won't yet be in ``self._procs``. By locking here and in 12906f32e7eSjoerg addProcess() we should be able to kill processes launched after 13006f32e7eSjoerg the initial call to _kill() 13106f32e7eSjoerg """ 13206f32e7eSjoerg with self._lock: 13306f32e7eSjoerg for p in self._procs: 13406f32e7eSjoerg lit.util.killProcessAndChildren(p.pid) 13506f32e7eSjoerg # Empty the list and note that we've done a pass over the list 13606f32e7eSjoerg self._procs = [] # Python2 doesn't have list.clear() 13706f32e7eSjoerg self._doneKillPass = True 13806f32e7eSjoerg 13906f32e7eSjoergclass ShellCommandResult(object): 14006f32e7eSjoerg """Captures the result of an individual command.""" 14106f32e7eSjoerg 14206f32e7eSjoerg def __init__(self, command, stdout, stderr, exitCode, timeoutReached, 14306f32e7eSjoerg outputFiles = []): 14406f32e7eSjoerg self.command = command 14506f32e7eSjoerg self.stdout = stdout 14606f32e7eSjoerg self.stderr = stderr 14706f32e7eSjoerg self.exitCode = exitCode 14806f32e7eSjoerg self.timeoutReached = timeoutReached 14906f32e7eSjoerg self.outputFiles = list(outputFiles) 15006f32e7eSjoerg 15106f32e7eSjoergdef executeShCmd(cmd, shenv, results, timeout=0): 15206f32e7eSjoerg """ 15306f32e7eSjoerg Wrapper around _executeShCmd that handles 15406f32e7eSjoerg timeout 15506f32e7eSjoerg """ 15606f32e7eSjoerg # Use the helper even when no timeout is required to make 15706f32e7eSjoerg # other code simpler (i.e. avoid bunch of ``!= None`` checks) 15806f32e7eSjoerg timeoutHelper = TimeoutHelper(timeout) 15906f32e7eSjoerg if timeout > 0: 16006f32e7eSjoerg timeoutHelper.startTimer() 16106f32e7eSjoerg finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper) 16206f32e7eSjoerg timeoutHelper.cancel() 16306f32e7eSjoerg timeoutInfo = None 16406f32e7eSjoerg if timeoutHelper.timeoutReached(): 16506f32e7eSjoerg timeoutInfo = 'Reached timeout of {} seconds'.format(timeout) 16606f32e7eSjoerg 16706f32e7eSjoerg return (finalExitCode, timeoutInfo) 16806f32e7eSjoerg 16906f32e7eSjoergdef expand_glob(arg, cwd): 17006f32e7eSjoerg if isinstance(arg, GlobItem): 17106f32e7eSjoerg return sorted(arg.resolve(cwd)) 17206f32e7eSjoerg return [arg] 17306f32e7eSjoerg 17406f32e7eSjoergdef expand_glob_expressions(args, cwd): 17506f32e7eSjoerg result = [args[0]] 17606f32e7eSjoerg for arg in args[1:]: 17706f32e7eSjoerg result.extend(expand_glob(arg, cwd)) 17806f32e7eSjoerg return result 17906f32e7eSjoerg 18006f32e7eSjoergdef quote_windows_command(seq): 18106f32e7eSjoerg """ 18206f32e7eSjoerg Reimplement Python's private subprocess.list2cmdline for MSys compatibility 18306f32e7eSjoerg 18406f32e7eSjoerg Based on CPython implementation here: 18506f32e7eSjoerg https://hg.python.org/cpython/file/849826a900d2/Lib/subprocess.py#l422 18606f32e7eSjoerg 18706f32e7eSjoerg Some core util distributions (MSys) don't tokenize command line arguments 18806f32e7eSjoerg the same way that MSVC CRT does. Lit rolls its own quoting logic similar to 18906f32e7eSjoerg the stock CPython logic to paper over these quoting and tokenization rule 19006f32e7eSjoerg differences. 19106f32e7eSjoerg 19206f32e7eSjoerg We use the same algorithm from MSDN as CPython 19306f32e7eSjoerg (http://msdn.microsoft.com/en-us/library/17w5ykft.aspx), but we treat more 194*da58b97aSjoerg characters as needing quoting, such as double quotes themselves, and square 195*da58b97aSjoerg brackets. 196*da58b97aSjoerg 197*da58b97aSjoerg For MSys based tools, this is very brittle though, because quoting an 198*da58b97aSjoerg argument makes the MSys based tool unescape backslashes where it shouldn't 199*da58b97aSjoerg (e.g. "a\b\\c\\\\d" becomes "a\b\c\\d" where it should stay as it was, 200*da58b97aSjoerg according to regular win32 command line parsing rules). 20106f32e7eSjoerg """ 20206f32e7eSjoerg result = [] 20306f32e7eSjoerg needquote = False 20406f32e7eSjoerg for arg in seq: 20506f32e7eSjoerg bs_buf = [] 20606f32e7eSjoerg 20706f32e7eSjoerg # Add a space to separate this argument from the others 20806f32e7eSjoerg if result: 20906f32e7eSjoerg result.append(' ') 21006f32e7eSjoerg 21106f32e7eSjoerg # This logic differs from upstream list2cmdline. 212*da58b97aSjoerg needquote = (" " in arg) or ("\t" in arg) or ("\"" in arg) or ("[" in arg) or not arg 21306f32e7eSjoerg if needquote: 21406f32e7eSjoerg result.append('"') 21506f32e7eSjoerg 21606f32e7eSjoerg for c in arg: 21706f32e7eSjoerg if c == '\\': 21806f32e7eSjoerg # Don't know if we need to double yet. 21906f32e7eSjoerg bs_buf.append(c) 22006f32e7eSjoerg elif c == '"': 22106f32e7eSjoerg # Double backslashes. 22206f32e7eSjoerg result.append('\\' * len(bs_buf)*2) 22306f32e7eSjoerg bs_buf = [] 22406f32e7eSjoerg result.append('\\"') 22506f32e7eSjoerg else: 22606f32e7eSjoerg # Normal char 22706f32e7eSjoerg if bs_buf: 22806f32e7eSjoerg result.extend(bs_buf) 22906f32e7eSjoerg bs_buf = [] 23006f32e7eSjoerg result.append(c) 23106f32e7eSjoerg 23206f32e7eSjoerg # Add remaining backslashes, if any. 23306f32e7eSjoerg if bs_buf: 23406f32e7eSjoerg result.extend(bs_buf) 23506f32e7eSjoerg 23606f32e7eSjoerg if needquote: 23706f32e7eSjoerg result.extend(bs_buf) 23806f32e7eSjoerg result.append('"') 23906f32e7eSjoerg 24006f32e7eSjoerg return ''.join(result) 24106f32e7eSjoerg 24206f32e7eSjoerg# args are from 'export' or 'env' command. 243*da58b97aSjoerg# Skips the command, and parses its arguments. 244*da58b97aSjoerg# Modifies env accordingly. 245*da58b97aSjoerg# Returns copy of args without the command or its arguments. 24606f32e7eSjoergdef updateEnv(env, args): 24706f32e7eSjoerg arg_idx_next = len(args) 24806f32e7eSjoerg unset_next_env_var = False 24906f32e7eSjoerg for arg_idx, arg in enumerate(args[1:]): 25006f32e7eSjoerg # Support for the -u flag (unsetting) for env command 25106f32e7eSjoerg # e.g., env -u FOO -u BAR will remove both FOO and BAR 25206f32e7eSjoerg # from the environment. 25306f32e7eSjoerg if arg == '-u': 25406f32e7eSjoerg unset_next_env_var = True 25506f32e7eSjoerg continue 25606f32e7eSjoerg if unset_next_env_var: 25706f32e7eSjoerg unset_next_env_var = False 25806f32e7eSjoerg if arg in env.env: 25906f32e7eSjoerg del env.env[arg] 26006f32e7eSjoerg continue 26106f32e7eSjoerg 26206f32e7eSjoerg # Partition the string into KEY=VALUE. 26306f32e7eSjoerg key, eq, val = arg.partition('=') 26406f32e7eSjoerg # Stop if there was no equals. 26506f32e7eSjoerg if eq == '': 26606f32e7eSjoerg arg_idx_next = arg_idx + 1 26706f32e7eSjoerg break 26806f32e7eSjoerg env.env[key] = val 26906f32e7eSjoerg return args[arg_idx_next:] 27006f32e7eSjoerg 271*da58b97aSjoergdef executeBuiltinCd(cmd, shenv): 272*da58b97aSjoerg """executeBuiltinCd - Change the current directory.""" 273*da58b97aSjoerg if len(cmd.args) != 2: 274*da58b97aSjoerg raise InternalShellError("'cd' supports only one argument") 275*da58b97aSjoerg newdir = cmd.args[1] 276*da58b97aSjoerg # Update the cwd in the parent environment. 277*da58b97aSjoerg if os.path.isabs(newdir): 278*da58b97aSjoerg shenv.cwd = newdir 279*da58b97aSjoerg else: 280*da58b97aSjoerg shenv.cwd = os.path.realpath(os.path.join(shenv.cwd, newdir)) 281*da58b97aSjoerg # The cd builtin always succeeds. If the directory does not exist, the 282*da58b97aSjoerg # following Popen calls will fail instead. 283*da58b97aSjoerg return ShellCommandResult(cmd, "", "", 0, False) 284*da58b97aSjoerg 285*da58b97aSjoergdef executeBuiltinExport(cmd, shenv): 286*da58b97aSjoerg """executeBuiltinExport - Set an environment variable.""" 287*da58b97aSjoerg if len(cmd.args) != 2: 288*da58b97aSjoerg raise InternalShellError("'export' supports only one argument") 289*da58b97aSjoerg updateEnv(shenv, cmd.args) 290*da58b97aSjoerg return ShellCommandResult(cmd, "", "", 0, False) 291*da58b97aSjoerg 29206f32e7eSjoergdef executeBuiltinEcho(cmd, shenv): 29306f32e7eSjoerg """Interpret a redirected echo command""" 29406f32e7eSjoerg opened_files = [] 29506f32e7eSjoerg stdin, stdout, stderr = processRedirects(cmd, subprocess.PIPE, shenv, 29606f32e7eSjoerg opened_files) 29706f32e7eSjoerg if stdin != subprocess.PIPE or stderr != subprocess.PIPE: 29806f32e7eSjoerg raise InternalShellError( 29906f32e7eSjoerg cmd, "stdin and stderr redirects not supported for echo") 30006f32e7eSjoerg 30106f32e7eSjoerg # Some tests have un-redirected echo commands to help debug test failures. 30206f32e7eSjoerg # Buffer our output and return it to the caller. 30306f32e7eSjoerg is_redirected = True 30406f32e7eSjoerg encode = lambda x : x 30506f32e7eSjoerg if stdout == subprocess.PIPE: 30606f32e7eSjoerg is_redirected = False 30706f32e7eSjoerg stdout = StringIO() 30806f32e7eSjoerg elif kIsWindows: 30906f32e7eSjoerg # Reopen stdout in binary mode to avoid CRLF translation. The versions 31006f32e7eSjoerg # of echo we are replacing on Windows all emit plain LF, and the LLVM 31106f32e7eSjoerg # tests now depend on this. 31206f32e7eSjoerg # When we open as binary, however, this also means that we have to write 31306f32e7eSjoerg # 'bytes' objects to stdout instead of 'str' objects. 31406f32e7eSjoerg encode = lit.util.to_bytes 31506f32e7eSjoerg stdout = open(stdout.name, stdout.mode + 'b') 31606f32e7eSjoerg opened_files.append((None, None, stdout, None)) 31706f32e7eSjoerg 31806f32e7eSjoerg # Implement echo flags. We only support -e and -n, and not yet in 31906f32e7eSjoerg # combination. We have to ignore unknown flags, because `echo "-D FOO"` 32006f32e7eSjoerg # prints the dash. 32106f32e7eSjoerg args = cmd.args[1:] 32206f32e7eSjoerg interpret_escapes = False 32306f32e7eSjoerg write_newline = True 32406f32e7eSjoerg while len(args) >= 1 and args[0] in ('-e', '-n'): 32506f32e7eSjoerg flag = args[0] 32606f32e7eSjoerg args = args[1:] 32706f32e7eSjoerg if flag == '-e': 32806f32e7eSjoerg interpret_escapes = True 32906f32e7eSjoerg elif flag == '-n': 33006f32e7eSjoerg write_newline = False 33106f32e7eSjoerg 33206f32e7eSjoerg def maybeUnescape(arg): 33306f32e7eSjoerg if not interpret_escapes: 33406f32e7eSjoerg return arg 33506f32e7eSjoerg 33606f32e7eSjoerg arg = lit.util.to_bytes(arg) 33706f32e7eSjoerg codec = 'string_escape' if sys.version_info < (3,0) else 'unicode_escape' 33806f32e7eSjoerg return arg.decode(codec) 33906f32e7eSjoerg 34006f32e7eSjoerg if args: 34106f32e7eSjoerg for arg in args[:-1]: 34206f32e7eSjoerg stdout.write(encode(maybeUnescape(arg))) 34306f32e7eSjoerg stdout.write(encode(' ')) 34406f32e7eSjoerg stdout.write(encode(maybeUnescape(args[-1]))) 34506f32e7eSjoerg if write_newline: 34606f32e7eSjoerg stdout.write(encode('\n')) 34706f32e7eSjoerg 34806f32e7eSjoerg for (name, mode, f, path) in opened_files: 34906f32e7eSjoerg f.close() 35006f32e7eSjoerg 351*da58b97aSjoerg output = "" if is_redirected else stdout.getvalue() 352*da58b97aSjoerg return ShellCommandResult(cmd, output, "", 0, False) 35306f32e7eSjoerg 35406f32e7eSjoergdef executeBuiltinMkdir(cmd, cmd_shenv): 35506f32e7eSjoerg """executeBuiltinMkdir - Create new directories.""" 35606f32e7eSjoerg args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:] 35706f32e7eSjoerg try: 35806f32e7eSjoerg opts, args = getopt.gnu_getopt(args, 'p') 35906f32e7eSjoerg except getopt.GetoptError as err: 36006f32e7eSjoerg raise InternalShellError(cmd, "Unsupported: 'mkdir': %s" % str(err)) 36106f32e7eSjoerg 36206f32e7eSjoerg parent = False 36306f32e7eSjoerg for o, a in opts: 36406f32e7eSjoerg if o == "-p": 36506f32e7eSjoerg parent = True 36606f32e7eSjoerg else: 36706f32e7eSjoerg assert False, "unhandled option" 36806f32e7eSjoerg 36906f32e7eSjoerg if len(args) == 0: 37006f32e7eSjoerg raise InternalShellError(cmd, "Error: 'mkdir' is missing an operand") 37106f32e7eSjoerg 37206f32e7eSjoerg stderr = StringIO() 37306f32e7eSjoerg exitCode = 0 37406f32e7eSjoerg for dir in args: 37506f32e7eSjoerg cwd = cmd_shenv.cwd 37606f32e7eSjoerg dir = to_unicode(dir) if kIsWindows else to_bytes(dir) 37706f32e7eSjoerg cwd = to_unicode(cwd) if kIsWindows else to_bytes(cwd) 37806f32e7eSjoerg if not os.path.isabs(dir): 37906f32e7eSjoerg dir = os.path.realpath(os.path.join(cwd, dir)) 38006f32e7eSjoerg if parent: 38106f32e7eSjoerg lit.util.mkdir_p(dir) 38206f32e7eSjoerg else: 38306f32e7eSjoerg try: 38406f32e7eSjoerg lit.util.mkdir(dir) 38506f32e7eSjoerg except OSError as err: 38606f32e7eSjoerg stderr.write("Error: 'mkdir' command failed, %s\n" % str(err)) 38706f32e7eSjoerg exitCode = 1 38806f32e7eSjoerg return ShellCommandResult(cmd, "", stderr.getvalue(), exitCode, False) 38906f32e7eSjoerg 39006f32e7eSjoergdef executeBuiltinRm(cmd, cmd_shenv): 39106f32e7eSjoerg """executeBuiltinRm - Removes (deletes) files or directories.""" 39206f32e7eSjoerg args = expand_glob_expressions(cmd.args, cmd_shenv.cwd)[1:] 39306f32e7eSjoerg try: 39406f32e7eSjoerg opts, args = getopt.gnu_getopt(args, "frR", ["--recursive"]) 39506f32e7eSjoerg except getopt.GetoptError as err: 39606f32e7eSjoerg raise InternalShellError(cmd, "Unsupported: 'rm': %s" % str(err)) 39706f32e7eSjoerg 39806f32e7eSjoerg force = False 39906f32e7eSjoerg recursive = False 40006f32e7eSjoerg for o, a in opts: 40106f32e7eSjoerg if o == "-f": 40206f32e7eSjoerg force = True 40306f32e7eSjoerg elif o in ("-r", "-R", "--recursive"): 40406f32e7eSjoerg recursive = True 40506f32e7eSjoerg else: 40606f32e7eSjoerg assert False, "unhandled option" 40706f32e7eSjoerg 40806f32e7eSjoerg if len(args) == 0: 40906f32e7eSjoerg raise InternalShellError(cmd, "Error: 'rm' is missing an operand") 41006f32e7eSjoerg 41106f32e7eSjoerg def on_rm_error(func, path, exc_info): 41206f32e7eSjoerg # path contains the path of the file that couldn't be removed 41306f32e7eSjoerg # let's just assume that it's read-only and remove it. 41406f32e7eSjoerg os.chmod(path, stat.S_IMODE( os.stat(path).st_mode) | stat.S_IWRITE) 41506f32e7eSjoerg os.remove(path) 41606f32e7eSjoerg 41706f32e7eSjoerg stderr = StringIO() 41806f32e7eSjoerg exitCode = 0 41906f32e7eSjoerg for path in args: 42006f32e7eSjoerg cwd = cmd_shenv.cwd 42106f32e7eSjoerg path = to_unicode(path) if kIsWindows else to_bytes(path) 42206f32e7eSjoerg cwd = to_unicode(cwd) if kIsWindows else to_bytes(cwd) 42306f32e7eSjoerg if not os.path.isabs(path): 42406f32e7eSjoerg path = os.path.realpath(os.path.join(cwd, path)) 42506f32e7eSjoerg if force and not os.path.exists(path): 42606f32e7eSjoerg continue 42706f32e7eSjoerg try: 42806f32e7eSjoerg if os.path.isdir(path): 42906f32e7eSjoerg if not recursive: 43006f32e7eSjoerg stderr.write("Error: %s is a directory\n" % path) 43106f32e7eSjoerg exitCode = 1 43206f32e7eSjoerg if platform.system() == 'Windows': 43306f32e7eSjoerg # NOTE: use ctypes to access `SHFileOperationsW` on Windows to 43406f32e7eSjoerg # use the NT style path to get access to long file paths which 43506f32e7eSjoerg # cannot be removed otherwise. 43606f32e7eSjoerg from ctypes.wintypes import BOOL, HWND, LPCWSTR, UINT, WORD 43706f32e7eSjoerg from ctypes import addressof, byref, c_void_p, create_unicode_buffer 43806f32e7eSjoerg from ctypes import Structure 43906f32e7eSjoerg from ctypes import windll, WinError, POINTER 44006f32e7eSjoerg 44106f32e7eSjoerg class SHFILEOPSTRUCTW(Structure): 44206f32e7eSjoerg _fields_ = [ 44306f32e7eSjoerg ('hWnd', HWND), 44406f32e7eSjoerg ('wFunc', UINT), 44506f32e7eSjoerg ('pFrom', LPCWSTR), 44606f32e7eSjoerg ('pTo', LPCWSTR), 44706f32e7eSjoerg ('fFlags', WORD), 44806f32e7eSjoerg ('fAnyOperationsAborted', BOOL), 44906f32e7eSjoerg ('hNameMappings', c_void_p), 45006f32e7eSjoerg ('lpszProgressTitle', LPCWSTR), 45106f32e7eSjoerg ] 45206f32e7eSjoerg 45306f32e7eSjoerg FO_MOVE, FO_COPY, FO_DELETE, FO_RENAME = range(1, 5) 45406f32e7eSjoerg 45506f32e7eSjoerg FOF_SILENT = 4 45606f32e7eSjoerg FOF_NOCONFIRMATION = 16 45706f32e7eSjoerg FOF_NOCONFIRMMKDIR = 512 45806f32e7eSjoerg FOF_NOERRORUI = 1024 45906f32e7eSjoerg 46006f32e7eSjoerg FOF_NO_UI = FOF_SILENT | FOF_NOCONFIRMATION | FOF_NOERRORUI | FOF_NOCONFIRMMKDIR 46106f32e7eSjoerg 46206f32e7eSjoerg SHFileOperationW = windll.shell32.SHFileOperationW 46306f32e7eSjoerg SHFileOperationW.argtypes = [POINTER(SHFILEOPSTRUCTW)] 46406f32e7eSjoerg 46506f32e7eSjoerg path = os.path.abspath(path) 46606f32e7eSjoerg 46706f32e7eSjoerg pFrom = create_unicode_buffer(path, len(path) + 2) 46806f32e7eSjoerg pFrom[len(path)] = pFrom[len(path) + 1] = '\0' 46906f32e7eSjoerg operation = SHFILEOPSTRUCTW(wFunc=UINT(FO_DELETE), 47006f32e7eSjoerg pFrom=LPCWSTR(addressof(pFrom)), 47106f32e7eSjoerg fFlags=FOF_NO_UI) 47206f32e7eSjoerg result = SHFileOperationW(byref(operation)) 47306f32e7eSjoerg if result: 47406f32e7eSjoerg raise WinError(result) 47506f32e7eSjoerg else: 47606f32e7eSjoerg shutil.rmtree(path, onerror = on_rm_error if force else None) 47706f32e7eSjoerg else: 47806f32e7eSjoerg if force and not os.access(path, os.W_OK): 47906f32e7eSjoerg os.chmod(path, 48006f32e7eSjoerg stat.S_IMODE(os.stat(path).st_mode) | stat.S_IWRITE) 48106f32e7eSjoerg os.remove(path) 48206f32e7eSjoerg except OSError as err: 48306f32e7eSjoerg stderr.write("Error: 'rm' command failed, %s" % str(err)) 48406f32e7eSjoerg exitCode = 1 48506f32e7eSjoerg return ShellCommandResult(cmd, "", stderr.getvalue(), exitCode, False) 48606f32e7eSjoerg 487*da58b97aSjoergdef executeBuiltinColon(cmd, cmd_shenv): 488*da58b97aSjoerg """executeBuiltinColon - Discard arguments and exit with status 0.""" 489*da58b97aSjoerg return ShellCommandResult(cmd, "", "", 0, False) 490*da58b97aSjoerg 49106f32e7eSjoergdef processRedirects(cmd, stdin_source, cmd_shenv, opened_files): 49206f32e7eSjoerg """Return the standard fds for cmd after applying redirects 49306f32e7eSjoerg 49406f32e7eSjoerg Returns the three standard file descriptors for the new child process. Each 49506f32e7eSjoerg fd may be an open, writable file object or a sentinel value from the 49606f32e7eSjoerg subprocess module. 49706f32e7eSjoerg """ 49806f32e7eSjoerg 49906f32e7eSjoerg # Apply the redirections, we use (N,) as a sentinel to indicate stdin, 50006f32e7eSjoerg # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or 50106f32e7eSjoerg # from a file are represented with a list [file, mode, file-object] 50206f32e7eSjoerg # where file-object is initially None. 50306f32e7eSjoerg redirects = [(0,), (1,), (2,)] 50406f32e7eSjoerg for (op, filename) in cmd.redirects: 50506f32e7eSjoerg if op == ('>',2): 50606f32e7eSjoerg redirects[2] = [filename, 'w', None] 50706f32e7eSjoerg elif op == ('>>',2): 50806f32e7eSjoerg redirects[2] = [filename, 'a', None] 50906f32e7eSjoerg elif op == ('>&',2) and filename in '012': 51006f32e7eSjoerg redirects[2] = redirects[int(filename)] 51106f32e7eSjoerg elif op == ('>&',) or op == ('&>',): 51206f32e7eSjoerg redirects[1] = redirects[2] = [filename, 'w', None] 51306f32e7eSjoerg elif op == ('>',): 51406f32e7eSjoerg redirects[1] = [filename, 'w', None] 51506f32e7eSjoerg elif op == ('>>',): 51606f32e7eSjoerg redirects[1] = [filename, 'a', None] 51706f32e7eSjoerg elif op == ('<',): 51806f32e7eSjoerg redirects[0] = [filename, 'r', None] 51906f32e7eSjoerg else: 52006f32e7eSjoerg raise InternalShellError(cmd, "Unsupported redirect: %r" % ((op, filename),)) 52106f32e7eSjoerg 52206f32e7eSjoerg # Open file descriptors in a second pass. 52306f32e7eSjoerg std_fds = [None, None, None] 52406f32e7eSjoerg for (index, r) in enumerate(redirects): 52506f32e7eSjoerg # Handle the sentinel values for defaults up front. 52606f32e7eSjoerg if isinstance(r, tuple): 52706f32e7eSjoerg if r == (0,): 52806f32e7eSjoerg fd = stdin_source 52906f32e7eSjoerg elif r == (1,): 53006f32e7eSjoerg if index == 0: 53106f32e7eSjoerg raise InternalShellError(cmd, "Unsupported redirect for stdin") 53206f32e7eSjoerg elif index == 1: 53306f32e7eSjoerg fd = subprocess.PIPE 53406f32e7eSjoerg else: 53506f32e7eSjoerg fd = subprocess.STDOUT 53606f32e7eSjoerg elif r == (2,): 53706f32e7eSjoerg if index != 2: 53806f32e7eSjoerg raise InternalShellError(cmd, "Unsupported redirect on stdout") 53906f32e7eSjoerg fd = subprocess.PIPE 54006f32e7eSjoerg else: 54106f32e7eSjoerg raise InternalShellError(cmd, "Bad redirect") 54206f32e7eSjoerg std_fds[index] = fd 54306f32e7eSjoerg continue 54406f32e7eSjoerg 54506f32e7eSjoerg (filename, mode, fd) = r 54606f32e7eSjoerg 54706f32e7eSjoerg # Check if we already have an open fd. This can happen if stdout and 54806f32e7eSjoerg # stderr go to the same place. 54906f32e7eSjoerg if fd is not None: 55006f32e7eSjoerg std_fds[index] = fd 55106f32e7eSjoerg continue 55206f32e7eSjoerg 55306f32e7eSjoerg redir_filename = None 55406f32e7eSjoerg name = expand_glob(filename, cmd_shenv.cwd) 55506f32e7eSjoerg if len(name) != 1: 55606f32e7eSjoerg raise InternalShellError(cmd, "Unsupported: glob in " 55706f32e7eSjoerg "redirect expanded to multiple files") 55806f32e7eSjoerg name = name[0] 55906f32e7eSjoerg if kAvoidDevNull and name == kDevNull: 56006f32e7eSjoerg fd = tempfile.TemporaryFile(mode=mode) 56106f32e7eSjoerg elif kIsWindows and name == '/dev/tty': 56206f32e7eSjoerg # Simulate /dev/tty on Windows. 56306f32e7eSjoerg # "CON" is a special filename for the console. 56406f32e7eSjoerg fd = open("CON", mode) 56506f32e7eSjoerg else: 56606f32e7eSjoerg # Make sure relative paths are relative to the cwd. 56706f32e7eSjoerg redir_filename = os.path.join(cmd_shenv.cwd, name) 56806f32e7eSjoerg redir_filename = to_unicode(redir_filename) \ 56906f32e7eSjoerg if kIsWindows else to_bytes(redir_filename) 57006f32e7eSjoerg fd = open(redir_filename, mode) 57106f32e7eSjoerg # Workaround a Win32 and/or subprocess bug when appending. 57206f32e7eSjoerg # 57306f32e7eSjoerg # FIXME: Actually, this is probably an instance of PR6753. 57406f32e7eSjoerg if mode == 'a': 57506f32e7eSjoerg fd.seek(0, 2) 57606f32e7eSjoerg # Mutate the underlying redirect list so that we can redirect stdout 57706f32e7eSjoerg # and stderr to the same place without opening the file twice. 57806f32e7eSjoerg r[2] = fd 57906f32e7eSjoerg opened_files.append((filename, mode, fd) + (redir_filename,)) 58006f32e7eSjoerg std_fds[index] = fd 58106f32e7eSjoerg 58206f32e7eSjoerg return std_fds 58306f32e7eSjoerg 58406f32e7eSjoergdef _executeShCmd(cmd, shenv, results, timeoutHelper): 58506f32e7eSjoerg if timeoutHelper.timeoutReached(): 58606f32e7eSjoerg # Prevent further recursion if the timeout has been hit 58706f32e7eSjoerg # as we should try avoid launching more processes. 58806f32e7eSjoerg return None 58906f32e7eSjoerg 59006f32e7eSjoerg if isinstance(cmd, ShUtil.Seq): 59106f32e7eSjoerg if cmd.op == ';': 59206f32e7eSjoerg res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper) 59306f32e7eSjoerg return _executeShCmd(cmd.rhs, shenv, results, timeoutHelper) 59406f32e7eSjoerg 59506f32e7eSjoerg if cmd.op == '&': 59606f32e7eSjoerg raise InternalShellError(cmd,"unsupported shell operator: '&'") 59706f32e7eSjoerg 59806f32e7eSjoerg if cmd.op == '||': 59906f32e7eSjoerg res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper) 60006f32e7eSjoerg if res != 0: 60106f32e7eSjoerg res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper) 60206f32e7eSjoerg return res 60306f32e7eSjoerg 60406f32e7eSjoerg if cmd.op == '&&': 60506f32e7eSjoerg res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper) 60606f32e7eSjoerg if res is None: 60706f32e7eSjoerg return res 60806f32e7eSjoerg 60906f32e7eSjoerg if res == 0: 61006f32e7eSjoerg res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper) 61106f32e7eSjoerg return res 61206f32e7eSjoerg 61306f32e7eSjoerg raise ValueError('Unknown shell command: %r' % cmd.op) 61406f32e7eSjoerg assert isinstance(cmd, ShUtil.Pipeline) 61506f32e7eSjoerg 61606f32e7eSjoerg procs = [] 617*da58b97aSjoerg proc_not_counts = [] 61806f32e7eSjoerg default_stdin = subprocess.PIPE 61906f32e7eSjoerg stderrTempFiles = [] 62006f32e7eSjoerg opened_files = [] 62106f32e7eSjoerg named_temp_files = [] 622*da58b97aSjoerg builtin_commands = set(['cat', 'diff']) 62306f32e7eSjoerg builtin_commands_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "builtin_commands") 624*da58b97aSjoerg inproc_builtins = {'cd': executeBuiltinCd, 625*da58b97aSjoerg 'export': executeBuiltinExport, 626*da58b97aSjoerg 'echo': executeBuiltinEcho, 627*da58b97aSjoerg 'mkdir': executeBuiltinMkdir, 628*da58b97aSjoerg 'rm': executeBuiltinRm, 629*da58b97aSjoerg ':': executeBuiltinColon} 63006f32e7eSjoerg # To avoid deadlock, we use a single stderr stream for piped 63106f32e7eSjoerg # output. This is null until we have seen some output using 63206f32e7eSjoerg # stderr. 63306f32e7eSjoerg for i,j in enumerate(cmd.commands): 63406f32e7eSjoerg # Reference the global environment by default. 63506f32e7eSjoerg cmd_shenv = shenv 63606f32e7eSjoerg args = list(j.args) 637*da58b97aSjoerg not_args = [] 638*da58b97aSjoerg not_count = 0 639*da58b97aSjoerg not_crash = False 640*da58b97aSjoerg while True: 641*da58b97aSjoerg if args[0] == 'env': 642*da58b97aSjoerg # Create a copy of the global environment and modify it for 643*da58b97aSjoerg # this one command. There might be multiple envs in a pipeline, 644*da58b97aSjoerg # and there might be multiple envs in a command (usually when 645*da58b97aSjoerg # one comes from a substitution): 64606f32e7eSjoerg # env FOO=1 llc < %s | env BAR=2 llvm-mc | FileCheck %s 647*da58b97aSjoerg # env FOO=1 %{another_env_plus_cmd} | FileCheck %s 648*da58b97aSjoerg if cmd_shenv is shenv: 64906f32e7eSjoerg cmd_shenv = ShellEnvironment(shenv.cwd, shenv.env) 650*da58b97aSjoerg args = updateEnv(cmd_shenv, args) 65106f32e7eSjoerg if not args: 652*da58b97aSjoerg raise InternalShellError(j, "Error: 'env' requires a" 653*da58b97aSjoerg " subcommand") 654*da58b97aSjoerg elif args[0] == 'not': 655*da58b97aSjoerg not_args.append(args.pop(0)) 656*da58b97aSjoerg not_count += 1 657*da58b97aSjoerg if args and args[0] == '--crash': 658*da58b97aSjoerg not_args.append(args.pop(0)) 659*da58b97aSjoerg not_crash = True 660*da58b97aSjoerg if not args: 661*da58b97aSjoerg raise InternalShellError(j, "Error: 'not' requires a" 662*da58b97aSjoerg " subcommand") 663*da58b97aSjoerg elif args[0] == '!': 664*da58b97aSjoerg not_args.append(args.pop(0)) 665*da58b97aSjoerg not_count += 1 666*da58b97aSjoerg if not args: 667*da58b97aSjoerg raise InternalShellError(j, "Error: '!' requires a" 668*da58b97aSjoerg " subcommand") 669*da58b97aSjoerg else: 670*da58b97aSjoerg break 671*da58b97aSjoerg 672*da58b97aSjoerg # Handle in-process builtins. 673*da58b97aSjoerg # 674*da58b97aSjoerg # Handle "echo" as a builtin if it is not part of a pipeline. This 675*da58b97aSjoerg # greatly speeds up tests that construct input files by repeatedly 676*da58b97aSjoerg # echo-appending to a file. 677*da58b97aSjoerg # FIXME: Standardize on the builtin echo implementation. We can use a 678*da58b97aSjoerg # temporary file to sidestep blocking pipe write issues. 679*da58b97aSjoerg inproc_builtin = inproc_builtins.get(args[0], None) 680*da58b97aSjoerg if inproc_builtin and (args[0] != 'echo' or len(cmd.commands) == 1): 681*da58b97aSjoerg # env calling an in-process builtin is useless, so we take the safe 682*da58b97aSjoerg # approach of complaining. 683*da58b97aSjoerg if not cmd_shenv is shenv: 684*da58b97aSjoerg raise InternalShellError(j, "Error: 'env' cannot call '{}'" 685*da58b97aSjoerg .format(args[0])) 686*da58b97aSjoerg if not_crash: 687*da58b97aSjoerg raise InternalShellError(j, "Error: 'not --crash' cannot call" 688*da58b97aSjoerg " '{}'".format(args[0])) 689*da58b97aSjoerg if len(cmd.commands) != 1: 690*da58b97aSjoerg raise InternalShellError(j, "Unsupported: '{}' cannot be part" 691*da58b97aSjoerg " of a pipeline".format(args[0])) 692*da58b97aSjoerg result = inproc_builtin(Command(args, j.redirects), cmd_shenv) 693*da58b97aSjoerg if not_count % 2: 694*da58b97aSjoerg result.exitCode = int(not result.exitCode) 695*da58b97aSjoerg result.command.args = j.args; 696*da58b97aSjoerg results.append(result) 697*da58b97aSjoerg return result.exitCode 698*da58b97aSjoerg 699*da58b97aSjoerg # Resolve any out-of-process builtin command before adding back 'not' 700*da58b97aSjoerg # commands. 701*da58b97aSjoerg if args[0] in builtin_commands: 702*da58b97aSjoerg args.insert(0, sys.executable) 703*da58b97aSjoerg cmd_shenv.env['PYTHONPATH'] = \ 704*da58b97aSjoerg os.path.dirname(os.path.abspath(__file__)) 705*da58b97aSjoerg args[1] = os.path.join(builtin_commands_dir, args[1] + ".py") 706*da58b97aSjoerg 707*da58b97aSjoerg # We had to search through the 'not' commands to find all the 'env' 708*da58b97aSjoerg # commands and any other in-process builtin command. We don't want to 709*da58b97aSjoerg # reimplement 'not' and its '--crash' here, so just push all 'not' 710*da58b97aSjoerg # commands back to be called as external commands. Because this 711*da58b97aSjoerg # approach effectively moves all 'env' commands up front, it relies on 712*da58b97aSjoerg # the assumptions that (1) environment variables are not intended to be 713*da58b97aSjoerg # relevant to 'not' commands and (2) the 'env' command should always 714*da58b97aSjoerg # blindly pass along the status it receives from any command it calls. 715*da58b97aSjoerg 716*da58b97aSjoerg # For plain negations, either 'not' without '--crash', or the shell 717*da58b97aSjoerg # operator '!', leave them out from the command to execute and 718*da58b97aSjoerg # invert the result code afterwards. 719*da58b97aSjoerg if not_crash: 720*da58b97aSjoerg args = not_args + args 721*da58b97aSjoerg not_count = 0 722*da58b97aSjoerg else: 723*da58b97aSjoerg not_args = [] 72406f32e7eSjoerg 72506f32e7eSjoerg stdin, stdout, stderr = processRedirects(j, default_stdin, cmd_shenv, 72606f32e7eSjoerg opened_files) 72706f32e7eSjoerg 72806f32e7eSjoerg # If stderr wants to come from stdout, but stdout isn't a pipe, then put 72906f32e7eSjoerg # stderr on a pipe and treat it as stdout. 73006f32e7eSjoerg if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE): 73106f32e7eSjoerg stderr = subprocess.PIPE 73206f32e7eSjoerg stderrIsStdout = True 73306f32e7eSjoerg else: 73406f32e7eSjoerg stderrIsStdout = False 73506f32e7eSjoerg 73606f32e7eSjoerg # Don't allow stderr on a PIPE except for the last 73706f32e7eSjoerg # process, this could deadlock. 73806f32e7eSjoerg # 73906f32e7eSjoerg # FIXME: This is slow, but so is deadlock. 74006f32e7eSjoerg if stderr == subprocess.PIPE and j != cmd.commands[-1]: 74106f32e7eSjoerg stderr = tempfile.TemporaryFile(mode='w+b') 74206f32e7eSjoerg stderrTempFiles.append((i, stderr)) 74306f32e7eSjoerg 74406f32e7eSjoerg # Resolve the executable path ourselves. 74506f32e7eSjoerg executable = None 74606f32e7eSjoerg # For paths relative to cwd, use the cwd of the shell environment. 74706f32e7eSjoerg if args[0].startswith('.'): 74806f32e7eSjoerg exe_in_cwd = os.path.join(cmd_shenv.cwd, args[0]) 74906f32e7eSjoerg if os.path.isfile(exe_in_cwd): 75006f32e7eSjoerg executable = exe_in_cwd 75106f32e7eSjoerg if not executable: 75206f32e7eSjoerg executable = lit.util.which(args[0], cmd_shenv.env['PATH']) 75306f32e7eSjoerg if not executable: 75406f32e7eSjoerg raise InternalShellError(j, '%r: command not found' % args[0]) 75506f32e7eSjoerg 75606f32e7eSjoerg # Replace uses of /dev/null with temporary files. 75706f32e7eSjoerg if kAvoidDevNull: 75806f32e7eSjoerg # In Python 2.x, basestring is the base class for all string (including unicode) 75906f32e7eSjoerg # In Python 3.x, basestring no longer exist and str is always unicode 76006f32e7eSjoerg try: 76106f32e7eSjoerg str_type = basestring 76206f32e7eSjoerg except NameError: 76306f32e7eSjoerg str_type = str 76406f32e7eSjoerg for i,arg in enumerate(args): 76506f32e7eSjoerg if isinstance(arg, str_type) and kDevNull in arg: 76606f32e7eSjoerg f = tempfile.NamedTemporaryFile(delete=False) 76706f32e7eSjoerg f.close() 76806f32e7eSjoerg named_temp_files.append(f.name) 76906f32e7eSjoerg args[i] = arg.replace(kDevNull, f.name) 77006f32e7eSjoerg 77106f32e7eSjoerg # Expand all glob expressions 77206f32e7eSjoerg args = expand_glob_expressions(args, cmd_shenv.cwd) 77306f32e7eSjoerg 77406f32e7eSjoerg # On Windows, do our own command line quoting for better compatibility 77506f32e7eSjoerg # with some core utility distributions. 77606f32e7eSjoerg if kIsWindows: 77706f32e7eSjoerg args = quote_windows_command(args) 77806f32e7eSjoerg 77906f32e7eSjoerg try: 78006f32e7eSjoerg procs.append(subprocess.Popen(args, cwd=cmd_shenv.cwd, 78106f32e7eSjoerg executable = executable, 78206f32e7eSjoerg stdin = stdin, 78306f32e7eSjoerg stdout = stdout, 78406f32e7eSjoerg stderr = stderr, 78506f32e7eSjoerg env = cmd_shenv.env, 78606f32e7eSjoerg close_fds = kUseCloseFDs)) 787*da58b97aSjoerg proc_not_counts.append(not_count) 78806f32e7eSjoerg # Let the helper know about this process 78906f32e7eSjoerg timeoutHelper.addProcess(procs[-1]) 79006f32e7eSjoerg except OSError as e: 79106f32e7eSjoerg raise InternalShellError(j, 'Could not create process ({}) due to {}'.format(executable, e)) 79206f32e7eSjoerg 79306f32e7eSjoerg # Immediately close stdin for any process taking stdin from us. 79406f32e7eSjoerg if stdin == subprocess.PIPE: 79506f32e7eSjoerg procs[-1].stdin.close() 79606f32e7eSjoerg procs[-1].stdin = None 79706f32e7eSjoerg 79806f32e7eSjoerg # Update the current stdin source. 79906f32e7eSjoerg if stdout == subprocess.PIPE: 80006f32e7eSjoerg default_stdin = procs[-1].stdout 80106f32e7eSjoerg elif stderrIsStdout: 80206f32e7eSjoerg default_stdin = procs[-1].stderr 80306f32e7eSjoerg else: 80406f32e7eSjoerg default_stdin = subprocess.PIPE 80506f32e7eSjoerg 80606f32e7eSjoerg # Explicitly close any redirected files. We need to do this now because we 80706f32e7eSjoerg # need to release any handles we may have on the temporary files (important 80806f32e7eSjoerg # on Win32, for example). Since we have already spawned the subprocess, our 80906f32e7eSjoerg # handles have already been transferred so we do not need them anymore. 81006f32e7eSjoerg for (name, mode, f, path) in opened_files: 81106f32e7eSjoerg f.close() 81206f32e7eSjoerg 81306f32e7eSjoerg # FIXME: There is probably still deadlock potential here. Yawn. 81406f32e7eSjoerg procData = [None] * len(procs) 81506f32e7eSjoerg procData[-1] = procs[-1].communicate() 81606f32e7eSjoerg 81706f32e7eSjoerg for i in range(len(procs) - 1): 81806f32e7eSjoerg if procs[i].stdout is not None: 81906f32e7eSjoerg out = procs[i].stdout.read() 82006f32e7eSjoerg else: 82106f32e7eSjoerg out = '' 82206f32e7eSjoerg if procs[i].stderr is not None: 82306f32e7eSjoerg err = procs[i].stderr.read() 82406f32e7eSjoerg else: 82506f32e7eSjoerg err = '' 82606f32e7eSjoerg procData[i] = (out,err) 82706f32e7eSjoerg 82806f32e7eSjoerg # Read stderr out of the temp files. 82906f32e7eSjoerg for i,f in stderrTempFiles: 83006f32e7eSjoerg f.seek(0, 0) 83106f32e7eSjoerg procData[i] = (procData[i][0], f.read()) 83206f32e7eSjoerg f.close() 83306f32e7eSjoerg 83406f32e7eSjoerg exitCode = None 83506f32e7eSjoerg for i,(out,err) in enumerate(procData): 83606f32e7eSjoerg res = procs[i].wait() 83706f32e7eSjoerg # Detect Ctrl-C in subprocess. 83806f32e7eSjoerg if res == -signal.SIGINT: 83906f32e7eSjoerg raise KeyboardInterrupt 840*da58b97aSjoerg if proc_not_counts[i] % 2: 841*da58b97aSjoerg res = not res 842*da58b97aSjoerg elif proc_not_counts[i] > 1: 843*da58b97aSjoerg res = 1 if res != 0 else 0 84406f32e7eSjoerg 84506f32e7eSjoerg # Ensure the resulting output is always of string type. 84606f32e7eSjoerg try: 84706f32e7eSjoerg if out is None: 84806f32e7eSjoerg out = '' 84906f32e7eSjoerg else: 85006f32e7eSjoerg out = to_string(out.decode('utf-8', errors='replace')) 85106f32e7eSjoerg except: 85206f32e7eSjoerg out = str(out) 85306f32e7eSjoerg try: 85406f32e7eSjoerg if err is None: 85506f32e7eSjoerg err = '' 85606f32e7eSjoerg else: 85706f32e7eSjoerg err = to_string(err.decode('utf-8', errors='replace')) 85806f32e7eSjoerg except: 85906f32e7eSjoerg err = str(err) 86006f32e7eSjoerg 86106f32e7eSjoerg # Gather the redirected output files for failed commands. 86206f32e7eSjoerg output_files = [] 86306f32e7eSjoerg if res != 0: 86406f32e7eSjoerg for (name, mode, f, path) in sorted(opened_files): 86506f32e7eSjoerg if path is not None and mode in ('w', 'a'): 86606f32e7eSjoerg try: 86706f32e7eSjoerg with open(path, 'rb') as f: 86806f32e7eSjoerg data = f.read() 86906f32e7eSjoerg except: 87006f32e7eSjoerg data = None 87106f32e7eSjoerg if data is not None: 87206f32e7eSjoerg output_files.append((name, path, data)) 87306f32e7eSjoerg 87406f32e7eSjoerg results.append(ShellCommandResult( 87506f32e7eSjoerg cmd.commands[i], out, err, res, timeoutHelper.timeoutReached(), 87606f32e7eSjoerg output_files)) 87706f32e7eSjoerg if cmd.pipe_err: 87806f32e7eSjoerg # Take the last failing exit code from the pipeline. 87906f32e7eSjoerg if not exitCode or res != 0: 88006f32e7eSjoerg exitCode = res 88106f32e7eSjoerg else: 88206f32e7eSjoerg exitCode = res 88306f32e7eSjoerg 88406f32e7eSjoerg # Remove any named temporary files we created. 88506f32e7eSjoerg for f in named_temp_files: 88606f32e7eSjoerg try: 88706f32e7eSjoerg os.remove(f) 88806f32e7eSjoerg except OSError: 88906f32e7eSjoerg pass 89006f32e7eSjoerg 89106f32e7eSjoerg if cmd.negate: 89206f32e7eSjoerg exitCode = not exitCode 89306f32e7eSjoerg 89406f32e7eSjoerg return exitCode 89506f32e7eSjoerg 89606f32e7eSjoergdef executeScriptInternal(test, litConfig, tmpBase, commands, cwd): 89706f32e7eSjoerg cmds = [] 89806f32e7eSjoerg for i, ln in enumerate(commands): 89906f32e7eSjoerg ln = commands[i] = re.sub(kPdbgRegex, ": '\\1'; ", ln) 90006f32e7eSjoerg try: 90106f32e7eSjoerg cmds.append(ShUtil.ShParser(ln, litConfig.isWindows, 90206f32e7eSjoerg test.config.pipefail).parse()) 90306f32e7eSjoerg except: 90406f32e7eSjoerg return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln) 90506f32e7eSjoerg 90606f32e7eSjoerg cmd = cmds[0] 90706f32e7eSjoerg for c in cmds[1:]: 90806f32e7eSjoerg cmd = ShUtil.Seq(cmd, '&&', c) 90906f32e7eSjoerg 91006f32e7eSjoerg results = [] 91106f32e7eSjoerg timeoutInfo = None 91206f32e7eSjoerg try: 91306f32e7eSjoerg shenv = ShellEnvironment(cwd, test.config.environment) 91406f32e7eSjoerg exitCode, timeoutInfo = executeShCmd(cmd, shenv, results, timeout=litConfig.maxIndividualTestTime) 91506f32e7eSjoerg except InternalShellError: 91606f32e7eSjoerg e = sys.exc_info()[1] 91706f32e7eSjoerg exitCode = 127 91806f32e7eSjoerg results.append( 91906f32e7eSjoerg ShellCommandResult(e.command, '', e.message, exitCode, False)) 92006f32e7eSjoerg 92106f32e7eSjoerg out = err = '' 92206f32e7eSjoerg for i,result in enumerate(results): 92306f32e7eSjoerg # Write the command line run. 92406f32e7eSjoerg out += '$ %s\n' % (' '.join('"%s"' % s 92506f32e7eSjoerg for s in result.command.args),) 92606f32e7eSjoerg 92706f32e7eSjoerg # If nothing interesting happened, move on. 92806f32e7eSjoerg if litConfig.maxIndividualTestTime == 0 and \ 92906f32e7eSjoerg result.exitCode == 0 and \ 93006f32e7eSjoerg not result.stdout.strip() and not result.stderr.strip(): 93106f32e7eSjoerg continue 93206f32e7eSjoerg 93306f32e7eSjoerg # Otherwise, something failed or was printed, show it. 93406f32e7eSjoerg 93506f32e7eSjoerg # Add the command output, if redirected. 93606f32e7eSjoerg for (name, path, data) in result.outputFiles: 93706f32e7eSjoerg if data.strip(): 93806f32e7eSjoerg out += "# redirected output from %r:\n" % (name,) 93906f32e7eSjoerg data = to_string(data.decode('utf-8', errors='replace')) 94006f32e7eSjoerg if len(data) > 1024: 94106f32e7eSjoerg out += data[:1024] + "\n...\n" 94206f32e7eSjoerg out += "note: data was truncated\n" 94306f32e7eSjoerg else: 94406f32e7eSjoerg out += data 94506f32e7eSjoerg out += "\n" 94606f32e7eSjoerg 94706f32e7eSjoerg if result.stdout.strip(): 94806f32e7eSjoerg out += '# command output:\n%s\n' % (result.stdout,) 94906f32e7eSjoerg if result.stderr.strip(): 95006f32e7eSjoerg out += '# command stderr:\n%s\n' % (result.stderr,) 95106f32e7eSjoerg if not result.stdout.strip() and not result.stderr.strip(): 95206f32e7eSjoerg out += "note: command had no output on stdout or stderr\n" 95306f32e7eSjoerg 95406f32e7eSjoerg # Show the error conditions: 95506f32e7eSjoerg if result.exitCode != 0: 95606f32e7eSjoerg # On Windows, a negative exit code indicates a signal, and those are 95706f32e7eSjoerg # easier to recognize or look up if we print them in hex. 95806f32e7eSjoerg if litConfig.isWindows and result.exitCode < 0: 95906f32e7eSjoerg codeStr = hex(int(result.exitCode & 0xFFFFFFFF)).rstrip("L") 96006f32e7eSjoerg else: 96106f32e7eSjoerg codeStr = str(result.exitCode) 96206f32e7eSjoerg out += "error: command failed with exit status: %s\n" % ( 96306f32e7eSjoerg codeStr,) 96406f32e7eSjoerg if litConfig.maxIndividualTestTime > 0 and result.timeoutReached: 96506f32e7eSjoerg out += 'error: command reached timeout: %s\n' % ( 96606f32e7eSjoerg str(result.timeoutReached),) 96706f32e7eSjoerg 96806f32e7eSjoerg return out, err, exitCode, timeoutInfo 96906f32e7eSjoerg 97006f32e7eSjoergdef executeScript(test, litConfig, tmpBase, commands, cwd): 97106f32e7eSjoerg bashPath = litConfig.getBashPath() 97206f32e7eSjoerg isWin32CMDEXE = (litConfig.isWindows and not bashPath) 97306f32e7eSjoerg script = tmpBase + '.script' 97406f32e7eSjoerg if isWin32CMDEXE: 97506f32e7eSjoerg script += '.bat' 97606f32e7eSjoerg 97706f32e7eSjoerg # Write script file 97806f32e7eSjoerg mode = 'w' 97906f32e7eSjoerg open_kwargs = {} 98006f32e7eSjoerg if litConfig.isWindows and not isWin32CMDEXE: 98106f32e7eSjoerg mode += 'b' # Avoid CRLFs when writing bash scripts. 98206f32e7eSjoerg elif sys.version_info > (3,0): 98306f32e7eSjoerg open_kwargs['encoding'] = 'utf-8' 98406f32e7eSjoerg f = open(script, mode, **open_kwargs) 98506f32e7eSjoerg if isWin32CMDEXE: 98606f32e7eSjoerg for i, ln in enumerate(commands): 98706f32e7eSjoerg commands[i] = re.sub(kPdbgRegex, "echo '\\1' > nul && ", ln) 98806f32e7eSjoerg if litConfig.echo_all_commands: 98906f32e7eSjoerg f.write('@echo on\n') 99006f32e7eSjoerg else: 99106f32e7eSjoerg f.write('@echo off\n') 99206f32e7eSjoerg f.write('\n@if %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) 99306f32e7eSjoerg else: 99406f32e7eSjoerg for i, ln in enumerate(commands): 99506f32e7eSjoerg commands[i] = re.sub(kPdbgRegex, ": '\\1'; ", ln) 99606f32e7eSjoerg if test.config.pipefail: 99706f32e7eSjoerg f.write(b'set -o pipefail;' if mode == 'wb' else 'set -o pipefail;') 99806f32e7eSjoerg if litConfig.echo_all_commands: 99906f32e7eSjoerg f.write(b'set -x;' if mode == 'wb' else 'set -x;') 100006f32e7eSjoerg if sys.version_info > (3,0) and mode == 'wb': 100106f32e7eSjoerg f.write(bytes('{ ' + '; } &&\n{ '.join(commands) + '; }', 'utf-8')) 100206f32e7eSjoerg else: 100306f32e7eSjoerg f.write('{ ' + '; } &&\n{ '.join(commands) + '; }') 100406f32e7eSjoerg f.write(b'\n' if mode == 'wb' else '\n') 100506f32e7eSjoerg f.close() 100606f32e7eSjoerg 100706f32e7eSjoerg if isWin32CMDEXE: 100806f32e7eSjoerg command = ['cmd','/c', script] 100906f32e7eSjoerg else: 101006f32e7eSjoerg if bashPath: 101106f32e7eSjoerg command = [bashPath, script] 101206f32e7eSjoerg else: 101306f32e7eSjoerg command = ['/bin/sh', script] 101406f32e7eSjoerg if litConfig.useValgrind: 101506f32e7eSjoerg # FIXME: Running valgrind on sh is overkill. We probably could just 101606f32e7eSjoerg # run on clang with no real loss. 101706f32e7eSjoerg command = litConfig.valgrindArgs + command 101806f32e7eSjoerg 101906f32e7eSjoerg try: 102006f32e7eSjoerg out, err, exitCode = lit.util.executeCommand(command, cwd=cwd, 102106f32e7eSjoerg env=test.config.environment, 102206f32e7eSjoerg timeout=litConfig.maxIndividualTestTime) 102306f32e7eSjoerg return (out, err, exitCode, None) 102406f32e7eSjoerg except lit.util.ExecuteCommandTimeoutException as e: 102506f32e7eSjoerg return (e.out, e.err, e.exitCode, e.msg) 102606f32e7eSjoerg 102706f32e7eSjoergdef parseIntegratedTestScriptCommands(source_path, keywords): 102806f32e7eSjoerg """ 102906f32e7eSjoerg parseIntegratedTestScriptCommands(source_path) -> commands 103006f32e7eSjoerg 103106f32e7eSjoerg Parse the commands in an integrated test script file into a list of 103206f32e7eSjoerg (line_number, command_type, line). 103306f32e7eSjoerg """ 103406f32e7eSjoerg 103506f32e7eSjoerg # This code is carefully written to be dual compatible with Python 2.5+ and 103606f32e7eSjoerg # Python 3 without requiring input files to always have valid codings. The 103706f32e7eSjoerg # trick we use is to open the file in binary mode and use the regular 103806f32e7eSjoerg # expression library to find the commands, with it scanning strings in 103906f32e7eSjoerg # Python2 and bytes in Python3. 104006f32e7eSjoerg # 104106f32e7eSjoerg # Once we find a match, we do require each script line to be decodable to 104206f32e7eSjoerg # UTF-8, so we convert the outputs to UTF-8 before returning. This way the 104306f32e7eSjoerg # remaining code can work with "strings" agnostic of the executing Python 104406f32e7eSjoerg # version. 104506f32e7eSjoerg 104606f32e7eSjoerg keywords_re = re.compile( 104706f32e7eSjoerg to_bytes("(%s)(.*)\n" % ("|".join(re.escape(k) for k in keywords),))) 104806f32e7eSjoerg 104906f32e7eSjoerg f = open(source_path, 'rb') 105006f32e7eSjoerg try: 105106f32e7eSjoerg # Read the entire file contents. 105206f32e7eSjoerg data = f.read() 105306f32e7eSjoerg 105406f32e7eSjoerg # Ensure the data ends with a newline. 105506f32e7eSjoerg if not data.endswith(to_bytes('\n')): 105606f32e7eSjoerg data = data + to_bytes('\n') 105706f32e7eSjoerg 105806f32e7eSjoerg # Iterate over the matches. 105906f32e7eSjoerg line_number = 1 106006f32e7eSjoerg last_match_position = 0 106106f32e7eSjoerg for match in keywords_re.finditer(data): 106206f32e7eSjoerg # Compute the updated line number by counting the intervening 106306f32e7eSjoerg # newlines. 106406f32e7eSjoerg match_position = match.start() 106506f32e7eSjoerg line_number += data.count(to_bytes('\n'), last_match_position, 106606f32e7eSjoerg match_position) 106706f32e7eSjoerg last_match_position = match_position 106806f32e7eSjoerg 106906f32e7eSjoerg # Convert the keyword and line to UTF-8 strings and yield the 107006f32e7eSjoerg # command. Note that we take care to return regular strings in 107106f32e7eSjoerg # Python 2, to avoid other code having to differentiate between the 107206f32e7eSjoerg # str and unicode types. 107306f32e7eSjoerg # 107406f32e7eSjoerg # Opening the file in binary mode prevented Windows \r newline 107506f32e7eSjoerg # characters from being converted to Unix \n newlines, so manually 107606f32e7eSjoerg # strip those from the yielded lines. 107706f32e7eSjoerg keyword,ln = match.groups() 107806f32e7eSjoerg yield (line_number, to_string(keyword.decode('utf-8')), 107906f32e7eSjoerg to_string(ln.decode('utf-8').rstrip('\r'))) 108006f32e7eSjoerg finally: 108106f32e7eSjoerg f.close() 108206f32e7eSjoerg 108306f32e7eSjoergdef getTempPaths(test): 108406f32e7eSjoerg """Get the temporary location, this is always relative to the test suite 108506f32e7eSjoerg root, not test source root.""" 108606f32e7eSjoerg execpath = test.getExecPath() 108706f32e7eSjoerg execdir,execbase = os.path.split(execpath) 108806f32e7eSjoerg tmpDir = os.path.join(execdir, 'Output') 108906f32e7eSjoerg tmpBase = os.path.join(tmpDir, execbase) 109006f32e7eSjoerg return tmpDir, tmpBase 109106f32e7eSjoerg 109206f32e7eSjoergdef colonNormalizePath(path): 109306f32e7eSjoerg if kIsWindows: 109406f32e7eSjoerg return re.sub(r'^(.):', r'\1', path.replace('\\', '/')) 109506f32e7eSjoerg else: 109606f32e7eSjoerg assert path[0] == '/' 109706f32e7eSjoerg return path[1:] 109806f32e7eSjoerg 109906f32e7eSjoergdef getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False): 110006f32e7eSjoerg sourcepath = test.getSourcePath() 110106f32e7eSjoerg sourcedir = os.path.dirname(sourcepath) 110206f32e7eSjoerg 110306f32e7eSjoerg # Normalize slashes, if requested. 110406f32e7eSjoerg if normalize_slashes: 110506f32e7eSjoerg sourcepath = sourcepath.replace('\\', '/') 110606f32e7eSjoerg sourcedir = sourcedir.replace('\\', '/') 110706f32e7eSjoerg tmpDir = tmpDir.replace('\\', '/') 110806f32e7eSjoerg tmpBase = tmpBase.replace('\\', '/') 110906f32e7eSjoerg 111006f32e7eSjoerg substitutions = [] 111106f32e7eSjoerg substitutions.extend(test.config.substitutions) 111206f32e7eSjoerg tmpName = tmpBase + '.tmp' 111306f32e7eSjoerg baseName = os.path.basename(tmpBase) 111406f32e7eSjoerg substitutions.extend([('%s', sourcepath), 111506f32e7eSjoerg ('%S', sourcedir), 111606f32e7eSjoerg ('%p', sourcedir), 111706f32e7eSjoerg ('%{pathsep}', os.pathsep), 111806f32e7eSjoerg ('%t', tmpName), 111906f32e7eSjoerg ('%basename_t', baseName), 1120*da58b97aSjoerg ('%T', tmpDir)]) 112106f32e7eSjoerg 112206f32e7eSjoerg # "%/[STpst]" should be normalized. 112306f32e7eSjoerg substitutions.extend([ 112406f32e7eSjoerg ('%/s', sourcepath.replace('\\', '/')), 112506f32e7eSjoerg ('%/S', sourcedir.replace('\\', '/')), 112606f32e7eSjoerg ('%/p', sourcedir.replace('\\', '/')), 112706f32e7eSjoerg ('%/t', tmpBase.replace('\\', '/') + '.tmp'), 112806f32e7eSjoerg ('%/T', tmpDir.replace('\\', '/')), 112906f32e7eSjoerg ]) 113006f32e7eSjoerg 1131*da58b97aSjoerg # "%{/[STpst]:regex_replacement}" should be normalized like "%/[STpst]" but we're 1132*da58b97aSjoerg # also in a regex replacement context of a s@@@ regex. 1133*da58b97aSjoerg def regex_escape(s): 1134*da58b97aSjoerg s = s.replace('@', r'\@') 1135*da58b97aSjoerg s = s.replace('&', r'\&') 1136*da58b97aSjoerg return s 1137*da58b97aSjoerg substitutions.extend([ 1138*da58b97aSjoerg ('%{/s:regex_replacement}', 1139*da58b97aSjoerg regex_escape(sourcepath.replace('\\', '/'))), 1140*da58b97aSjoerg ('%{/S:regex_replacement}', 1141*da58b97aSjoerg regex_escape(sourcedir.replace('\\', '/'))), 1142*da58b97aSjoerg ('%{/p:regex_replacement}', 1143*da58b97aSjoerg regex_escape(sourcedir.replace('\\', '/'))), 1144*da58b97aSjoerg ('%{/t:regex_replacement}', 1145*da58b97aSjoerg regex_escape(tmpBase.replace('\\', '/')) + '.tmp'), 1146*da58b97aSjoerg ('%{/T:regex_replacement}', 1147*da58b97aSjoerg regex_escape(tmpDir.replace('\\', '/'))), 1148*da58b97aSjoerg ]) 1149*da58b97aSjoerg 115006f32e7eSjoerg # "%:[STpst]" are normalized paths without colons and without a leading 115106f32e7eSjoerg # slash. 115206f32e7eSjoerg substitutions.extend([ 115306f32e7eSjoerg ('%:s', colonNormalizePath(sourcepath)), 115406f32e7eSjoerg ('%:S', colonNormalizePath(sourcedir)), 115506f32e7eSjoerg ('%:p', colonNormalizePath(sourcedir)), 115606f32e7eSjoerg ('%:t', colonNormalizePath(tmpBase + '.tmp')), 115706f32e7eSjoerg ('%:T', colonNormalizePath(tmpDir)), 115806f32e7eSjoerg ]) 115906f32e7eSjoerg return substitutions 116006f32e7eSjoerg 1161*da58b97aSjoergdef _memoize(f): 1162*da58b97aSjoerg cache = {} # Intentionally unbounded, see applySubstitutions() 1163*da58b97aSjoerg def memoized(x): 1164*da58b97aSjoerg if x not in cache: 1165*da58b97aSjoerg cache[x] = f(x) 1166*da58b97aSjoerg return cache[x] 1167*da58b97aSjoerg return memoized 1168*da58b97aSjoerg 1169*da58b97aSjoerg@_memoize 1170*da58b97aSjoergdef _caching_re_compile(r): 1171*da58b97aSjoerg return re.compile(r) 1172*da58b97aSjoerg 1173*da58b97aSjoergdef applySubstitutions(script, substitutions, recursion_limit=None): 1174*da58b97aSjoerg """ 1175*da58b97aSjoerg Apply substitutions to the script. Allow full regular expression syntax. 117606f32e7eSjoerg Replace each matching occurrence of regular expression pattern a with 1177*da58b97aSjoerg substitution b in line ln. 1178*da58b97aSjoerg 1179*da58b97aSjoerg If a substitution expands into another substitution, it is expanded 1180*da58b97aSjoerg recursively until the line has no more expandable substitutions. If 1181*da58b97aSjoerg the line can still can be substituted after being substituted 1182*da58b97aSjoerg `recursion_limit` times, it is an error. If the `recursion_limit` is 1183*da58b97aSjoerg `None` (the default), no recursive substitution is performed at all. 1184*da58b97aSjoerg """ 1185*da58b97aSjoerg 1186*da58b97aSjoerg # We use #_MARKER_# to hide %% while we do the other substitutions. 1187*da58b97aSjoerg def escape(ln): 1188*da58b97aSjoerg return _caching_re_compile('%%').sub('#_MARKER_#', ln) 1189*da58b97aSjoerg 1190*da58b97aSjoerg def unescape(ln): 1191*da58b97aSjoerg return _caching_re_compile('#_MARKER_#').sub('%', ln) 1192*da58b97aSjoerg 119306f32e7eSjoerg def processLine(ln): 119406f32e7eSjoerg # Apply substitutions 119506f32e7eSjoerg for a,b in substitutions: 119606f32e7eSjoerg if kIsWindows: 119706f32e7eSjoerg b = b.replace("\\","\\\\") 1198*da58b97aSjoerg # re.compile() has a built-in LRU cache with 512 entries. In some 1199*da58b97aSjoerg # test suites lit ends up thrashing that cache, which made e.g. 1200*da58b97aSjoerg # check-llvm run 50% slower. Use an explicit, unbounded cache 1201*da58b97aSjoerg # to prevent that from happening. Since lit is fairly 1202*da58b97aSjoerg # short-lived, since the set of substitutions is fairly small, and 1203*da58b97aSjoerg # since thrashing has such bad consequences, not bounding the cache 1204*da58b97aSjoerg # seems reasonable. 1205*da58b97aSjoerg ln = _caching_re_compile(a).sub(str(b), escape(ln)) 120606f32e7eSjoerg 120706f32e7eSjoerg # Strip the trailing newline and any extra whitespace. 120806f32e7eSjoerg return ln.strip() 1209*da58b97aSjoerg 1210*da58b97aSjoerg def processLineToFixedPoint(ln): 1211*da58b97aSjoerg assert isinstance(recursion_limit, int) and recursion_limit >= 0 1212*da58b97aSjoerg origLine = ln 1213*da58b97aSjoerg steps = 0 1214*da58b97aSjoerg processed = processLine(ln) 1215*da58b97aSjoerg while processed != ln and steps < recursion_limit: 1216*da58b97aSjoerg ln = processed 1217*da58b97aSjoerg processed = processLine(ln) 1218*da58b97aSjoerg steps += 1 1219*da58b97aSjoerg 1220*da58b97aSjoerg if processed != ln: 1221*da58b97aSjoerg raise ValueError("Recursive substitution of '%s' did not complete " 1222*da58b97aSjoerg "in the provided recursion limit (%s)" % \ 1223*da58b97aSjoerg (origLine, recursion_limit)) 1224*da58b97aSjoerg 1225*da58b97aSjoerg return processed 1226*da58b97aSjoerg 1227*da58b97aSjoerg process = processLine if recursion_limit is None else processLineToFixedPoint 1228*da58b97aSjoerg 1229*da58b97aSjoerg return [unescape(process(ln)) for ln in script] 123006f32e7eSjoerg 123106f32e7eSjoerg 123206f32e7eSjoergclass ParserKind(object): 123306f32e7eSjoerg """ 123406f32e7eSjoerg An enumeration representing the style of an integrated test keyword or 123506f32e7eSjoerg command. 123606f32e7eSjoerg 123706f32e7eSjoerg TAG: A keyword taking no value. Ex 'END.' 123806f32e7eSjoerg COMMAND: A keyword taking a list of shell commands. Ex 'RUN:' 123906f32e7eSjoerg LIST: A keyword taking a comma-separated list of values. 124006f32e7eSjoerg BOOLEAN_EXPR: A keyword taking a comma-separated list of 124106f32e7eSjoerg boolean expressions. Ex 'XFAIL:' 1242*da58b97aSjoerg INTEGER: A keyword taking a single integer. Ex 'ALLOW_RETRIES:' 124306f32e7eSjoerg CUSTOM: A keyword with custom parsing semantics. 124406f32e7eSjoerg """ 124506f32e7eSjoerg TAG = 0 124606f32e7eSjoerg COMMAND = 1 124706f32e7eSjoerg LIST = 2 124806f32e7eSjoerg BOOLEAN_EXPR = 3 1249*da58b97aSjoerg INTEGER = 4 1250*da58b97aSjoerg CUSTOM = 5 125106f32e7eSjoerg 125206f32e7eSjoerg @staticmethod 125306f32e7eSjoerg def allowedKeywordSuffixes(value): 125406f32e7eSjoerg return { ParserKind.TAG: ['.'], 125506f32e7eSjoerg ParserKind.COMMAND: [':'], 125606f32e7eSjoerg ParserKind.LIST: [':'], 125706f32e7eSjoerg ParserKind.BOOLEAN_EXPR: [':'], 1258*da58b97aSjoerg ParserKind.INTEGER: [':'], 125906f32e7eSjoerg ParserKind.CUSTOM: [':', '.'] 126006f32e7eSjoerg } [value] 126106f32e7eSjoerg 126206f32e7eSjoerg @staticmethod 126306f32e7eSjoerg def str(value): 126406f32e7eSjoerg return { ParserKind.TAG: 'TAG', 126506f32e7eSjoerg ParserKind.COMMAND: 'COMMAND', 126606f32e7eSjoerg ParserKind.LIST: 'LIST', 126706f32e7eSjoerg ParserKind.BOOLEAN_EXPR: 'BOOLEAN_EXPR', 1268*da58b97aSjoerg ParserKind.INTEGER: 'INTEGER', 126906f32e7eSjoerg ParserKind.CUSTOM: 'CUSTOM' 127006f32e7eSjoerg } [value] 127106f32e7eSjoerg 127206f32e7eSjoerg 127306f32e7eSjoergclass IntegratedTestKeywordParser(object): 127406f32e7eSjoerg """A parser for LLVM/Clang style integrated test scripts. 127506f32e7eSjoerg 127606f32e7eSjoerg keyword: The keyword to parse for. It must end in either '.' or ':'. 127706f32e7eSjoerg kind: An value of ParserKind. 127806f32e7eSjoerg parser: A custom parser. This value may only be specified with 127906f32e7eSjoerg ParserKind.CUSTOM. 128006f32e7eSjoerg """ 128106f32e7eSjoerg def __init__(self, keyword, kind, parser=None, initial_value=None): 128206f32e7eSjoerg allowedSuffixes = ParserKind.allowedKeywordSuffixes(kind) 128306f32e7eSjoerg if len(keyword) == 0 or keyword[-1] not in allowedSuffixes: 128406f32e7eSjoerg if len(allowedSuffixes) == 1: 128506f32e7eSjoerg raise ValueError("Keyword '%s' of kind '%s' must end in '%s'" 128606f32e7eSjoerg % (keyword, ParserKind.str(kind), 128706f32e7eSjoerg allowedSuffixes[0])) 128806f32e7eSjoerg else: 128906f32e7eSjoerg raise ValueError("Keyword '%s' of kind '%s' must end in " 129006f32e7eSjoerg " one of '%s'" 129106f32e7eSjoerg % (keyword, ParserKind.str(kind), 129206f32e7eSjoerg ' '.join(allowedSuffixes))) 129306f32e7eSjoerg 129406f32e7eSjoerg if parser is not None and kind != ParserKind.CUSTOM: 129506f32e7eSjoerg raise ValueError("custom parsers can only be specified with " 129606f32e7eSjoerg "ParserKind.CUSTOM") 129706f32e7eSjoerg self.keyword = keyword 129806f32e7eSjoerg self.kind = kind 129906f32e7eSjoerg self.parsed_lines = [] 130006f32e7eSjoerg self.value = initial_value 130106f32e7eSjoerg self.parser = parser 130206f32e7eSjoerg 130306f32e7eSjoerg if kind == ParserKind.COMMAND: 130406f32e7eSjoerg self.parser = lambda line_number, line, output: \ 130506f32e7eSjoerg self._handleCommand(line_number, line, output, 130606f32e7eSjoerg self.keyword) 130706f32e7eSjoerg elif kind == ParserKind.LIST: 130806f32e7eSjoerg self.parser = self._handleList 130906f32e7eSjoerg elif kind == ParserKind.BOOLEAN_EXPR: 131006f32e7eSjoerg self.parser = self._handleBooleanExpr 1311*da58b97aSjoerg elif kind == ParserKind.INTEGER: 1312*da58b97aSjoerg self.parser = self._handleSingleInteger 131306f32e7eSjoerg elif kind == ParserKind.TAG: 131406f32e7eSjoerg self.parser = self._handleTag 131506f32e7eSjoerg elif kind == ParserKind.CUSTOM: 131606f32e7eSjoerg if parser is None: 131706f32e7eSjoerg raise ValueError("ParserKind.CUSTOM requires a custom parser") 131806f32e7eSjoerg self.parser = parser 131906f32e7eSjoerg else: 132006f32e7eSjoerg raise ValueError("Unknown kind '%s'" % kind) 132106f32e7eSjoerg 132206f32e7eSjoerg def parseLine(self, line_number, line): 132306f32e7eSjoerg try: 132406f32e7eSjoerg self.parsed_lines += [(line_number, line)] 132506f32e7eSjoerg self.value = self.parser(line_number, line, self.value) 132606f32e7eSjoerg except ValueError as e: 132706f32e7eSjoerg raise ValueError(str(e) + ("\nin %s directive on test line %d" % 132806f32e7eSjoerg (self.keyword, line_number))) 132906f32e7eSjoerg 133006f32e7eSjoerg def getValue(self): 133106f32e7eSjoerg return self.value 133206f32e7eSjoerg 133306f32e7eSjoerg @staticmethod 133406f32e7eSjoerg def _handleTag(line_number, line, output): 133506f32e7eSjoerg """A helper for parsing TAG type keywords""" 133606f32e7eSjoerg return (not line.strip() or output) 133706f32e7eSjoerg 133806f32e7eSjoerg @staticmethod 133906f32e7eSjoerg def _handleCommand(line_number, line, output, keyword): 134006f32e7eSjoerg """A helper for parsing COMMAND type keywords""" 134106f32e7eSjoerg # Trim trailing whitespace. 134206f32e7eSjoerg line = line.rstrip() 134306f32e7eSjoerg # Substitute line number expressions 134406f32e7eSjoerg line = re.sub(r'%\(line\)', str(line_number), line) 134506f32e7eSjoerg 134606f32e7eSjoerg def replace_line_number(match): 134706f32e7eSjoerg if match.group(1) == '+': 134806f32e7eSjoerg return str(line_number + int(match.group(2))) 134906f32e7eSjoerg if match.group(1) == '-': 135006f32e7eSjoerg return str(line_number - int(match.group(2))) 135106f32e7eSjoerg line = re.sub(r'%\(line *([\+-]) *(\d+)\)', replace_line_number, line) 135206f32e7eSjoerg # Collapse lines with trailing '\\'. 135306f32e7eSjoerg if output and output[-1][-1] == '\\': 135406f32e7eSjoerg output[-1] = output[-1][:-1] + line 135506f32e7eSjoerg else: 135606f32e7eSjoerg if output is None: 135706f32e7eSjoerg output = [] 135806f32e7eSjoerg pdbg = "%dbg({keyword} at line {line_number})".format( 135906f32e7eSjoerg keyword=keyword, 136006f32e7eSjoerg line_number=line_number) 136106f32e7eSjoerg assert re.match(kPdbgRegex + "$", pdbg), \ 136206f32e7eSjoerg "kPdbgRegex expected to match actual %dbg usage" 136306f32e7eSjoerg line = "{pdbg} {real_command}".format( 136406f32e7eSjoerg pdbg=pdbg, 136506f32e7eSjoerg real_command=line) 136606f32e7eSjoerg output.append(line) 136706f32e7eSjoerg return output 136806f32e7eSjoerg 136906f32e7eSjoerg @staticmethod 137006f32e7eSjoerg def _handleList(line_number, line, output): 137106f32e7eSjoerg """A parser for LIST type keywords""" 137206f32e7eSjoerg if output is None: 137306f32e7eSjoerg output = [] 137406f32e7eSjoerg output.extend([s.strip() for s in line.split(',')]) 137506f32e7eSjoerg return output 137606f32e7eSjoerg 137706f32e7eSjoerg @staticmethod 1378*da58b97aSjoerg def _handleSingleInteger(line_number, line, output): 1379*da58b97aSjoerg """A parser for INTEGER type keywords""" 1380*da58b97aSjoerg if output is None: 1381*da58b97aSjoerg output = [] 1382*da58b97aSjoerg try: 1383*da58b97aSjoerg n = int(line) 1384*da58b97aSjoerg except ValueError: 1385*da58b97aSjoerg raise ValueError("INTEGER parser requires the input to be an integer (got {})".format(line)) 1386*da58b97aSjoerg output.append(n) 1387*da58b97aSjoerg return output 1388*da58b97aSjoerg 1389*da58b97aSjoerg @staticmethod 139006f32e7eSjoerg def _handleBooleanExpr(line_number, line, output): 139106f32e7eSjoerg """A parser for BOOLEAN_EXPR type keywords""" 139206f32e7eSjoerg parts = [s.strip() for s in line.split(',') if s.strip() != ''] 139306f32e7eSjoerg if output and output[-1][-1] == '\\': 139406f32e7eSjoerg output[-1] = output[-1][:-1] + parts[0] 139506f32e7eSjoerg del parts[0] 139606f32e7eSjoerg if output is None: 139706f32e7eSjoerg output = [] 139806f32e7eSjoerg output.extend(parts) 139906f32e7eSjoerg # Evaluate each expression to verify syntax. 140006f32e7eSjoerg # We don't want any results, just the raised ValueError. 140106f32e7eSjoerg for s in output: 140206f32e7eSjoerg if s != '*' and not s.endswith('\\'): 140306f32e7eSjoerg BooleanExpression.evaluate(s, []) 140406f32e7eSjoerg return output 140506f32e7eSjoerg 140606f32e7eSjoerg 1407*da58b97aSjoergdef _parseKeywords(sourcepath, additional_parsers=[], 140806f32e7eSjoerg require_script=True): 1409*da58b97aSjoerg """_parseKeywords 141006f32e7eSjoerg 1411*da58b97aSjoerg Scan an LLVM/Clang style integrated test script and extract all the lines 1412*da58b97aSjoerg pertaining to a special parser. This includes 'RUN', 'XFAIL', 'REQUIRES', 1413*da58b97aSjoerg 'UNSUPPORTED' and 'ALLOW_RETRIES', as well as other specified custom 1414*da58b97aSjoerg parsers. 141506f32e7eSjoerg 1416*da58b97aSjoerg Returns a dictionary mapping each custom parser to its value after 1417*da58b97aSjoerg parsing the test. 141806f32e7eSjoerg """ 141906f32e7eSjoerg # Install the built-in keyword parsers. 142006f32e7eSjoerg script = [] 142106f32e7eSjoerg builtin_parsers = [ 1422*da58b97aSjoerg IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND, initial_value=script), 1423*da58b97aSjoerg IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR), 1424*da58b97aSjoerg IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR), 1425*da58b97aSjoerg IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR), 1426*da58b97aSjoerg IntegratedTestKeywordParser('ALLOW_RETRIES:', ParserKind.INTEGER), 142706f32e7eSjoerg IntegratedTestKeywordParser('END.', ParserKind.TAG) 142806f32e7eSjoerg ] 142906f32e7eSjoerg keyword_parsers = {p.keyword: p for p in builtin_parsers} 143006f32e7eSjoerg 143106f32e7eSjoerg # Install user-defined additional parsers. 143206f32e7eSjoerg for parser in additional_parsers: 143306f32e7eSjoerg if not isinstance(parser, IntegratedTestKeywordParser): 1434*da58b97aSjoerg raise ValueError('Additional parser must be an instance of ' 143506f32e7eSjoerg 'IntegratedTestKeywordParser') 143606f32e7eSjoerg if parser.keyword in keyword_parsers: 143706f32e7eSjoerg raise ValueError("Parser for keyword '%s' already exists" 143806f32e7eSjoerg % parser.keyword) 143906f32e7eSjoerg keyword_parsers[parser.keyword] = parser 144006f32e7eSjoerg 144106f32e7eSjoerg # Collect the test lines from the script. 144206f32e7eSjoerg for line_number, command_type, ln in \ 144306f32e7eSjoerg parseIntegratedTestScriptCommands(sourcepath, 144406f32e7eSjoerg keyword_parsers.keys()): 144506f32e7eSjoerg parser = keyword_parsers[command_type] 144606f32e7eSjoerg parser.parseLine(line_number, ln) 144706f32e7eSjoerg if command_type == 'END.' and parser.getValue() is True: 144806f32e7eSjoerg break 144906f32e7eSjoerg 145006f32e7eSjoerg # Verify the script contains a run line. 145106f32e7eSjoerg if require_script and not script: 1452*da58b97aSjoerg raise ValueError("Test has no 'RUN:' line") 145306f32e7eSjoerg 145406f32e7eSjoerg # Check for unterminated run lines. 145506f32e7eSjoerg if script and script[-1][-1] == '\\': 1456*da58b97aSjoerg raise ValueError("Test has unterminated 'RUN:' lines (with '\\')") 145706f32e7eSjoerg 145806f32e7eSjoerg # Check boolean expressions for unterminated lines. 145906f32e7eSjoerg for key in keyword_parsers: 146006f32e7eSjoerg kp = keyword_parsers[key] 146106f32e7eSjoerg if kp.kind != ParserKind.BOOLEAN_EXPR: 146206f32e7eSjoerg continue 146306f32e7eSjoerg value = kp.getValue() 146406f32e7eSjoerg if value and value[-1][-1] == '\\': 1465*da58b97aSjoerg raise ValueError("Test has unterminated '{key}' lines (with '\\')" 1466*da58b97aSjoerg .format(key=key)) 1467*da58b97aSjoerg 1468*da58b97aSjoerg # Make sure there's at most one ALLOW_RETRIES: line 1469*da58b97aSjoerg allowed_retries = keyword_parsers['ALLOW_RETRIES:'].getValue() 1470*da58b97aSjoerg if allowed_retries and len(allowed_retries) > 1: 1471*da58b97aSjoerg raise ValueError("Test has more than one ALLOW_RETRIES lines") 1472*da58b97aSjoerg 1473*da58b97aSjoerg return {p.keyword: p.getValue() for p in keyword_parsers.values()} 1474*da58b97aSjoerg 1475*da58b97aSjoerg 1476*da58b97aSjoergdef parseIntegratedTestScript(test, additional_parsers=[], 1477*da58b97aSjoerg require_script=True): 1478*da58b97aSjoerg """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test 1479*da58b97aSjoerg script and extract the lines to 'RUN' as well as 'XFAIL', 'REQUIRES', 1480*da58b97aSjoerg 'UNSUPPORTED' and 'ALLOW_RETRIES' information into the given test. 1481*da58b97aSjoerg 1482*da58b97aSjoerg If additional parsers are specified then the test is also scanned for the 1483*da58b97aSjoerg keywords they specify and all matches are passed to the custom parser. 1484*da58b97aSjoerg 1485*da58b97aSjoerg If 'require_script' is False an empty script 1486*da58b97aSjoerg may be returned. This can be used for test formats where the actual script 1487*da58b97aSjoerg is optional or ignored. 1488*da58b97aSjoerg """ 1489*da58b97aSjoerg # Parse the test sources and extract test properties 1490*da58b97aSjoerg try: 1491*da58b97aSjoerg parsed = _parseKeywords(test.getSourcePath(), additional_parsers, 1492*da58b97aSjoerg require_script) 1493*da58b97aSjoerg except ValueError as e: 1494*da58b97aSjoerg return lit.Test.Result(Test.UNRESOLVED, str(e)) 1495*da58b97aSjoerg script = parsed['RUN:'] or [] 1496*da58b97aSjoerg test.xfails += parsed['XFAIL:'] or [] 1497*da58b97aSjoerg test.requires += parsed['REQUIRES:'] or [] 1498*da58b97aSjoerg test.unsupported += parsed['UNSUPPORTED:'] or [] 1499*da58b97aSjoerg if parsed['ALLOW_RETRIES:']: 1500*da58b97aSjoerg test.allowed_retries = parsed['ALLOW_RETRIES:'][0] 150106f32e7eSjoerg 150206f32e7eSjoerg # Enforce REQUIRES: 150306f32e7eSjoerg missing_required_features = test.getMissingRequiredFeatures() 150406f32e7eSjoerg if missing_required_features: 150506f32e7eSjoerg msg = ', '.join(missing_required_features) 150606f32e7eSjoerg return lit.Test.Result(Test.UNSUPPORTED, 150706f32e7eSjoerg "Test requires the following unavailable " 150806f32e7eSjoerg "features: %s" % msg) 150906f32e7eSjoerg 151006f32e7eSjoerg # Enforce UNSUPPORTED: 151106f32e7eSjoerg unsupported_features = test.getUnsupportedFeatures() 151206f32e7eSjoerg if unsupported_features: 151306f32e7eSjoerg msg = ', '.join(unsupported_features) 151406f32e7eSjoerg return lit.Test.Result( 151506f32e7eSjoerg Test.UNSUPPORTED, 151606f32e7eSjoerg "Test does not support the following features " 151706f32e7eSjoerg "and/or targets: %s" % msg) 151806f32e7eSjoerg 151906f32e7eSjoerg # Enforce limit_to_features. 152006f32e7eSjoerg if not test.isWithinFeatureLimits(): 152106f32e7eSjoerg msg = ', '.join(test.config.limit_to_features) 152206f32e7eSjoerg return lit.Test.Result(Test.UNSUPPORTED, 152306f32e7eSjoerg "Test does not require any of the features " 152406f32e7eSjoerg "specified in limit_to_features: %s" % msg) 152506f32e7eSjoerg 152606f32e7eSjoerg return script 152706f32e7eSjoerg 152806f32e7eSjoerg 152906f32e7eSjoergdef _runShTest(test, litConfig, useExternalSh, script, tmpBase): 1530*da58b97aSjoerg def runOnce(execdir): 153106f32e7eSjoerg if useExternalSh: 153206f32e7eSjoerg res = executeScript(test, litConfig, tmpBase, script, execdir) 153306f32e7eSjoerg else: 153406f32e7eSjoerg res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) 153506f32e7eSjoerg if isinstance(res, lit.Test.Result): 153606f32e7eSjoerg return res 153706f32e7eSjoerg 153806f32e7eSjoerg out,err,exitCode,timeoutInfo = res 153906f32e7eSjoerg if exitCode == 0: 154006f32e7eSjoerg status = Test.PASS 154106f32e7eSjoerg else: 154206f32e7eSjoerg if timeoutInfo is None: 154306f32e7eSjoerg status = Test.FAIL 154406f32e7eSjoerg else: 154506f32e7eSjoerg status = Test.TIMEOUT 1546*da58b97aSjoerg return out,err,exitCode,timeoutInfo,status 1547*da58b97aSjoerg 1548*da58b97aSjoerg # Create the output directory if it does not already exist. 1549*da58b97aSjoerg lit.util.mkdir_p(os.path.dirname(tmpBase)) 1550*da58b97aSjoerg 1551*da58b97aSjoerg # Re-run failed tests up to test.allowed_retries times. 1552*da58b97aSjoerg execdir = os.path.dirname(test.getExecPath()) 1553*da58b97aSjoerg attempts = test.allowed_retries + 1 1554*da58b97aSjoerg for i in range(attempts): 1555*da58b97aSjoerg res = runOnce(execdir) 1556*da58b97aSjoerg if isinstance(res, lit.Test.Result): 1557*da58b97aSjoerg return res 1558*da58b97aSjoerg 1559*da58b97aSjoerg out,err,exitCode,timeoutInfo,status = res 1560*da58b97aSjoerg if status != Test.FAIL: 1561*da58b97aSjoerg break 1562*da58b97aSjoerg 1563*da58b97aSjoerg # If we had to run the test more than once, count it as a flaky pass. These 1564*da58b97aSjoerg # will be printed separately in the test summary. 1565*da58b97aSjoerg if i > 0 and status == Test.PASS: 1566*da58b97aSjoerg status = Test.FLAKYPASS 156706f32e7eSjoerg 156806f32e7eSjoerg # Form the output log. 156906f32e7eSjoerg output = """Script:\n--\n%s\n--\nExit Code: %d\n""" % ( 157006f32e7eSjoerg '\n'.join(script), exitCode) 157106f32e7eSjoerg 157206f32e7eSjoerg if timeoutInfo is not None: 157306f32e7eSjoerg output += """Timeout: %s\n""" % (timeoutInfo,) 157406f32e7eSjoerg output += "\n" 157506f32e7eSjoerg 157606f32e7eSjoerg # Append the outputs, if present. 157706f32e7eSjoerg if out: 157806f32e7eSjoerg output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,) 157906f32e7eSjoerg if err: 158006f32e7eSjoerg output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,) 158106f32e7eSjoerg 158206f32e7eSjoerg return lit.Test.Result(status, output) 158306f32e7eSjoerg 158406f32e7eSjoerg 158506f32e7eSjoergdef executeShTest(test, litConfig, useExternalSh, 1586*da58b97aSjoerg extra_substitutions=[], 1587*da58b97aSjoerg preamble_commands=[]): 158806f32e7eSjoerg if test.config.unsupported: 158906f32e7eSjoerg return lit.Test.Result(Test.UNSUPPORTED, 'Test is unsupported') 159006f32e7eSjoerg 1591*da58b97aSjoerg script = list(preamble_commands) 1592*da58b97aSjoerg parsed = parseIntegratedTestScript(test, require_script=not script) 1593*da58b97aSjoerg if isinstance(parsed, lit.Test.Result): 1594*da58b97aSjoerg return parsed 1595*da58b97aSjoerg script += parsed 1596*da58b97aSjoerg 159706f32e7eSjoerg if litConfig.noExecute: 159806f32e7eSjoerg return lit.Test.Result(Test.PASS) 159906f32e7eSjoerg 160006f32e7eSjoerg tmpDir, tmpBase = getTempPaths(test) 160106f32e7eSjoerg substitutions = list(extra_substitutions) 160206f32e7eSjoerg substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase, 160306f32e7eSjoerg normalize_slashes=useExternalSh) 1604*da58b97aSjoerg script = applySubstitutions(script, substitutions, 1605*da58b97aSjoerg recursion_limit=test.config.recursiveExpansionLimit) 160606f32e7eSjoerg 1607*da58b97aSjoerg return _runShTest(test, litConfig, useExternalSh, script, tmpBase) 1608