1from __future__ import absolute_import
2import os, signal, subprocess, sys
3import re
4import platform
5import tempfile
6
7import lit.ShUtil as ShUtil
8import lit.Test as Test
9import lit.util
10
11class InternalShellError(Exception):
12    def __init__(self, command, message):
13        self.command = command
14        self.message = message
15
16kIsWindows = platform.system() == 'Windows'
17
18# Don't use close_fds on Windows.
19kUseCloseFDs = not kIsWindows
20
21# Use temporary files to replace /dev/null on Windows.
22kAvoidDevNull = kIsWindows
23
24def executeShCmd(cmd, cfg, cwd, results):
25    if isinstance(cmd, ShUtil.Seq):
26        if cmd.op == ';':
27            res = executeShCmd(cmd.lhs, cfg, cwd, results)
28            return executeShCmd(cmd.rhs, cfg, cwd, results)
29
30        if cmd.op == '&':
31            raise InternalShellError(cmd,"unsupported shell operator: '&'")
32
33        if cmd.op == '||':
34            res = executeShCmd(cmd.lhs, cfg, cwd, results)
35            if res != 0:
36                res = executeShCmd(cmd.rhs, cfg, cwd, results)
37            return res
38
39        if cmd.op == '&&':
40            res = executeShCmd(cmd.lhs, cfg, cwd, results)
41            if res is None:
42                return res
43
44            if res == 0:
45                res = executeShCmd(cmd.rhs, cfg, cwd, results)
46            return res
47
48        raise ValueError('Unknown shell command: %r' % cmd.op)
49
50    assert isinstance(cmd, ShUtil.Pipeline)
51    procs = []
52    input = subprocess.PIPE
53    stderrTempFiles = []
54    opened_files = []
55    named_temp_files = []
56    # To avoid deadlock, we use a single stderr stream for piped
57    # output. This is null until we have seen some output using
58    # stderr.
59    for i,j in enumerate(cmd.commands):
60        # Apply the redirections, we use (N,) as a sentinel to indicate stdin,
61        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
62        # from a file are represented with a list [file, mode, file-object]
63        # where file-object is initially None.
64        redirects = [(0,), (1,), (2,)]
65        for r in j.redirects:
66            if r[0] == ('>',2):
67                redirects[2] = [r[1], 'w', None]
68            elif r[0] == ('>>',2):
69                redirects[2] = [r[1], 'a', None]
70            elif r[0] == ('>&',2) and r[1] in '012':
71                redirects[2] = redirects[int(r[1])]
72            elif r[0] == ('>&',) or r[0] == ('&>',):
73                redirects[1] = redirects[2] = [r[1], 'w', None]
74            elif r[0] == ('>',):
75                redirects[1] = [r[1], 'w', None]
76            elif r[0] == ('>>',):
77                redirects[1] = [r[1], 'a', None]
78            elif r[0] == ('<',):
79                redirects[0] = [r[1], 'r', None]
80            else:
81                raise InternalShellError(j,"Unsupported redirect: %r" % (r,))
82
83        # Map from the final redirections to something subprocess can handle.
84        final_redirects = []
85        for index,r in enumerate(redirects):
86            if r == (0,):
87                result = input
88            elif r == (1,):
89                if index == 0:
90                    raise InternalShellError(j,"Unsupported redirect for stdin")
91                elif index == 1:
92                    result = subprocess.PIPE
93                else:
94                    result = subprocess.STDOUT
95            elif r == (2,):
96                if index != 2:
97                    raise InternalShellError(j,"Unsupported redirect on stdout")
98                result = subprocess.PIPE
99            else:
100                if r[2] is None:
101                    if kAvoidDevNull and r[0] == '/dev/null':
102                        r[2] = tempfile.TemporaryFile(mode=r[1])
103                    else:
104                        r[2] = open(r[0], r[1])
105                    # Workaround a Win32 and/or subprocess bug when appending.
106                    #
107                    # FIXME: Actually, this is probably an instance of PR6753.
108                    if r[1] == 'a':
109                        r[2].seek(0, 2)
110                    opened_files.append(r[2])
111                result = r[2]
112            final_redirects.append(result)
113
114        stdin, stdout, stderr = final_redirects
115
116        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
117        # stderr on a pipe and treat it as stdout.
118        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
119            stderr = subprocess.PIPE
120            stderrIsStdout = True
121        else:
122            stderrIsStdout = False
123
124            # Don't allow stderr on a PIPE except for the last
125            # process, this could deadlock.
126            #
127            # FIXME: This is slow, but so is deadlock.
128            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
129                stderr = tempfile.TemporaryFile(mode='w+b')
130                stderrTempFiles.append((i, stderr))
131
132        # Resolve the executable path ourselves.
133        args = list(j.args)
134        executable = lit.util.which(args[0], cfg.environment['PATH'])
135        if not executable:
136            raise InternalShellError(j, '%r: command not found' % j.args[0])
137
138        # Replace uses of /dev/null with temporary files.
139        if kAvoidDevNull:
140            for i,arg in enumerate(args):
141                if arg == "/dev/null":
142                    f = tempfile.NamedTemporaryFile(delete=False)
143                    f.close()
144                    named_temp_files.append(f.name)
145                    args[i] = f.name
146
147        procs.append(subprocess.Popen(args, cwd=cwd,
148                                      executable = executable,
149                                      stdin = stdin,
150                                      stdout = stdout,
151                                      stderr = stderr,
152                                      env = cfg.environment,
153                                      close_fds = kUseCloseFDs))
154
155        # Immediately close stdin for any process taking stdin from us.
156        if stdin == subprocess.PIPE:
157            procs[-1].stdin.close()
158            procs[-1].stdin = None
159
160        # Update the current stdin source.
161        if stdout == subprocess.PIPE:
162            input = procs[-1].stdout
163        elif stderrIsStdout:
164            input = procs[-1].stderr
165        else:
166            input = subprocess.PIPE
167
168    # Explicitly close any redirected files. We need to do this now because we
169    # need to release any handles we may have on the temporary files (important
170    # on Win32, for example). Since we have already spawned the subprocess, our
171    # handles have already been transferred so we do not need them anymore.
172    for f in opened_files:
173        f.close()
174
175    # FIXME: There is probably still deadlock potential here. Yawn.
176    procData = [None] * len(procs)
177    procData[-1] = procs[-1].communicate()
178
179    for i in range(len(procs) - 1):
180        if procs[i].stdout is not None:
181            out = procs[i].stdout.read()
182        else:
183            out = ''
184        if procs[i].stderr is not None:
185            err = procs[i].stderr.read()
186        else:
187            err = ''
188        procData[i] = (out,err)
189
190    # Read stderr out of the temp files.
191    for i,f in stderrTempFiles:
192        f.seek(0, 0)
193        procData[i] = (procData[i][0], f.read())
194
195    exitCode = None
196    for i,(out,err) in enumerate(procData):
197        res = procs[i].wait()
198        # Detect Ctrl-C in subprocess.
199        if res == -signal.SIGINT:
200            raise KeyboardInterrupt
201
202        # Ensure the resulting output is always of string type.
203        try:
204            out = str(out.decode('ascii'))
205        except:
206            out = str(out)
207        try:
208            err = str(err.decode('ascii'))
209        except:
210            err = str(err)
211
212        results.append((cmd.commands[i], out, err, res))
213        if cmd.pipe_err:
214            # Python treats the exit code as a signed char.
215            if exitCode is None:
216                exitCode = res
217            elif res < 0:
218                exitCode = min(exitCode, res)
219            else:
220                exitCode = max(exitCode, res)
221        else:
222            exitCode = res
223
224    # Remove any named temporary files we created.
225    for f in named_temp_files:
226        try:
227            os.remove(f)
228        except OSError:
229            pass
230
231    if cmd.negate:
232        exitCode = not exitCode
233
234    return exitCode
235
236def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
237    cmds = []
238    for ln in commands:
239        try:
240            cmds.append(ShUtil.ShParser(ln, litConfig.isWindows,
241                                        test.config.pipefail).parse())
242        except:
243            return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln)
244
245    cmd = cmds[0]
246    for c in cmds[1:]:
247        cmd = ShUtil.Seq(cmd, '&&', c)
248
249    results = []
250    try:
251        exitCode = executeShCmd(cmd, test.config, cwd, results)
252    except InternalShellError:
253        e = sys.exc_info()[1]
254        exitCode = 127
255        results.append((e.command, '', e.message, exitCode))
256
257    out = err = ''
258    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
259        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
260        out += 'Command %d Result: %r\n' % (i, res)
261        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
262        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
263
264    return out, err, exitCode
265
266def executeScript(test, litConfig, tmpBase, commands, cwd):
267    bashPath = litConfig.getBashPath();
268    isWin32CMDEXE = (litConfig.isWindows and not bashPath)
269    script = tmpBase + '.script'
270    if isWin32CMDEXE:
271        script += '.bat'
272
273    # Write script file
274    mode = 'w'
275    if litConfig.isWindows and not isWin32CMDEXE:
276      mode += 'b'  # Avoid CRLFs when writing bash scripts.
277    f = open(script, mode)
278    if isWin32CMDEXE:
279        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
280    else:
281        if test.config.pipefail:
282            f.write('set -o pipefail;')
283        f.write('{ ' + '; } &&\n{ '.join(commands) + '; }')
284    f.write('\n')
285    f.close()
286
287    if isWin32CMDEXE:
288        command = ['cmd','/c', script]
289    else:
290        if bashPath:
291            command = [bashPath, script]
292        else:
293            command = ['/bin/sh', script]
294        if litConfig.useValgrind:
295            # FIXME: Running valgrind on sh is overkill. We probably could just
296            # run on clang with no real loss.
297            command = litConfig.valgrindArgs + command
298
299    return lit.util.executeCommand(command, cwd=cwd,
300                                   env=test.config.environment)
301
302def parseIntegratedTestScriptCommands(source_path):
303    """
304    parseIntegratedTestScriptCommands(source_path) -> commands
305
306    Parse the commands in an integrated test script file into a list of
307    (line_number, command_type, line).
308    """
309
310    # This code is carefully written to be dual compatible with Python 2.5+ and
311    # Python 3 without requiring input files to always have valid codings. The
312    # trick we use is to open the file in binary mode and use the regular
313    # expression library to find the commands, with it scanning strings in
314    # Python2 and bytes in Python3.
315    #
316    # Once we find a match, we do require each script line to be decodable to
317    # ascii, so we convert the outputs to ascii before returning. This way the
318    # remaining code can work with "strings" agnostic of the executing Python
319    # version.
320
321    def to_bytes(str):
322        # Encode to Latin1 to get binary data.
323        return str.encode('ISO-8859-1')
324    keywords = ('RUN:', 'XFAIL:', 'REQUIRES:', 'END.')
325    keywords_re = re.compile(
326        to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),)))
327
328    f = open(source_path, 'rb')
329    try:
330        # Read the entire file contents.
331        data = f.read()
332
333        # Iterate over the matches.
334        line_number = 1
335        last_match_position = 0
336        for match in keywords_re.finditer(data):
337            # Compute the updated line number by counting the intervening
338            # newlines.
339            match_position = match.start()
340            line_number += data.count(to_bytes('\n'), last_match_position,
341                                      match_position)
342            last_match_position = match_position
343
344            # Convert the keyword and line to ascii strings and yield the
345            # command. Note that we take care to return regular strings in
346            # Python 2, to avoid other code having to differentiate between the
347            # str and unicode types.
348            keyword,ln = match.groups()
349            yield (line_number, str(keyword[:-1].decode('ascii')),
350                   str(ln.decode('ascii')))
351    finally:
352        f.close()
353
354def parseIntegratedTestScript(test, normalize_slashes=False,
355                              extra_substitutions=[]):
356    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
357    script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES'
358    information. The RUN lines also will have variable substitution performed.
359    """
360
361    # Get the temporary location, this is always relative to the test suite
362    # root, not test source root.
363    #
364    # FIXME: This should not be here?
365    sourcepath = test.getSourcePath()
366    sourcedir = os.path.dirname(sourcepath)
367    execpath = test.getExecPath()
368    execdir,execbase = os.path.split(execpath)
369    tmpDir = os.path.join(execdir, 'Output')
370    tmpBase = os.path.join(tmpDir, execbase)
371
372    # Normalize slashes, if requested.
373    if normalize_slashes:
374        sourcepath = sourcepath.replace('\\', '/')
375        sourcedir = sourcedir.replace('\\', '/')
376        tmpDir = tmpDir.replace('\\', '/')
377        tmpBase = tmpBase.replace('\\', '/')
378
379    # We use #_MARKER_# to hide %% while we do the other substitutions.
380    substitutions = list(extra_substitutions)
381    substitutions.extend([('%%', '#_MARKER_#')])
382    substitutions.extend(test.config.substitutions)
383    substitutions.extend([('%s', sourcepath),
384                          ('%S', sourcedir),
385                          ('%p', sourcedir),
386                          ('%{pathsep}', os.pathsep),
387                          ('%t', tmpBase + '.tmp'),
388                          ('%T', tmpDir),
389                          ('#_MARKER_#', '%')])
390
391    # "%/[STpst]" should be normalized.
392    substitutions.extend([
393            ('%/s', sourcepath.replace('\\', '/')),
394            ('%/S', sourcedir.replace('\\', '/')),
395            ('%/p', sourcedir.replace('\\', '/')),
396            ('%/t', tmpBase.replace('\\', '/') + '.tmp'),
397            ('%/T', tmpDir.replace('\\', '/')),
398            ])
399
400    # Collect the test lines from the script.
401    script = []
402    requires = []
403    for line_number, command_type, ln in \
404            parseIntegratedTestScriptCommands(sourcepath):
405        if command_type == 'RUN':
406            # Trim trailing whitespace.
407            ln = ln.rstrip()
408
409            # Substitute line number expressions
410            ln = re.sub('%\(line\)', str(line_number), ln)
411            def replace_line_number(match):
412                if match.group(1) == '+':
413                    return str(line_number + int(match.group(2)))
414                if match.group(1) == '-':
415                    return str(line_number - int(match.group(2)))
416            ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln)
417
418            # Collapse lines with trailing '\\'.
419            if script and script[-1][-1] == '\\':
420                script[-1] = script[-1][:-1] + ln
421            else:
422                script.append(ln)
423        elif command_type == 'XFAIL':
424            test.xfails.extend([s.strip() for s in ln.split(',')])
425        elif command_type == 'REQUIRES':
426            requires.extend([s.strip() for s in ln.split(',')])
427        elif command_type == 'END':
428            # END commands are only honored if the rest of the line is empty.
429            if not ln.strip():
430                break
431        else:
432            raise ValueError("unknown script command type: %r" % (
433                    command_type,))
434
435    # Apply substitutions to the script.  Allow full regular
436    # expression syntax.  Replace each matching occurrence of regular
437    # expression pattern a with substitution b in line ln.
438    def processLine(ln):
439        # Apply substitutions
440        for a,b in substitutions:
441            if kIsWindows:
442                b = b.replace("\\","\\\\")
443            ln = re.sub(a, b, ln)
444
445        # Strip the trailing newline and any extra whitespace.
446        return ln.strip()
447    script = [processLine(ln)
448              for ln in script]
449
450    # Verify the script contains a run line.
451    if not script:
452        return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!")
453
454    # Check for unterminated run lines.
455    if script[-1][-1] == '\\':
456        return lit.Test.Result(Test.UNRESOLVED,
457                               "Test has unterminated run lines (with '\\')")
458
459    # Check that we have the required features:
460    missing_required_features = [f for f in requires
461                                 if f not in test.config.available_features]
462    if missing_required_features:
463        msg = ', '.join(missing_required_features)
464        return lit.Test.Result(Test.UNSUPPORTED,
465                               "Test requires the following features: %s" % msg)
466
467    return script,tmpBase,execdir
468
469def executeShTest(test, litConfig, useExternalSh,
470                  extra_substitutions=[]):
471    if test.config.unsupported:
472        return (Test.UNSUPPORTED, 'Test is unsupported')
473
474    res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions)
475    if isinstance(res, lit.Test.Result):
476        return res
477    if litConfig.noExecute:
478        return lit.Test.Result(Test.PASS)
479
480    script, tmpBase, execdir = res
481
482    # Create the output directory if it does not already exist.
483    lit.util.mkdir_p(os.path.dirname(tmpBase))
484
485    if useExternalSh:
486        res = executeScript(test, litConfig, tmpBase, script, execdir)
487    else:
488        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
489    if isinstance(res, lit.Test.Result):
490        return res
491
492    out,err,exitCode = res
493    if exitCode == 0:
494        status = Test.PASS
495    else:
496        status = Test.FAIL
497
498    # Form the output log.
499    output = """Script:\n--\n%s\n--\nExit Code: %d\n\n""" % (
500        '\n'.join(script), exitCode)
501
502    # Append the outputs, if present.
503    if out:
504        output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,)
505    if err:
506        output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
507
508    return lit.Test.Result(status, output)
509