1#!/usr/bin/env python
2# This Source Code Form is subject to the terms of the Mozilla Public
3# License, v. 2.0. If a copy of the MPL was not distributed with this
4# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
6# run-tests.py -- Python harness for GDB SpiderMonkey support
7
8import os, re, subprocess, sys, traceback
9from threading import Thread
10
11# From this directory:
12import progressbar
13from taskpool import TaskPool, get_cpu_count
14
15# Backported from Python 3.1 posixpath.py
16def _relpath(path, start=None):
17    """Return a relative version of a path"""
18
19    if not path:
20        raise ValueError("no path specified")
21
22    if start is None:
23        start = os.curdir
24
25    start_list = os.path.abspath(start).split(os.sep)
26    path_list = os.path.abspath(path).split(os.sep)
27
28    # Work out how much of the filepath is shared by start and path.
29    i = len(os.path.commonprefix([start_list, path_list]))
30
31    rel_list = [os.pardir] * (len(start_list)-i) + path_list[i:]
32    if not rel_list:
33        return os.curdir
34    return os.path.join(*rel_list)
35
36os.path.relpath = _relpath
37
38# Characters that need to be escaped when used in shell words.
39shell_need_escapes = re.compile('[^\w\d%+,-./:=@\'"]', re.DOTALL)
40# Characters that need to be escaped within double-quoted strings.
41shell_dquote_escapes = re.compile('[^\w\d%+,-./:=@"]', re.DOTALL)
42def make_shell_cmd(l):
43    def quote(s):
44        if shell_need_escapes.search(s):
45            if s.find("'") < 0:
46                return "'" + s + "'"
47            return '"' + shell_dquote_escapes.sub('\\g<0>', s) + '"'
48        return s
49
50    return ' '.join([quote(_) for _ in l])
51
52# An instance of this class collects the lists of passing, failing, and
53# timing-out tests, runs the progress bar, and prints a summary at the end.
54class Summary(object):
55
56    class SummaryBar(progressbar.ProgressBar):
57        def __init__(self, limit):
58            super(Summary.SummaryBar, self).__init__('', limit, 24)
59        def start(self):
60            self.label = '[starting           ]'
61            self.update(0)
62        def counts(self, run, failures, timeouts):
63            self.label = '[%4d|%4d|%4d|%4d]' % (run - failures, failures, timeouts, run)
64            self.update(run)
65
66    def __init__(self, num_tests):
67        self.run = 0
68        self.failures = []              # kind of judgemental; "unexpecteds"?
69        self.timeouts = []
70        if not OPTIONS.hide_progress:
71            self.bar = Summary.SummaryBar(num_tests)
72
73    # Progress bar control.
74    def start(self):
75        if not OPTIONS.hide_progress:
76            self.bar.start()
77    def update(self):
78        if not OPTIONS.hide_progress:
79            self.bar.counts(self.run, len(self.failures), len(self.timeouts))
80    # Call 'thunk' to show some output, while getting the progress bar out of the way.
81    def interleave_output(self, thunk):
82        if not OPTIONS.hide_progress:
83            self.bar.clear()
84        thunk()
85        self.update()
86
87    def passed(self, test):
88        self.run += 1
89        self.update()
90
91    def failed(self, test):
92        self.run += 1
93        self.failures.append(test)
94        self.update()
95
96    def timeout(self, test):
97        self.run += 1
98        self.timeouts.append(test)
99        self.update()
100
101    def finish(self):
102        if not OPTIONS.hide_progress:
103            self.bar.finish()
104
105        if self.failures:
106
107            print "tests failed:"
108            for test in self.failures:
109                test.show(sys.stdout)
110
111            if OPTIONS.worklist:
112                try:
113                    with open(OPTIONS.worklist) as out:
114                        for test in self.failures:
115                            out.write(test.name + '\n')
116                except IOError as err:
117                    sys.stderr.write("Error writing worklist file '%s': %s"
118                                     % (OPTIONS.worklist, err))
119                    sys.exit(1)
120
121            if OPTIONS.write_failures:
122                try:
123                    with open(OPTIONS.write_failures) as out:
124                        for test in self.failures:
125                            test.show(out)
126                except IOError as err:
127                    sys.stderr.write("Error writing worklist file '%s': %s"
128                                     % (OPTIONS.write_failures, err))
129                    sys.exit(1)
130
131        if self.timeouts:
132            print "tests timed out:"
133            for test in self.timeouts:
134                test.show(sys.stdout)
135
136        if self.failures or self.timeouts:
137            sys.exit(2)
138
139class Test(TaskPool.Task):
140    def __init__(self, path, summary):
141        super(Test, self).__init__()
142        self.test_path = path           # path to .py test file
143        self.summary = summary
144
145        # test.name is the name of the test relative to the top of the test
146        # directory. This is what we use to report failures and timeouts,
147        # and when writing test lists.
148        self.name = os.path.relpath(self.test_path, OPTIONS.testdir)
149
150        self.stdout = ''
151        self.stderr = ''
152        self.returncode = None
153
154    def cmd(self):
155        testlibdir = os.path.normpath(os.path.join(OPTIONS.testdir, '..', 'lib-for-tests'))
156        return [OPTIONS.gdb_executable,
157                '-nw',          # Don't create a window (unnecessary?)
158                '-nx',          # Don't read .gdbinit.
159                '--ex', 'add-auto-load-safe-path %s' % (OPTIONS.builddir,),
160                '--ex', 'set env LD_LIBRARY_PATH %s' % os.path.join(OPTIONS.objdir, 'js', 'src'),
161                '--ex', 'file %s' % (os.path.join(OPTIONS.builddir, 'gdb-tests'),),
162                '--eval-command', 'python testlibdir=%r' % (testlibdir,),
163                '--eval-command', 'python testscript=%r' % (self.test_path,),
164                '--eval-command', 'python exec(open(%r).read())' % os.path.join(testlibdir, 'catcher.py')]
165
166    def start(self, pipe, deadline):
167        super(Test, self).start(pipe, deadline)
168        if OPTIONS.show_cmd:
169            self.summary.interleave_output(lambda: self.show_cmd(sys.stdout))
170
171    def onStdout(self, text):
172        self.stdout += text
173
174    def onStderr(self, text):
175        self.stderr += text
176
177    def onFinished(self, returncode):
178        self.returncode = returncode
179        if OPTIONS.show_output:
180            self.summary.interleave_output(lambda: self.show_output(sys.stdout))
181        if returncode != 0:
182            self.summary.failed(self)
183        else:
184            self.summary.passed(self)
185
186    def onTimeout(self):
187        self.summary.timeout(self)
188
189    def show_cmd(self, out):
190        print "Command: ", make_shell_cmd(self.cmd())
191
192    def show_output(self, out):
193        if self.stdout:
194            out.write('Standard output:')
195            out.write('\n' + self.stdout + '\n')
196        if self.stderr:
197            out.write('Standard error:')
198            out.write('\n' + self.stderr + '\n')
199
200    def show(self, out):
201        out.write(self.name + '\n')
202        if OPTIONS.write_failure_output:
203            out.write('Command: %s\n' % (make_shell_cmd(self.cmd()),))
204            self.show_output(out)
205            out.write('GDB exit code: %r\n' % (self.returncode,))
206
207def find_tests(dir, substring = None):
208    ans = []
209    for dirpath, dirnames, filenames in os.walk(dir):
210        if dirpath == '.':
211            continue
212        for filename in filenames:
213            if not filename.endswith('.py'):
214                continue
215            test = os.path.join(dirpath, filename)
216            if substring is None or substring in os.path.relpath(test, dir):
217                ans.append(test)
218    return ans
219
220def build_test_exec(builddir):
221    p = subprocess.check_call(['make', 'gdb-tests'], cwd=builddir)
222
223def run_tests(tests, summary):
224    pool = TaskPool(tests, job_limit=OPTIONS.workercount, timeout=OPTIONS.timeout)
225    pool.run_all()
226
227OPTIONS = None
228def main(argv):
229    global OPTIONS
230    script_path = os.path.abspath(__file__)
231    script_dir = os.path.dirname(script_path)
232
233    # OBJDIR is a standalone SpiderMonkey build directory. This is where we
234    # find the SpiderMonkey shared library to link against.
235    #
236    # The [TESTS] optional arguments are paths of test files relative
237    # to the jit-test/tests directory.
238    from optparse import OptionParser
239    op = OptionParser(usage='%prog [options] OBJDIR [TESTS...]')
240    op.add_option('-s', '--show-cmd', dest='show_cmd', action='store_true',
241                  help='show GDB shell command run')
242    op.add_option('-o', '--show-output', dest='show_output', action='store_true',
243                  help='show output from GDB')
244    op.add_option('-x', '--exclude', dest='exclude', action='append',
245                  help='exclude given test dir or path')
246    op.add_option('-t', '--timeout', dest='timeout',  type=float, default=150.0,
247                  help='set test timeout in seconds')
248    op.add_option('-j', '--worker-count', dest='workercount', type=int,
249                  help='Run [WORKERCOUNT] tests at a time')
250    op.add_option('--no-progress', dest='hide_progress', action='store_true',
251                  help='hide progress bar')
252    op.add_option('--worklist', dest='worklist', metavar='FILE',
253                  help='Read tests to run from [FILE] (or run all if [FILE] not found);\n'
254                       'write failures back to [FILE]')
255    op.add_option('-r', '--read-tests', dest='read_tests', metavar='FILE',
256                  help='Run test files listed in [FILE]')
257    op.add_option('-w', '--write-failures', dest='write_failures', metavar='FILE',
258                  help='Write failing tests to [FILE]')
259    op.add_option('--write-failure-output', dest='write_failure_output', action='store_true',
260                  help='With --write-failures=FILE, additionally write the output of failed tests to [FILE]')
261    op.add_option('--gdb', dest='gdb_executable', metavar='EXECUTABLE', default='gdb',
262                  help='Run tests with [EXECUTABLE], rather than plain \'gdb\'.')
263    op.add_option('--srcdir', dest='srcdir',
264                  default=os.path.abspath(os.path.join(script_dir, '..')),
265                  help='Use SpiderMonkey sources in [SRCDIR].')
266    op.add_option('--testdir', dest='testdir', default=os.path.join(script_dir, 'tests'),
267                  help='Find tests in [TESTDIR].')
268    op.add_option('--builddir', dest='builddir',
269                  help='Build test executable in [BUILDDIR].')
270    (OPTIONS, args) = op.parse_args(argv)
271    if len(args) < 1:
272        op.error('missing OBJDIR argument')
273    OPTIONS.objdir = os.path.abspath(args[0])
274
275    test_args = args[1:]
276
277    if not OPTIONS.workercount:
278        OPTIONS.workercount = get_cpu_count()
279
280    # Compute default for OPTIONS.builddir now, since we've computed OPTIONS.objdir.
281    if not OPTIONS.builddir:
282        OPTIONS.builddir = os.path.join(OPTIONS.objdir, 'js', 'src', 'gdb')
283
284    test_set = set()
285
286    # All the various sources of test names accumulate.
287    if test_args:
288        for arg in test_args:
289            test_set.update(find_tests(OPTIONS.testdir, arg))
290    if OPTIONS.worklist:
291        try:
292            with open(OPTIONS.worklist) as f:
293                for line in f:
294                    test_set.update(os.path.join(test_dir, line.strip('\n')))
295        except IOError:
296            # With worklist, a missing file means to start the process with
297            # the complete list of tests.
298            sys.stderr.write("Couldn't read worklist file '%s'; running all tests\n"
299                             % (OPTIONS.worklist,))
300            test_set = set(find_tests(OPTIONS.testdir))
301    if OPTIONS.read_tests:
302        try:
303            with open(OPTIONS.read_tests) as f:
304                for line in f:
305                    test_set.update(os.path.join(test_dir, line.strip('\n')))
306        except IOError as err:
307            sys.stderr.write("Error trying to read test file '%s': %s\n"
308                             % (OPTIONS.read_tests, err))
309            sys.exit(1)
310
311    # If none of the above options were passed, and no tests were listed
312    # explicitly, use the complete set.
313    if not test_args and not OPTIONS.worklist and not OPTIONS.read_tests:
314        test_set = set(find_tests(OPTIONS.testdir))
315
316    if OPTIONS.exclude:
317        exclude_set = set()
318        for exclude in OPTIONS.exclude:
319            exclude_set.update(find_tests(test_dir, exclude))
320        test_set -= exclude_set
321
322    if not test_set:
323        sys.stderr.write("No tests found matching command line arguments.\n")
324        sys.exit(1)
325
326    summary = Summary(len(test_set))
327    test_list = [ Test(_, summary) for _ in sorted(test_set) ]
328
329    # Build the test executable from all the .cpp files found in the test
330    # directory tree.
331    try:
332        build_test_exec(OPTIONS.builddir)
333    except subprocess.CalledProcessError as err:
334        sys.stderr.write("Error building test executable: %s\n" % (err,))
335        sys.exit(1)
336
337    # Run the tests.
338    try:
339        summary.start()
340        run_tests(test_list, summary)
341        summary.finish()
342    except OSError as err:
343        sys.stderr.write("Error running tests: %s\n" % (err,))
344        sys.exit(1)
345
346    sys.exit(0)
347
348if __name__ == '__main__':
349    main(sys.argv[1:])
350