1#!/usr/bin/env python 2# This Source Code Form is subject to the terms of the Mozilla Public 3# License, v. 2.0. If a copy of the MPL was not distributed with this 4# file, You can obtain one at http://mozilla.org/MPL/2.0/. 5 6# run-tests.py -- Python harness for GDB SpiderMonkey support 7 8import os, re, subprocess, sys, traceback 9from threading import Thread 10 11# From this directory: 12import progressbar 13from taskpool import TaskPool, get_cpu_count 14 15# Backported from Python 3.1 posixpath.py 16def _relpath(path, start=None): 17 """Return a relative version of a path""" 18 19 if not path: 20 raise ValueError("no path specified") 21 22 if start is None: 23 start = os.curdir 24 25 start_list = os.path.abspath(start).split(os.sep) 26 path_list = os.path.abspath(path).split(os.sep) 27 28 # Work out how much of the filepath is shared by start and path. 29 i = len(os.path.commonprefix([start_list, path_list])) 30 31 rel_list = [os.pardir] * (len(start_list)-i) + path_list[i:] 32 if not rel_list: 33 return os.curdir 34 return os.path.join(*rel_list) 35 36os.path.relpath = _relpath 37 38# Characters that need to be escaped when used in shell words. 39shell_need_escapes = re.compile('[^\w\d%+,-./:=@\'"]', re.DOTALL) 40# Characters that need to be escaped within double-quoted strings. 41shell_dquote_escapes = re.compile('[^\w\d%+,-./:=@"]', re.DOTALL) 42def make_shell_cmd(l): 43 def quote(s): 44 if shell_need_escapes.search(s): 45 if s.find("'") < 0: 46 return "'" + s + "'" 47 return '"' + shell_dquote_escapes.sub('\\g<0>', s) + '"' 48 return s 49 50 return ' '.join([quote(_) for _ in l]) 51 52# An instance of this class collects the lists of passing, failing, and 53# timing-out tests, runs the progress bar, and prints a summary at the end. 54class Summary(object): 55 56 class SummaryBar(progressbar.ProgressBar): 57 def __init__(self, limit): 58 super(Summary.SummaryBar, self).__init__('', limit, 24) 59 def start(self): 60 self.label = '[starting ]' 61 self.update(0) 62 def counts(self, run, failures, timeouts): 63 self.label = '[%4d|%4d|%4d|%4d]' % (run - failures, failures, timeouts, run) 64 self.update(run) 65 66 def __init__(self, num_tests): 67 self.run = 0 68 self.failures = [] # kind of judgemental; "unexpecteds"? 69 self.timeouts = [] 70 if not OPTIONS.hide_progress: 71 self.bar = Summary.SummaryBar(num_tests) 72 73 # Progress bar control. 74 def start(self): 75 if not OPTIONS.hide_progress: 76 self.bar.start() 77 def update(self): 78 if not OPTIONS.hide_progress: 79 self.bar.counts(self.run, len(self.failures), len(self.timeouts)) 80 # Call 'thunk' to show some output, while getting the progress bar out of the way. 81 def interleave_output(self, thunk): 82 if not OPTIONS.hide_progress: 83 self.bar.clear() 84 thunk() 85 self.update() 86 87 def passed(self, test): 88 self.run += 1 89 self.update() 90 91 def failed(self, test): 92 self.run += 1 93 self.failures.append(test) 94 self.update() 95 96 def timeout(self, test): 97 self.run += 1 98 self.timeouts.append(test) 99 self.update() 100 101 def finish(self): 102 if not OPTIONS.hide_progress: 103 self.bar.finish() 104 105 if self.failures: 106 107 print "tests failed:" 108 for test in self.failures: 109 test.show(sys.stdout) 110 111 if OPTIONS.worklist: 112 try: 113 with open(OPTIONS.worklist) as out: 114 for test in self.failures: 115 out.write(test.name + '\n') 116 except IOError as err: 117 sys.stderr.write("Error writing worklist file '%s': %s" 118 % (OPTIONS.worklist, err)) 119 sys.exit(1) 120 121 if OPTIONS.write_failures: 122 try: 123 with open(OPTIONS.write_failures) as out: 124 for test in self.failures: 125 test.show(out) 126 except IOError as err: 127 sys.stderr.write("Error writing worklist file '%s': %s" 128 % (OPTIONS.write_failures, err)) 129 sys.exit(1) 130 131 if self.timeouts: 132 print "tests timed out:" 133 for test in self.timeouts: 134 test.show(sys.stdout) 135 136 if self.failures or self.timeouts: 137 sys.exit(2) 138 139class Test(TaskPool.Task): 140 def __init__(self, path, summary): 141 super(Test, self).__init__() 142 self.test_path = path # path to .py test file 143 self.summary = summary 144 145 # test.name is the name of the test relative to the top of the test 146 # directory. This is what we use to report failures and timeouts, 147 # and when writing test lists. 148 self.name = os.path.relpath(self.test_path, OPTIONS.testdir) 149 150 self.stdout = '' 151 self.stderr = '' 152 self.returncode = None 153 154 def cmd(self): 155 testlibdir = os.path.normpath(os.path.join(OPTIONS.testdir, '..', 'lib-for-tests')) 156 return [OPTIONS.gdb_executable, 157 '-nw', # Don't create a window (unnecessary?) 158 '-nx', # Don't read .gdbinit. 159 '--ex', 'add-auto-load-safe-path %s' % (OPTIONS.builddir,), 160 '--ex', 'set env LD_LIBRARY_PATH %s' % os.path.join(OPTIONS.objdir, 'js', 'src'), 161 '--ex', 'file %s' % (os.path.join(OPTIONS.builddir, 'gdb-tests'),), 162 '--eval-command', 'python testlibdir=%r' % (testlibdir,), 163 '--eval-command', 'python testscript=%r' % (self.test_path,), 164 '--eval-command', 'python exec(open(%r).read())' % os.path.join(testlibdir, 'catcher.py')] 165 166 def start(self, pipe, deadline): 167 super(Test, self).start(pipe, deadline) 168 if OPTIONS.show_cmd: 169 self.summary.interleave_output(lambda: self.show_cmd(sys.stdout)) 170 171 def onStdout(self, text): 172 self.stdout += text 173 174 def onStderr(self, text): 175 self.stderr += text 176 177 def onFinished(self, returncode): 178 self.returncode = returncode 179 if OPTIONS.show_output: 180 self.summary.interleave_output(lambda: self.show_output(sys.stdout)) 181 if returncode != 0: 182 self.summary.failed(self) 183 else: 184 self.summary.passed(self) 185 186 def onTimeout(self): 187 self.summary.timeout(self) 188 189 def show_cmd(self, out): 190 print "Command: ", make_shell_cmd(self.cmd()) 191 192 def show_output(self, out): 193 if self.stdout: 194 out.write('Standard output:') 195 out.write('\n' + self.stdout + '\n') 196 if self.stderr: 197 out.write('Standard error:') 198 out.write('\n' + self.stderr + '\n') 199 200 def show(self, out): 201 out.write(self.name + '\n') 202 if OPTIONS.write_failure_output: 203 out.write('Command: %s\n' % (make_shell_cmd(self.cmd()),)) 204 self.show_output(out) 205 out.write('GDB exit code: %r\n' % (self.returncode,)) 206 207def find_tests(dir, substring = None): 208 ans = [] 209 for dirpath, dirnames, filenames in os.walk(dir): 210 if dirpath == '.': 211 continue 212 for filename in filenames: 213 if not filename.endswith('.py'): 214 continue 215 test = os.path.join(dirpath, filename) 216 if substring is None or substring in os.path.relpath(test, dir): 217 ans.append(test) 218 return ans 219 220def build_test_exec(builddir): 221 p = subprocess.check_call(['make', 'gdb-tests'], cwd=builddir) 222 223def run_tests(tests, summary): 224 pool = TaskPool(tests, job_limit=OPTIONS.workercount, timeout=OPTIONS.timeout) 225 pool.run_all() 226 227OPTIONS = None 228def main(argv): 229 global OPTIONS 230 script_path = os.path.abspath(__file__) 231 script_dir = os.path.dirname(script_path) 232 233 # OBJDIR is a standalone SpiderMonkey build directory. This is where we 234 # find the SpiderMonkey shared library to link against. 235 # 236 # The [TESTS] optional arguments are paths of test files relative 237 # to the jit-test/tests directory. 238 from optparse import OptionParser 239 op = OptionParser(usage='%prog [options] OBJDIR [TESTS...]') 240 op.add_option('-s', '--show-cmd', dest='show_cmd', action='store_true', 241 help='show GDB shell command run') 242 op.add_option('-o', '--show-output', dest='show_output', action='store_true', 243 help='show output from GDB') 244 op.add_option('-x', '--exclude', dest='exclude', action='append', 245 help='exclude given test dir or path') 246 op.add_option('-t', '--timeout', dest='timeout', type=float, default=150.0, 247 help='set test timeout in seconds') 248 op.add_option('-j', '--worker-count', dest='workercount', type=int, 249 help='Run [WORKERCOUNT] tests at a time') 250 op.add_option('--no-progress', dest='hide_progress', action='store_true', 251 help='hide progress bar') 252 op.add_option('--worklist', dest='worklist', metavar='FILE', 253 help='Read tests to run from [FILE] (or run all if [FILE] not found);\n' 254 'write failures back to [FILE]') 255 op.add_option('-r', '--read-tests', dest='read_tests', metavar='FILE', 256 help='Run test files listed in [FILE]') 257 op.add_option('-w', '--write-failures', dest='write_failures', metavar='FILE', 258 help='Write failing tests to [FILE]') 259 op.add_option('--write-failure-output', dest='write_failure_output', action='store_true', 260 help='With --write-failures=FILE, additionally write the output of failed tests to [FILE]') 261 op.add_option('--gdb', dest='gdb_executable', metavar='EXECUTABLE', default='gdb', 262 help='Run tests with [EXECUTABLE], rather than plain \'gdb\'.') 263 op.add_option('--srcdir', dest='srcdir', 264 default=os.path.abspath(os.path.join(script_dir, '..')), 265 help='Use SpiderMonkey sources in [SRCDIR].') 266 op.add_option('--testdir', dest='testdir', default=os.path.join(script_dir, 'tests'), 267 help='Find tests in [TESTDIR].') 268 op.add_option('--builddir', dest='builddir', 269 help='Build test executable in [BUILDDIR].') 270 (OPTIONS, args) = op.parse_args(argv) 271 if len(args) < 1: 272 op.error('missing OBJDIR argument') 273 OPTIONS.objdir = os.path.abspath(args[0]) 274 275 test_args = args[1:] 276 277 if not OPTIONS.workercount: 278 OPTIONS.workercount = get_cpu_count() 279 280 # Compute default for OPTIONS.builddir now, since we've computed OPTIONS.objdir. 281 if not OPTIONS.builddir: 282 OPTIONS.builddir = os.path.join(OPTIONS.objdir, 'js', 'src', 'gdb') 283 284 test_set = set() 285 286 # All the various sources of test names accumulate. 287 if test_args: 288 for arg in test_args: 289 test_set.update(find_tests(OPTIONS.testdir, arg)) 290 if OPTIONS.worklist: 291 try: 292 with open(OPTIONS.worklist) as f: 293 for line in f: 294 test_set.update(os.path.join(test_dir, line.strip('\n'))) 295 except IOError: 296 # With worklist, a missing file means to start the process with 297 # the complete list of tests. 298 sys.stderr.write("Couldn't read worklist file '%s'; running all tests\n" 299 % (OPTIONS.worklist,)) 300 test_set = set(find_tests(OPTIONS.testdir)) 301 if OPTIONS.read_tests: 302 try: 303 with open(OPTIONS.read_tests) as f: 304 for line in f: 305 test_set.update(os.path.join(test_dir, line.strip('\n'))) 306 except IOError as err: 307 sys.stderr.write("Error trying to read test file '%s': %s\n" 308 % (OPTIONS.read_tests, err)) 309 sys.exit(1) 310 311 # If none of the above options were passed, and no tests were listed 312 # explicitly, use the complete set. 313 if not test_args and not OPTIONS.worklist and not OPTIONS.read_tests: 314 test_set = set(find_tests(OPTIONS.testdir)) 315 316 if OPTIONS.exclude: 317 exclude_set = set() 318 for exclude in OPTIONS.exclude: 319 exclude_set.update(find_tests(test_dir, exclude)) 320 test_set -= exclude_set 321 322 if not test_set: 323 sys.stderr.write("No tests found matching command line arguments.\n") 324 sys.exit(1) 325 326 summary = Summary(len(test_set)) 327 test_list = [ Test(_, summary) for _ in sorted(test_set) ] 328 329 # Build the test executable from all the .cpp files found in the test 330 # directory tree. 331 try: 332 build_test_exec(OPTIONS.builddir) 333 except subprocess.CalledProcessError as err: 334 sys.stderr.write("Error building test executable: %s\n" % (err,)) 335 sys.exit(1) 336 337 # Run the tests. 338 try: 339 summary.start() 340 run_tests(test_list, summary) 341 summary.finish() 342 except OSError as err: 343 sys.stderr.write("Error running tests: %s\n" % (err,)) 344 sys.exit(1) 345 346 sys.exit(0) 347 348if __name__ == '__main__': 349 main(sys.argv[1:]) 350