1#!/usr/bin/env python 2# 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this 5# file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7from __future__ import absolute_import, print_function, with_statement 8import sys 9import os 10from optparse import OptionParser 11from os import environ as env 12import manifestparser 13import mozprocess 14import mozinfo 15import mozcrash 16import mozfile 17import mozlog 18import mozrunner.utils 19 20SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__))) 21 22# Export directory js/src for tests that need it. 23env["CPP_UNIT_TESTS_DIR_JS_SRC"] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..")) 24 25 26class CPPUnitTests(object): 27 # Time (seconds) to wait for test process to complete 28 TEST_PROC_TIMEOUT = 900 29 # Time (seconds) in which process will be killed if it produces no output. 30 TEST_PROC_NO_OUTPUT_TIMEOUT = 300 31 32 def run_one_test( 33 self, prog, env, symbols_path=None, interactive=False, timeout_factor=1 34 ): 35 """ 36 Run a single C++ unit test program. 37 38 Arguments: 39 * prog: The path to the test program to run. 40 * env: The environment to use for running the program. 41 * symbols_path: A path to a directory containing Breakpad-formatted 42 symbol files for producing stack traces on crash. 43 * timeout_factor: An optional test-specific timeout multiplier. 44 45 Return True if the program exits with a zero status, False otherwise. 46 """ 47 basename = os.path.basename(prog) 48 self.log.test_start(basename) 49 with mozfile.TemporaryDirectory() as tempdir: 50 if interactive: 51 # For tests run locally, via mach, print output directly 52 proc = mozprocess.ProcessHandler( 53 [prog], 54 cwd=tempdir, 55 env=env, 56 storeOutput=False, 57 universal_newlines=True, 58 ) 59 else: 60 proc = mozprocess.ProcessHandler( 61 [prog], 62 cwd=tempdir, 63 env=env, 64 storeOutput=True, 65 processOutputLine=lambda _: None, 66 universal_newlines=True, 67 ) 68 # TODO: After bug 811320 is fixed, don't let .run() kill the process, 69 # instead use a timeout in .wait() and then kill to get a stack. 70 test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor 71 proc.run( 72 timeout=test_timeout, 73 outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT, 74 ) 75 proc.wait() 76 if proc.output: 77 if self.fix_stack: 78 procOutput = [self.fix_stack(l) for l in proc.output] 79 else: 80 procOutput = proc.output 81 82 output = "\n%s" % "\n".join(procOutput) 83 self.log.process_output(proc.pid, output, command=[prog]) 84 if proc.timedOut: 85 message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT 86 self.log.test_end( 87 basename, status="TIMEOUT", expected="PASS", message=message 88 ) 89 return False 90 if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename): 91 self.log.test_end(basename, status="CRASH", expected="PASS") 92 return False 93 result = proc.proc.returncode == 0 94 if not result: 95 self.log.test_end( 96 basename, 97 status="FAIL", 98 expected="PASS", 99 message=("test failed with return code %d" % proc.proc.returncode), 100 ) 101 else: 102 self.log.test_end(basename, status="PASS", expected="PASS") 103 return result 104 105 def build_core_environment(self, env): 106 """ 107 Add environment variables likely to be used across all platforms, including remote systems. 108 """ 109 env["MOZ_XRE_DIR"] = self.xre_path 110 # TODO: switch this to just abort once all C++ unit tests have 111 # been fixed to enable crash reporting 112 env["XPCOM_DEBUG_BREAK"] = "stack-and-abort" 113 env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" 114 env["MOZ_CRASHREPORTER"] = "1" 115 return env 116 117 def build_environment(self): 118 """ 119 Create and return a dictionary of all the appropriate env variables and values. 120 On a remote system, we overload this to set different values and are missing things 121 like os.environ and PATH. 122 """ 123 if not os.path.isdir(self.xre_path): 124 raise Exception("xre_path does not exist: %s", self.xre_path) 125 env = dict(os.environ) 126 env = self.build_core_environment(env) 127 pathvar = "" 128 libpath = self.xre_path 129 if mozinfo.os == "linux": 130 pathvar = "LD_LIBRARY_PATH" 131 elif mozinfo.os == "mac": 132 applibpath = os.path.join(os.path.dirname(libpath), "MacOS") 133 if os.path.exists(applibpath): 134 # Set the library load path to Contents/MacOS if we're run from 135 # the app bundle. 136 libpath = applibpath 137 pathvar = "DYLD_LIBRARY_PATH" 138 elif mozinfo.os == "win": 139 pathvar = "PATH" 140 if pathvar: 141 if pathvar in env: 142 env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar]) 143 else: 144 env[pathvar] = libpath 145 146 if mozinfo.info["asan"]: 147 # Use llvm-symbolizer for ASan if available/required 148 llvmsym = os.path.join( 149 self.xre_path, "llvm-symbolizer" + mozinfo.info["bin_suffix"] 150 ) 151 if os.path.isfile(llvmsym): 152 env["ASAN_SYMBOLIZER_PATH"] = llvmsym 153 self.log.info("ASan using symbolizer at %s" % llvmsym) 154 else: 155 self.log.info("Failed to find ASan symbolizer at %s" % llvmsym) 156 157 # dom/media/webrtc/transport tests statically link in NSS, which 158 # causes ODR violations. See bug 1215679. 159 assert "ASAN_OPTIONS" not in env 160 env["ASAN_OPTIONS"] = "detect_leaks=0:detect_odr_violation=0" 161 162 return env 163 164 def run_tests( 165 self, 166 programs, 167 xre_path, 168 symbols_path=None, 169 utility_path=None, 170 interactive=False, 171 ): 172 """ 173 Run a set of C++ unit test programs. 174 175 Arguments: 176 * programs: An iterable containing (test path, test timeout factor) tuples 177 * xre_path: A path to a directory containing a XUL Runtime Environment. 178 * symbols_path: A path to a directory containing Breakpad-formatted 179 symbol files for producing stack traces on crash. 180 * utility_path: A path to a directory containing utility programs 181 (xpcshell et al) 182 183 Returns True if all test programs exited with a zero status, False 184 otherwise. 185 """ 186 self.xre_path = xre_path 187 self.log = mozlog.get_default_logger() 188 if utility_path: 189 self.fix_stack = mozrunner.utils.get_stack_fixer_function( 190 utility_path, symbols_path 191 ) 192 self.log.suite_start(programs, name="cppunittest") 193 env = self.build_environment() 194 pass_count = 0 195 fail_count = 0 196 for prog in programs: 197 test_path = prog[0] 198 timeout_factor = prog[1] 199 single_result = self.run_one_test( 200 test_path, env, symbols_path, interactive, timeout_factor 201 ) 202 if single_result: 203 pass_count += 1 204 else: 205 fail_count += 1 206 self.log.suite_end() 207 208 # Mozharness-parseable summary formatting. 209 self.log.info("Result summary:") 210 self.log.info("cppunittests INFO | Passed: %d" % pass_count) 211 self.log.info("cppunittests INFO | Failed: %d" % fail_count) 212 return fail_count == 0 213 214 215class CPPUnittestOptions(OptionParser): 216 def __init__(self): 217 OptionParser.__init__(self) 218 self.add_option( 219 "--xre-path", 220 action="store", 221 type="string", 222 dest="xre_path", 223 default=None, 224 help="absolute path to directory containing XRE (probably xulrunner)", 225 ) 226 self.add_option( 227 "--symbols-path", 228 action="store", 229 type="string", 230 dest="symbols_path", 231 default=None, 232 help="absolute path to directory containing breakpad symbols, or " 233 "the URL of a zip file containing symbols", 234 ) 235 self.add_option( 236 "--manifest-path", 237 action="store", 238 type="string", 239 dest="manifest_path", 240 default=None, 241 help="path to test manifest, if different from the path to test binaries", 242 ) 243 self.add_option( 244 "--utility-path", 245 action="store", 246 type="string", 247 dest="utility_path", 248 default=None, 249 help="path to directory containing utility programs", 250 ) 251 252 253def extract_unittests_from_args(args, environ, manifest_path): 254 """Extract unittests from args, expanding directories as needed""" 255 mp = manifestparser.TestManifest(strict=True) 256 tests = [] 257 binary_path = None 258 259 if manifest_path: 260 mp.read(manifest_path) 261 binary_path = os.path.abspath(args[0]) 262 else: 263 for p in args: 264 if os.path.isdir(p): 265 try: 266 mp.read(os.path.join(p, "cppunittest.ini")) 267 except IOError: 268 files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)] 269 tests.extend( 270 (f, 1) for f in files if os.access(f, os.R_OK | os.X_OK) 271 ) 272 else: 273 tests.append((os.path.abspath(p), 1)) 274 275 # we skip the existence check here because not all tests are built 276 # for all platforms (and it will fail on Windows anyway) 277 active_tests = mp.active_tests(exists=False, disabled=False, **environ) 278 suffix = ".exe" if mozinfo.isWin else "" 279 if binary_path: 280 tests.extend( 281 [ 282 ( 283 os.path.join(binary_path, test["relpath"] + suffix), 284 int(test.get("requesttimeoutfactor", 1)), 285 ) 286 for test in active_tests 287 ] 288 ) 289 else: 290 tests.extend( 291 [ 292 (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1))) 293 for test in active_tests 294 ] 295 ) 296 297 # skip and warn for any tests in the manifest that are not found 298 final_tests = [] 299 log = mozlog.get_default_logger() 300 for test in tests: 301 if os.path.isfile(test[0]): 302 final_tests.append(test) 303 else: 304 log.warning("test file not found: %s - skipped" % test[0]) 305 306 return final_tests 307 308 309def update_mozinfo(): 310 """walk up directories to find mozinfo.json update the info""" 311 path = SCRIPT_DIR 312 dirs = set() 313 while path != os.path.expanduser("~"): 314 if path in dirs: 315 break 316 dirs.add(path) 317 path = os.path.split(path)[0] 318 mozinfo.find_and_update_from_json(*dirs) 319 320 321def run_test_harness(options, args): 322 update_mozinfo() 323 progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path) 324 options.xre_path = os.path.abspath(options.xre_path) 325 options.utility_path = os.path.abspath(options.utility_path) 326 tester = CPPUnitTests() 327 result = tester.run_tests( 328 progs, 329 options.xre_path, 330 options.symbols_path, 331 options.utility_path, 332 ) 333 334 return result 335 336 337def main(): 338 parser = CPPUnittestOptions() 339 mozlog.commandline.add_logging_group(parser) 340 options, args = parser.parse_args() 341 if not args: 342 print( 343 """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0], 344 file=sys.stderr, 345 ) 346 sys.exit(1) 347 if not options.xre_path: 348 print("""Error: --xre-path is required""", file=sys.stderr) 349 sys.exit(1) 350 if options.manifest_path and len(args) > 1: 351 print( 352 "Error: multiple arguments not supported with --test-manifest", 353 file=sys.stderr, 354 ) 355 sys.exit(1) 356 log = mozlog.commandline.setup_logging( 357 "cppunittests", options, {"tbpl": sys.stdout} 358 ) 359 try: 360 result = run_test_harness(options, args) 361 except Exception as e: 362 log.error(str(e)) 363 result = False 364 365 sys.exit(0 if result else 1) 366 367 368if __name__ == "__main__": 369 main() 370