1#!/usr/bin/env python
2#
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this
5# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7from __future__ import absolute_import, print_function, with_statement
8import sys
9import os
10from optparse import OptionParser
11from os import environ as env
12import manifestparser
13import mozprocess
14import mozinfo
15import mozcrash
16import mozfile
17import mozlog
18import mozrunner.utils
19
20SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
21
22# Export directory js/src for tests that need it.
23env["CPP_UNIT_TESTS_DIR_JS_SRC"] = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
24
25
26class CPPUnitTests(object):
27    # Time (seconds) to wait for test process to complete
28    TEST_PROC_TIMEOUT = 900
29    # Time (seconds) in which process will be killed if it produces no output.
30    TEST_PROC_NO_OUTPUT_TIMEOUT = 300
31
32    def run_one_test(
33        self, prog, env, symbols_path=None, interactive=False, timeout_factor=1
34    ):
35        """
36        Run a single C++ unit test program.
37
38        Arguments:
39        * prog: The path to the test program to run.
40        * env: The environment to use for running the program.
41        * symbols_path: A path to a directory containing Breakpad-formatted
42                        symbol files for producing stack traces on crash.
43        * timeout_factor: An optional test-specific timeout multiplier.
44
45        Return True if the program exits with a zero status, False otherwise.
46        """
47        basename = os.path.basename(prog)
48        self.log.test_start(basename)
49        with mozfile.TemporaryDirectory() as tempdir:
50            if interactive:
51                # For tests run locally, via mach, print output directly
52                proc = mozprocess.ProcessHandler(
53                    [prog],
54                    cwd=tempdir,
55                    env=env,
56                    storeOutput=False,
57                    universal_newlines=True,
58                )
59            else:
60                proc = mozprocess.ProcessHandler(
61                    [prog],
62                    cwd=tempdir,
63                    env=env,
64                    storeOutput=True,
65                    processOutputLine=lambda _: None,
66                    universal_newlines=True,
67                )
68            # TODO: After bug 811320 is fixed, don't let .run() kill the process,
69            # instead use a timeout in .wait() and then kill to get a stack.
70            test_timeout = CPPUnitTests.TEST_PROC_TIMEOUT * timeout_factor
71            proc.run(
72                timeout=test_timeout,
73                outputTimeout=CPPUnitTests.TEST_PROC_NO_OUTPUT_TIMEOUT,
74            )
75            proc.wait()
76            if proc.output:
77                if self.fix_stack:
78                    procOutput = [self.fix_stack(l) for l in proc.output]
79                else:
80                    procOutput = proc.output
81
82                output = "\n%s" % "\n".join(procOutput)
83                self.log.process_output(proc.pid, output, command=[prog])
84            if proc.timedOut:
85                message = "timed out after %d seconds" % CPPUnitTests.TEST_PROC_TIMEOUT
86                self.log.test_end(
87                    basename, status="TIMEOUT", expected="PASS", message=message
88                )
89                return False
90            if mozcrash.check_for_crashes(tempdir, symbols_path, test_name=basename):
91                self.log.test_end(basename, status="CRASH", expected="PASS")
92                return False
93            result = proc.proc.returncode == 0
94            if not result:
95                self.log.test_end(
96                    basename,
97                    status="FAIL",
98                    expected="PASS",
99                    message=("test failed with return code %d" % proc.proc.returncode),
100                )
101            else:
102                self.log.test_end(basename, status="PASS", expected="PASS")
103            return result
104
105    def build_core_environment(self, env):
106        """
107        Add environment variables likely to be used across all platforms, including remote systems.
108        """
109        env["MOZ_XRE_DIR"] = self.xre_path
110        # TODO: switch this to just abort once all C++ unit tests have
111        # been fixed to enable crash reporting
112        env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
113        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
114        env["MOZ_CRASHREPORTER"] = "1"
115        return env
116
117    def build_environment(self):
118        """
119        Create and return a dictionary of all the appropriate env variables and values.
120        On a remote system, we overload this to set different values and are missing things
121        like os.environ and PATH.
122        """
123        if not os.path.isdir(self.xre_path):
124            raise Exception("xre_path does not exist: %s", self.xre_path)
125        env = dict(os.environ)
126        env = self.build_core_environment(env)
127        pathvar = ""
128        libpath = self.xre_path
129        if mozinfo.os == "linux":
130            pathvar = "LD_LIBRARY_PATH"
131        elif mozinfo.os == "mac":
132            applibpath = os.path.join(os.path.dirname(libpath), "MacOS")
133            if os.path.exists(applibpath):
134                # Set the library load path to Contents/MacOS if we're run from
135                # the app bundle.
136                libpath = applibpath
137            pathvar = "DYLD_LIBRARY_PATH"
138        elif mozinfo.os == "win":
139            pathvar = "PATH"
140        if pathvar:
141            if pathvar in env:
142                env[pathvar] = "%s%s%s" % (libpath, os.pathsep, env[pathvar])
143            else:
144                env[pathvar] = libpath
145
146        if mozinfo.info["asan"]:
147            # Use llvm-symbolizer for ASan if available/required
148            llvmsym = os.path.join(
149                self.xre_path, "llvm-symbolizer" + mozinfo.info["bin_suffix"]
150            )
151            if os.path.isfile(llvmsym):
152                env["ASAN_SYMBOLIZER_PATH"] = llvmsym
153                self.log.info("ASan using symbolizer at %s" % llvmsym)
154            else:
155                self.log.info("Failed to find ASan symbolizer at %s" % llvmsym)
156
157            # dom/media/webrtc/transport tests statically link in NSS, which
158            # causes ODR violations. See bug 1215679.
159            assert "ASAN_OPTIONS" not in env
160            env["ASAN_OPTIONS"] = "detect_leaks=0:detect_odr_violation=0"
161
162        return env
163
164    def run_tests(
165        self,
166        programs,
167        xre_path,
168        symbols_path=None,
169        utility_path=None,
170        interactive=False,
171    ):
172        """
173        Run a set of C++ unit test programs.
174
175        Arguments:
176        * programs: An iterable containing (test path, test timeout factor) tuples
177        * xre_path: A path to a directory containing a XUL Runtime Environment.
178        * symbols_path: A path to a directory containing Breakpad-formatted
179                        symbol files for producing stack traces on crash.
180        * utility_path: A path to a directory containing utility programs
181                        (xpcshell et al)
182
183        Returns True if all test programs exited with a zero status, False
184        otherwise.
185        """
186        self.xre_path = xre_path
187        self.log = mozlog.get_default_logger()
188        if utility_path:
189            self.fix_stack = mozrunner.utils.get_stack_fixer_function(
190                utility_path, symbols_path
191            )
192        self.log.suite_start(programs, name="cppunittest")
193        env = self.build_environment()
194        pass_count = 0
195        fail_count = 0
196        for prog in programs:
197            test_path = prog[0]
198            timeout_factor = prog[1]
199            single_result = self.run_one_test(
200                test_path, env, symbols_path, interactive, timeout_factor
201            )
202            if single_result:
203                pass_count += 1
204            else:
205                fail_count += 1
206        self.log.suite_end()
207
208        # Mozharness-parseable summary formatting.
209        self.log.info("Result summary:")
210        self.log.info("cppunittests INFO | Passed: %d" % pass_count)
211        self.log.info("cppunittests INFO | Failed: %d" % fail_count)
212        return fail_count == 0
213
214
215class CPPUnittestOptions(OptionParser):
216    def __init__(self):
217        OptionParser.__init__(self)
218        self.add_option(
219            "--xre-path",
220            action="store",
221            type="string",
222            dest="xre_path",
223            default=None,
224            help="absolute path to directory containing XRE (probably xulrunner)",
225        )
226        self.add_option(
227            "--symbols-path",
228            action="store",
229            type="string",
230            dest="symbols_path",
231            default=None,
232            help="absolute path to directory containing breakpad symbols, or "
233            "the URL of a zip file containing symbols",
234        )
235        self.add_option(
236            "--manifest-path",
237            action="store",
238            type="string",
239            dest="manifest_path",
240            default=None,
241            help="path to test manifest, if different from the path to test binaries",
242        )
243        self.add_option(
244            "--utility-path",
245            action="store",
246            type="string",
247            dest="utility_path",
248            default=None,
249            help="path to directory containing utility programs",
250        )
251
252
253def extract_unittests_from_args(args, environ, manifest_path):
254    """Extract unittests from args, expanding directories as needed"""
255    mp = manifestparser.TestManifest(strict=True)
256    tests = []
257    binary_path = None
258
259    if manifest_path:
260        mp.read(manifest_path)
261        binary_path = os.path.abspath(args[0])
262    else:
263        for p in args:
264            if os.path.isdir(p):
265                try:
266                    mp.read(os.path.join(p, "cppunittest.ini"))
267                except IOError:
268                    files = [os.path.abspath(os.path.join(p, x)) for x in os.listdir(p)]
269                    tests.extend(
270                        (f, 1) for f in files if os.access(f, os.R_OK | os.X_OK)
271                    )
272            else:
273                tests.append((os.path.abspath(p), 1))
274
275    # we skip the existence check here because not all tests are built
276    # for all platforms (and it will fail on Windows anyway)
277    active_tests = mp.active_tests(exists=False, disabled=False, **environ)
278    suffix = ".exe" if mozinfo.isWin else ""
279    if binary_path:
280        tests.extend(
281            [
282                (
283                    os.path.join(binary_path, test["relpath"] + suffix),
284                    int(test.get("requesttimeoutfactor", 1)),
285                )
286                for test in active_tests
287            ]
288        )
289    else:
290        tests.extend(
291            [
292                (test["path"] + suffix, int(test.get("requesttimeoutfactor", 1)))
293                for test in active_tests
294            ]
295        )
296
297    # skip and warn for any tests in the manifest that are not found
298    final_tests = []
299    log = mozlog.get_default_logger()
300    for test in tests:
301        if os.path.isfile(test[0]):
302            final_tests.append(test)
303        else:
304            log.warning("test file not found: %s - skipped" % test[0])
305
306    return final_tests
307
308
309def update_mozinfo():
310    """walk up directories to find mozinfo.json update the info"""
311    path = SCRIPT_DIR
312    dirs = set()
313    while path != os.path.expanduser("~"):
314        if path in dirs:
315            break
316        dirs.add(path)
317        path = os.path.split(path)[0]
318    mozinfo.find_and_update_from_json(*dirs)
319
320
321def run_test_harness(options, args):
322    update_mozinfo()
323    progs = extract_unittests_from_args(args, mozinfo.info, options.manifest_path)
324    options.xre_path = os.path.abspath(options.xre_path)
325    options.utility_path = os.path.abspath(options.utility_path)
326    tester = CPPUnitTests()
327    result = tester.run_tests(
328        progs,
329        options.xre_path,
330        options.symbols_path,
331        options.utility_path,
332    )
333
334    return result
335
336
337def main():
338    parser = CPPUnittestOptions()
339    mozlog.commandline.add_logging_group(parser)
340    options, args = parser.parse_args()
341    if not args:
342        print(
343            """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0],
344            file=sys.stderr,
345        )
346        sys.exit(1)
347    if not options.xre_path:
348        print("""Error: --xre-path is required""", file=sys.stderr)
349        sys.exit(1)
350    if options.manifest_path and len(args) > 1:
351        print(
352            "Error: multiple arguments not supported with --test-manifest",
353            file=sys.stderr,
354        )
355        sys.exit(1)
356    log = mozlog.commandline.setup_logging(
357        "cppunittests", options, {"tbpl": sys.stdout}
358    )
359    try:
360        result = run_test_harness(options, args)
361    except Exception as e:
362        log.error(str(e))
363        result = False
364
365    sys.exit(0 if result else 1)
366
367
368if __name__ == "__main__":
369    main()
370