1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import
6
7import os
8from functools import partial
9
10from manifestparser import TestManifest
11
12import mozunit
13import pytest
14from moztest.selftest.output import get_mozharness_status, filter_action
15from conftest import setup_args
16
17from mozharness.base.log import INFO, WARNING, ERROR
18from mozharness.mozilla.automation import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
19
20
21here = os.path.abspath(os.path.dirname(__file__))
22get_mozharness_status = partial(get_mozharness_status, "mochitest")
23
24
25@pytest.fixture
26def test_name(request):
27    flavor = request.getfixturevalue("flavor")
28
29    def inner(name):
30        if flavor == "plain":
31            return f"test_{name}.html"
32        elif flavor == "browser-chrome":
33            return f"browser_{name}.js"
34
35    return inner
36
37
38@pytest.fixture
39def test_manifest(setup_test_harness, request):
40    flavor = request.getfixturevalue("flavor")
41    test_root = setup_test_harness(*setup_args, flavor=flavor)
42    assert test_root
43
44    def inner(manifestFileNames):
45        return TestManifest(
46            manifests=[os.path.join(test_root, name) for name in manifestFileNames],
47            strict=False,
48            rootdir=test_root,
49        )
50
51    return inner
52
53
54@pytest.mark.parametrize("runFailures", ["selftest", ""])
55@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
56def test_output_pass(flavor, runFailures, runtests, test_name):
57    extra_opts = {}
58    results = {
59        "status": 1 if runFailures else 0,
60        "tbpl_status": TBPL_WARNING if runFailures else TBPL_SUCCESS,
61        "log_level": (INFO, WARNING),
62        "lines": 2 if runFailures else 1,
63        "line_status": "PASS",
64    }
65    if runFailures:
66        extra_opts["runFailures"] = runFailures
67        extra_opts["crashAsPass"] = True
68        extra_opts["timeoutAsPass"] = True
69
70    status, lines = runtests(test_name("pass"), **extra_opts)
71    assert status == results["status"]
72
73    tbpl_status, log_level, summary = get_mozharness_status(lines, status)
74    assert tbpl_status == results["tbpl_status"]
75    assert log_level in results["log_level"]
76
77    lines = filter_action("test_status", lines)
78    assert len(lines) == results["lines"]
79    assert lines[0]["status"] == results["line_status"]
80
81
82@pytest.mark.parametrize("runFailures", ["selftest", ""])
83@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
84def test_output_fail(flavor, runFailures, runtests, test_name):
85    extra_opts = {}
86    results = {
87        "status": 0 if runFailures else 1,
88        "tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING,
89        "log_level": (INFO, WARNING),
90        "lines": 1,
91        "line_status": "PASS" if runFailures else "FAIL",
92    }
93    if runFailures:
94        extra_opts["runFailures"] = runFailures
95        extra_opts["crashAsPass"] = True
96        extra_opts["timeoutAsPass"] = True
97
98    status, lines = runtests(test_name("fail"), **extra_opts)
99    assert status == results["status"]
100
101    tbpl_status, log_level, summary = get_mozharness_status(lines, status)
102    assert tbpl_status == results["tbpl_status"]
103    assert log_level in results["log_level"]
104
105    lines = filter_action("test_status", lines)
106    assert len(lines) == results["lines"]
107    assert lines[0]["status"] == results["line_status"]
108
109
110@pytest.mark.skip_mozinfo("!crashreporter")
111@pytest.mark.parametrize("runFailures", ["selftest", ""])
112@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
113def test_output_crash(flavor, runFailures, runtests, test_name):
114    extra_opts = {}
115    results = {
116        "status": 0 if runFailures else 1,
117        "tbpl_status": TBPL_FAILURE,
118        "log_level": ERROR,
119        "lines": 1 if runFailures else 0,
120    }
121    if runFailures:
122        extra_opts["runFailures"] = runFailures
123        extra_opts["crashAsPass"] = True
124        extra_opts["timeoutAsPass"] = True
125        # bug 1443327 - we do not set MOZ_CRASHREPORTER_SHUTDOWN for browser-chrome
126        # the error regex's don't pick this up as a failure
127        if flavor == "browser-chrome":
128            results["tbpl_status"] = TBPL_SUCCESS
129            results["log_level"] = (INFO, WARNING)
130
131    status, lines = runtests(
132        test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts
133    )
134    assert status == results["status"]
135
136    tbpl_status, log_level, summary = get_mozharness_status(lines, status)
137    assert tbpl_status == results["tbpl_status"]
138    assert log_level in results["log_level"]
139
140    if not runFailures:
141        crash = filter_action("crash", lines)
142        assert len(crash) == 1
143        assert crash[0]["action"] == "crash"
144        assert crash[0]["signature"]
145        assert crash[0]["minidump_path"]
146
147    lines = filter_action("test_end", lines)
148    assert len(lines) == results["lines"]
149
150
151@pytest.mark.skip_mozinfo("!asan")
152@pytest.mark.parametrize("runFailures", [""])
153@pytest.mark.parametrize("flavor", ["plain"])
154def test_output_asan(flavor, runFailures, runtests, test_name):
155    extra_opts = {}
156    results = {"status": 1, "tbpl_status": TBPL_FAILURE, "log_level": ERROR, "lines": 0}
157
158    status, lines = runtests(
159        test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts
160    )
161    assert status == results["status"]
162
163    tbpl_status, log_level, summary = get_mozharness_status(lines, status)
164    assert tbpl_status == results["tbpl_status"]
165    assert log_level == results["log_level"]
166
167    crash = filter_action("crash", lines)
168    assert len(crash) == results["lines"]
169
170    process_output = filter_action("process_output", lines)
171    assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output)
172
173
174@pytest.mark.skip_mozinfo("!debug")
175@pytest.mark.parametrize("runFailures", [""])
176@pytest.mark.parametrize("flavor", ["plain"])
177def test_output_assertion(flavor, runFailures, runtests, test_name):
178    extra_opts = {}
179    results = {
180        "status": 0,
181        "tbpl_status": TBPL_WARNING,
182        "log_level": WARNING,
183        "lines": 1,
184        "assertions": 1,
185    }
186
187    status, lines = runtests(test_name("assertion"), **extra_opts)
188    # TODO: mochitest should return non-zero here
189    assert status == results["status"]
190
191    tbpl_status, log_level, summary = get_mozharness_status(lines, status)
192    assert tbpl_status == results["tbpl_status"]
193    assert log_level == results["log_level"]
194
195    test_end = filter_action("test_end", lines)
196    assert len(test_end) == results["lines"]
197    # TODO: this should be ASSERT, but moving the assertion check before
198    # the test_end action caused a bunch of failures.
199    assert test_end[0]["status"] == "OK"
200
201    assertions = filter_action("assertion_count", lines)
202    assert len(assertions) == results["assertions"]
203    assert assertions[0]["count"] == results["assertions"]
204
205
206@pytest.mark.skip_mozinfo("!debug")
207@pytest.mark.parametrize("runFailures", [""])
208@pytest.mark.parametrize("flavor", ["plain"])
209def test_output_leak(flavor, runFailures, runtests, test_name):
210    extra_opts = {}
211    results = {"status": 0, "tbpl_status": TBPL_WARNING, "log_level": WARNING}
212
213    status, lines = runtests(test_name("leak"), **extra_opts)
214    # TODO: mochitest should return non-zero here
215    assert status == results["status"]
216
217    tbpl_status, log_level, summary = get_mozharness_status(lines, status)
218    assert tbpl_status == results["tbpl_status"]
219    assert log_level == results["log_level"]
220
221    leak_totals = filter_action("mozleak_total", lines)
222    found_leaks = False
223    for lt in leak_totals:
224        if lt["bytes"] == 0:
225            # No leaks in this process.
226            assert len(lt["objects"]) == 0
227            continue
228
229        assert not found_leaks, "Only one process should have leaked"
230        found_leaks = True
231        assert lt["process"] == "tab"
232        assert lt["bytes"] == 1
233        assert lt["objects"] == ["IntentionallyLeakedObject"]
234
235    assert found_leaks, "At least one process should have leaked"
236
237
238@pytest.mark.parametrize("flavor", ["plain"])
239def test_output_testfile_in_dupe_manifests(flavor, runtests, test_name, test_manifest):
240    results = {
241        "status": 0,
242        "tbpl_status": TBPL_SUCCESS,
243        "log_level": (INFO, WARNING),
244        "line_status": "PASS",
245        # We expect the test to be executed exactly 2 times,
246        # once for each manifest where the test file has been included.
247        "lines": 2,
248    }
249
250    # Explicitly provide a manifestFile property that includes the
251    # two manifest files that share the same test file.
252    extra_opts = {
253        "manifestFile": test_manifest(
254            [
255                "mochitest-dupemanifest-1.ini",
256                "mochitest-dupemanifest-2.ini",
257            ]
258        ),
259        "runByManifest": True,
260    }
261
262    # Execute mochitest by explicitly request the test file listed
263    # in two manifest files to be executed.
264    status, lines = runtests(test_name("pass"), **extra_opts)
265    assert status == results["status"]
266
267    tbpl_status, log_level, summary = get_mozharness_status(lines, status)
268    assert tbpl_status == results["tbpl_status"]
269    assert log_level in results["log_level"]
270
271    lines = filter_action("test_status", lines)
272    assert len(lines) == results["lines"]
273    assert lines[0]["status"] == results["line_status"]
274
275
276if __name__ == "__main__":
277    mozunit.main()
278