1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import 6 7import os 8from functools import partial 9 10from manifestparser import TestManifest 11 12import mozunit 13import pytest 14from moztest.selftest.output import get_mozharness_status, filter_action 15from conftest import setup_args 16 17from mozharness.base.log import INFO, WARNING, ERROR 18from mozharness.mozilla.automation import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE 19 20 21here = os.path.abspath(os.path.dirname(__file__)) 22get_mozharness_status = partial(get_mozharness_status, "mochitest") 23 24 25@pytest.fixture 26def test_name(request): 27 flavor = request.getfixturevalue("flavor") 28 29 def inner(name): 30 if flavor == "plain": 31 return f"test_{name}.html" 32 elif flavor == "browser-chrome": 33 return f"browser_{name}.js" 34 35 return inner 36 37 38@pytest.fixture 39def test_manifest(setup_test_harness, request): 40 flavor = request.getfixturevalue("flavor") 41 test_root = setup_test_harness(*setup_args, flavor=flavor) 42 assert test_root 43 44 def inner(manifestFileNames): 45 return TestManifest( 46 manifests=[os.path.join(test_root, name) for name in manifestFileNames], 47 strict=False, 48 rootdir=test_root, 49 ) 50 51 return inner 52 53 54@pytest.mark.parametrize("runFailures", ["selftest", ""]) 55@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) 56def test_output_pass(flavor, runFailures, runtests, test_name): 57 extra_opts = {} 58 results = { 59 "status": 1 if runFailures else 0, 60 "tbpl_status": TBPL_WARNING if runFailures else TBPL_SUCCESS, 61 "log_level": (INFO, WARNING), 62 "lines": 2 if runFailures else 1, 63 "line_status": "PASS", 64 } 65 if runFailures: 66 extra_opts["runFailures"] = runFailures 67 extra_opts["crashAsPass"] = True 68 extra_opts["timeoutAsPass"] = True 69 70 status, lines = runtests(test_name("pass"), **extra_opts) 71 assert status == results["status"] 72 73 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 74 assert tbpl_status == results["tbpl_status"] 75 assert log_level in results["log_level"] 76 77 lines = filter_action("test_status", lines) 78 assert len(lines) == results["lines"] 79 assert lines[0]["status"] == results["line_status"] 80 81 82@pytest.mark.parametrize("runFailures", ["selftest", ""]) 83@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) 84def test_output_fail(flavor, runFailures, runtests, test_name): 85 extra_opts = {} 86 results = { 87 "status": 0 if runFailures else 1, 88 "tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING, 89 "log_level": (INFO, WARNING), 90 "lines": 1, 91 "line_status": "PASS" if runFailures else "FAIL", 92 } 93 if runFailures: 94 extra_opts["runFailures"] = runFailures 95 extra_opts["crashAsPass"] = True 96 extra_opts["timeoutAsPass"] = True 97 98 status, lines = runtests(test_name("fail"), **extra_opts) 99 assert status == results["status"] 100 101 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 102 assert tbpl_status == results["tbpl_status"] 103 assert log_level in results["log_level"] 104 105 lines = filter_action("test_status", lines) 106 assert len(lines) == results["lines"] 107 assert lines[0]["status"] == results["line_status"] 108 109 110@pytest.mark.skip_mozinfo("!crashreporter") 111@pytest.mark.parametrize("runFailures", ["selftest", ""]) 112@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) 113def test_output_crash(flavor, runFailures, runtests, test_name): 114 extra_opts = {} 115 results = { 116 "status": 0 if runFailures else 1, 117 "tbpl_status": TBPL_FAILURE, 118 "log_level": ERROR, 119 "lines": 1 if runFailures else 0, 120 } 121 if runFailures: 122 extra_opts["runFailures"] = runFailures 123 extra_opts["crashAsPass"] = True 124 extra_opts["timeoutAsPass"] = True 125 # bug 1443327 - we do not set MOZ_CRASHREPORTER_SHUTDOWN for browser-chrome 126 # the error regex's don't pick this up as a failure 127 if flavor == "browser-chrome": 128 results["tbpl_status"] = TBPL_SUCCESS 129 results["log_level"] = (INFO, WARNING) 130 131 status, lines = runtests( 132 test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts 133 ) 134 assert status == results["status"] 135 136 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 137 assert tbpl_status == results["tbpl_status"] 138 assert log_level in results["log_level"] 139 140 if not runFailures: 141 crash = filter_action("crash", lines) 142 assert len(crash) == 1 143 assert crash[0]["action"] == "crash" 144 assert crash[0]["signature"] 145 assert crash[0]["minidump_path"] 146 147 lines = filter_action("test_end", lines) 148 assert len(lines) == results["lines"] 149 150 151@pytest.mark.skip_mozinfo("!asan") 152@pytest.mark.parametrize("runFailures", [""]) 153@pytest.mark.parametrize("flavor", ["plain"]) 154def test_output_asan(flavor, runFailures, runtests, test_name): 155 extra_opts = {} 156 results = {"status": 1, "tbpl_status": TBPL_FAILURE, "log_level": ERROR, "lines": 0} 157 158 status, lines = runtests( 159 test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts 160 ) 161 assert status == results["status"] 162 163 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 164 assert tbpl_status == results["tbpl_status"] 165 assert log_level == results["log_level"] 166 167 crash = filter_action("crash", lines) 168 assert len(crash) == results["lines"] 169 170 process_output = filter_action("process_output", lines) 171 assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output) 172 173 174@pytest.mark.skip_mozinfo("!debug") 175@pytest.mark.parametrize("runFailures", [""]) 176@pytest.mark.parametrize("flavor", ["plain"]) 177def test_output_assertion(flavor, runFailures, runtests, test_name): 178 extra_opts = {} 179 results = { 180 "status": 0, 181 "tbpl_status": TBPL_WARNING, 182 "log_level": WARNING, 183 "lines": 1, 184 "assertions": 1, 185 } 186 187 status, lines = runtests(test_name("assertion"), **extra_opts) 188 # TODO: mochitest should return non-zero here 189 assert status == results["status"] 190 191 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 192 assert tbpl_status == results["tbpl_status"] 193 assert log_level == results["log_level"] 194 195 test_end = filter_action("test_end", lines) 196 assert len(test_end) == results["lines"] 197 # TODO: this should be ASSERT, but moving the assertion check before 198 # the test_end action caused a bunch of failures. 199 assert test_end[0]["status"] == "OK" 200 201 assertions = filter_action("assertion_count", lines) 202 assert len(assertions) == results["assertions"] 203 assert assertions[0]["count"] == results["assertions"] 204 205 206@pytest.mark.skip_mozinfo("!debug") 207@pytest.mark.parametrize("runFailures", [""]) 208@pytest.mark.parametrize("flavor", ["plain"]) 209def test_output_leak(flavor, runFailures, runtests, test_name): 210 extra_opts = {} 211 results = {"status": 0, "tbpl_status": TBPL_WARNING, "log_level": WARNING} 212 213 status, lines = runtests(test_name("leak"), **extra_opts) 214 # TODO: mochitest should return non-zero here 215 assert status == results["status"] 216 217 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 218 assert tbpl_status == results["tbpl_status"] 219 assert log_level == results["log_level"] 220 221 leak_totals = filter_action("mozleak_total", lines) 222 found_leaks = False 223 for lt in leak_totals: 224 if lt["bytes"] == 0: 225 # No leaks in this process. 226 assert len(lt["objects"]) == 0 227 continue 228 229 assert not found_leaks, "Only one process should have leaked" 230 found_leaks = True 231 assert lt["process"] == "tab" 232 assert lt["bytes"] == 1 233 assert lt["objects"] == ["IntentionallyLeakedObject"] 234 235 assert found_leaks, "At least one process should have leaked" 236 237 238@pytest.mark.parametrize("flavor", ["plain"]) 239def test_output_testfile_in_dupe_manifests(flavor, runtests, test_name, test_manifest): 240 results = { 241 "status": 0, 242 "tbpl_status": TBPL_SUCCESS, 243 "log_level": (INFO, WARNING), 244 "line_status": "PASS", 245 # We expect the test to be executed exactly 2 times, 246 # once for each manifest where the test file has been included. 247 "lines": 2, 248 } 249 250 # Explicitly provide a manifestFile property that includes the 251 # two manifest files that share the same test file. 252 extra_opts = { 253 "manifestFile": test_manifest( 254 [ 255 "mochitest-dupemanifest-1.ini", 256 "mochitest-dupemanifest-2.ini", 257 ] 258 ), 259 "runByManifest": True, 260 } 261 262 # Execute mochitest by explicitly request the test file listed 263 # in two manifest files to be executed. 264 status, lines = runtests(test_name("pass"), **extra_opts) 265 assert status == results["status"] 266 267 tbpl_status, log_level, summary = get_mozharness_status(lines, status) 268 assert tbpl_status == results["tbpl_status"] 269 assert log_level in results["log_level"] 270 271 lines = filter_action("test_status", lines) 272 assert len(lines) == results["lines"] 273 assert lines[0]["status"] == results["line_status"] 274 275 276if __name__ == "__main__": 277 mozunit.main() 278