1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4# flake8: noqa: E501 5 6from __future__ import absolute_import, print_function, unicode_literals 7 8try: 9 import cPickle as pickle 10except ImportError: 11 import pickle 12 13import json 14import os 15import re 16import shutil 17import tempfile 18from collections import defaultdict 19 20import pytest 21import mozpack.path as mozpath 22import mozunit 23from mozbuild.base import MozbuildObject 24from mozbuild.frontend.reader import BuildReader 25from mozbuild.test.common import MockConfig 26from mozfile import NamedTemporaryFile 27 28from moztest.resolve import ( 29 BuildBackendLoader, 30 TestManifestLoader, 31 TestResolver, 32 TEST_SUITES, 33) 34 35here = os.path.abspath(os.path.dirname(__file__)) 36data_path = os.path.join(here, 'data') 37 38 39@pytest.fixture(scope='module') 40def topsrcdir(): 41 return mozpath.join(data_path, 'srcdir') 42 43 44@pytest.fixture(scope='module') 45def create_tests(topsrcdir): 46 47 def inner(*paths, **defaults): 48 tests = defaultdict(list) 49 for path in paths: 50 if isinstance(path, tuple): 51 path, kwargs = path 52 else: 53 kwargs = {} 54 55 path = mozpath.normpath(path) 56 manifest_name = kwargs.get('flavor', defaults.get('flavor', 'manifest')) 57 manifest = kwargs.pop('manifest', defaults.pop('manifest', 58 mozpath.join(mozpath.dirname(path), manifest_name + '.ini'))) 59 60 manifest_abspath = mozpath.join(topsrcdir, manifest) 61 relpath = mozpath.relpath(path, mozpath.dirname(manifest)) 62 test = { 63 'name': relpath, 64 'path': mozpath.join(topsrcdir, path), 65 'relpath': relpath, 66 'file_relpath': path, 67 'dir_relpath': mozpath.dirname(path), 68 'here': mozpath.dirname(manifest_abspath), 69 'manifest': manifest_abspath, 70 'manifest_relpath': manifest, 71 } 72 test.update(**defaults) 73 test.update(**kwargs) 74 75 # Normalize paths to ensure that the fixture matches reality. 76 for k in ['ancestor_manifest', 'manifest', 'manifest_relpath', 'path', 'relpath']: 77 p = test.get(k) 78 if p: 79 test[k] = p.replace('/', os.path.sep) 80 81 tests[path].append(test) 82 83 # dump tests to stdout for easier debugging on failure 84 print("The 'create_tests' fixture returned:") 85 print(json.dumps(dict(tests), indent=2, sort_keys=True)) 86 return tests 87 88 return inner 89 90 91@pytest.fixture(scope='module') 92def all_tests(create_tests): 93 return create_tests(*[ 94 ("apple/test_a11y.html", { 95 "expected": "pass", 96 "flavor": "a11y", 97 }), 98 ("banana/currant/test_xpcshell_A.js", { 99 "firefox-appdir": "browser", 100 "flavor": "xpcshell", 101 "head": "head_global.js head_helpers.js head_http.js", 102 }), 103 ("banana/currant/test_xpcshell_B.js", { 104 "firefox-appdir": "browser", 105 "flavor": "xpcshell", 106 "head": "head_global.js head_helpers.js head_http.js", 107 }), 108 ("carrot/test_included.js", { 109 "ancestor_manifest": "carrot/xpcshell-one.ini", 110 "manifest": "carrot/xpcshell-shared.ini", 111 "flavor": "xpcshell", 112 "stick": "one", 113 }), 114 ("carrot/test_included.js", { 115 "ancestor_manifest": "carrot/xpcshell-two.ini", 116 "manifest": "carrot/xpcshell-shared.ini", 117 "flavor": "xpcshell", 118 "stick": "two", 119 }), 120 ("dragonfruit/elderberry/test_xpcshell_C.js", { 121 "flavor": "xpcshell", 122 "generated-files": "head_update.js", 123 "head": "head_update.js", 124 "manifest": "dragonfruit/xpcshell.ini", 125 "reason": "busted", 126 "run-sequentially": "Launches application.", 127 "skip-if": "os == 'android'", 128 }), 129 ("dragonfruit/elderberry/test_xpcshell_C.js", { 130 "flavor": "xpcshell", 131 "generated-files": "head_update.js", 132 "head": "head_update.js head2.js", 133 "manifest": "dragonfruit/elderberry/xpcshell_updater.ini", 134 "reason": "don't work", 135 "run-sequentially": "Launches application.", 136 "skip-if": "os == 'android'", 137 }), 138 ("fig/grape/src/TestInstrumentationA.java", { 139 "flavor": "instrumentation", 140 "manifest": "fig/grape/instrumentation.ini", 141 "subsuite": "background", 142 }), 143 ("fig/huckleberry/src/TestInstrumentationB.java", { 144 "flavor": "instrumentation", 145 "manifest": "fig/huckleberry/instrumentation.ini", 146 "subsuite": "browser", 147 }), 148 ("juniper/browser_chrome.js", { 149 "flavor": "browser-chrome", 150 "manifest": "juniper/browser.ini", 151 "skip-if": "e10s # broken", 152 }), 153 ("kiwi/browser_devtools.js", { 154 "flavor": "browser-chrome", 155 "manifest": "kiwi/browser.ini", 156 "subsuite": "devtools", 157 "tags": "devtools", 158 }), 159 ]) 160 161 162@pytest.fixture(scope='module') 163def defaults(topsrcdir): 164 def to_abspath(relpath): 165 # test-defaults.pkl uses absolute paths with platform-specific path separators. 166 # Use platform-specific separators if needed to avoid regressing on bug 1644223. 167 return os.path.normpath(os.path.join(topsrcdir, relpath)) 168 169 return { 170 (to_abspath("dragonfruit/elderberry/xpcshell_updater.ini")): { 171 "support-files": "\ndata/**\nxpcshell_updater.ini" 172 }, 173 (to_abspath("carrot/xpcshell-one.ini"), to_abspath("carrot/xpcshell-shared.ini")): { 174 "head": "head_one.js", 175 }, 176 (to_abspath("carrot/xpcshell-two.ini"), to_abspath("carrot/xpcshell-shared.ini")): { 177 "head": "head_two.js", 178 }, 179 } 180 181 182@pytest.fixture(params=[BuildBackendLoader, TestManifestLoader]) 183def resolver(request, tmpdir, topsrcdir, all_tests, defaults): 184 topobjdir = tmpdir.mkdir("objdir").strpath 185 loader_cls = request.param 186 187 if loader_cls == BuildBackendLoader: 188 with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh: 189 pickle.dump(all_tests, fh) 190 with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh: 191 pickle.dump(defaults, fh) 192 193 # The mock data already exists, so prevent BuildBackendLoader from regenerating 194 # the build information from the whole gecko tree... 195 class BuildBackendLoaderNeverOutOfDate(BuildBackendLoader): 196 def backend_out_of_date(self, backend_file): 197 return False 198 loader_cls = BuildBackendLoaderNeverOutOfDate 199 200 resolver = TestResolver(topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls) 201 resolver._puppeteer_loaded = True 202 resolver._wpt_loaded = True 203 204 if loader_cls == TestManifestLoader: 205 config = MockConfig(topsrcdir) 206 resolver.load_tests.reader = BuildReader(config) 207 return resolver 208 209 210def test_load(resolver): 211 assert len(resolver.tests_by_path) == 9 212 213 assert len(resolver.tests_by_flavor['xpcshell']) == 4 214 assert len(resolver.tests_by_flavor['mochitest-plain']) == 0 215 216 217def test_resolve_all(resolver): 218 assert len(list(resolver._resolve())) == 11 219 220 221def test_resolve_filter_flavor(resolver): 222 assert len(list(resolver._resolve(flavor='xpcshell'))) == 6 223 224 225def test_resolve_by_dir(resolver): 226 assert len(list(resolver._resolve(paths=['banana/currant']))) == 2 227 228 229def test_resolve_under_path(resolver): 230 assert len(list(resolver._resolve(under_path='banana'))) == 2 231 assert len(list(resolver._resolve(flavor='xpcshell', under_path='banana'))) == 2 232 233 234def test_resolve_multiple_paths(resolver): 235 result = list(resolver.resolve_tests(paths=['banana', 'dragonfruit'])) 236 assert len(result) == 4 237 238 239def test_resolve_support_files(resolver): 240 expected_support_files = "\ndata/**\nxpcshell_updater.ini" 241 tests = list(resolver.resolve_tests(paths=['dragonfruit'])) 242 assert len(tests) == 2 243 244 for test in tests: 245 if test['manifest'].endswith('xpcshell_updater.ini'): 246 assert test['support-files'] == expected_support_files 247 else: 248 assert 'support-files' not in test 249 250 251def test_resolve_path_prefix(resolver): 252 tests = list(resolver._resolve(paths=['juniper'])) 253 assert len(tests) == 1 254 255 # relative manifest 256 tests = list(resolver._resolve(paths=['apple/a11y.ini'])) 257 assert len(tests) == 1 258 assert tests[0]['name'] == 'test_a11y.html' 259 260 # absolute manifest 261 tests = list(resolver._resolve(paths=[os.path.join(resolver.topsrcdir, 'apple/a11y.ini')])) 262 assert len(tests) == 1 263 assert tests[0]['name'] == 'test_a11y.html' 264 265 266def test_cwd_children_only(resolver): 267 """If cwd is defined, only resolve tests under the specified cwd.""" 268 # Pretend we're under '/services' and ask for 'common'. This should 269 # pick up all tests from '/services/common' 270 tests = list(resolver.resolve_tests(paths=['currant'], cwd=os.path.join(resolver.topsrcdir, 271 'banana'))) 272 273 assert len(tests) == 2 274 275 # Tests should be rewritten to objdir. 276 for t in tests: 277 assert t['here'] == mozpath.join(resolver.topobjdir, 278 '_tests/xpcshell/banana/currant') 279 280def test_various_cwd(resolver): 281 """Test various cwd conditions are all equal.""" 282 expected = list(resolver.resolve_tests(paths=['banana'])) 283 actual = list(resolver.resolve_tests(paths=['banana'], cwd='/')) 284 assert actual == expected 285 286 actual = list(resolver.resolve_tests(paths=['banana'], cwd=resolver.topsrcdir)) 287 assert actual == expected 288 289 actual = list(resolver.resolve_tests(paths=['banana'], cwd=resolver.topobjdir)) 290 assert actual == expected 291 292 293def test_subsuites(resolver): 294 """Test filtering by subsuite.""" 295 tests = list(resolver.resolve_tests(paths=['fig'])) 296 assert len(tests) == 2 297 298 tests = list(resolver.resolve_tests(paths=['fig'], subsuite='browser')) 299 assert len(tests) == 1 300 assert tests[0]['name'] == 'src/TestInstrumentationB.java' 301 302 tests = list(resolver.resolve_tests(paths=['fig'], subsuite='background')) 303 assert len(tests) == 1 304 assert tests[0]['name'] == 'src/TestInstrumentationA.java' 305 306 # Resolve tests *without* a subsuite. 307 tests = list(resolver.resolve_tests(flavor='browser-chrome', subsuite='undefined')) 308 assert len(tests) == 1 309 assert tests[0]['name'] == 'browser_chrome.js' 310 311 312def test_wildcard_patterns(resolver): 313 """Test matching paths by wildcard.""" 314 tests = list(resolver.resolve_tests(paths=['fig/**'])) 315 assert len(tests) == 2 316 for t in tests: 317 assert t['file_relpath'].startswith('fig') 318 319 tests = list(resolver.resolve_tests(paths=['**/**.js', 'apple/**'])) 320 assert len(tests) == 9 321 for t in tests: 322 path = t['file_relpath'] 323 assert path.startswith('apple') or path.endswith('.js') 324 325 326def test_resolve_metadata(resolver): 327 """Test finding metadata from outgoing files.""" 328 suites, tests = resolver.resolve_metadata(['bc']) 329 assert suites == {'mochitest-browser-chrome'} 330 assert tests == [] 331 332 suites, tests = resolver.resolve_metadata(['mochitest-a11y', '/browser', 'xpcshell']) 333 assert suites == {'mochitest-a11y', 'xpcshell'} 334 assert sorted(t['file_relpath'] for t in tests) == [ 335 'juniper/browser_chrome.js', 336 'kiwi/browser_devtools.js', 337 ] 338 339def test_ancestor_manifest_defaults(resolver, topsrcdir, defaults): 340 """Test that defaults from ancestor manifests are found.""" 341 tests = list(resolver._resolve(paths=['carrot/test_included.js'])) 342 assert len(tests) == 2 343 344 if tests[0]['ancestor_manifest'] == os.path.join('carrot', 'xpcshell-one.ini'): 345 [testOne, testTwo] = tests 346 else: 347 [testTwo, testOne] = tests 348 349 assert testOne['ancestor_manifest'] == os.path.join('carrot', 'xpcshell-one.ini') 350 assert testOne['manifest_relpath'] == os.path.join('carrot', 'xpcshell-shared.ini') 351 assert testOne['head'] == 'head_one.js' 352 assert testOne['stick'] == 'one' 353 354 assert testTwo['ancestor_manifest'] == os.path.join('carrot', 'xpcshell-two.ini') 355 assert testTwo['manifest_relpath'] == os.path.join('carrot', 'xpcshell-shared.ini') 356 assert testTwo['head'] == 'head_two.js' 357 assert testTwo['stick'] == 'two' 358 359 360def test_task_regexes(): 361 """Test the task_regexes defined in TEST_SUITES.""" 362 task_labels = [ 363 'test-linux64/opt-browser-screenshots-1', 364 'test-linux64/opt-browser-screenshots-e10s-1', 365 'test-linux64/opt-marionette', 366 'test-linux64/opt-mochitest-plain', 367 'test-linux64/debug-mochitest-plain-e10s', 368 'test-linux64/opt-mochitest-a11y', 369 'test-linux64/opt-mochitest-browser', 370 'test-linux64/opt-mochitest-browser-chrome', 371 'test-linux64/opt-mochitest-browser-chrome-e10s', 372 'test-linux64/opt-mochitest-browser-chrome-e10s-11', 373 'test-linux64/opt-mochitest-chrome', 374 'test-linux64/opt-mochitest-devtools', 375 'test-linux64/opt-mochitest-devtools-chrome', 376 'test-linux64/opt-mochitest-gpu', 377 'test-linux64/opt-mochitest-gpu-e10s', 378 'test-linux64/opt-mochitest-media-e10s-1', 379 'test-linux64/opt-mochitest-media-e10s-11', 380 'test-linux64/opt-mochitest-screenshots-1', 381 'test-linux64/opt-reftest', 382 'test-linux64/debug-reftest-e10s-1', 383 'test-linux64/debug-reftest-e10s-11', 384 'test-linux64/opt-robocop', 385 'test-linux64/opt-robocop-1', 386 'test-linux64/opt-robocop-e10s', 387 'test-linux64/opt-robocop-e10s-1', 388 'test-linux64/opt-robocop-e10s-11', 389 'test-linux64/opt-web-platform-tests-e10s-1', 390 'test-linux64/opt-web-platform-tests-reftest-e10s-1', 391 'test-linux64/opt-web-platform-tests-wdspec-e10s-1', 392 'test-linux64/opt-web-platform-tests-1', 393 'test-linux64/opt-web-platform-test-e10s-1', 394 'test-linux64/opt-xpcshell', 395 'test-linux64/opt-xpcshell-1', 396 'test-linux64/opt-xpcshell-2', 397 ] 398 399 test_cases = { 400 'mochitest-browser-chrome': [ 401 'test-linux64/opt-mochitest-browser-chrome', 402 'test-linux64/opt-mochitest-browser-chrome-e10s', 403 ], 404 'mochitest-chrome': [ 405 'test-linux64/opt-mochitest-chrome', 406 ], 407 'mochitest-devtools-chrome': [ 408 'test-linux64/opt-mochitest-devtools-chrome', 409 ], 410 'mochitest-media': [ 411 'test-linux64/opt-mochitest-media-e10s-1', 412 ], 413 'mochitest-plain': [ 414 'test-linux64/opt-mochitest-plain', 415 'test-linux64/debug-mochitest-plain-e10s', 416 ], 417 'mochitest-plain-gpu': [ 418 'test-linux64/opt-mochitest-gpu', 419 'test-linux64/opt-mochitest-gpu-e10s', 420 ], 421 'mochitest-browser-chrome-screenshots': [ 422 'test-linux64/opt-browser-screenshots-1', 423 'test-linux64/opt-browser-screenshots-e10s-1', 424 ], 425 'reftest': [ 426 'test-linux64/opt-reftest', 427 'test-linux64/debug-reftest-e10s-1', 428 ], 429 'robocop': [ 430 'test-linux64/opt-robocop', 431 'test-linux64/opt-robocop-1', 432 'test-linux64/opt-robocop-e10s', 433 'test-linux64/opt-robocop-e10s-1', 434 ], 435 'web-platform-tests': [ 436 'test-linux64/opt-web-platform-tests-e10s-1', 437 'test-linux64/opt-web-platform-tests-reftest-e10s-1', 438 'test-linux64/opt-web-platform-tests-wdspec-e10s-1', 439 'test-linux64/opt-web-platform-tests-1', 440 ], 441 'web-platform-tests-reftest': [ 442 'test-linux64/opt-web-platform-tests-reftest-e10s-1', 443 ], 444 'web-platform-tests-wdspec': [ 445 'test-linux64/opt-web-platform-tests-wdspec-e10s-1', 446 ], 447 'xpcshell': [ 448 'test-linux64/opt-xpcshell', 449 'test-linux64/opt-xpcshell-1', 450 ], 451 } 452 453 regexes = [] 454 455 def match_task(task): 456 return any(re.search(pattern, task) for pattern in regexes) 457 458 for suite, expected in sorted(test_cases.items()): 459 print(suite) 460 regexes = TEST_SUITES[suite]['task_regex'] 461 assert set(filter(match_task, task_labels)) == set(expected) 462 463 464if __name__ == '__main__': 465 mozunit.main() 466