1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4# flake8: noqa: E501
5
6from __future__ import absolute_import, print_function, unicode_literals
7
8try:
9    import cPickle as pickle
10except ImportError:
11    import pickle
12
13import json
14import os
15import re
16import shutil
17import tempfile
18from collections import defaultdict
19
20import pytest
21import mozpack.path as mozpath
22import mozunit
23from mozbuild.base import MozbuildObject
24from mozbuild.frontend.reader import BuildReader
25from mozbuild.test.common import MockConfig
26from mozfile import NamedTemporaryFile
27
28from moztest.resolve import (
29    BuildBackendLoader,
30    TestManifestLoader,
31    TestResolver,
32    TEST_SUITES,
33)
34
35here = os.path.abspath(os.path.dirname(__file__))
36data_path = os.path.join(here, 'data')
37
38
39@pytest.fixture(scope='module')
40def topsrcdir():
41    return mozpath.join(data_path, 'srcdir')
42
43
44@pytest.fixture(scope='module')
45def create_tests(topsrcdir):
46
47    def inner(*paths, **defaults):
48        tests = defaultdict(list)
49        for path in paths:
50            if isinstance(path, tuple):
51                path, kwargs = path
52            else:
53                kwargs = {}
54
55            path = mozpath.normpath(path)
56            manifest_name = kwargs.get('flavor', defaults.get('flavor', 'manifest'))
57            manifest = kwargs.pop('manifest', defaults.pop('manifest',
58                                  mozpath.join(mozpath.dirname(path), manifest_name + '.ini')))
59
60            manifest_abspath = mozpath.join(topsrcdir, manifest)
61            relpath = mozpath.relpath(path, mozpath.dirname(manifest))
62            test = {
63                'name': relpath,
64                'path': mozpath.join(topsrcdir, path),
65                'relpath': relpath,
66                'file_relpath': path,
67                'dir_relpath': mozpath.dirname(path),
68                'here': mozpath.dirname(manifest_abspath),
69                'manifest': manifest_abspath,
70                'manifest_relpath': manifest,
71            }
72            test.update(**defaults)
73            test.update(**kwargs)
74
75            # Normalize paths to ensure that the fixture matches reality.
76            for k in ['ancestor_manifest', 'manifest', 'manifest_relpath', 'path', 'relpath']:
77                p = test.get(k)
78                if p:
79                    test[k] = p.replace('/', os.path.sep)
80
81            tests[path].append(test)
82
83        # dump tests to stdout for easier debugging on failure
84        print("The 'create_tests' fixture returned:")
85        print(json.dumps(dict(tests), indent=2, sort_keys=True))
86        return tests
87
88    return inner
89
90
91@pytest.fixture(scope='module')
92def all_tests(create_tests):
93    return create_tests(*[
94        ("apple/test_a11y.html", {
95             "expected": "pass",
96             "flavor": "a11y",
97         }),
98        ("banana/currant/test_xpcshell_A.js", {
99            "firefox-appdir": "browser",
100            "flavor": "xpcshell",
101            "head": "head_global.js head_helpers.js head_http.js",
102         }),
103        ("banana/currant/test_xpcshell_B.js", {
104            "firefox-appdir": "browser",
105            "flavor": "xpcshell",
106            "head": "head_global.js head_helpers.js head_http.js",
107         }),
108        ("carrot/test_included.js", {
109            "ancestor_manifest": "carrot/xpcshell-one.ini",
110            "manifest": "carrot/xpcshell-shared.ini",
111            "flavor": "xpcshell",
112            "stick": "one",
113         }),
114        ("carrot/test_included.js", {
115            "ancestor_manifest": "carrot/xpcshell-two.ini",
116            "manifest": "carrot/xpcshell-shared.ini",
117            "flavor": "xpcshell",
118            "stick": "two",
119         }),
120        ("dragonfruit/elderberry/test_xpcshell_C.js", {
121            "flavor": "xpcshell",
122            "generated-files": "head_update.js",
123            "head": "head_update.js",
124            "manifest": "dragonfruit/xpcshell.ini",
125            "reason": "busted",
126            "run-sequentially": "Launches application.",
127            "skip-if": "os == 'android'",
128         }),
129        ("dragonfruit/elderberry/test_xpcshell_C.js", {
130            "flavor": "xpcshell",
131            "generated-files": "head_update.js",
132            "head": "head_update.js head2.js",
133            "manifest": "dragonfruit/elderberry/xpcshell_updater.ini",
134            "reason": "don't work",
135            "run-sequentially": "Launches application.",
136            "skip-if": "os == 'android'",
137         }),
138        ("fig/grape/src/TestInstrumentationA.java", {
139            "flavor": "instrumentation",
140            "manifest": "fig/grape/instrumentation.ini",
141            "subsuite": "background",
142         }),
143        ("fig/huckleberry/src/TestInstrumentationB.java", {
144            "flavor": "instrumentation",
145            "manifest": "fig/huckleberry/instrumentation.ini",
146            "subsuite": "browser",
147         }),
148        ("juniper/browser_chrome.js", {
149            "flavor": "browser-chrome",
150            "manifest": "juniper/browser.ini",
151            "skip-if": "e10s  # broken",
152         }),
153        ("kiwi/browser_devtools.js", {
154            "flavor": "browser-chrome",
155            "manifest": "kiwi/browser.ini",
156            "subsuite": "devtools",
157            "tags": "devtools",
158         }),
159    ])
160
161
162@pytest.fixture(scope='module')
163def defaults(topsrcdir):
164    def to_abspath(relpath):
165        # test-defaults.pkl uses absolute paths with platform-specific path separators.
166        # Use platform-specific separators if needed to avoid regressing on bug 1644223.
167        return os.path.normpath(os.path.join(topsrcdir, relpath))
168
169    return {
170        (to_abspath("dragonfruit/elderberry/xpcshell_updater.ini")): {
171            "support-files": "\ndata/**\nxpcshell_updater.ini"
172        },
173        (to_abspath("carrot/xpcshell-one.ini"), to_abspath("carrot/xpcshell-shared.ini")): {
174            "head": "head_one.js",
175        },
176        (to_abspath("carrot/xpcshell-two.ini"), to_abspath("carrot/xpcshell-shared.ini")): {
177            "head": "head_two.js",
178        },
179    }
180
181
182@pytest.fixture(params=[BuildBackendLoader, TestManifestLoader])
183def resolver(request, tmpdir, topsrcdir, all_tests, defaults):
184    topobjdir = tmpdir.mkdir("objdir").strpath
185    loader_cls = request.param
186
187    if loader_cls == BuildBackendLoader:
188        with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
189            pickle.dump(all_tests, fh)
190        with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
191            pickle.dump(defaults, fh)
192
193        # The mock data already exists, so prevent BuildBackendLoader from regenerating
194        # the build information from the whole gecko tree...
195        class BuildBackendLoaderNeverOutOfDate(BuildBackendLoader):
196            def backend_out_of_date(self, backend_file):
197                return False
198        loader_cls = BuildBackendLoaderNeverOutOfDate
199
200    resolver = TestResolver(topsrcdir, None, None, topobjdir=topobjdir, loader_cls=loader_cls)
201    resolver._puppeteer_loaded = True
202    resolver._wpt_loaded = True
203
204    if loader_cls == TestManifestLoader:
205        config = MockConfig(topsrcdir)
206        resolver.load_tests.reader = BuildReader(config)
207    return resolver
208
209
210def test_load(resolver):
211    assert len(resolver.tests_by_path) == 9
212
213    assert len(resolver.tests_by_flavor['xpcshell']) == 4
214    assert len(resolver.tests_by_flavor['mochitest-plain']) == 0
215
216
217def test_resolve_all(resolver):
218    assert len(list(resolver._resolve())) == 11
219
220
221def test_resolve_filter_flavor(resolver):
222    assert len(list(resolver._resolve(flavor='xpcshell'))) == 6
223
224
225def test_resolve_by_dir(resolver):
226    assert len(list(resolver._resolve(paths=['banana/currant']))) == 2
227
228
229def test_resolve_under_path(resolver):
230    assert len(list(resolver._resolve(under_path='banana'))) == 2
231    assert len(list(resolver._resolve(flavor='xpcshell', under_path='banana'))) == 2
232
233
234def test_resolve_multiple_paths(resolver):
235    result = list(resolver.resolve_tests(paths=['banana', 'dragonfruit']))
236    assert len(result) == 4
237
238
239def test_resolve_support_files(resolver):
240    expected_support_files = "\ndata/**\nxpcshell_updater.ini"
241    tests = list(resolver.resolve_tests(paths=['dragonfruit']))
242    assert len(tests) == 2
243
244    for test in tests:
245        if test['manifest'].endswith('xpcshell_updater.ini'):
246            assert test['support-files'] == expected_support_files
247        else:
248            assert 'support-files' not in test
249
250
251def test_resolve_path_prefix(resolver):
252    tests = list(resolver._resolve(paths=['juniper']))
253    assert len(tests) == 1
254
255    # relative manifest
256    tests = list(resolver._resolve(paths=['apple/a11y.ini']))
257    assert len(tests) == 1
258    assert tests[0]['name'] == 'test_a11y.html'
259
260    # absolute manifest
261    tests = list(resolver._resolve(paths=[os.path.join(resolver.topsrcdir, 'apple/a11y.ini')]))
262    assert len(tests) == 1
263    assert tests[0]['name'] == 'test_a11y.html'
264
265
266def test_cwd_children_only(resolver):
267    """If cwd is defined, only resolve tests under the specified cwd."""
268    # Pretend we're under '/services' and ask for 'common'. This should
269    # pick up all tests from '/services/common'
270    tests = list(resolver.resolve_tests(paths=['currant'], cwd=os.path.join(resolver.topsrcdir,
271        'banana')))
272
273    assert len(tests) == 2
274
275    # Tests should be rewritten to objdir.
276    for t in tests:
277        assert t['here'] == mozpath.join(resolver.topobjdir,
278                                         '_tests/xpcshell/banana/currant')
279
280def test_various_cwd(resolver):
281    """Test various cwd conditions are all equal."""
282    expected = list(resolver.resolve_tests(paths=['banana']))
283    actual = list(resolver.resolve_tests(paths=['banana'], cwd='/'))
284    assert actual == expected
285
286    actual = list(resolver.resolve_tests(paths=['banana'], cwd=resolver.topsrcdir))
287    assert actual == expected
288
289    actual = list(resolver.resolve_tests(paths=['banana'], cwd=resolver.topobjdir))
290    assert actual == expected
291
292
293def test_subsuites(resolver):
294    """Test filtering by subsuite."""
295    tests = list(resolver.resolve_tests(paths=['fig']))
296    assert len(tests) == 2
297
298    tests = list(resolver.resolve_tests(paths=['fig'], subsuite='browser'))
299    assert len(tests) == 1
300    assert tests[0]['name'] == 'src/TestInstrumentationB.java'
301
302    tests = list(resolver.resolve_tests(paths=['fig'], subsuite='background'))
303    assert len(tests) == 1
304    assert tests[0]['name'] == 'src/TestInstrumentationA.java'
305
306    # Resolve tests *without* a subsuite.
307    tests = list(resolver.resolve_tests(flavor='browser-chrome', subsuite='undefined'))
308    assert len(tests) == 1
309    assert tests[0]['name'] == 'browser_chrome.js'
310
311
312def test_wildcard_patterns(resolver):
313    """Test matching paths by wildcard."""
314    tests = list(resolver.resolve_tests(paths=['fig/**']))
315    assert len(tests) == 2
316    for t in tests:
317        assert t['file_relpath'].startswith('fig')
318
319    tests = list(resolver.resolve_tests(paths=['**/**.js', 'apple/**']))
320    assert len(tests) == 9
321    for t in tests:
322        path = t['file_relpath']
323        assert path.startswith('apple') or path.endswith('.js')
324
325
326def test_resolve_metadata(resolver):
327    """Test finding metadata from outgoing files."""
328    suites, tests = resolver.resolve_metadata(['bc'])
329    assert suites == {'mochitest-browser-chrome'}
330    assert tests == []
331
332    suites, tests = resolver.resolve_metadata(['mochitest-a11y', '/browser', 'xpcshell'])
333    assert suites == {'mochitest-a11y', 'xpcshell'}
334    assert sorted(t['file_relpath'] for t in tests) == [
335        'juniper/browser_chrome.js',
336        'kiwi/browser_devtools.js',
337    ]
338
339def test_ancestor_manifest_defaults(resolver, topsrcdir, defaults):
340    """Test that defaults from ancestor manifests are found."""
341    tests = list(resolver._resolve(paths=['carrot/test_included.js']))
342    assert len(tests) == 2
343
344    if tests[0]['ancestor_manifest'] == os.path.join('carrot', 'xpcshell-one.ini'):
345        [testOne, testTwo] = tests
346    else:
347        [testTwo, testOne] = tests
348
349    assert testOne['ancestor_manifest'] == os.path.join('carrot', 'xpcshell-one.ini')
350    assert testOne['manifest_relpath'] == os.path.join('carrot', 'xpcshell-shared.ini')
351    assert testOne['head'] == 'head_one.js'
352    assert testOne['stick'] == 'one'
353
354    assert testTwo['ancestor_manifest'] == os.path.join('carrot', 'xpcshell-two.ini')
355    assert testTwo['manifest_relpath'] == os.path.join('carrot', 'xpcshell-shared.ini')
356    assert testTwo['head'] == 'head_two.js'
357    assert testTwo['stick'] == 'two'
358
359
360def test_task_regexes():
361    """Test the task_regexes defined in TEST_SUITES."""
362    task_labels = [
363        'test-linux64/opt-browser-screenshots-1',
364        'test-linux64/opt-browser-screenshots-e10s-1',
365        'test-linux64/opt-marionette',
366        'test-linux64/opt-mochitest-plain',
367        'test-linux64/debug-mochitest-plain-e10s',
368        'test-linux64/opt-mochitest-a11y',
369        'test-linux64/opt-mochitest-browser',
370        'test-linux64/opt-mochitest-browser-chrome',
371        'test-linux64/opt-mochitest-browser-chrome-e10s',
372        'test-linux64/opt-mochitest-browser-chrome-e10s-11',
373        'test-linux64/opt-mochitest-chrome',
374        'test-linux64/opt-mochitest-devtools',
375        'test-linux64/opt-mochitest-devtools-chrome',
376        'test-linux64/opt-mochitest-gpu',
377        'test-linux64/opt-mochitest-gpu-e10s',
378        'test-linux64/opt-mochitest-media-e10s-1',
379        'test-linux64/opt-mochitest-media-e10s-11',
380        'test-linux64/opt-mochitest-screenshots-1',
381        'test-linux64/opt-reftest',
382        'test-linux64/debug-reftest-e10s-1',
383        'test-linux64/debug-reftest-e10s-11',
384        'test-linux64/opt-robocop',
385        'test-linux64/opt-robocop-1',
386        'test-linux64/opt-robocop-e10s',
387        'test-linux64/opt-robocop-e10s-1',
388        'test-linux64/opt-robocop-e10s-11',
389        'test-linux64/opt-web-platform-tests-e10s-1',
390        'test-linux64/opt-web-platform-tests-reftest-e10s-1',
391        'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
392        'test-linux64/opt-web-platform-tests-1',
393        'test-linux64/opt-web-platform-test-e10s-1',
394        'test-linux64/opt-xpcshell',
395        'test-linux64/opt-xpcshell-1',
396        'test-linux64/opt-xpcshell-2',
397    ]
398
399    test_cases = {
400        'mochitest-browser-chrome': [
401            'test-linux64/opt-mochitest-browser-chrome',
402            'test-linux64/opt-mochitest-browser-chrome-e10s',
403        ],
404        'mochitest-chrome': [
405            'test-linux64/opt-mochitest-chrome',
406        ],
407        'mochitest-devtools-chrome': [
408            'test-linux64/opt-mochitest-devtools-chrome',
409        ],
410        'mochitest-media': [
411            'test-linux64/opt-mochitest-media-e10s-1',
412        ],
413        'mochitest-plain': [
414            'test-linux64/opt-mochitest-plain',
415            'test-linux64/debug-mochitest-plain-e10s',
416        ],
417        'mochitest-plain-gpu': [
418            'test-linux64/opt-mochitest-gpu',
419            'test-linux64/opt-mochitest-gpu-e10s',
420        ],
421        'mochitest-browser-chrome-screenshots': [
422            'test-linux64/opt-browser-screenshots-1',
423            'test-linux64/opt-browser-screenshots-e10s-1',
424        ],
425        'reftest': [
426            'test-linux64/opt-reftest',
427            'test-linux64/debug-reftest-e10s-1',
428        ],
429        'robocop': [
430            'test-linux64/opt-robocop',
431            'test-linux64/opt-robocop-1',
432            'test-linux64/opt-robocop-e10s',
433            'test-linux64/opt-robocop-e10s-1',
434        ],
435        'web-platform-tests': [
436            'test-linux64/opt-web-platform-tests-e10s-1',
437            'test-linux64/opt-web-platform-tests-reftest-e10s-1',
438            'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
439            'test-linux64/opt-web-platform-tests-1',
440        ],
441        'web-platform-tests-reftest': [
442            'test-linux64/opt-web-platform-tests-reftest-e10s-1',
443        ],
444        'web-platform-tests-wdspec': [
445            'test-linux64/opt-web-platform-tests-wdspec-e10s-1',
446        ],
447        'xpcshell': [
448            'test-linux64/opt-xpcshell',
449            'test-linux64/opt-xpcshell-1',
450        ],
451    }
452
453    regexes = []
454
455    def match_task(task):
456        return any(re.search(pattern, task) for pattern in regexes)
457
458    for suite, expected in sorted(test_cases.items()):
459        print(suite)
460        regexes = TEST_SUITES[suite]['task_regex']
461        assert set(filter(match_task, task_labels)) == set(expected)
462
463
464if __name__ == '__main__':
465    mozunit.main()
466