1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, unicode_literals
6
7import cPickle as pickle
8import os
9import sys
10
11import mozpack.path as mozpath
12
13from mozpack.copier import FileCopier
14from mozpack.manifests import InstallManifest
15
16import manifestparser
17
18
19# These definitions provide a single source of truth for modules attempting
20# to get a view of all tests for a build. Used by the emitter to figure out
21# how to read/install manifests and by test dependency annotations in Files()
22# entries to enumerate test flavors.
23
24# While there are multiple test manifests, the behavior is very similar
25# across them. We enforce this by having common handling of all
26# manifests and outputting a single class type with the differences
27# described inside the instance.
28#
29# Keys are variable prefixes and values are tuples describing how these
30# manifests should be handled:
31#
32#    (flavor, install_root, install_subdir, package_tests)
33#
34# flavor identifies the flavor of this test.
35# install_root is the path prefix to install the files starting from the root
36#     directory and not as specified by the manifest location. (bug 972168)
37# install_subdir is the path of where to install the files in
38#     the tests directory.
39# package_tests indicates whether to package test files into the test
40#     package; suites that compile the test files should not install
41#     them into the test package.
42#
43TEST_MANIFESTS = dict(
44    A11Y=('a11y', 'testing/mochitest', 'a11y', True),
45    BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
46    ANDROID_INSTRUMENTATION=('instrumentation', 'instrumentation', '.', False),
47    FIREFOX_UI_FUNCTIONAL=('firefox-ui-functional', 'firefox-ui', '.', False),
48    FIREFOX_UI_UPDATE=('firefox-ui-update', 'firefox-ui', '.', False),
49    PUPPETEER_FIREFOX=('firefox-ui-functional', 'firefox-ui', '.', False),
50    PYTHON_UNITTEST=('python', 'python', '.', False),
51    CRAMTEST=('cram', 'cram', '.', False),
52
53    # marionette tests are run from the srcdir
54    # TODO(ato): make packaging work as for other test suites
55    MARIONETTE=('marionette', 'marionette', '.', False),
56    MARIONETTE_UNIT=('marionette', 'marionette', '.', False),
57    MARIONETTE_WEBAPI=('marionette', 'marionette', '.', False),
58
59    METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
60    MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
61    MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
62    WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
63    XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True),
64)
65
66# Reftests have their own manifest format and are processed separately.
67REFTEST_FLAVORS = ('crashtest', 'reftest')
68
69# Web platform tests have their own manifest format and are processed separately.
70WEB_PLATFORM_TESTS_FLAVORS = ('web-platform-tests',)
71
72def all_test_flavors():
73    return ([v[0] for v in TEST_MANIFESTS.values()] +
74            list(REFTEST_FLAVORS) +
75            list(WEB_PLATFORM_TESTS_FLAVORS))
76
77class TestInstallInfo(object):
78    def __init__(self):
79        self.seen = set()
80        self.pattern_installs = []
81        self.installs = []
82        self.external_installs = set()
83        self.deferred_installs = set()
84
85    def __ior__(self, other):
86        self.pattern_installs.extend(other.pattern_installs)
87        self.installs.extend(other.installs)
88        self.external_installs |= other.external_installs
89        self.deferred_installs |= other.deferred_installs
90        return self
91
92class SupportFilesConverter(object):
93    """Processes a "support-files" entry from a test object, either from
94    a parsed object from a test manifests or its representation in
95    moz.build and returns the installs to perform for this test object.
96
97    Processing the same support files multiple times will not have any further
98    effect, and the structure of the parsed objects from manifests will have a
99    lot of repeated entries, so this class takes care of memoizing.
100    """
101    def __init__(self):
102        self._fields = (('head', set()),
103                        ('support-files', set()),
104                        ('generated-files', set()))
105
106    def convert_support_files(self, test, install_root, manifest_dir, out_dir):
107        # Arguments:
108        #  test - The test object to process.
109        #  install_root - The directory under $objdir/_tests that will contain
110        #                 the tests for this harness (examples are "testing/mochitest",
111        #                 "xpcshell").
112        #  manifest_dir - Absoulute path to the (srcdir) directory containing the
113        #                 manifest that included this test
114        #  out_dir - The path relative to $objdir/_tests used as the destination for the
115        #            test, based on the relative path to the manifest in the srcdir,
116        #            the install_root, and 'install-to-subdir', if present in the manifest.
117        info = TestInstallInfo()
118        for field, seen in self._fields:
119            value = test.get(field, '')
120            for pattern in value.split():
121
122                # We track uniqueness locally (per test) where duplicates are forbidden,
123                # and globally, where they are permitted. If a support file appears multiple
124                # times for a single test, there are unnecessary entries in the manifest. But
125                # many entries will be shared across tests that share defaults.
126                # We need to memoize on the basis of both the path and the output
127                # directory for the benefit of tests specifying 'install-to-subdir'.
128                key = field, pattern, out_dir
129                if key in info.seen:
130                    raise ValueError("%s appears multiple times in a test manifest under a %s field,"
131                                     " please omit the duplicate entry." % (pattern, field))
132                info.seen.add(key)
133                if key in seen:
134                    continue
135                seen.add(key)
136
137                if field == 'generated-files':
138                    info.external_installs.add(mozpath.normpath(mozpath.join(out_dir, pattern)))
139                # '!' indicates our syntax for inter-directory support file
140                # dependencies. These receive special handling in the backend.
141                elif pattern[0] == '!':
142                    info.deferred_installs.add(pattern)
143                # We only support globbing on support-files because
144                # the harness doesn't support * for head.
145                elif '*' in pattern and field == 'support-files':
146                    info.pattern_installs.append((manifest_dir, pattern, out_dir))
147                # "absolute" paths identify files that are to be
148                # placed in the install_root directory (no globs)
149                elif pattern[0] == '/':
150                    full = mozpath.normpath(mozpath.join(manifest_dir,
151                                                         mozpath.basename(pattern)))
152                    info.installs.append((full, mozpath.join(install_root, pattern[1:])))
153                else:
154                    full = mozpath.normpath(mozpath.join(manifest_dir, pattern))
155                    dest_path = mozpath.join(out_dir, pattern)
156
157                    # If the path resolves to a different directory
158                    # tree, we take special behavior depending on the
159                    # entry type.
160                    if not full.startswith(manifest_dir):
161                        # If it's a support file, we install the file
162                        # into the current destination directory.
163                        # This implementation makes installing things
164                        # with custom prefixes impossible. If this is
165                        # needed, we can add support for that via a
166                        # special syntax later.
167                        if field == 'support-files':
168                            dest_path = mozpath.join(out_dir,
169                                                     os.path.basename(pattern))
170                        # If it's not a support file, we ignore it.
171                        # This preserves old behavior so things like
172                        # head files doesn't get installed multiple
173                        # times.
174                        else:
175                            continue
176                    info.installs.append((full, mozpath.normpath(dest_path)))
177        return info
178
179def _resolve_installs(paths, topobjdir, manifest):
180    """Using the given paths as keys, find any unresolved installs noted
181    by the build backend corresponding to those keys, and add them
182    to the given manifest.
183    """
184    filename = os.path.join(topobjdir, 'test-installs.pkl')
185    with open(filename, 'rb') as fh:
186        resolved_installs = pickle.load(fh)
187
188    for path in paths:
189        path = path[2:]
190        if path not in resolved_installs:
191            raise Exception('A cross-directory support file path noted in a '
192                'test manifest does not appear in any other manifest.\n "%s" '
193                'must appear in another test manifest to specify an install '
194                'for "!/%s".' % (path, path))
195        installs = resolved_installs[path]
196        for install_info in installs:
197            try:
198                if len(install_info) == 3:
199                    manifest.add_pattern_link(*install_info)
200                if len(install_info) == 2:
201                    manifest.add_link(*install_info)
202            except ValueError:
203                # A duplicate value here is pretty likely when running
204                # multiple directories at once, and harmless.
205                pass
206
207def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
208    """Installs the requested test files to the objdir. This is invoked by
209    test runners to avoid installing tens of thousands of test files when
210    only a few tests need to be run.
211    """
212    flavor_info = {flavor: (root, prefix, install)
213                   for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}
214    objdir_dest = mozpath.join(topobjdir, tests_root)
215
216    converter = SupportFilesConverter()
217    install_info = TestInstallInfo()
218    for o in test_objs:
219        flavor = o['flavor']
220        if flavor not in flavor_info:
221            # This is a test flavor that isn't installed by the build system.
222            continue
223        root, prefix, install = flavor_info[flavor]
224        if not install:
225            # This flavor isn't installed to the objdir.
226            continue
227
228        manifest_path = o['manifest']
229        manifest_dir = mozpath.dirname(manifest_path)
230
231        out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
232        file_relpath = o['file_relpath']
233        source = mozpath.join(topsrcdir, file_relpath)
234        dest = mozpath.join(root, prefix, file_relpath)
235        if 'install-to-subdir' in o:
236            out_dir = mozpath.join(out_dir, o['install-to-subdir'])
237            manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
238            dest = mozpath.join(out_dir, manifest_relpath)
239
240        install_info.installs.append((source, dest))
241        install_info |= converter.convert_support_files(o, root,
242                                                        manifest_dir,
243                                                        out_dir)
244
245    manifest = InstallManifest()
246
247    for source, dest in set(install_info.installs):
248        if dest in install_info.external_installs:
249            continue
250        manifest.add_link(source, dest)
251    for base, pattern, dest in install_info.pattern_installs:
252        manifest.add_pattern_link(base, pattern, dest)
253
254    _resolve_installs(install_info.deferred_installs, topobjdir, manifest)
255
256    # Harness files are treated as a monolith and installed each time we run tests.
257    # Fortunately there are not very many.
258    manifest |= InstallManifest(mozpath.join(topobjdir,
259                                             '_build_manifests',
260                                             'install', tests_root))
261    copier = FileCopier()
262    manifest.populate_registry(copier)
263    copier.copy(objdir_dest,
264                remove_unaccounted=False)
265
266
267# Convenience methods for test manifest reading.
268def read_manifestparser_manifest(context, manifest_path):
269    path = manifest_path.full_path
270    return manifestparser.TestManifest(manifests=[path], strict=True,
271                                       rootdir=context.config.topsrcdir,
272                                       finder=context._finder,
273                                       handle_defaults=False)
274
275def read_reftest_manifest(context, manifest_path):
276    import reftest
277    path = manifest_path.full_path
278    manifest = reftest.ReftestManifest(finder=context._finder)
279    manifest.load(path)
280    return manifest
281
282def read_wpt_manifest(context, paths):
283    manifest_path, tests_root = paths
284    full_path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path))
285    old_path = sys.path[:]
286    try:
287        # Setup sys.path to include all the dependencies required to import
288        # the web-platform-tests manifest parser. web-platform-tests provides
289        # a the localpaths.py to do the path manipulation, which we load,
290        # providing the __file__ variable so it can resolve the relative
291        # paths correctly.
292        paths_file = os.path.join(context.config.topsrcdir, "testing",
293                                  "web-platform", "tests", "tools", "localpaths.py")
294        _globals = {"__file__": paths_file}
295        execfile(paths_file, _globals)
296        import manifest as wptmanifest
297    finally:
298        sys.path = old_path
299        f = context._finder.get(full_path)
300        return wptmanifest.manifest.load(tests_root, f)
301