1# Library for JSTest manifests.
2#
3# This includes classes for representing and parsing JS manifests.
4
5from __future__ import print_function
6
7import os, re, sys
8from subprocess import Popen, PIPE
9
10from tests import RefTestCase
11
12
13def split_path_into_dirs(path):
14    dirs = [path]
15
16    while True:
17        path, tail = os.path.split(path)
18        if not tail:
19            break
20        dirs.append(path)
21    return dirs
22
23class XULInfo:
24    def __init__(self, abi, os, isdebug):
25        self.abi = abi
26        self.os = os
27        self.isdebug = isdebug
28        self.browserIsRemote = False
29
30    def as_js(self):
31        """Return JS that when executed sets up variables so that JS expression
32        predicates on XUL build info evaluate properly."""
33
34        return ('var xulRuntime = {{ OS: "{}", XPCOMABI: "{}", shell: true }};'
35                'var isDebugBuild={}; var Android={}; '
36                'var browserIsRemote={}'.format(
37                    self.os,
38                    self.abi,
39                    str(self.isdebug).lower(),
40                    str(self.os == "Android").lower(),
41                    str(self.browserIsRemote).lower()))
42
43    @classmethod
44    def create(cls, jsdir):
45        """Create a XULInfo based on the current platform's characteristics."""
46
47        # Our strategy is to find the autoconf.mk generated for the build and
48        # read the values from there.
49
50        # Find config/autoconf.mk.
51        dirs = split_path_into_dirs(os.getcwd()) + split_path_into_dirs(jsdir)
52
53        path = None
54        for dir in dirs:
55            _path = os.path.join(dir, 'config/autoconf.mk')
56            if os.path.isfile(_path):
57                path = _path
58                break
59
60        if path == None:
61            print("Can't find config/autoconf.mk on a directory containing"
62                  " the JS shell (searched from {})".format(jsdir))
63            sys.exit(1)
64
65        # Read the values.
66        val_re = re.compile(r'(TARGET_XPCOM_ABI|OS_TARGET|MOZ_DEBUG)\s*=\s*(.*)')
67        kw = {'isdebug': False}
68        for line in open(path):
69            m = val_re.match(line)
70            if m:
71                key, val = m.groups()
72                val = val.rstrip()
73                if key == 'TARGET_XPCOM_ABI':
74                    kw['abi'] = val
75                if key == 'OS_TARGET':
76                    kw['os'] = val
77                if key == 'MOZ_DEBUG':
78                    kw['isdebug'] = (val == '1')
79        return cls(**kw)
80
81class XULInfoTester:
82    def __init__(self, xulinfo, js_bin):
83        self.js_prologue = xulinfo.as_js()
84        self.js_bin = js_bin
85        # Maps JS expr to evaluation result.
86        self.cache = {}
87
88    def test(self, cond):
89        """Test a XUL predicate condition against this local info."""
90        ans = self.cache.get(cond, None)
91        if ans is None:
92            cmd = [
93                self.js_bin,
94                # run in safe configuration, since it is hard to debug
95                # crashes when running code here. In particular, msan will
96                # error out if the jit is active.
97                '--no-baseline',
98                '-e', self.js_prologue,
99                '-e', 'print(!!({}))'.format(cond)
100            ]
101            p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
102            out, err = p.communicate()
103            if out in ('true\n', 'true\r\n'):
104                ans = True
105            elif out in ('false\n', 'false\r\n'):
106                ans = False
107            else:
108                raise Exception("Failed to test XUL condition {!r};"
109                                " output was {!r}, stderr was {!r}".format(
110                                    cond, out, err))
111            self.cache[cond] = ans
112        return ans
113
114class NullXULInfoTester:
115    """Can be used to parse manifests without a JS shell."""
116    def test(self, cond):
117        return False
118
119def _parse_one(testcase, xul_tester):
120    pos = 0
121    parts = testcase.terms.split()
122    while pos < len(parts):
123        if parts[pos] == 'fails':
124            testcase.expect = False
125            pos += 1
126        elif parts[pos] == 'skip':
127            testcase.expect = testcase.enable = False
128            pos += 1
129        elif parts[pos] == 'random':
130            testcase.random = True
131            pos += 1
132        elif parts[pos].startswith('fails-if'):
133            cond = parts[pos][len('fails-if('):-1]
134            if xul_tester.test(cond):
135                testcase.expect = False
136            pos += 1
137        elif parts[pos].startswith('asserts-if'):
138            # This directive means we may flunk some number of
139            # NS_ASSERTIONs in the browser. For the shell, ignore it.
140            pos += 1
141        elif parts[pos].startswith('skip-if'):
142            cond = parts[pos][len('skip-if('):-1]
143            if xul_tester.test(cond):
144                testcase.expect = testcase.enable = False
145            pos += 1
146        elif parts[pos].startswith('random-if'):
147            cond = parts[pos][len('random-if('):-1]
148            if xul_tester.test(cond):
149                testcase.random = True
150            pos += 1
151        elif parts[pos] == 'slow':
152            testcase.slow = True
153            pos += 1
154        elif parts[pos] == 'silentfail':
155            # silentfails use tons of memory, and Darwin doesn't support ulimit.
156            if xul_tester.test("xulRuntime.OS == 'Darwin'"):
157                testcase.expect = testcase.enable = False
158            pos += 1
159        else:
160            print('warning: invalid manifest line element "{}"'.format(
161                parts[pos]))
162            pos += 1
163
164def _build_manifest_script_entry(script_name, test):
165    line = []
166    if test.terms:
167        line.append(test.terms)
168    line.append("script")
169    line.append(script_name)
170    if test.comment:
171        line.append("#")
172        line.append(test.comment)
173    return ' '.join(line)
174
175def _map_prefixes_left(test_gen):
176    """
177    Splits tests into a dictionary keyed on the first component of the test
178    path, aggregating tests with a common base path into a list.
179    """
180    byprefix = {}
181    for t in test_gen:
182        left, sep, remainder = t.path.partition(os.sep)
183        if left not in byprefix:
184            byprefix[left] = []
185        if remainder:
186            t.path = remainder
187        byprefix[left].append(t)
188    return byprefix
189
190def _emit_manifest_at(location, relative, test_gen, depth):
191    """
192    location  - str: absolute path where we want to write the manifest
193    relative  - str: relative path from topmost manifest directory to current
194    test_gen  - (str): generator of all test paths and directorys
195    depth     - int: number of dirs we are below the topmost manifest dir
196    """
197    manifests = _map_prefixes_left(test_gen)
198
199    filename = os.path.join(location, 'jstests.list')
200    manifest = []
201    numTestFiles = 0
202    for k, test_list in manifests.iteritems():
203        fullpath = os.path.join(location, k)
204        if os.path.isdir(fullpath):
205            manifest.append("include " + k + "/jstests.list")
206            relpath = os.path.join(relative, k)
207            _emit_manifest_at(fullpath, relpath, test_list, depth + 1)
208        else:
209            numTestFiles += 1
210            if len(test_list) != 1:
211                import pdb; pdb.set_trace()
212            assert len(test_list) == 1
213            line = _build_manifest_script_entry(k, test_list[0])
214            manifest.append(line)
215
216    # Always present our manifest in sorted order.
217    manifest.sort()
218
219    # If we have tests, we have to set the url-prefix so reftest can find them.
220    if numTestFiles > 0:
221        manifest = ["url-prefix {}jsreftest.html?test={}/".format(
222            '../' * depth, relative)] + manifest
223
224    fp = open(filename, 'w')
225    try:
226        fp.write('\n'.join(manifest) + '\n')
227    finally:
228        fp.close()
229
230def make_manifests(location, test_gen):
231    _emit_manifest_at(location, '', test_gen, 0)
232
233def _find_all_js_files(base, location):
234    for root, dirs, files in os.walk(location):
235        root = root[len(base) + 1:]
236        for fn in files:
237            if fn.endswith('.js'):
238                yield root, fn
239
240TEST_HEADER_PATTERN_INLINE = re.compile(r'//\s*\|(.*?)\|\s*(.*?)\s*(--\s*(.*))?$')
241TEST_HEADER_PATTERN_MULTI  = re.compile(r'/\*\s*\|(.*?)\|\s*(.*?)\s*(--\s*(.*))?\*/')
242
243def _parse_test_header(fullpath, testcase, xul_tester):
244    """
245    This looks a bit weird.  The reason is that it needs to be efficient, since
246    it has to be done on every test
247    """
248    fp = open(fullpath, 'r')
249    try:
250        buf = fp.read(512)
251    finally:
252        fp.close()
253
254    # Bail early if we do not start with a single comment.
255    if not buf.startswith("//"):
256        return
257
258    # Extract the token.
259    buf, _, _ = buf.partition('\n')
260    matches = TEST_HEADER_PATTERN_INLINE.match(buf)
261
262    if not matches:
263        matches = TEST_HEADER_PATTERN_MULTI.match(buf)
264        if not matches:
265            return
266
267    testcase.tag = matches.group(1)
268    testcase.terms = matches.group(2)
269    testcase.comment = matches.group(4)
270
271    _parse_one(testcase, xul_tester)
272
273def _parse_external_manifest(filename, relpath):
274    """
275    Reads an external manifest file for test suites whose individual test cases
276    can't be decorated with reftest comments.
277    filename - str: name of the manifest file
278    relpath - str: relative path of the directory containing the manifest
279                   within the test suite
280    """
281    entries = []
282
283    with open(filename, 'r') as fp:
284        manifest_re = re.compile(r'^\s*(.*)\s+(include|script)\s+(\S+)$')
285        for line in fp:
286            line, _, comment = line.partition('#')
287            line = line.strip()
288            if not line:
289                continue
290            matches = manifest_re.match(line)
291            if not matches:
292                print('warning: unrecognized line in jstests.list:'
293                      ' {0}'.format(line))
294                continue
295
296            path = os.path.normpath(os.path.join(relpath, matches.group(3)))
297            if matches.group(2) == 'include':
298                # The manifest spec wants a reference to another manifest here,
299                # but we need just the directory. We do need the trailing
300                # separator so we don't accidentally match other paths of which
301                # this one is a prefix.
302                assert(path.endswith('jstests.list'))
303                path = path[:-len('jstests.list')]
304
305            entries.append({'path': path, 'terms': matches.group(1),
306                            'comment': comment.strip()})
307
308    # if one directory name is a prefix of another, we want the shorter one
309    # first
310    entries.sort(key=lambda x: x["path"])
311    return entries
312
313def _apply_external_manifests(filename, testcase, entries, xul_tester):
314    for entry in entries:
315        if filename.startswith(entry["path"]):
316            # The reftest spec would require combining the terms (failure types)
317            # that may already be defined in the test case with the terms
318            # specified in entry; for example, a skip overrides a random, which
319            # overrides a fails. Since we don't necessarily know yet in which
320            # environment the test cases will be run, we'd also have to
321            # consider skip-if, random-if, and fails-if with as-yet unresolved
322            # conditions.
323            # At this point, we use external manifests only for test cases
324            # that can't have their own failure type comments, so we simply
325            # use the terms for the most specific path.
326            testcase.terms = entry["terms"]
327            testcase.comment = entry["comment"]
328            _parse_one(testcase, xul_tester)
329
330def _is_test_file(path_from_root, basename, filename, requested_paths,
331                  excluded_paths):
332    # Any file whose basename matches something in this set is ignored.
333    EXCLUDED = set(('browser.js', 'shell.js', 'template.js',
334                    'user.js', 'sta.js',
335                    'test262-browser.js', 'test262-shell.js',
336                    'test402-browser.js', 'test402-shell.js',
337                    'testBuiltInObject.js', 'testIntl.js',
338                    'js-test-driver-begin.js', 'js-test-driver-end.js'))
339
340    # Skip js files in the root test directory.
341    if not path_from_root:
342        return False
343
344    # Skip files that we know are not tests.
345    if basename in EXCLUDED:
346        return False
347
348    # If any tests are requested by name, skip tests that do not match.
349    if requested_paths \
350        and not any(req in filename for req in requested_paths):
351        return False
352
353    # Skip excluded tests.
354    if filename in excluded_paths:
355        return False
356
357    return True
358
359
360def count_tests(location, requested_paths, excluded_paths):
361    count = 0
362    for root, basename in _find_all_js_files(location, location):
363        filename = os.path.join(root, basename)
364        if _is_test_file(root, basename, filename, requested_paths, excluded_paths):
365            count += 1
366    return count
367
368
369def load_reftests(location, requested_paths, excluded_paths, xul_tester, reldir=''):
370    """
371    Locates all tests by walking the filesystem starting at |location|.
372    Uses xul_tester to evaluate any test conditions in the test header.
373    Failure type and comment for a test case can come from
374    - an external manifest entry for the test case,
375    - an external manifest entry for a containing directory,
376    - most commonly: the header of the test case itself.
377    """
378    manifestFile = os.path.join(location, 'jstests.list')
379    externalManifestEntries = _parse_external_manifest(manifestFile, '')
380
381    for root, basename in _find_all_js_files(location, location):
382        # Get the full path and relative location of the file.
383        filename = os.path.join(root, basename)
384        if not _is_test_file(root, basename, filename, requested_paths, excluded_paths):
385            continue
386
387        # Skip empty files.
388        fullpath = os.path.join(location, filename)
389        statbuf = os.stat(fullpath)
390
391        testcase = RefTestCase(os.path.join(reldir, filename))
392        _apply_external_manifests(filename, testcase, externalManifestEntries,
393                                  xul_tester)
394        _parse_test_header(fullpath, testcase, xul_tester)
395        yield testcase
396