1# Library for JSTest manifests. 2# 3# This includes classes for representing and parsing JS manifests. 4 5from __future__ import print_function 6 7import os, re, sys 8from subprocess import Popen, PIPE 9 10from tests import RefTestCase 11 12 13def split_path_into_dirs(path): 14 dirs = [path] 15 16 while True: 17 path, tail = os.path.split(path) 18 if not tail: 19 break 20 dirs.append(path) 21 return dirs 22 23class XULInfo: 24 def __init__(self, abi, os, isdebug): 25 self.abi = abi 26 self.os = os 27 self.isdebug = isdebug 28 self.browserIsRemote = False 29 30 def as_js(self): 31 """Return JS that when executed sets up variables so that JS expression 32 predicates on XUL build info evaluate properly.""" 33 34 return ('var xulRuntime = {{ OS: "{}", XPCOMABI: "{}", shell: true }};' 35 'var isDebugBuild={}; var Android={}; ' 36 'var browserIsRemote={}'.format( 37 self.os, 38 self.abi, 39 str(self.isdebug).lower(), 40 str(self.os == "Android").lower(), 41 str(self.browserIsRemote).lower())) 42 43 @classmethod 44 def create(cls, jsdir): 45 """Create a XULInfo based on the current platform's characteristics.""" 46 47 # Our strategy is to find the autoconf.mk generated for the build and 48 # read the values from there. 49 50 # Find config/autoconf.mk. 51 dirs = split_path_into_dirs(os.getcwd()) + split_path_into_dirs(jsdir) 52 53 path = None 54 for dir in dirs: 55 _path = os.path.join(dir, 'config/autoconf.mk') 56 if os.path.isfile(_path): 57 path = _path 58 break 59 60 if path == None: 61 print("Can't find config/autoconf.mk on a directory containing" 62 " the JS shell (searched from {})".format(jsdir)) 63 sys.exit(1) 64 65 # Read the values. 66 val_re = re.compile(r'(TARGET_XPCOM_ABI|OS_TARGET|MOZ_DEBUG)\s*=\s*(.*)') 67 kw = {'isdebug': False} 68 for line in open(path): 69 m = val_re.match(line) 70 if m: 71 key, val = m.groups() 72 val = val.rstrip() 73 if key == 'TARGET_XPCOM_ABI': 74 kw['abi'] = val 75 if key == 'OS_TARGET': 76 kw['os'] = val 77 if key == 'MOZ_DEBUG': 78 kw['isdebug'] = (val == '1') 79 return cls(**kw) 80 81class XULInfoTester: 82 def __init__(self, xulinfo, js_bin): 83 self.js_prologue = xulinfo.as_js() 84 self.js_bin = js_bin 85 # Maps JS expr to evaluation result. 86 self.cache = {} 87 88 def test(self, cond): 89 """Test a XUL predicate condition against this local info.""" 90 ans = self.cache.get(cond, None) 91 if ans is None: 92 cmd = [ 93 self.js_bin, 94 # run in safe configuration, since it is hard to debug 95 # crashes when running code here. In particular, msan will 96 # error out if the jit is active. 97 '--no-baseline', 98 '-e', self.js_prologue, 99 '-e', 'print(!!({}))'.format(cond) 100 ] 101 p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) 102 out, err = p.communicate() 103 if out in ('true\n', 'true\r\n'): 104 ans = True 105 elif out in ('false\n', 'false\r\n'): 106 ans = False 107 else: 108 raise Exception("Failed to test XUL condition {!r};" 109 " output was {!r}, stderr was {!r}".format( 110 cond, out, err)) 111 self.cache[cond] = ans 112 return ans 113 114class NullXULInfoTester: 115 """Can be used to parse manifests without a JS shell.""" 116 def test(self, cond): 117 return False 118 119def _parse_one(testcase, xul_tester): 120 pos = 0 121 parts = testcase.terms.split() 122 while pos < len(parts): 123 if parts[pos] == 'fails': 124 testcase.expect = False 125 pos += 1 126 elif parts[pos] == 'skip': 127 testcase.expect = testcase.enable = False 128 pos += 1 129 elif parts[pos] == 'random': 130 testcase.random = True 131 pos += 1 132 elif parts[pos].startswith('fails-if'): 133 cond = parts[pos][len('fails-if('):-1] 134 if xul_tester.test(cond): 135 testcase.expect = False 136 pos += 1 137 elif parts[pos].startswith('asserts-if'): 138 # This directive means we may flunk some number of 139 # NS_ASSERTIONs in the browser. For the shell, ignore it. 140 pos += 1 141 elif parts[pos].startswith('skip-if'): 142 cond = parts[pos][len('skip-if('):-1] 143 if xul_tester.test(cond): 144 testcase.expect = testcase.enable = False 145 pos += 1 146 elif parts[pos].startswith('random-if'): 147 cond = parts[pos][len('random-if('):-1] 148 if xul_tester.test(cond): 149 testcase.random = True 150 pos += 1 151 elif parts[pos] == 'slow': 152 testcase.slow = True 153 pos += 1 154 elif parts[pos] == 'silentfail': 155 # silentfails use tons of memory, and Darwin doesn't support ulimit. 156 if xul_tester.test("xulRuntime.OS == 'Darwin'"): 157 testcase.expect = testcase.enable = False 158 pos += 1 159 else: 160 print('warning: invalid manifest line element "{}"'.format( 161 parts[pos])) 162 pos += 1 163 164def _build_manifest_script_entry(script_name, test): 165 line = [] 166 if test.terms: 167 line.append(test.terms) 168 line.append("script") 169 line.append(script_name) 170 if test.comment: 171 line.append("#") 172 line.append(test.comment) 173 return ' '.join(line) 174 175def _map_prefixes_left(test_gen): 176 """ 177 Splits tests into a dictionary keyed on the first component of the test 178 path, aggregating tests with a common base path into a list. 179 """ 180 byprefix = {} 181 for t in test_gen: 182 left, sep, remainder = t.path.partition(os.sep) 183 if left not in byprefix: 184 byprefix[left] = [] 185 if remainder: 186 t.path = remainder 187 byprefix[left].append(t) 188 return byprefix 189 190def _emit_manifest_at(location, relative, test_gen, depth): 191 """ 192 location - str: absolute path where we want to write the manifest 193 relative - str: relative path from topmost manifest directory to current 194 test_gen - (str): generator of all test paths and directorys 195 depth - int: number of dirs we are below the topmost manifest dir 196 """ 197 manifests = _map_prefixes_left(test_gen) 198 199 filename = os.path.join(location, 'jstests.list') 200 manifest = [] 201 numTestFiles = 0 202 for k, test_list in manifests.iteritems(): 203 fullpath = os.path.join(location, k) 204 if os.path.isdir(fullpath): 205 manifest.append("include " + k + "/jstests.list") 206 relpath = os.path.join(relative, k) 207 _emit_manifest_at(fullpath, relpath, test_list, depth + 1) 208 else: 209 numTestFiles += 1 210 if len(test_list) != 1: 211 import pdb; pdb.set_trace() 212 assert len(test_list) == 1 213 line = _build_manifest_script_entry(k, test_list[0]) 214 manifest.append(line) 215 216 # Always present our manifest in sorted order. 217 manifest.sort() 218 219 # If we have tests, we have to set the url-prefix so reftest can find them. 220 if numTestFiles > 0: 221 manifest = ["url-prefix {}jsreftest.html?test={}/".format( 222 '../' * depth, relative)] + manifest 223 224 fp = open(filename, 'w') 225 try: 226 fp.write('\n'.join(manifest) + '\n') 227 finally: 228 fp.close() 229 230def make_manifests(location, test_gen): 231 _emit_manifest_at(location, '', test_gen, 0) 232 233def _find_all_js_files(base, location): 234 for root, dirs, files in os.walk(location): 235 root = root[len(base) + 1:] 236 for fn in files: 237 if fn.endswith('.js'): 238 yield root, fn 239 240TEST_HEADER_PATTERN_INLINE = re.compile(r'//\s*\|(.*?)\|\s*(.*?)\s*(--\s*(.*))?$') 241TEST_HEADER_PATTERN_MULTI = re.compile(r'/\*\s*\|(.*?)\|\s*(.*?)\s*(--\s*(.*))?\*/') 242 243def _parse_test_header(fullpath, testcase, xul_tester): 244 """ 245 This looks a bit weird. The reason is that it needs to be efficient, since 246 it has to be done on every test 247 """ 248 fp = open(fullpath, 'r') 249 try: 250 buf = fp.read(512) 251 finally: 252 fp.close() 253 254 # Bail early if we do not start with a single comment. 255 if not buf.startswith("//"): 256 return 257 258 # Extract the token. 259 buf, _, _ = buf.partition('\n') 260 matches = TEST_HEADER_PATTERN_INLINE.match(buf) 261 262 if not matches: 263 matches = TEST_HEADER_PATTERN_MULTI.match(buf) 264 if not matches: 265 return 266 267 testcase.tag = matches.group(1) 268 testcase.terms = matches.group(2) 269 testcase.comment = matches.group(4) 270 271 _parse_one(testcase, xul_tester) 272 273def _parse_external_manifest(filename, relpath): 274 """ 275 Reads an external manifest file for test suites whose individual test cases 276 can't be decorated with reftest comments. 277 filename - str: name of the manifest file 278 relpath - str: relative path of the directory containing the manifest 279 within the test suite 280 """ 281 entries = [] 282 283 with open(filename, 'r') as fp: 284 manifest_re = re.compile(r'^\s*(.*)\s+(include|script)\s+(\S+)$') 285 for line in fp: 286 line, _, comment = line.partition('#') 287 line = line.strip() 288 if not line: 289 continue 290 matches = manifest_re.match(line) 291 if not matches: 292 print('warning: unrecognized line in jstests.list:' 293 ' {0}'.format(line)) 294 continue 295 296 path = os.path.normpath(os.path.join(relpath, matches.group(3))) 297 if matches.group(2) == 'include': 298 # The manifest spec wants a reference to another manifest here, 299 # but we need just the directory. We do need the trailing 300 # separator so we don't accidentally match other paths of which 301 # this one is a prefix. 302 assert(path.endswith('jstests.list')) 303 path = path[:-len('jstests.list')] 304 305 entries.append({'path': path, 'terms': matches.group(1), 306 'comment': comment.strip()}) 307 308 # if one directory name is a prefix of another, we want the shorter one 309 # first 310 entries.sort(key=lambda x: x["path"]) 311 return entries 312 313def _apply_external_manifests(filename, testcase, entries, xul_tester): 314 for entry in entries: 315 if filename.startswith(entry["path"]): 316 # The reftest spec would require combining the terms (failure types) 317 # that may already be defined in the test case with the terms 318 # specified in entry; for example, a skip overrides a random, which 319 # overrides a fails. Since we don't necessarily know yet in which 320 # environment the test cases will be run, we'd also have to 321 # consider skip-if, random-if, and fails-if with as-yet unresolved 322 # conditions. 323 # At this point, we use external manifests only for test cases 324 # that can't have their own failure type comments, so we simply 325 # use the terms for the most specific path. 326 testcase.terms = entry["terms"] 327 testcase.comment = entry["comment"] 328 _parse_one(testcase, xul_tester) 329 330def _is_test_file(path_from_root, basename, filename, requested_paths, 331 excluded_paths): 332 # Any file whose basename matches something in this set is ignored. 333 EXCLUDED = set(('browser.js', 'shell.js', 'template.js', 334 'user.js', 'sta.js', 335 'test262-browser.js', 'test262-shell.js', 336 'test402-browser.js', 'test402-shell.js', 337 'testBuiltInObject.js', 'testIntl.js', 338 'js-test-driver-begin.js', 'js-test-driver-end.js')) 339 340 # Skip js files in the root test directory. 341 if not path_from_root: 342 return False 343 344 # Skip files that we know are not tests. 345 if basename in EXCLUDED: 346 return False 347 348 # If any tests are requested by name, skip tests that do not match. 349 if requested_paths \ 350 and not any(req in filename for req in requested_paths): 351 return False 352 353 # Skip excluded tests. 354 if filename in excluded_paths: 355 return False 356 357 return True 358 359 360def count_tests(location, requested_paths, excluded_paths): 361 count = 0 362 for root, basename in _find_all_js_files(location, location): 363 filename = os.path.join(root, basename) 364 if _is_test_file(root, basename, filename, requested_paths, excluded_paths): 365 count += 1 366 return count 367 368 369def load_reftests(location, requested_paths, excluded_paths, xul_tester, reldir=''): 370 """ 371 Locates all tests by walking the filesystem starting at |location|. 372 Uses xul_tester to evaluate any test conditions in the test header. 373 Failure type and comment for a test case can come from 374 - an external manifest entry for the test case, 375 - an external manifest entry for a containing directory, 376 - most commonly: the header of the test case itself. 377 """ 378 manifestFile = os.path.join(location, 'jstests.list') 379 externalManifestEntries = _parse_external_manifest(manifestFile, '') 380 381 for root, basename in _find_all_js_files(location, location): 382 # Get the full path and relative location of the file. 383 filename = os.path.join(root, basename) 384 if not _is_test_file(root, basename, filename, requested_paths, excluded_paths): 385 continue 386 387 # Skip empty files. 388 fullpath = os.path.join(location, filename) 389 statbuf = os.stat(fullpath) 390 391 testcase = RefTestCase(os.path.join(reldir, filename)) 392 _apply_external_manifests(filename, testcase, externalManifestEntries, 393 xul_tester) 394 _parse_test_header(fullpath, testcase, xul_tester) 395 yield testcase 396