1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5import argparse 6import json 7import logging 8import os 9import re 10from urllib import parse as urlparse 11from collections import defaultdict 12 13import manifestupdate 14 15from wptrunner import expected 16from wptrunner.wptmanifest.serializer import serialize 17from wptrunner.wptmanifest.backends import base 18 19here = os.path.dirname(__file__) 20logger = logging.getLogger(__name__) 21yaml = None 22 23 24class Compiler(base.Compiler): 25 def visit_KeyValueNode(self, node): 26 key_name = node.data 27 values = [] 28 for child in node.children: 29 values.append(self.visit(child)) 30 31 self.output_node.set(key_name, values) 32 33 def visit_ConditionalNode(self, node): 34 assert len(node.children) == 2 35 # For conditional nodes, just return the subtree 36 return node.children[0], self.visit(node.children[1]) 37 38 def visit_UnaryExpressionNode(self, node): 39 raise NotImplementedError 40 41 def visit_BinaryExpressionNode(self, node): 42 raise NotImplementedError 43 44 def visit_UnaryOperatorNode(self, node): 45 raise NotImplementedError 46 47 def visit_BinaryOperatorNode(self, node): 48 raise NotImplementedError 49 50 51class ExpectedManifest(base.ManifestItem): 52 def __init__(self, node, test_path, url_base): 53 """Object representing all the tests in a particular manifest 54 55 :param name: Name of the AST Node associated with this object. 56 Should always be None since this should always be associated with 57 the root node of the AST. 58 :param test_path: Path of the test file associated with this manifest. 59 :param url_base: Base url for serving the tests in this manifest 60 """ 61 if test_path is None: 62 raise ValueError("ExpectedManifest requires a test path") 63 if url_base is None: 64 raise ValueError("ExpectedManifest requires a base url") 65 base.ManifestItem.__init__(self, node) 66 self.child_map = {} 67 self.test_path = test_path 68 self.url_base = url_base 69 70 def append(self, child): 71 """Add a test to the manifest""" 72 base.ManifestItem.append(self, child) 73 self.child_map[child.id] = child 74 75 @property 76 def url(self): 77 return urlparse.urljoin( 78 self.url_base, "/".join(self.test_path.split(os.path.sep)) 79 ) 80 81 82class DirectoryManifest(base.ManifestItem): 83 pass 84 85 86class TestManifestItem(base.ManifestItem): 87 def __init__(self, node, **kwargs): 88 """Tree node associated with a particular test in a manifest 89 90 :param name: name of the test""" 91 base.ManifestItem.__init__(self, node) 92 self.subtests = {} 93 94 @property 95 def id(self): 96 return urlparse.urljoin(self.parent.url, self.name) 97 98 def append(self, node): 99 """Add a subtest to the current test 100 101 :param node: AST Node associated with the subtest""" 102 child = base.ManifestItem.append(self, node) 103 self.subtests[child.name] = child 104 105 def get_subtest(self, name): 106 """Get the SubtestNode corresponding to a particular subtest, by name 107 108 :param name: Name of the node to return""" 109 if name in self.subtests: 110 return self.subtests[name] 111 return None 112 113 114class SubtestManifestItem(TestManifestItem): 115 pass 116 117 118def data_cls_getter(output_node, visited_node): 119 # visited_node is intentionally unused 120 if output_node is None: 121 return ExpectedManifest 122 if isinstance(output_node, ExpectedManifest): 123 return TestManifestItem 124 if isinstance(output_node, TestManifestItem): 125 return SubtestManifestItem 126 raise ValueError 127 128 129def get_manifest(metadata_root, test_path, url_base): 130 """Get the ExpectedManifest for a particular test path, or None if there is no 131 metadata stored for that test path. 132 133 :param metadata_root: Absolute path to the root of the metadata directory 134 :param test_path: Path to the test(s) relative to the test root 135 :param url_base: Base url for serving the tests in this manifest 136 :param run_info: Dictionary of properties of the test run for which the expectation 137 values should be computed. 138 """ 139 manifest_path = expected.expected_path(metadata_root, test_path) 140 try: 141 with open(manifest_path, "rb") as f: 142 return compile( 143 f, 144 data_cls_getter=data_cls_getter, 145 test_path=test_path, 146 url_base=url_base, 147 ) 148 except IOError: 149 return None 150 151 152def get_dir_manifest(path): 153 """Get the ExpectedManifest for a particular test path, or None if there is no 154 metadata stored for that test path. 155 156 :param path: Full path to the ini file 157 :param run_info: Dictionary of properties of the test run for which the expectation 158 values should be computed. 159 """ 160 try: 161 with open(path, "rb") as f: 162 return compile(f, data_cls_getter=lambda x, y: DirectoryManifest) 163 except IOError: 164 return None 165 166 167def compile(stream, data_cls_getter=None, **kwargs): 168 return base.compile(Compiler, stream, data_cls_getter=data_cls_getter, **kwargs) 169 170 171def create_parser(): 172 parser = argparse.ArgumentParser() 173 parser.add_argument("--out-dir", help="Directory to store output files") 174 parser.add_argument( 175 "--meta-dir", help="Directory containing wpt-metadata " "checkout to update." 176 ) 177 return parser 178 179 180def run(src_root, obj_root, logger_=None, **kwargs): 181 logger_obj = logger_ if logger_ is not None else logger 182 183 manifests = manifestupdate.run(src_root, obj_root, logger_obj, **kwargs) 184 185 rv = {} 186 dirs_seen = set() 187 188 for meta_root, test_path, test_metadata in iter_tests(manifests): 189 for dir_path in get_dir_paths(meta_root, test_path): 190 if dir_path not in dirs_seen: 191 dirs_seen.add(dir_path) 192 dir_manifest = get_dir_manifest(dir_path) 193 rel_path = os.path.relpath(dir_path, meta_root) 194 if dir_manifest: 195 add_manifest(rv, rel_path, dir_manifest) 196 else: 197 break 198 add_manifest(rv, test_path, test_metadata) 199 200 if kwargs["out_dir"]: 201 if not os.path.exists(kwargs["out_dir"]): 202 os.makedirs(kwargs["out_dir"]) 203 out_path = os.path.join(kwargs["out_dir"], "summary.json") 204 with open(out_path, "w") as f: 205 json.dump(rv, f) 206 else: 207 print(json.dumps(rv, indent=2)) 208 209 if kwargs["meta_dir"]: 210 update_wpt_meta(logger_obj, kwargs["meta_dir"], rv) 211 212 213def get_dir_paths(test_root, test_path): 214 if not os.path.isabs(test_path): 215 test_path = os.path.join(test_root, test_path) 216 dir_path = os.path.dirname(test_path) 217 while dir_path != test_root: 218 yield os.path.join(dir_path, "__dir__.ini") 219 dir_path = os.path.dirname(dir_path) 220 assert len(dir_path) >= len(test_root) 221 222 223def iter_tests(manifests): 224 for manifest in manifests.keys(): 225 for test_type, test_path, tests in manifest: 226 url_base = manifests[manifest]["url_base"] 227 metadata_base = manifests[manifest]["metadata_path"] 228 expected_manifest = get_manifest(metadata_base, test_path, url_base) 229 if expected_manifest: 230 yield metadata_base, test_path, expected_manifest 231 232 233def add_manifest(target, path, metadata): 234 dir_name = os.path.dirname(path) 235 key = [dir_name] 236 237 add_metadata(target, key, metadata) 238 239 key.append("_tests") 240 241 for test_metadata in metadata.children: 242 key.append(test_metadata.name) 243 add_metadata(target, key, test_metadata) 244 key.append("_subtests") 245 for subtest_metadata in test_metadata.children: 246 key.append(subtest_metadata.name) 247 add_metadata(target, key, subtest_metadata) 248 key.pop() 249 key.pop() 250 key.pop() 251 252 253simple_props = [ 254 "disabled", 255 "min-asserts", 256 "max-asserts", 257 "lsan-allowed", 258 "leak-allowed", 259 "bug", 260] 261statuses = set(["CRASH"]) 262 263 264def add_metadata(target, key, metadata): 265 if not is_interesting(metadata): 266 return 267 268 for part in key: 269 if part not in target: 270 target[part] = {} 271 target = target[part] 272 273 for prop in simple_props: 274 if metadata.has_key(prop): # noqa W601 275 target[prop] = get_condition_value_list(metadata, prop) 276 277 if metadata.has_key("expected"): # noqa W601 278 intermittent = [] 279 values = metadata.get("expected") 280 by_status = defaultdict(list) 281 for item in values: 282 if isinstance(item, tuple): 283 condition, status = item 284 else: 285 condition = None 286 status = item 287 if isinstance(status, list): 288 intermittent.append((condition, status)) 289 expected_status = status[0] 290 else: 291 expected_status = status 292 by_status[expected_status].append(condition) 293 for status in statuses: 294 if status in by_status: 295 target["expected_%s" % status] = [ 296 serialize(item) if item else None for item in by_status[status] 297 ] 298 if intermittent: 299 target["intermittent"] = [ 300 [serialize(cond) if cond else None, intermittent_statuses] 301 for cond, intermittent_statuses in intermittent 302 ] 303 304 305def get_condition_value_list(metadata, key): 306 conditions = [] 307 for item in metadata.get(key): 308 if isinstance(item, tuple): 309 assert len(item) == 2 310 conditions.append((serialize(item[0]), item[1])) 311 else: 312 conditions.append((None, item)) 313 return conditions 314 315 316def is_interesting(metadata): 317 if any(metadata.has_key(prop) for prop in simple_props): # noqa W601 318 return True 319 320 if metadata.has_key("expected"): # noqa W601 321 for expected_value in metadata.get("expected"): 322 # Include both expected and known intermittent values 323 if isinstance(expected_value, tuple): 324 expected_value = expected_value[1] 325 if isinstance(expected_value, list): 326 return True 327 if expected_value in statuses: 328 return True 329 return True 330 return False 331 332 333def update_wpt_meta(logger, meta_root, data): 334 global yaml 335 import yaml 336 337 if not os.path.exists(meta_root) or not os.path.isdir(meta_root): 338 raise ValueError("%s is not a directory" % (meta_root,)) 339 340 with WptMetaCollection(meta_root) as wpt_meta: 341 for dir_path, dir_data in sorted(data.items()): 342 for test, test_data in dir_data.get("_tests", {}).items(): 343 add_test_data(logger, wpt_meta, dir_path, test, None, test_data) 344 for subtest, subtest_data in test_data.get("_subtests", {}).items(): 345 add_test_data( 346 logger, wpt_meta, dir_path, test, subtest, subtest_data 347 ) 348 349 350def add_test_data(logger, wpt_meta, dir_path, test, subtest, test_data): 351 triage_keys = ["bug"] 352 353 for key in triage_keys: 354 if key in test_data: 355 value = test_data[key] 356 for cond_value in value: 357 if cond_value[0] is not None: 358 logger.info("Skipping conditional metadata") 359 continue 360 cond_value = cond_value[1] 361 if not isinstance(cond_value, list): 362 cond_value = [cond_value] 363 for bug_value in cond_value: 364 bug_link = get_bug_link(bug_value) 365 if bug_link is None: 366 logger.info("Could not extract bug: %s" % value) 367 continue 368 meta = wpt_meta.get(dir_path) 369 meta.set(test, subtest, product="firefox", bug_url=bug_link) 370 371 372bugzilla_re = re.compile("https://bugzilla\.mozilla\.org/show_bug\.cgi\?id=\d+") 373bug_re = re.compile("(?:[Bb][Uu][Gg])?\s*(\d+)") 374 375 376def get_bug_link(value): 377 value = value.strip() 378 m = bugzilla_re.match(value) 379 if m: 380 return m.group(0) 381 m = bug_re.match(value) 382 if m: 383 return "https://bugzilla.mozilla.org/show_bug.cgi?id=%s" % m.group(1) 384 385 386class WptMetaCollection(object): 387 def __init__(self, root): 388 self.root = root 389 self.loaded = {} 390 391 def __enter__(self): 392 return self 393 394 def __exit__(self, *args, **kwargs): 395 for item in self.loaded.itervalues(): 396 item.write(self.root) 397 self.loaded = {} 398 399 def get(self, dir_path): 400 if dir_path not in self.loaded: 401 meta = WptMeta.get_or_create(self.root, dir_path) 402 self.loaded[dir_path] = meta 403 return self.loaded[dir_path] 404 405 406class WptMeta(object): 407 def __init__(self, dir_path, data): 408 assert "links" in data and isinstance(data["links"], list) 409 self.dir_path = dir_path 410 self.data = data 411 412 @staticmethod 413 def meta_path(meta_root, dir_path): 414 return os.path.join(meta_root, dir_path, "META.yml") 415 416 def path(self, meta_root): 417 return self.meta_path(meta_root, self.dir_path) 418 419 @classmethod 420 def get_or_create(cls, meta_root, dir_path): 421 if os.path.exists(cls.meta_path(meta_root, dir_path)): 422 return cls.load(meta_root, dir_path) 423 return cls(dir_path, {"links": []}) 424 425 @classmethod 426 def load(cls, meta_root, dir_path): 427 with open(cls.meta_path(meta_root, dir_path), "r") as f: 428 data = yaml.safe_load(f) 429 return cls(dir_path, data) 430 431 def set(self, test, subtest, product, bug_url): 432 target_link = None 433 for link in self.data["links"]: 434 link_product = link.get("product") 435 if link_product: 436 link_product = link_product.split("-", 1)[0] 437 if link_product is None or link_product == product: 438 if link["url"] == bug_url: 439 target_link = link 440 break 441 442 if target_link is None: 443 target_link = { 444 "product": product.encode("utf8"), 445 "url": bug_url.encode("utf8"), 446 "results": [], 447 } 448 self.data["links"].append(target_link) 449 450 if "results" not in target_link: 451 target_link["results"] = [] 452 453 has_result = any( 454 (result["test"] == test and result.get("subtest") == subtest) 455 for result in target_link["results"] 456 ) 457 if not has_result: 458 data = {"test": test.encode("utf8")} 459 if subtest: 460 data["subtest"] = subtest.encode("utf8") 461 target_link["results"].append(data) 462 463 def write(self, meta_root): 464 path = self.path(meta_root) 465 dirname = os.path.dirname(path) 466 if not os.path.exists(dirname): 467 os.makedirs(dirname) 468 with open(path, "wb") as f: 469 yaml.safe_dump(self.data, f, default_flow_style=False, allow_unicode=True) 470