1import argparse 2import json 3import logging 4import os 5import re 6import urlparse 7from collections import defaultdict 8 9import manifestupdate 10 11from wptrunner import expected 12from wptrunner.wptmanifest.serializer import serialize 13from wptrunner.wptmanifest.backends import base 14 15here = os.path.dirname(__file__) 16logger = logging.getLogger(__name__) 17yaml = None 18 19 20class Compiler(base.Compiler): 21 def visit_KeyValueNode(self, node): 22 key_name = node.data 23 values = [] 24 for child in node.children: 25 values.append(self.visit(child)) 26 27 self.output_node.set(key_name, values) 28 29 def visit_ConditionalNode(self, node): 30 assert len(node.children) == 2 31 # For conditional nodes, just return the subtree 32 return node.children[0], self.visit(node.children[1]) 33 34 def visit_UnaryExpressionNode(self, node): 35 raise NotImplementedError 36 37 def visit_BinaryExpressionNode(self, node): 38 raise NotImplementedError 39 40 def visit_UnaryOperatorNode(self, node): 41 raise NotImplementedError 42 43 def visit_BinaryOperatorNode(self, node): 44 raise NotImplementedError 45 46 47class ExpectedManifest(base.ManifestItem): 48 def __init__(self, node, test_path, url_base): 49 """Object representing all the tests in a particular manifest 50 51 :param name: Name of the AST Node associated with this object. 52 Should always be None since this should always be associated with 53 the root node of the AST. 54 :param test_path: Path of the test file associated with this manifest. 55 :param url_base: Base url for serving the tests in this manifest 56 """ 57 if test_path is None: 58 raise ValueError("ExpectedManifest requires a test path") 59 if url_base is None: 60 raise ValueError("ExpectedManifest requires a base url") 61 base.ManifestItem.__init__(self, node) 62 self.child_map = {} 63 self.test_path = test_path 64 self.url_base = url_base 65 66 def append(self, child): 67 """Add a test to the manifest""" 68 base.ManifestItem.append(self, child) 69 self.child_map[child.id] = child 70 71 @property 72 def url(self): 73 return urlparse.urljoin(self.url_base, 74 "/".join(self.test_path.split(os.path.sep))) 75 76 77class DirectoryManifest(base.ManifestItem): 78 pass 79 80 81class TestManifestItem(base.ManifestItem): 82 def __init__(self, node, **kwargs): 83 """Tree node associated with a particular test in a manifest 84 85 :param name: name of the test""" 86 base.ManifestItem.__init__(self, node) 87 self.subtests = {} 88 89 @property 90 def id(self): 91 return urlparse.urljoin(self.parent.url, self.name) 92 93 def append(self, node): 94 """Add a subtest to the current test 95 96 :param node: AST Node associated with the subtest""" 97 child = base.ManifestItem.append(self, node) 98 self.subtests[child.name] = child 99 100 def get_subtest(self, name): 101 """Get the SubtestNode corresponding to a particular subtest, by name 102 103 :param name: Name of the node to return""" 104 if name in self.subtests: 105 return self.subtests[name] 106 return None 107 108 109class SubtestManifestItem(TestManifestItem): 110 pass 111 112 113def data_cls_getter(output_node, visited_node): 114 # visited_node is intentionally unused 115 if output_node is None: 116 return ExpectedManifest 117 if isinstance(output_node, ExpectedManifest): 118 return TestManifestItem 119 if isinstance(output_node, TestManifestItem): 120 return SubtestManifestItem 121 raise ValueError 122 123 124def get_manifest(metadata_root, test_path, url_base): 125 """Get the ExpectedManifest for a particular test path, or None if there is no 126 metadata stored for that test path. 127 128 :param metadata_root: Absolute path to the root of the metadata directory 129 :param test_path: Path to the test(s) relative to the test root 130 :param url_base: Base url for serving the tests in this manifest 131 :param run_info: Dictionary of properties of the test run for which the expectation 132 values should be computed. 133 """ 134 manifest_path = expected.expected_path(metadata_root, test_path) 135 try: 136 with open(manifest_path) as f: 137 return compile(f, 138 data_cls_getter=data_cls_getter, 139 test_path=test_path, 140 url_base=url_base) 141 except IOError: 142 return None 143 144 145def get_dir_manifest(path): 146 """Get the ExpectedManifest for a particular test path, or None if there is no 147 metadata stored for that test path. 148 149 :param path: Full path to the ini file 150 :param run_info: Dictionary of properties of the test run for which the expectation 151 values should be computed. 152 """ 153 try: 154 with open(path) as f: 155 return compile(f, data_cls_getter=lambda x,y: DirectoryManifest) 156 except IOError: 157 return None 158 159 160def compile(stream, data_cls_getter=None, **kwargs): 161 return base.compile(Compiler, 162 stream, 163 data_cls_getter=data_cls_getter, 164 **kwargs) 165 166 167def create_parser(): 168 parser = argparse.ArgumentParser() 169 parser.add_argument("--out-dir", help="Directory to store output files") 170 parser.add_argument("--meta-dir", help="Directory containing wpt-metadata " 171 "checkout to update.") 172 return parser 173 174 175def run(src_root, obj_root, logger_=None, **kwargs): 176 logger_obj = logger_ if logger_ is not None else logger 177 178 manifests = manifestupdate.run(src_root, obj_root, logger_obj, **kwargs) 179 180 rv = {} 181 dirs_seen = set() 182 183 for meta_root, test_path, test_metadata in iter_tests(manifests): 184 for dir_path in get_dir_paths(meta_root, test_path): 185 if dir_path not in dirs_seen: 186 dirs_seen.add(dir_path) 187 dir_manifest = get_dir_manifest(dir_path) 188 rel_path = os.path.relpath(dir_path, meta_root) 189 if dir_manifest: 190 add_manifest(rv, rel_path, dir_manifest) 191 else: 192 break 193 add_manifest(rv, test_path, test_metadata) 194 195 if kwargs["out_dir"]: 196 if not os.path.exists(kwargs["out_dir"]): 197 os.makedirs(kwargs["out_dir"]) 198 out_path = os.path.join(kwargs["out_dir"], "summary.json") 199 with open(out_path, "w") as f: 200 json.dump(rv, f) 201 else: 202 print(json.dumps(rv, indent=2)) 203 204 if kwargs["meta_dir"]: 205 update_wpt_meta(logger_obj, kwargs["meta_dir"], rv) 206 207 208def get_dir_paths(test_root, test_path): 209 if not os.path.isabs(test_path): 210 test_path = os.path.join(test_root, test_path) 211 dir_path = os.path.dirname(test_path) 212 while dir_path != test_root: 213 yield os.path.join(dir_path, "__dir__.ini") 214 dir_path = os.path.dirname(dir_path) 215 assert len(dir_path) >= len(test_root) 216 217 218def iter_tests(manifests): 219 for manifest in manifests.iterkeys(): 220 for test_type, test_path, tests in manifest: 221 url_base = manifests[manifest]["url_base"] 222 metadata_base = manifests[manifest]["metadata_path"] 223 expected_manifest = get_manifest(metadata_base, test_path, url_base) 224 if expected_manifest: 225 yield metadata_base, test_path, expected_manifest 226 227 228def add_manifest(target, path, metadata): 229 dir_name = os.path.dirname(path) 230 key = [dir_name] 231 232 add_metadata(target, key, metadata) 233 234 key.append("_tests") 235 236 for test_metadata in metadata.children: 237 key.append(test_metadata.name) 238 add_metadata(target, key, test_metadata) 239 key.append("_subtests") 240 for subtest_metadata in test_metadata.children: 241 key.append(subtest_metadata.name) 242 add_metadata(target, 243 key, 244 subtest_metadata) 245 key.pop() 246 key.pop() 247 key.pop() 248 249 250simple_props = ["disabled", "min-asserts", "max-asserts", "lsan-allowed", 251 "leak-allowed", "bug"] 252statuses = set(["CRASH"]) 253 254 255def add_metadata(target, key, metadata): 256 if not is_interesting(metadata): 257 return 258 259 for part in key: 260 if part not in target: 261 target[part] = {} 262 target = target[part] 263 264 for prop in simple_props: 265 if metadata.has_key(prop): 266 target[prop] = get_condition_value_list(metadata, prop) 267 268 if metadata.has_key("expected"): 269 intermittent = [] 270 values = metadata.get("expected") 271 by_status = defaultdict(list) 272 for item in values: 273 if isinstance(item, tuple): 274 condition, status = item 275 else: 276 condition = None 277 status = item 278 if isinstance(status, list): 279 intermittent.append((condition, status)) 280 expected_status = status[0] 281 else: 282 expected_status = status 283 by_status[expected_status].append(condition) 284 for status in statuses: 285 if status in by_status: 286 target["expected_%s" % status] = [serialize(item) if item else None 287 for item in by_status[status]] 288 if intermittent: 289 target["intermittent"] = [[serialize(cond) if cond else None, intermittent_statuses] 290 for cond, intermittent_statuses in intermittent] 291 292 293def get_condition_value_list(metadata, key): 294 conditions = [] 295 for item in metadata.get(key): 296 if isinstance(item, tuple): 297 assert len(item) == 2 298 conditions.append((serialize(item[0]), item[1])) 299 else: 300 conditions.append((None, item)) 301 return conditions 302 303 304def is_interesting(metadata): 305 if any(metadata.has_key(prop) for prop in simple_props): 306 return True 307 308 if metadata.has_key("expected"): 309 for expected_value in metadata.get("expected"): 310 # Include both expected and known intermittent values 311 if isinstance(expected_value, tuple): 312 expected_value = expected_value[1] 313 if isinstance(expected_value, list): 314 return True 315 if expected_value in statuses: 316 return True 317 return True 318 return False 319 320 321def update_wpt_meta(logger, meta_root, data): 322 global yaml 323 import yaml 324 325 if not os.path.exists(meta_root) or not os.path.isdir(meta_root): 326 raise ValueError("%s is not a directory" % (meta_root,)) 327 328 with WptMetaCollection(meta_root) as wpt_meta: 329 for dir_path, dir_data in sorted(data.iteritems()): 330 for test, test_data in dir_data.get("_tests", {}).iteritems(): 331 add_test_data(logger, wpt_meta, dir_path, test, None, test_data) 332 for subtest, subtest_data in test_data.get("_subtests", {}).iteritems(): 333 add_test_data(logger, wpt_meta, dir_path, test, subtest, subtest_data) 334 335def add_test_data(logger, wpt_meta, dir_path, test, subtest, test_data): 336 triage_keys = ["bug"] 337 338 for key in triage_keys: 339 if key in test_data: 340 value = test_data[key] 341 for cond_value in value: 342 if cond_value[0] is not None: 343 logger.info("Skipping conditional metadata") 344 continue 345 cond_value = cond_value[1] 346 if not isinstance(cond_value, list): 347 cond_value = [cond_value] 348 for bug_value in cond_value: 349 bug_link = get_bug_link(bug_value) 350 if bug_link is None: 351 logger.info("Could not extract bug: %s" % value) 352 continue 353 meta = wpt_meta.get(dir_path) 354 meta.set(test, 355 subtest, 356 product="firefox", 357 bug_url=bug_link) 358 359 360bugzilla_re = re.compile("https://bugzilla\.mozilla\.org/show_bug\.cgi\?id=\d+") 361bug_re = re.compile("(?:[Bb][Uu][Gg])?\s*(\d+)") 362 363 364def get_bug_link(value): 365 value = value.strip() 366 m = bugzilla_re.match(value) 367 if m: 368 return m.group(0) 369 m = bug_re.match(value) 370 if m: 371 return "https://bugzilla.mozilla.org/show_bug.cgi?id=%s" % m.group(1) 372 373 374class WptMetaCollection(object): 375 def __init__(self, root): 376 self.root = root 377 self.loaded = {} 378 379 def __enter__(self): 380 return self 381 382 def __exit__(self, *args, **kwargs): 383 for item in self.loaded.itervalues(): 384 item.write(self.root) 385 self.loaded = {} 386 387 def get(self, dir_path): 388 if dir_path not in self.loaded: 389 meta = WptMeta.get_or_create(self.root, dir_path) 390 self.loaded[dir_path] = meta 391 return self.loaded[dir_path] 392 393 394class WptMeta(object): 395 def __init__(self, dir_path, data): 396 assert "links" in data and isinstance(data["links"], list) 397 self.dir_path = dir_path 398 self.data = data 399 400 @staticmethod 401 def meta_path(meta_root, dir_path): 402 return os.path.join(meta_root, dir_path, "META.yml") 403 404 def path(self, meta_root): 405 return self.meta_path(meta_root, self.dir_path) 406 407 @classmethod 408 def get_or_create(cls, meta_root, dir_path): 409 if os.path.exists(cls.meta_path(meta_root, dir_path)): 410 return cls.load(meta_root, dir_path) 411 return cls(dir_path, {"links": []}) 412 413 @classmethod 414 def load(cls, meta_root, dir_path): 415 with open(cls.meta_path(meta_root, dir_path), "r") as f: 416 data = yaml.safe_load(f) 417 return cls(dir_path, data) 418 419 def set(self, test, subtest, product, bug_url): 420 target_link = None 421 for link in self.data["links"]: 422 link_product = link.get("product") 423 if link_product: 424 link_product = link_product.split("-", 1)[0] 425 if link_product is None or link_product == product: 426 if link["url"] == bug_url: 427 target_link = link 428 break 429 430 if target_link is None: 431 target_link = {"product": product.encode("utf8"), 432 "url": bug_url.encode("utf8"), 433 "results": []} 434 self.data["links"].append(target_link) 435 436 if not "results" in target_link: 437 target_link["results"] = [] 438 439 has_result = any((result["test"] == test and result.get("subtest") == subtest) 440 for result in target_link["results"]) 441 if not has_result: 442 data = {"test": test.encode("utf8")} 443 if subtest: 444 data["subtest"] = subtest.encode("utf8") 445 target_link["results"].append(data) 446 447 def write(self, meta_root): 448 path = self.path(meta_root) 449 dirname = os.path.dirname(path) 450 if not os.path.exists(dirname): 451 os.makedirs(dirname) 452 with open(path, "wb") as f: 453 yaml.safe_dump(self.data, f, 454 default_flow_style=False, 455 allow_unicode=True) 456