1import os 2from collections import deque 3from urllib.parse import urljoin 4 5from .wptmanifest.backends import static 6from .wptmanifest.backends.base import ManifestItem 7 8from . import expected 9 10"""Manifest structure used to store expected results of a test. 11 12Each manifest file is represented by an ExpectedManifest that 13has one or more TestNode children, one per test in the manifest. 14Each TestNode has zero or more SubtestNode children, one for each 15known subtest of the test. 16""" 17 18 19def data_cls_getter(output_node, visited_node): 20 # visited_node is intentionally unused 21 if output_node is None: 22 return ExpectedManifest 23 if isinstance(output_node, ExpectedManifest): 24 return TestNode 25 if isinstance(output_node, TestNode): 26 return SubtestNode 27 raise ValueError 28 29 30def bool_prop(name, node): 31 """Boolean property""" 32 try: 33 return bool(node.get(name)) 34 except KeyError: 35 return None 36 37 38def int_prop(name, node): 39 """Boolean property""" 40 try: 41 return int(node.get(name)) 42 except KeyError: 43 return None 44 45 46def list_prop(name, node): 47 """List property""" 48 try: 49 list_prop = node.get(name) 50 if isinstance(list_prop, str): 51 return [list_prop] 52 return list(list_prop) 53 except KeyError: 54 return [] 55 56 57def str_prop(name, node): 58 try: 59 prop = node.get(name) 60 if not isinstance(prop, str): 61 raise ValueError 62 return prop 63 except KeyError: 64 return None 65 66 67def tags(node): 68 """Set of tags that have been applied to the test""" 69 try: 70 value = node.get("tags") 71 if isinstance(value, str): 72 return {value} 73 return set(value) 74 except KeyError: 75 return set() 76 77 78def prefs(node): 79 def value(ini_value): 80 if isinstance(ini_value, str): 81 return tuple(pref_piece.strip() for pref_piece in ini_value.split(':', 1)) 82 else: 83 # this should be things like @Reset, which are apparently type 'object' 84 return (ini_value, None) 85 86 try: 87 node_prefs = node.get("prefs") 88 if isinstance(node_prefs, str): 89 rv = dict(value(node_prefs)) 90 else: 91 rv = dict(value(item) for item in node_prefs) 92 except KeyError: 93 rv = {} 94 return rv 95 96 97def set_prop(name, node): 98 try: 99 node_items = node.get(name) 100 if isinstance(node_items, str): 101 rv = {node_items} 102 else: 103 rv = set(node_items) 104 except KeyError: 105 rv = set() 106 return rv 107 108 109def leak_threshold(node): 110 rv = {} 111 try: 112 node_items = node.get("leak-threshold") 113 if isinstance(node_items, str): 114 node_items = [node_items] 115 for item in node_items: 116 process, value = item.rsplit(":", 1) 117 rv[process.strip()] = int(value.strip()) 118 except KeyError: 119 pass 120 return rv 121 122 123def fuzzy_prop(node): 124 """Fuzzy reftest match 125 126 This can either be a list of strings or a single string. When a list is 127 supplied, the format of each item matches the description below. 128 129 The general format is 130 fuzzy = [key ":"] <prop> ";" <prop> 131 key = <test name> [reftype <reference name>] 132 reftype = "==" | "!=" 133 prop = [propName "=" ] range 134 propName = "maxDifferences" | "totalPixels" 135 range = <digits> ["-" <digits>] 136 137 So for example: 138 maxDifferences=10;totalPixels=10-20 139 140 specifies that for any test/ref pair for which no other rule is supplied, 141 there must be a maximum pixel difference of exactly 10, and between 10 and 142 20 total pixels different. 143 144 test.html==ref.htm:10;20 145 146 specifies that for a equality comparison between test.html and ref.htm, 147 resolved relative to the test path, there can be a maximum difference 148 of 10 in the pixel value for any channel and 20 pixels total difference. 149 150 ref.html:10;20 151 152 is just like the above but applies to any comparison involving ref.html 153 on the right hand side. 154 155 The return format is [(key, (maxDifferenceRange, totalPixelsRange))], where 156 the key is either None where no specific reference is specified, the reference 157 name where there is only one component or a tuple (test, ref, reftype) when the 158 exact comparison is specified. maxDifferenceRange and totalPixelsRange are tuples 159 of integers indicating the inclusive range of allowed values. 160""" 161 rv = [] 162 args = ["maxDifference", "totalPixels"] 163 try: 164 value = node.get("fuzzy") 165 except KeyError: 166 return rv 167 if not isinstance(value, list): 168 value = [value] 169 for item in value: 170 if not isinstance(item, str): 171 rv.append(item) 172 continue 173 parts = item.rsplit(":", 1) 174 if len(parts) == 1: 175 key = None 176 fuzzy_values = parts[0] 177 else: 178 key, fuzzy_values = parts 179 for reftype in ["==", "!="]: 180 if reftype in key: 181 key = key.split(reftype) 182 key.append(reftype) 183 key = tuple(key) 184 ranges = fuzzy_values.split(";") 185 if len(ranges) != 2: 186 raise ValueError("Malformed fuzzy value %s" % item) 187 arg_values = {None: deque()} 188 for range_str_value in ranges: 189 if "=" in range_str_value: 190 name, range_str_value = [part.strip() 191 for part in range_str_value.split("=", 1)] 192 if name not in args: 193 raise ValueError("%s is not a valid fuzzy property" % name) 194 if arg_values.get(name): 195 raise ValueError("Got multiple values for argument %s" % name) 196 else: 197 name = None 198 if "-" in range_str_value: 199 range_min, range_max = range_str_value.split("-") 200 else: 201 range_min = range_str_value 202 range_max = range_str_value 203 try: 204 range_value = tuple(int(item.strip()) for item in (range_min, range_max)) 205 except ValueError: 206 raise ValueError("Fuzzy value %s must be a range of integers" % range_str_value) 207 if name is None: 208 arg_values[None].append(range_value) 209 else: 210 arg_values[name] = range_value 211 range_values = [] 212 for arg_name in args: 213 if arg_values.get(arg_name): 214 value = arg_values.pop(arg_name) 215 else: 216 value = arg_values[None].popleft() 217 range_values.append(value) 218 rv.append((key, tuple(range_values))) 219 return rv 220 221 222class ExpectedManifest(ManifestItem): 223 def __init__(self, node, test_path, url_base): 224 """Object representing all the tests in a particular manifest 225 226 :param name: Name of the AST Node associated with this object. 227 Should always be None since this should always be associated with 228 the root node of the AST. 229 :param test_path: Path of the test file associated with this manifest. 230 :param url_base: Base url for serving the tests in this manifest 231 """ 232 name = node.data 233 if name is not None: 234 raise ValueError("ExpectedManifest should represent the root node") 235 if test_path is None: 236 raise ValueError("ExpectedManifest requires a test path") 237 if url_base is None: 238 raise ValueError("ExpectedManifest requires a base url") 239 ManifestItem.__init__(self, node) 240 self.child_map = {} 241 self.test_path = test_path 242 self.url_base = url_base 243 244 def append(self, child): 245 """Add a test to the manifest""" 246 ManifestItem.append(self, child) 247 self.child_map[child.id] = child 248 249 def _remove_child(self, child): 250 del self.child_map[child.id] 251 ManifestItem.remove_child(self, child) 252 assert len(self.child_map) == len(self.children) 253 254 def get_test(self, test_id): 255 """Get a test from the manifest by ID 256 257 :param test_id: ID of the test to return.""" 258 return self.child_map.get(test_id) 259 260 @property 261 def url(self): 262 return urljoin(self.url_base, 263 "/".join(self.test_path.split(os.path.sep))) 264 265 @property 266 def disabled(self): 267 return bool_prop("disabled", self) 268 269 @property 270 def restart_after(self): 271 return bool_prop("restart-after", self) 272 273 @property 274 def leaks(self): 275 return bool_prop("leaks", self) 276 277 @property 278 def min_assertion_count(self): 279 return int_prop("min-asserts", self) 280 281 @property 282 def max_assertion_count(self): 283 return int_prop("max-asserts", self) 284 285 @property 286 def tags(self): 287 return tags(self) 288 289 @property 290 def prefs(self): 291 return prefs(self) 292 293 @property 294 def lsan_disabled(self): 295 return bool_prop("lsan-disabled", self) 296 297 @property 298 def lsan_allowed(self): 299 return set_prop("lsan-allowed", self) 300 301 @property 302 def leak_allowed(self): 303 return set_prop("leak-allowed", self) 304 305 @property 306 def leak_threshold(self): 307 return leak_threshold(self) 308 309 @property 310 def lsan_max_stack_depth(self): 311 return int_prop("lsan-max-stack-depth", self) 312 313 @property 314 def fuzzy(self): 315 return fuzzy_prop(self) 316 317 @property 318 def expected(self): 319 return list_prop("expected", self)[0] 320 321 @property 322 def known_intermittent(self): 323 return list_prop("expected", self)[1:] 324 325 @property 326 def implementation_status(self): 327 return str_prop("implementation-status", self) 328 329 330class DirectoryManifest(ManifestItem): 331 @property 332 def disabled(self): 333 return bool_prop("disabled", self) 334 335 @property 336 def restart_after(self): 337 return bool_prop("restart-after", self) 338 339 @property 340 def leaks(self): 341 return bool_prop("leaks", self) 342 343 @property 344 def min_assertion_count(self): 345 return int_prop("min-asserts", self) 346 347 @property 348 def max_assertion_count(self): 349 return int_prop("max-asserts", self) 350 351 @property 352 def tags(self): 353 return tags(self) 354 355 @property 356 def prefs(self): 357 return prefs(self) 358 359 @property 360 def lsan_disabled(self): 361 return bool_prop("lsan-disabled", self) 362 363 @property 364 def lsan_allowed(self): 365 return set_prop("lsan-allowed", self) 366 367 @property 368 def leak_allowed(self): 369 return set_prop("leak-allowed", self) 370 371 @property 372 def leak_threshold(self): 373 return leak_threshold(self) 374 375 @property 376 def lsan_max_stack_depth(self): 377 return int_prop("lsan-max-stack-depth", self) 378 379 @property 380 def fuzzy(self): 381 return fuzzy_prop(self) 382 383 @property 384 def implementation_status(self): 385 return str_prop("implementation-status", self) 386 387 388class TestNode(ManifestItem): 389 def __init__(self, node, **kwargs): 390 """Tree node associated with a particular test in a manifest 391 392 :param name: name of the test""" 393 assert node.data is not None 394 ManifestItem.__init__(self, node, **kwargs) 395 self.updated_expected = [] 396 self.new_expected = [] 397 self.subtests = {} 398 self.default_status = None 399 self._from_file = True 400 401 @property 402 def is_empty(self): 403 required_keys = {"type"} 404 if set(self._data.keys()) != required_keys: 405 return False 406 return all(child.is_empty for child in self.children) 407 408 @property 409 def test_type(self): 410 return self.get("type") 411 412 @property 413 def id(self): 414 return urljoin(self.parent.url, self.name) 415 416 @property 417 def disabled(self): 418 return bool_prop("disabled", self) 419 420 @property 421 def restart_after(self): 422 return bool_prop("restart-after", self) 423 424 @property 425 def leaks(self): 426 return bool_prop("leaks", self) 427 428 @property 429 def min_assertion_count(self): 430 return int_prop("min-asserts", self) 431 432 @property 433 def max_assertion_count(self): 434 return int_prop("max-asserts", self) 435 436 @property 437 def tags(self): 438 return tags(self) 439 440 @property 441 def prefs(self): 442 return prefs(self) 443 444 @property 445 def lsan_disabled(self): 446 return bool_prop("lsan-disabled", self) 447 448 @property 449 def lsan_allowed(self): 450 return set_prop("lsan-allowed", self) 451 452 @property 453 def leak_allowed(self): 454 return set_prop("leak-allowed", self) 455 456 @property 457 def leak_threshold(self): 458 return leak_threshold(self) 459 460 @property 461 def lsan_max_stack_depth(self): 462 return int_prop("lsan-max-stack-depth", self) 463 464 @property 465 def fuzzy(self): 466 return fuzzy_prop(self) 467 468 @property 469 def expected(self): 470 return list_prop("expected", self)[0] 471 472 @property 473 def known_intermittent(self): 474 return list_prop("expected", self)[1:] 475 476 @property 477 def implementation_status(self): 478 return str_prop("implementation-status", self) 479 480 def append(self, node): 481 """Add a subtest to the current test 482 483 :param node: AST Node associated with the subtest""" 484 child = ManifestItem.append(self, node) 485 self.subtests[child.name] = child 486 487 def get_subtest(self, name): 488 """Get the SubtestNode corresponding to a particular subtest, by name 489 490 :param name: Name of the node to return""" 491 if name in self.subtests: 492 return self.subtests[name] 493 return None 494 495 496class SubtestNode(TestNode): 497 @property 498 def is_empty(self): 499 if self._data: 500 return False 501 return True 502 503 504def get_manifest(metadata_root, test_path, url_base, run_info): 505 """Get the ExpectedManifest for a particular test path, or None if there is no 506 metadata stored for that test path. 507 508 :param metadata_root: Absolute path to the root of the metadata directory 509 :param test_path: Path to the test(s) relative to the test root 510 :param url_base: Base url for serving the tests in this manifest 511 :param run_info: Dictionary of properties of the test run for which the expectation 512 values should be computed. 513 """ 514 manifest_path = expected.expected_path(metadata_root, test_path) 515 try: 516 with open(manifest_path, "rb") as f: 517 return static.compile(f, 518 run_info, 519 data_cls_getter=data_cls_getter, 520 test_path=test_path, 521 url_base=url_base) 522 except IOError: 523 return None 524 525 526def get_dir_manifest(path, run_info): 527 """Get the ExpectedManifest for a particular test path, or None if there is no 528 metadata stored for that test path. 529 530 :param path: Full path to the ini file 531 :param run_info: Dictionary of properties of the test run for which the expectation 532 values should be computed. 533 """ 534 try: 535 with open(path, "rb") as f: 536 return static.compile(f, 537 run_info, 538 data_cls_getter=lambda x,y: DirectoryManifest) 539 except IOError: 540 return None 541