1import os 2import subprocess 3import sys 4from collections import defaultdict 5from typing import Any, ClassVar, Dict, Type 6from urllib.parse import urljoin 7 8from .wptmanifest.parser import atoms 9 10atom_reset = atoms["Reset"] 11enabled_tests = {"testharness", "reftest", "wdspec", "crashtest", "print-reftest"} 12 13 14class Result(object): 15 def __init__(self, 16 status, 17 message, 18 expected=None, 19 extra=None, 20 stack=None, 21 known_intermittent=None): 22 if status not in self.statuses: 23 raise ValueError("Unrecognised status %s" % status) 24 self.status = status 25 self.message = message 26 self.expected = expected 27 self.known_intermittent = known_intermittent if known_intermittent is not None else [] 28 self.extra = extra if extra is not None else {} 29 self.stack = stack 30 31 def __repr__(self): 32 return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.status) 33 34 35class SubtestResult(object): 36 def __init__(self, name, status, message, stack=None, expected=None, known_intermittent=None): 37 self.name = name 38 if status not in self.statuses: 39 raise ValueError("Unrecognised status %s" % status) 40 self.status = status 41 self.message = message 42 self.stack = stack 43 self.expected = expected 44 self.known_intermittent = known_intermittent if known_intermittent is not None else [] 45 46 def __repr__(self): 47 return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.name, self.status) 48 49 50class TestharnessResult(Result): 51 default_expected = "OK" 52 statuses = {"OK", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH", "PRECONDITION_FAILED"} 53 54 55class TestharnessSubtestResult(SubtestResult): 56 default_expected = "PASS" 57 statuses = {"PASS", "FAIL", "TIMEOUT", "NOTRUN", "PRECONDITION_FAILED"} 58 59 60class ReftestResult(Result): 61 default_expected = "PASS" 62 statuses = {"PASS", "FAIL", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", 63 "CRASH"} 64 65 66class WdspecResult(Result): 67 default_expected = "OK" 68 statuses = {"OK", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"} 69 70 71class WdspecSubtestResult(SubtestResult): 72 default_expected = "PASS" 73 statuses = {"PASS", "FAIL", "ERROR"} 74 75 76class CrashtestResult(Result): 77 default_expected = "PASS" 78 statuses = {"PASS", "ERROR", "INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", 79 "CRASH"} 80 81 82def get_run_info(metadata_root, product, **kwargs): 83 return RunInfo(metadata_root, product, **kwargs) 84 85 86class RunInfo(Dict[str, Any]): 87 def __init__(self, metadata_root, product, debug, 88 browser_version=None, 89 browser_channel=None, 90 verify=None, 91 extras=None, 92 enable_webrender=False): 93 import mozinfo 94 self._update_mozinfo(metadata_root) 95 self.update(mozinfo.info) 96 97 from .update.tree import GitTree 98 try: 99 # GitTree.__init__ throws if we are not in a git tree. 100 rev = GitTree(log_error=False).rev 101 except (OSError, subprocess.CalledProcessError): 102 rev = None 103 if rev: 104 self["revision"] = rev.decode("utf-8") 105 106 self["python_version"] = sys.version_info.major 107 self["product"] = product 108 if debug is not None: 109 self["debug"] = debug 110 elif "debug" not in self: 111 # Default to release 112 self["debug"] = False 113 if browser_version: 114 self["browser_version"] = browser_version 115 if browser_channel: 116 self["browser_channel"] = browser_channel 117 118 self["verify"] = verify 119 if "wasm" not in self: 120 self["wasm"] = False 121 if extras is not None: 122 self.update(extras) 123 124 self["headless"] = extras.get("headless", False) 125 self["webrender"] = enable_webrender 126 127 def _update_mozinfo(self, metadata_root): 128 """Add extra build information from a mozinfo.json file in a parent 129 directory""" 130 import mozinfo 131 132 path = metadata_root 133 dirs = set() 134 while path != os.path.expanduser('~'): 135 if path in dirs: 136 break 137 dirs.add(str(path)) 138 path = os.path.dirname(path) 139 140 mozinfo.find_and_update_from_json(*dirs) 141 142 143def server_protocol(manifest_item): 144 if hasattr(manifest_item, "h2") and manifest_item.h2: 145 return "h2" 146 if hasattr(manifest_item, "https") and manifest_item.https: 147 return "https" 148 return "http" 149 150 151class Test(object): 152 153 result_cls = None # type: ClassVar[Type[Result]] 154 subtest_result_cls = None # type: ClassVar[Type[SubtestResult]] 155 test_type = None # type: ClassVar[str] 156 157 default_timeout = 10 # seconds 158 long_timeout = 60 # seconds 159 160 def __init__(self, url_base, tests_root, url, inherit_metadata, test_metadata, 161 timeout=None, path=None, protocol="http", subdomain=False, 162 quic=False): 163 self.url_base = url_base 164 self.tests_root = tests_root 165 self.url = url 166 self._inherit_metadata = inherit_metadata 167 self._test_metadata = test_metadata 168 self.timeout = timeout if timeout is not None else self.default_timeout 169 self.path = path 170 171 self.subdomain = subdomain 172 self.environment = {"url_base": url_base, 173 "protocol": protocol, 174 "prefs": self.prefs, 175 "quic": quic} 176 177 def __eq__(self, other): 178 if not isinstance(other, Test): 179 return False 180 return self.id == other.id 181 182 # Python 2 does not have this delegation, while Python 3 does. 183 def __ne__(self, other): 184 return not self.__eq__(other) 185 186 def update_metadata(self, metadata=None): 187 if metadata is None: 188 metadata = {} 189 return metadata 190 191 @classmethod 192 def from_manifest(cls, manifest_file, manifest_item, inherit_metadata, test_metadata): 193 timeout = cls.long_timeout if manifest_item.timeout == "long" else cls.default_timeout 194 return cls(manifest_file.url_base, 195 manifest_file.tests_root, 196 manifest_item.url, 197 inherit_metadata, 198 test_metadata, 199 timeout=timeout, 200 path=os.path.join(manifest_file.tests_root, manifest_item.path), 201 protocol=server_protocol(manifest_item), 202 subdomain=manifest_item.subdomain) 203 204 @property 205 def id(self): 206 return self.url 207 208 @property 209 def keys(self): 210 return tuple() 211 212 @property 213 def abs_path(self): 214 return os.path.join(self.tests_root, self.path) 215 216 def _get_metadata(self, subtest=None): 217 if self._test_metadata is not None and subtest is not None: 218 return self._test_metadata.get_subtest(subtest) 219 else: 220 return self._test_metadata 221 222 def itermeta(self, subtest=None): 223 if self._test_metadata is not None: 224 if subtest is not None: 225 subtest_meta = self._get_metadata(subtest) 226 if subtest_meta is not None: 227 yield subtest_meta 228 yield self._get_metadata() 229 for metadata in reversed(self._inherit_metadata): 230 yield metadata 231 232 def disabled(self, subtest=None): 233 for meta in self.itermeta(subtest): 234 disabled = meta.disabled 235 if disabled is not None: 236 return disabled 237 return None 238 239 @property 240 def restart_after(self): 241 for meta in self.itermeta(None): 242 restart_after = meta.restart_after 243 if restart_after is not None: 244 return True 245 return False 246 247 @property 248 def leaks(self): 249 for meta in self.itermeta(None): 250 leaks = meta.leaks 251 if leaks is not None: 252 return leaks 253 return False 254 255 @property 256 def min_assertion_count(self): 257 for meta in self.itermeta(None): 258 count = meta.min_assertion_count 259 if count is not None: 260 return count 261 return 0 262 263 @property 264 def max_assertion_count(self): 265 for meta in self.itermeta(None): 266 count = meta.max_assertion_count 267 if count is not None: 268 return count 269 return 0 270 271 @property 272 def lsan_disabled(self): 273 for meta in self.itermeta(): 274 if meta.lsan_disabled is not None: 275 return meta.lsan_disabled 276 return False 277 278 @property 279 def lsan_allowed(self): 280 lsan_allowed = set() 281 for meta in self.itermeta(): 282 lsan_allowed |= meta.lsan_allowed 283 if atom_reset in lsan_allowed: 284 lsan_allowed.remove(atom_reset) 285 break 286 return lsan_allowed 287 288 @property 289 def lsan_max_stack_depth(self): 290 for meta in self.itermeta(None): 291 depth = meta.lsan_max_stack_depth 292 if depth is not None: 293 return depth 294 return None 295 296 @property 297 def mozleak_allowed(self): 298 mozleak_allowed = set() 299 for meta in self.itermeta(): 300 mozleak_allowed |= meta.leak_allowed 301 if atom_reset in mozleak_allowed: 302 mozleak_allowed.remove(atom_reset) 303 break 304 return mozleak_allowed 305 306 @property 307 def mozleak_threshold(self): 308 rv = {} 309 for meta in self.itermeta(None): 310 threshold = meta.leak_threshold 311 for key, value in threshold.items(): 312 if key not in rv: 313 rv[key] = value 314 return rv 315 316 @property 317 def tags(self): 318 tags = set() 319 for meta in self.itermeta(): 320 meta_tags = meta.tags 321 tags |= meta_tags 322 if atom_reset in meta_tags: 323 tags.remove(atom_reset) 324 break 325 326 tags.add("dir:%s" % self.id.lstrip("/").split("/")[0]) 327 328 return tags 329 330 @property 331 def prefs(self): 332 prefs = {} 333 for meta in reversed(list(self.itermeta())): 334 meta_prefs = meta.prefs 335 if atom_reset in meta_prefs: 336 del meta_prefs[atom_reset] 337 prefs = {} 338 prefs.update(meta_prefs) 339 return prefs 340 341 def expected(self, subtest=None): 342 if subtest is None: 343 default = self.result_cls.default_expected 344 else: 345 default = self.subtest_result_cls.default_expected 346 347 metadata = self._get_metadata(subtest) 348 if metadata is None: 349 return default 350 351 try: 352 expected = metadata.get("expected") 353 if isinstance(expected, str): 354 return expected 355 elif isinstance(expected, list): 356 return expected[0] 357 elif expected is None: 358 return default 359 except KeyError: 360 return default 361 362 def implementation_status(self): 363 implementation_status = None 364 for meta in self.itermeta(): 365 implementation_status = meta.implementation_status 366 if implementation_status: 367 return implementation_status 368 369 # assuming no specific case, we are implementing it 370 return "implementing" 371 372 def known_intermittent(self, subtest=None): 373 metadata = self._get_metadata(subtest) 374 if metadata is None: 375 return [] 376 377 try: 378 expected = metadata.get("expected") 379 if isinstance(expected, list): 380 return expected[1:] 381 return [] 382 except KeyError: 383 return [] 384 385 def expect_any_subtest_status(self): 386 metadata = self._get_metadata() 387 if metadata is None: 388 return False 389 try: 390 # This key is used by the Blink CI to ignore subtest statuses 391 metadata.get("blink_expect_any_subtest_status") 392 return True 393 except KeyError: 394 return False 395 396 def __repr__(self): 397 return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id) 398 399 400class TestharnessTest(Test): 401 result_cls = TestharnessResult 402 subtest_result_cls = TestharnessSubtestResult 403 test_type = "testharness" 404 405 def __init__(self, url_base, tests_root, url, inherit_metadata, test_metadata, 406 timeout=None, path=None, protocol="http", testdriver=False, 407 jsshell=False, scripts=None, subdomain=False, quic=False): 408 Test.__init__(self, url_base, tests_root, url, inherit_metadata, test_metadata, timeout, 409 path, protocol, subdomain, quic) 410 411 self.testdriver = testdriver 412 self.jsshell = jsshell 413 self.scripts = scripts or [] 414 415 @classmethod 416 def from_manifest(cls, manifest_file, manifest_item, inherit_metadata, test_metadata): 417 timeout = cls.long_timeout if manifest_item.timeout == "long" else cls.default_timeout 418 testdriver = manifest_item.testdriver if hasattr(manifest_item, "testdriver") else False 419 jsshell = manifest_item.jsshell if hasattr(manifest_item, "jsshell") else False 420 quic = manifest_item.quic if hasattr(manifest_item, "quic") else False 421 script_metadata = manifest_item.script_metadata or [] 422 scripts = [v for (k, v) in script_metadata 423 if k == "script"] 424 return cls(manifest_file.url_base, 425 manifest_file.tests_root, 426 manifest_item.url, 427 inherit_metadata, 428 test_metadata, 429 timeout=timeout, 430 path=os.path.join(manifest_file.tests_root, manifest_item.path), 431 protocol=server_protocol(manifest_item), 432 testdriver=testdriver, 433 jsshell=jsshell, 434 scripts=scripts, 435 subdomain=manifest_item.subdomain, 436 quic=quic) 437 438 @property 439 def id(self): 440 return self.url 441 442 443class ManualTest(Test): 444 test_type = "manual" 445 446 @property 447 def id(self): 448 return self.url 449 450 451class ReftestTest(Test): 452 """A reftest 453 454 A reftest should be considered to pass if one of its references matches (see below) *and* the 455 reference passes if it has any references recursively. 456 457 Attributes: 458 references (List[Tuple[str, str]]): a list of alternate references, where one must match for the test to pass 459 viewport_size (Optional[Tuple[int, int]]): size of the viewport for this test, if not default 460 dpi (Optional[int]): dpi to use when rendering this test, if not default 461 462 """ 463 result_cls = ReftestResult 464 test_type = "reftest" 465 466 def __init__(self, url_base, tests_root, url, inherit_metadata, test_metadata, references, 467 timeout=None, path=None, viewport_size=None, dpi=None, fuzzy=None, 468 protocol="http", subdomain=False, quic=False): 469 Test.__init__(self, url_base, tests_root, url, inherit_metadata, test_metadata, timeout, 470 path, protocol, subdomain, quic) 471 472 for _, ref_type in references: 473 if ref_type not in ("==", "!="): 474 raise ValueError 475 476 self.references = references 477 self.viewport_size = self.get_viewport_size(viewport_size) 478 self.dpi = dpi 479 self._fuzzy = fuzzy or {} 480 481 @classmethod 482 def cls_kwargs(cls, manifest_test): 483 return {"viewport_size": manifest_test.viewport_size, 484 "dpi": manifest_test.dpi, 485 "protocol": server_protocol(manifest_test), 486 "fuzzy": manifest_test.fuzzy} 487 488 @classmethod 489 def from_manifest(cls, 490 manifest_file, 491 manifest_test, 492 inherit_metadata, 493 test_metadata): 494 495 timeout = cls.long_timeout if manifest_test.timeout == "long" else cls.default_timeout 496 quic = manifest_test.quic if hasattr(manifest_test, "quic") else False 497 498 url = manifest_test.url 499 500 node = cls(manifest_file.url_base, 501 manifest_file.tests_root, 502 manifest_test.url, 503 inherit_metadata, 504 test_metadata, 505 [], 506 timeout=timeout, 507 path=manifest_test.path, 508 subdomain=manifest_test.subdomain, 509 quic=quic, 510 **cls.cls_kwargs(manifest_test)) 511 512 refs_by_type = defaultdict(list) 513 514 for ref_url, ref_type in manifest_test.references: 515 refs_by_type[ref_type].append(ref_url) 516 517 # Construct a list of all the mismatches, where we end up with mismatch_1 != url != 518 # mismatch_2 != url != mismatch_3 etc. 519 # 520 # Per the logic documented above, this means that none of the mismatches provided match, 521 mismatch_walk = None 522 if refs_by_type["!="]: 523 mismatch_walk = ReftestTest(manifest_file.url_base, 524 manifest_file.tests_root, 525 refs_by_type["!="][0], 526 [], 527 None, 528 []) 529 cmp_ref = mismatch_walk 530 for ref_url in refs_by_type["!="][1:]: 531 cmp_self = ReftestTest(manifest_file.url_base, 532 manifest_file.tests_root, 533 url, 534 [], 535 None, 536 []) 537 cmp_ref.references.append((cmp_self, "!=")) 538 cmp_ref = ReftestTest(manifest_file.url_base, 539 manifest_file.tests_root, 540 ref_url, 541 [], 542 None, 543 []) 544 cmp_self.references.append((cmp_ref, "!=")) 545 546 if mismatch_walk is None: 547 mismatch_refs = [] 548 else: 549 mismatch_refs = [(mismatch_walk, "!=")] 550 551 if refs_by_type["=="]: 552 # For each == ref, add a reference to this node whose tail is the mismatch list. 553 # Per the logic documented above, this means any one of the matches must pass plus all the mismatches. 554 for ref_url in refs_by_type["=="]: 555 ref = ReftestTest(manifest_file.url_base, 556 manifest_file.tests_root, 557 ref_url, 558 [], 559 None, 560 mismatch_refs) 561 node.references.append((ref, "==")) 562 else: 563 # Otherwise, we just add the mismatches directly as we are immediately into the 564 # mismatch chain with no alternates. 565 node.references.extend(mismatch_refs) 566 567 return node 568 569 def update_metadata(self, metadata): 570 if "url_count" not in metadata: 571 metadata["url_count"] = defaultdict(int) 572 for reference, _ in self.references: 573 # We assume a naive implementation in which a url with multiple 574 # possible screenshots will need to take both the lhs and rhs screenshots 575 # for each possible match 576 metadata["url_count"][(self.environment["protocol"], reference.url)] += 1 577 reference.update_metadata(metadata) 578 return metadata 579 580 def get_viewport_size(self, override): 581 return override 582 583 @property 584 def id(self): 585 return self.url 586 587 @property 588 def keys(self): 589 return ("reftype", "refurl") 590 591 @property 592 def fuzzy(self): 593 return self._fuzzy 594 595 @property 596 def fuzzy_override(self): 597 values = {} 598 for meta in reversed(list(self.itermeta(None))): 599 value = meta.fuzzy 600 if not value: 601 continue 602 if atom_reset in value: 603 value.remove(atom_reset) 604 values = {} 605 for key, data in value: 606 if isinstance(key, (tuple, list)): 607 key = list(key) 608 key[0] = urljoin(self.url, key[0]) 609 key[1] = urljoin(self.url, key[1]) 610 key = tuple(key) 611 elif key: 612 # Key is just a relative url to a ref 613 key = urljoin(self.url, key) 614 values[key] = data 615 return values 616 617 @property 618 def page_ranges(self): 619 return {} 620 621 622class PrintReftestTest(ReftestTest): 623 test_type = "print-reftest" 624 625 def __init__(self, url_base, tests_root, url, inherit_metadata, test_metadata, references, 626 timeout=None, path=None, viewport_size=None, dpi=None, fuzzy=None, 627 page_ranges=None, protocol="http", subdomain=False, quic=False): 628 super(PrintReftestTest, self).__init__(url_base, tests_root, url, inherit_metadata, test_metadata, 629 references, timeout, path, viewport_size, dpi, 630 fuzzy, protocol, subdomain=subdomain, quic=quic) 631 self._page_ranges = page_ranges 632 633 @classmethod 634 def cls_kwargs(cls, manifest_test): 635 rv = super(PrintReftestTest, cls).cls_kwargs(manifest_test) 636 rv["page_ranges"] = manifest_test.page_ranges 637 return rv 638 639 def get_viewport_size(self, override): 640 assert override is None 641 return (5*2.54, 3*2.54) 642 643 @property 644 def page_ranges(self): 645 return self._page_ranges 646 647 648class WdspecTest(Test): 649 result_cls = WdspecResult 650 subtest_result_cls = WdspecSubtestResult 651 test_type = "wdspec" 652 653 default_timeout = 25 654 long_timeout = 180 # 3 minutes 655 656 657class CrashTest(Test): 658 result_cls = CrashtestResult 659 test_type = "crashtest" 660 661 662manifest_test_cls = {"reftest": ReftestTest, 663 "print-reftest": PrintReftestTest, 664 "testharness": TestharnessTest, 665 "manual": ManualTest, 666 "wdspec": WdspecTest, 667 "crashtest": CrashTest} 668 669 670def from_manifest(manifest_file, manifest_test, inherit_metadata, test_metadata): 671 test_cls = manifest_test_cls[manifest_test.item_type] 672 return test_cls.from_manifest(manifest_file, manifest_test, inherit_metadata, test_metadata) 673