1import json 2import mock 3import os 4import pytest 5import sys 6from io import BytesIO 7 8from .. import metadata, manifestupdate 9from ..update.update import WPTUpdate 10from ..update.base import StepRunner, Step 11from mozlog import structuredlog, handlers, formatters 12 13here = os.path.dirname(__file__) 14sys.path.insert(0, os.path.join(here, os.pardir, os.pardir, os.pardir)) 15from manifest import manifest, item as manifest_item, utils 16 17 18def rel_path_to_test_url(rel_path): 19 assert not os.path.isabs(rel_path) 20 return rel_path.replace(os.sep, "/") 21 22 23def SourceFileWithTest(path, hash, cls, *args): 24 path_parts = tuple(path.split("/")) 25 path = utils.to_os_path(path) 26 s = mock.Mock(rel_path=path, rel_path_parts=path_parts, hash=hash) 27 test = cls("/foobar", path, "/", rel_path_to_test_url(path), *args) 28 s.manifest_items = mock.Mock(return_value=(cls.item_type, [test])) 29 return s 30 31 32item_classes = {"testharness": manifest_item.TestharnessTest, 33 "reftest": manifest_item.RefTest, 34 "manual": manifest_item.ManualTest, 35 "wdspec": manifest_item.WebDriverSpecTest, 36 "conformancechecker": manifest_item.ConformanceCheckerTest, 37 "visual": manifest_item.VisualTest, 38 "support": manifest_item.SupportFile} 39 40 41default_run_info = {"debug": False, "os": "linux", "version": "18.04", "processor": "x86_64", "bits": 64} 42test_id = "/path/to/test.htm" 43dir_id = "path/to/__dir__" 44 45 46def reset_globals(): 47 metadata.prop_intern.clear() 48 metadata.run_info_intern.clear() 49 metadata.status_intern.clear() 50 51 52def get_run_info(overrides): 53 run_info = default_run_info.copy() 54 run_info.update(overrides) 55 return run_info 56 57 58def update(tests, *logs, **kwargs): 59 full_update = kwargs.pop("full_update", False) 60 disable_intermittent = kwargs.pop("disable_intermittent", False) 61 update_intermittent = kwargs.pop("update_intermittent", False) 62 remove_intermittent = kwargs.pop("remove_intermittent", False) 63 assert not kwargs 64 id_test_map, updater = create_updater(tests) 65 66 for log in logs: 67 log = create_log(log) 68 updater.update_from_log(log) 69 70 update_properties = (["debug", "os", "version", "processor"], 71 {"os": ["version"], "processor": ["bits"]}) 72 73 expected_data = {} 74 metadata.load_expected = lambda _, __, test_path, *args: expected_data.get(test_path) 75 for test_path, test_ids, test_type, manifest_str in tests: 76 test_path = utils.to_os_path(test_path) 77 expected_data[test_path] = manifestupdate.compile(BytesIO(manifest_str), 78 test_path, 79 "/", 80 update_properties, 81 update_intermittent, 82 remove_intermittent) 83 84 return list(metadata.update_results(id_test_map, 85 update_properties, 86 full_update, 87 disable_intermittent, 88 update_intermittent, 89 remove_intermittent)) 90 91 92def create_updater(tests, url_base="/", **kwargs): 93 id_test_map = {} 94 m = create_test_manifest(tests, url_base) 95 96 reset_globals() 97 id_test_map = metadata.create_test_tree(None, m) 98 99 return id_test_map, metadata.ExpectedUpdater(id_test_map, **kwargs) 100 101 102def create_log(entries): 103 data = BytesIO() 104 if isinstance(entries, list): 105 logger = structuredlog.StructuredLogger("expected_test") 106 handler = handlers.StreamHandler(data, formatters.JSONFormatter()) 107 logger.add_handler(handler) 108 109 for item in entries: 110 action, kwargs = item 111 getattr(logger, action)(**kwargs) 112 logger.remove_handler(handler) 113 else: 114 data.write(json.dumps(entries).encode()) 115 data.seek(0) 116 return data 117 118 119def suite_log(entries, run_info=None): 120 _run_info = default_run_info.copy() 121 if run_info: 122 _run_info.update(run_info) 123 return ([("suite_start", {"tests": [], "run_info": _run_info})] + 124 entries + 125 [("suite_end", {})]) 126 127 128def create_test_manifest(tests, url_base="/"): 129 source_files = [] 130 for i, (test, _, test_type, _) in enumerate(tests): 131 if test_type: 132 source_files.append((SourceFileWithTest(test, str(i) * 40, item_classes[test_type]), True)) 133 m = manifest.Manifest() 134 m.update(source_files) 135 return m 136 137 138def test_update_0(): 139 tests = [("path/to/test.htm", [test_id], "testharness", 140 b"""[test.htm] 141 [test1] 142 expected: FAIL""")] 143 144 log = suite_log([("test_start", {"test": "/path/to/test.htm"}), 145 ("test_status", {"test": "/path/to/test.htm", 146 "subtest": "test1", 147 "status": "PASS", 148 "expected": "FAIL"}), 149 ("test_end", {"test": "/path/to/test.htm", 150 "status": "OK"})]) 151 152 updated = update(tests, log) 153 154 assert len(updated) == 1 155 assert updated[0][1].is_empty 156 157 158def test_update_1(): 159 tests = [("path/to/test.htm", [test_id], "testharness", 160 b"""[test.htm] 161 [test1] 162 expected: ERROR""")] 163 164 log = suite_log([("test_start", {"test": test_id}), 165 ("test_status", {"test": test_id, 166 "subtest": "test1", 167 "status": "FAIL", 168 "expected": "ERROR"}), 169 ("test_end", {"test": test_id, 170 "status": "OK"})]) 171 172 updated = update(tests, log) 173 174 new_manifest = updated[0][1] 175 assert not new_manifest.is_empty 176 assert new_manifest.get_test(test_id).children[0].get("expected", default_run_info) == "FAIL" 177 178 179def test_update_known_intermittent_1(): 180 tests = [("path/to/test.htm", [test_id], "testharness", 181 b"""[test.htm] 182 [test1] 183 expected: PASS""")] 184 185 log_0 = suite_log([("test_start", {"test": test_id}), 186 ("test_status", {"test": test_id, 187 "subtest": "test1", 188 "status": "FAIL", 189 "expected": "PASS"}), 190 ("test_end", {"test": test_id, 191 "status": "OK"})]) 192 193 log_1 = suite_log([("test_start", {"test": test_id}), 194 ("test_status", {"test": test_id, 195 "subtest": "test1", 196 "status": "PASS", 197 "expected": "PASS"}), 198 ("test_end", {"test": test_id, 199 "status": "OK"})]) 200 201 log_2 = suite_log([("test_start", {"test": test_id}), 202 ("test_status", {"test": test_id, 203 "subtest": "test1", 204 "status": "PASS", 205 "expected": "PASS"}), 206 ("test_end", {"test": test_id, 207 "status": "OK"})]) 208 209 updated = update(tests, log_0, log_1, log_2, update_intermittent=True) 210 211 new_manifest = updated[0][1] 212 213 assert not new_manifest.is_empty 214 assert new_manifest.get_test(test_id).children[0].get( 215 "expected", default_run_info) == ["PASS", "FAIL"] 216 217 218def test_update_known_intermittent_2(): 219 tests = [("path/to/test.htm", [test_id], "testharness", 220 b"""[test.htm] 221 [test1] 222 expected: PASS""")] 223 224 log_0 = suite_log([("test_start", {"test": test_id}), 225 ("test_status", {"test": test_id, 226 "subtest": "test1", 227 "status": "FAIL", 228 "expected": "PASS"}), 229 ("test_end", {"test": test_id, 230 "status": "OK"})]) 231 232 updated = update(tests, log_0, update_intermittent=True) 233 234 new_manifest = updated[0][1] 235 236 assert not new_manifest.is_empty 237 assert new_manifest.get_test(test_id).children[0].get( 238 "expected", default_run_info) == "FAIL" 239 240 241def test_update_existing_known_intermittent(): 242 tests = [("path/to/test.htm", [test_id], "testharness", 243 b"""[test.htm] 244 [test1] 245 expected: [PASS, FAIL]""")] 246 247 log_0 = suite_log([("test_start", {"test": test_id}), 248 ("test_status", {"test": test_id, 249 "subtest": "test1", 250 "status": "ERROR", 251 "expected": "PASS", 252 "known_intermittent": ["FAIL"]}), 253 ("test_end", {"test": test_id, 254 "status": "OK"})]) 255 256 log_1 = suite_log([("test_start", {"test": test_id}), 257 ("test_status", {"test": test_id, 258 "subtest": "test1", 259 "status": "PASS", 260 "expected": "PASS", 261 "known_intermittent": ["FAIL"]}), 262 ("test_end", {"test": test_id, 263 "status": "OK"})]) 264 265 log_2 = suite_log([("test_start", {"test": test_id}), 266 ("test_status", {"test": test_id, 267 "subtest": "test1", 268 "status": "PASS", 269 "expected": "PASS", 270 "known_intermittent": ["FAIL"]}), 271 ("test_end", {"test": test_id, 272 "status": "OK"})]) 273 274 updated = update(tests, log_0, log_1, log_2, update_intermittent=True) 275 276 new_manifest = updated[0][1] 277 assert not new_manifest.is_empty 278 assert new_manifest.get_test(test_id).children[0].get( 279 "expected", default_run_info) == ["PASS", "ERROR", "FAIL"] 280 281 282def test_update_remove_previous_intermittent(): 283 tests = [("path/to/test.htm", [test_id], "testharness", 284 b"""[test.htm] 285 [test1] 286 expected: [PASS, FAIL]""")] 287 288 log_0 = suite_log([("test_start", {"test": test_id}), 289 ("test_status", {"test": test_id, 290 "subtest": "test1", 291 "status": "ERROR", 292 "expected": "PASS", 293 "known_intermittent": ["FAIL"]}), 294 ("test_end", {"test": test_id, 295 "status": "OK"})]) 296 297 log_1 = suite_log([("test_start", {"test": test_id}), 298 ("test_status", {"test": test_id, 299 "subtest": "test1", 300 "status": "PASS", 301 "expected": "PASS", 302 "known_intermittent": ["FAIL"]}), 303 ("test_end", {"test": test_id, 304 "status": "OK"})]) 305 306 log_2 = suite_log([("test_start", {"test": test_id}), 307 ("test_status", {"test": test_id, 308 "subtest": "test1", 309 "status": "PASS", 310 "expected": "PASS", 311 "known_intermittent": ["FAIL"]}), 312 ("test_end", {"test": test_id, 313 "status": "OK"})]) 314 315 updated = update(tests, 316 log_0, 317 log_1, 318 log_2, 319 update_intermittent=True, 320 remove_intermittent=True) 321 322 new_manifest = updated[0][1] 323 assert not new_manifest.is_empty 324 assert new_manifest.get_test(test_id).children[0].get( 325 "expected", default_run_info) == ["PASS", "ERROR"] 326 327 328def test_update_new_test_with_intermittent(): 329 tests = [("path/to/test.htm", [test_id], "testharness", None)] 330 331 log_0 = suite_log([("test_start", {"test": test_id}), 332 ("test_status", {"test": test_id, 333 "subtest": "test1", 334 "status": "PASS", 335 "expected": "PASS"}), 336 ("test_end", {"test": test_id, 337 "status": "OK"})]) 338 339 log_1 = suite_log([("test_start", {"test": test_id}), 340 ("test_status", {"test": test_id, 341 "subtest": "test1", 342 "status": "PASS", 343 "expected": "PASS"}), 344 ("test_end", {"test": test_id, 345 "status": "OK"})]) 346 347 log_2 = suite_log([("test_start", {"test": test_id}), 348 ("test_status", {"test": test_id, 349 "subtest": "test1", 350 "status": "FAIL", 351 "expected": "PASS"}), 352 ("test_end", {"test": test_id, 353 "status": "OK"})]) 354 355 updated = update(tests, log_0, log_1, log_2, update_intermittent=True) 356 new_manifest = updated[0][1] 357 358 assert not new_manifest.is_empty 359 assert new_manifest.get_test("test.htm") is None 360 assert len(new_manifest.get_test(test_id).children) == 1 361 assert new_manifest.get_test(test_id).children[0].get( 362 "expected", default_run_info) == ["PASS", "FAIL"] 363 364 365def test_update_expected_tie_resolution(): 366 tests = [("path/to/test.htm", [test_id], "testharness", None)] 367 368 log_0 = suite_log([("test_start", {"test": test_id}), 369 ("test_status", {"test": test_id, 370 "subtest": "test1", 371 "status": "PASS", 372 "expected": "PASS"}), 373 ("test_end", {"test": test_id, 374 "status": "OK"})]) 375 376 log_1 = suite_log([("test_start", {"test": test_id}), 377 ("test_status", {"test": test_id, 378 "subtest": "test1", 379 "status": "FAIL", 380 "expected": "PASS"}), 381 ("test_end", {"test": test_id, 382 "status": "OK"})]) 383 384 updated = update(tests, log_0, log_1, update_intermittent=True) 385 new_manifest = updated[0][1] 386 387 assert not new_manifest.is_empty 388 assert new_manifest.get_test(test_id).children[0].get( 389 "expected", default_run_info) == ["PASS", "FAIL"] 390 391 392def test_update_reorder_expected(): 393 tests = [("path/to/test.htm", [test_id], "testharness", 394 b"""[test.htm] 395 [test1] 396 expected: [PASS, FAIL]""")] 397 398 log_0 = suite_log([("test_start", {"test": test_id}), 399 ("test_status", {"test": test_id, 400 "subtest": "test1", 401 "status": "FAIL", 402 "expected": "PASS", 403 "known_intermittent": ["FAIL"]}), 404 ("test_end", {"test": test_id, 405 "status": "OK"})]) 406 407 log_1 = suite_log([("test_start", {"test": test_id}), 408 ("test_status", {"test": test_id, 409 "subtest": "test1", 410 "status": "FAIL", 411 "expected": "PASS", 412 "known_intermittent": ["FAIL"]}), 413 ("test_end", {"test": test_id, 414 "status": "OK"})]) 415 416 log_2 = suite_log([("test_start", {"test": test_id}), 417 ("test_status", {"test": test_id, 418 "subtest": "test1", 419 "status": "PASS", 420 "expected": "PASS", 421 "known_intermittent": ["FAIL"]}), 422 ("test_end", {"test": test_id, 423 "status": "OK"})]) 424 425 updated = update(tests, log_0, log_1, log_2, update_intermittent=True) 426 new_manifest = updated[0][1] 427 428 assert not new_manifest.is_empty 429 assert new_manifest.get_test(test_id).children[0].get( 430 "expected", default_run_info) == ["FAIL", "PASS"] 431 432 433def test_update_and_preserve_unchanged_expected_intermittent(): 434 tests = [("path/to/test.htm", [test_id], "testharness", b""" 435[test.htm] 436 expected: 437 if os == "android": [PASS, FAIL] 438 FAIL""")] 439 440 log_0 = suite_log([("test_start", {"test": test_id}), 441 ("test_end", {"test": test_id, 442 "status": "FAIL", 443 "expected": "PASS", 444 "known_intermittent": ["FAIL"]})], 445 run_info={"os": "android"}) 446 447 log_1 = suite_log([("test_start", {"test": test_id}), 448 ("test_end", {"test": test_id, 449 "status": "PASS", 450 "expected": "PASS", 451 "known_intermittent": ["FAIL"]})], 452 run_info={"os": "android"}) 453 454 log_2 = suite_log([("test_start", {"test": test_id}), 455 ("test_end", {"test": test_id, 456 "status": "PASS", 457 "expected": "FAIL"})]) 458 459 updated = update(tests, log_0, log_1, log_2) 460 new_manifest = updated[0][1] 461 462 assert not new_manifest.is_empty 463 464 run_info_1 = default_run_info.copy() 465 run_info_1.update({"os": "android"}) 466 467 assert not new_manifest.is_empty 468 assert new_manifest.get_test(test_id).get( 469 "expected", run_info_1) == ["PASS", "FAIL"] 470 assert new_manifest.get_test(test_id).get( 471 "expected", default_run_info) == "PASS" 472 473 474def test_update_test_with_intermittent_to_one_expected_status(): 475 tests = [("path/to/test.htm", [test_id], "testharness", 476 b"""[test.htm] 477 [test1] 478 expected: [PASS, FAIL]""")] 479 480 log_0 = suite_log([("test_start", {"test": test_id}), 481 ("test_status", {"test": test_id, 482 "subtest": "test1", 483 "status": "ERROR", 484 "expected": "PASS", 485 "known_intermittent": ["FAIL"]}), 486 ("test_end", {"test": test_id, 487 "status": "OK"})]) 488 489 updated = update(tests, log_0) 490 491 new_manifest = updated[0][1] 492 assert not new_manifest.is_empty 493 assert new_manifest.get_test(test_id).children[0].get( 494 "expected", default_run_info) == "ERROR" 495 496 497def test_update_intermittent_with_conditions(): 498 tests = [("path/to/test.htm", [test_id], "testharness", b""" 499[test.htm] 500 expected: 501 if os == "android": [PASS, FAIL]""")] 502 503 log_0 = suite_log([("test_start", {"test": test_id}), 504 ("test_end", {"test": test_id, 505 "status": "TIMEOUT", 506 "expected": "PASS", 507 "known_intermittent": ["FAIL"]})], 508 run_info={"os": "android"}) 509 510 log_1 = suite_log([("test_start", {"test": test_id}), 511 ("test_end", {"test": test_id, 512 "status": "PASS", 513 "expected": "PASS", 514 "known_intermittent": ["FAIL"]})], 515 run_info={"os": "android"}) 516 517 updated = update(tests, log_0, log_1, update_intermittent=True) 518 new_manifest = updated[0][1] 519 520 assert not new_manifest.is_empty 521 522 run_info_1 = default_run_info.copy() 523 run_info_1.update({"os": "android"}) 524 525 assert not new_manifest.is_empty 526 assert new_manifest.get_test(test_id).get( 527 "expected", run_info_1) == ["PASS", "TIMEOUT", "FAIL"] 528 529 530def test_update_and_remove_intermittent_with_conditions(): 531 tests = [("path/to/test.htm", [test_id], "testharness", b""" 532[test.htm] 533 expected: 534 if os == "android": [PASS, FAIL]""")] 535 536 log_0 = suite_log([("test_start", {"test": test_id}), 537 ("test_end", {"test": test_id, 538 "status": "TIMEOUT", 539 "expected": "PASS", 540 "known_intermittent": ["FAIL"]})], 541 run_info={"os": "android"}) 542 543 log_1 = suite_log([("test_start", {"test": test_id}), 544 ("test_end", {"test": test_id, 545 "status": "PASS", 546 "expected": "PASS", 547 "known_intermittent": ["FAIL"]})], 548 run_info={"os": "android"}) 549 550 updated = update(tests, log_0, log_1, update_intermittent=True, remove_intermittent=True) 551 new_manifest = updated[0][1] 552 553 assert not new_manifest.is_empty 554 555 run_info_1 = default_run_info.copy() 556 run_info_1.update({"os": "android"}) 557 558 assert not new_manifest.is_empty 559 assert new_manifest.get_test(test_id).get( 560 "expected", run_info_1) == ["PASS", "TIMEOUT"] 561 562 563def test_update_intermittent_full(): 564 tests = [("path/to/test.htm", [test_id], "testharness", 565 b"""[test.htm] 566 [test1] 567 expected: 568 if os == "mac": [FAIL, TIMEOUT] 569 FAIL""")] 570 571 log_0 = suite_log([("test_start", {"test": test_id}), 572 ("test_status", {"test": test_id, 573 "subtest": "test1", 574 "status": "FAIL", 575 "expected": "FAIL", 576 "known_intermittent": ["TIMEOUT"]}), 577 ("test_end", {"test": test_id, 578 "status": "OK"})], 579 run_info={"os": "mac"}) 580 581 log_1 = suite_log([("test_start", {"test": test_id}), 582 ("test_status", {"test": test_id, 583 "subtest": "test1", 584 "status": "FAIL"}), 585 ("test_end", {"test": test_id, 586 "status": "OK"})]) 587 588 updated = update(tests, log_0, log_1, update_intermittent=True, full_update=True) 589 590 new_manifest = updated[0][1] 591 592 assert not new_manifest.is_empty 593 run_info_1 = default_run_info.copy() 594 run_info_1.update({"os": "mac"}) 595 assert new_manifest.get_test(test_id).children[0].get( 596 "expected", run_info_1) == ["FAIL", "TIMEOUT"] 597 assert new_manifest.get_test(test_id).children[0].get( 598 "expected", default_run_info) == "FAIL" 599 600 601def test_update_intermittent_full_remove(): 602 tests = [("path/to/test.htm", [test_id], "testharness", 603 b"""[test.htm] 604 [test1] 605 expected: 606 if os == "mac": [FAIL, TIMEOUT, PASS] 607 FAIL""")] 608 609 log_0 = suite_log([("test_start", {"test": test_id}), 610 ("test_status", {"test": test_id, 611 "subtest": "test1", 612 "status": "FAIL", 613 "expected": "FAIL", 614 "known_intermittent": ["TIMEOUT", "PASS"]}), 615 ("test_end", {"test": test_id, 616 "status": "OK"})], 617 run_info={"os": "mac"}) 618 619 log_1 = suite_log([("test_start", {"test": test_id}), 620 ("test_status", {"test": test_id, 621 "subtest": "test1", 622 "status": "TIMEOUT", 623 "expected": "FAIL", 624 "known_intermittent": ["TIMEOUT", "PASS"]}), 625 ("test_end", {"test": test_id, 626 "status": "OK"})], 627 run_info={"os": "mac"}) 628 629 log_2 = suite_log([("test_start", {"test": test_id}), 630 ("test_status", {"test": test_id, 631 "subtest": "test1", 632 "status": "FAIL"}), 633 ("test_end", {"test": test_id, 634 "status": "OK"})]) 635 636 updated = update(tests, log_0, log_1, log_2, update_intermittent=True, 637 full_update=True, remove_intermittent=True) 638 639 new_manifest = updated[0][1] 640 641 assert not new_manifest.is_empty 642 run_info_1 = default_run_info.copy() 643 run_info_1.update({"os": "mac"}) 644 assert new_manifest.get_test(test_id).children[0].get( 645 "expected", run_info_1) == ["FAIL", "TIMEOUT"] 646 assert new_manifest.get_test(test_id).children[0].get( 647 "expected", default_run_info) == "FAIL" 648 649 650def test_full_update(): 651 tests = [("path/to/test.htm", [test_id], "testharness", 652 b"""[test.htm] 653 [test1] 654 expected: 655 if os == "mac": [FAIL, TIMEOUT] 656 FAIL""")] 657 658 log_0 = suite_log([("test_start", {"test": test_id}), 659 ("test_status", {"test": test_id, 660 "subtest": "test1", 661 "status": "FAIL", 662 "expected": "FAIL", 663 "known_intermittent": ["TIMEOUT"]}), 664 ("test_end", {"test": test_id, 665 "status": "OK"})], 666 run_info={"os": "mac"}) 667 668 log_1 = suite_log([("test_start", {"test": test_id}), 669 ("test_status", {"test": test_id, 670 "subtest": "test1", 671 "status": "FAIL"}), 672 ("test_end", {"test": test_id, 673 "status": "OK"})]) 674 675 updated = update(tests, log_0, log_1, full_update=True) 676 677 new_manifest = updated[0][1] 678 679 assert not new_manifest.is_empty 680 run_info_1 = default_run_info.copy() 681 run_info_1.update({"os": "mac"}) 682 assert new_manifest.get_test(test_id).children[0].get( 683 "expected", run_info_1) == "FAIL" 684 assert new_manifest.get_test(test_id).children[0].get( 685 "expected", default_run_info) == "FAIL" 686 687 688def test_full_orphan(): 689 tests = [("path/to/test.htm", [test_id], "testharness", 690 b"""[test.htm] 691 [test1] 692 expected: FAIL 693 [subsub test] 694 expected: TIMEOUT 695 [test2] 696 expected: ERROR 697""")] 698 699 log_0 = suite_log([("test_start", {"test": test_id}), 700 ("test_status", {"test": test_id, 701 "subtest": "test1", 702 "status": "FAIL", 703 "expected": "FAIL"}), 704 ("test_end", {"test": test_id, 705 "status": "OK"})]) 706 707 708 updated = update(tests, log_0, full_update=True) 709 710 new_manifest = updated[0][1] 711 712 assert not new_manifest.is_empty 713 assert len(new_manifest.get_test(test_id).children[0].children) == 0 714 assert new_manifest.get_test(test_id).children[0].get( 715 "expected", default_run_info) == "FAIL" 716 assert len(new_manifest.get_test(test_id).children) == 1 717 718 719def test_update_reorder_expected_full_conditions(): 720 tests = [("path/to/test.htm", [test_id], "testharness", 721 b"""[test.htm] 722 [test1] 723 expected: 724 if os == "mac": [FAIL, TIMEOUT] 725 [FAIL, PASS]""")] 726 727 log_0 = suite_log([("test_start", {"test": test_id}), 728 ("test_status", {"test": test_id, 729 "subtest": "test1", 730 "status": "TIMEOUT", 731 "expected": "FAIL", 732 "known_intermittent": ["TIMEOUT"]}), 733 ("test_end", {"test": test_id, 734 "status": "OK"})], 735 run_info={"os": "mac"}) 736 737 log_1 = suite_log([("test_start", {"test": test_id}), 738 ("test_status", {"test": test_id, 739 "subtest": "test1", 740 "status": "TIMEOUT", 741 "expected": "FAIL", 742 "known_intermittent": ["TIMEOUT"]}), 743 ("test_end", {"test": test_id, 744 "status": "OK"})], 745 run_info={"os": "mac"}) 746 747 log_2 = suite_log([("test_start", {"test": test_id}), 748 ("test_status", {"test": test_id, 749 "subtest": "test1", 750 "status": "PASS", 751 "expected": "FAIL", 752 "known_intermittent": ["PASS"]}), 753 ("test_end", {"test": test_id, 754 "status": "OK"})]) 755 756 log_3 = suite_log([("test_start", {"test": test_id}), 757 ("test_status", {"test": test_id, 758 "subtest": "test1", 759 "status": "PASS", 760 "expected": "FAIL", 761 "known_intermittent": ["PASS"]}), 762 ("test_end", {"test": test_id, 763 "status": "OK"})]) 764 765 updated = update(tests, log_0, log_1, log_2, log_3, update_intermittent=True, full_update=True) 766 767 new_manifest = updated[0][1] 768 769 assert not new_manifest.is_empty 770 run_info_1 = default_run_info.copy() 771 run_info_1.update({"os": "mac"}) 772 assert new_manifest.get_test(test_id).children[0].get( 773 "expected", run_info_1) == ["TIMEOUT", "FAIL"] 774 assert new_manifest.get_test(test_id).children[0].get( 775 "expected", default_run_info) == ["PASS", "FAIL"] 776 777 778def test_skip_0(): 779 tests = [("path/to/test.htm", [test_id], "testharness", 780 b"""[test.htm] 781 [test1] 782 expected: FAIL""")] 783 784 log = suite_log([("test_start", {"test": test_id}), 785 ("test_status", {"test": test_id, 786 "subtest": "test1", 787 "status": "FAIL", 788 "expected": "FAIL"}), 789 ("test_end", {"test": test_id, 790 "status": "OK"})]) 791 792 updated = update(tests, log) 793 assert not updated 794 795 796def test_new_subtest(): 797 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 798 [test1] 799 expected: FAIL""")] 800 801 log = suite_log([("test_start", {"test": test_id}), 802 ("test_status", {"test": test_id, 803 "subtest": "test1", 804 "status": "FAIL", 805 "expected": "FAIL"}), 806 ("test_status", {"test": test_id, 807 "subtest": "test2", 808 "status": "FAIL", 809 "expected": "PASS"}), 810 ("test_end", {"test": test_id, 811 "status": "OK"})]) 812 updated = update(tests, log) 813 new_manifest = updated[0][1] 814 assert not new_manifest.is_empty 815 assert new_manifest.get_test(test_id).children[0].get("expected", default_run_info) == "FAIL" 816 assert new_manifest.get_test(test_id).children[1].get("expected", default_run_info) == "FAIL" 817 818 819def test_update_multiple_0(): 820 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 821 [test1] 822 expected: FAIL""")] 823 824 log_0 = suite_log([("test_start", {"test": test_id}), 825 ("test_status", {"test": test_id, 826 "subtest": "test1", 827 "status": "FAIL", 828 "expected": "FAIL"}), 829 ("test_end", {"test": test_id, 830 "status": "OK"})], 831 run_info={"debug": False, "os": "osx"}) 832 833 log_1 = suite_log([("test_start", {"test": test_id}), 834 ("test_status", {"test": test_id, 835 "subtest": "test1", 836 "status": "TIMEOUT", 837 "expected": "FAIL"}), 838 ("test_end", {"test": test_id, 839 "status": "OK"})], 840 run_info={"debug": False, "os": "linux"}) 841 842 updated = update(tests, log_0, log_1) 843 new_manifest = updated[0][1] 844 845 assert not new_manifest.is_empty 846 run_info_1 = default_run_info.copy() 847 run_info_1.update({"debug": False, "os": "osx"}) 848 run_info_2 = default_run_info.copy() 849 run_info_2.update({"debug": False, "os": "linux"}) 850 assert new_manifest.get_test(test_id).children[0].get( 851 "expected", run_info_1) == "FAIL" 852 assert new_manifest.get_test(test_id).children[0].get( 853 "expected", {"debug": False, "os": "linux"}) == "TIMEOUT" 854 855 856def test_update_multiple_1(): 857 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 858 [test1] 859 expected: FAIL""")] 860 861 log_0 = suite_log([("test_start", {"test": test_id}), 862 ("test_status", {"test": test_id, 863 "subtest": "test1", 864 "status": "FAIL", 865 "expected": "FAIL"}), 866 ("test_end", {"test": test_id, 867 "status": "OK"})], 868 run_info={"os": "osx"}) 869 870 log_1 = suite_log([("test_start", {"test": test_id}), 871 ("test_status", {"test": test_id, 872 "subtest": "test1", 873 "status": "TIMEOUT", 874 "expected": "FAIL"}), 875 ("test_end", {"test": test_id, 876 "status": "OK"})], 877 run_info={"os": "linux"}) 878 879 updated = update(tests, log_0, log_1) 880 new_manifest = updated[0][1] 881 882 assert not new_manifest.is_empty 883 run_info_1 = default_run_info.copy() 884 run_info_1.update({"os": "osx"}) 885 run_info_2 = default_run_info.copy() 886 run_info_2.update({"os": "linux"}) 887 run_info_3 = default_run_info.copy() 888 run_info_3.update({"os": "win"}) 889 890 assert new_manifest.get_test(test_id).children[0].get( 891 "expected", run_info_1) == "FAIL" 892 assert new_manifest.get_test(test_id).children[0].get( 893 "expected", run_info_2) == "TIMEOUT" 894 assert new_manifest.get_test(test_id).children[0].get( 895 "expected", run_info_3) == "FAIL" 896 897 898def test_update_multiple_2(): 899 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 900 [test1] 901 expected: FAIL""")] 902 903 log_0 = suite_log([("test_start", {"test": test_id}), 904 ("test_status", {"test": test_id, 905 "subtest": "test1", 906 "status": "FAIL", 907 "expected": "FAIL"}), 908 ("test_end", {"test": test_id, 909 "status": "OK"})], 910 run_info={"debug": False, "os": "osx"}) 911 912 log_1 = suite_log([("test_start", {"test": test_id}), 913 ("test_status", {"test": test_id, 914 "subtest": "test1", 915 "status": "TIMEOUT", 916 "expected": "FAIL"}), 917 ("test_end", {"test": test_id, 918 "status": "OK"})], 919 run_info={"debug": True, "os": "osx"}) 920 921 updated = update(tests, log_0, log_1) 922 new_manifest = updated[0][1] 923 924 run_info_1 = default_run_info.copy() 925 run_info_1.update({"debug": False, "os": "osx"}) 926 run_info_2 = default_run_info.copy() 927 run_info_2.update({"debug": True, "os": "osx"}) 928 929 assert not new_manifest.is_empty 930 assert new_manifest.get_test(test_id).children[0].get( 931 "expected", run_info_1) == "FAIL" 932 assert new_manifest.get_test(test_id).children[0].get( 933 "expected", run_info_2) == "TIMEOUT" 934 935 936def test_update_multiple_3(): 937 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 938 [test1] 939 expected: 940 if debug: FAIL 941 if not debug and os == "osx": TIMEOUT""")] 942 943 log_0 = suite_log([("test_start", {"test": test_id}), 944 ("test_status", {"test": test_id, 945 "subtest": "test1", 946 "status": "FAIL", 947 "expected": "FAIL"}), 948 ("test_end", {"test": test_id, 949 "status": "OK"})], 950 run_info={"debug": False, "os": "osx"}) 951 952 log_1 = suite_log([("test_start", {"test": test_id}), 953 ("test_status", {"test": test_id, 954 "subtest": "test1", 955 "status": "TIMEOUT", 956 "expected": "FAIL"}), 957 ("test_end", {"test": test_id, 958 "status": "OK"})], 959 run_info={"debug": True, "os": "osx"}) 960 961 updated = update(tests, log_0, log_1) 962 new_manifest = updated[0][1] 963 964 run_info_1 = default_run_info.copy() 965 run_info_1.update({"debug": False, "os": "osx"}) 966 run_info_2 = default_run_info.copy() 967 run_info_2.update({"debug": True, "os": "osx"}) 968 969 assert not new_manifest.is_empty 970 assert new_manifest.get_test(test_id).children[0].get( 971 "expected", run_info_1) == "FAIL" 972 assert new_manifest.get_test(test_id).children[0].get( 973 "expected", run_info_2) == "TIMEOUT" 974 975 976def test_update_ignore_existing(): 977 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 978 [test1] 979 expected: 980 if debug: TIMEOUT 981 if not debug and os == "osx": NOTRUN""")] 982 983 log_0 = suite_log([("test_start", {"test": test_id}), 984 ("test_status", {"test": test_id, 985 "subtest": "test1", 986 "status": "FAIL", 987 "expected": "PASS"}), 988 ("test_end", {"test": test_id, 989 "status": "OK"})], 990 run_info={"debug": False, "os": "linux"}) 991 992 log_1 = suite_log([("test_start", {"test": test_id}), 993 ("test_status", {"test": test_id, 994 "subtest": "test1", 995 "status": "FAIL", 996 "expected": "PASS"}), 997 ("test_end", {"test": test_id, 998 "status": "OK"})], 999 run_info={"debug": True, "os": "windows"}) 1000 1001 updated = update(tests, log_0, log_1) 1002 new_manifest = updated[0][1] 1003 1004 run_info_1 = default_run_info.copy() 1005 run_info_1.update({"debug": False, "os": "linux"}) 1006 run_info_2 = default_run_info.copy() 1007 run_info_2.update({"debug": False, "os": "osx"}) 1008 1009 assert not new_manifest.is_empty 1010 assert new_manifest.get_test(test_id).children[0].get( 1011 "expected", run_info_1) == "FAIL" 1012 assert new_manifest.get_test(test_id).children[0].get( 1013 "expected", run_info_2) == "NOTRUN" 1014 1015 1016def test_update_new_test(): 1017 tests = [("path/to/test.htm", [test_id], "testharness", None)] 1018 1019 log_0 = suite_log([("test_start", {"test": test_id}), 1020 ("test_status", {"test": test_id, 1021 "subtest": "test1", 1022 "status": "FAIL", 1023 "expected": "PASS"}), 1024 ("test_end", {"test": test_id, 1025 "status": "OK"})]) 1026 updated = update(tests, log_0) 1027 new_manifest = updated[0][1] 1028 1029 run_info_1 = default_run_info.copy() 1030 1031 assert not new_manifest.is_empty 1032 assert new_manifest.get_test("test.htm") is None 1033 assert len(new_manifest.get_test(test_id).children) == 1 1034 assert new_manifest.get_test(test_id).children[0].get( 1035 "expected", run_info_1) == "FAIL" 1036 1037 1038def test_update_duplicate(): 1039 tests = [("path/to/test.htm", [test_id], "testharness", b""" 1040[test.htm] 1041 expected: ERROR""")] 1042 1043 log_0 = suite_log([("test_start", {"test": test_id}), 1044 ("test_end", {"test": test_id, 1045 "status": "PASS"})]) 1046 log_1 = suite_log([("test_start", {"test": test_id}), 1047 ("test_end", {"test": test_id, 1048 "status": "FAIL"})]) 1049 1050 updated = update(tests, log_0, log_1) 1051 new_manifest = updated[0][1] 1052 run_info_1 = default_run_info.copy() 1053 1054 assert new_manifest.get_test(test_id).get( 1055 "expected", run_info_1) == "ERROR" 1056 1057 1058def test_update_disable_intermittent(): 1059 tests = [("path/to/test.htm", [test_id], "testharness", b""" 1060[test.htm] 1061 expected: ERROR""")] 1062 1063 log_0 = suite_log([("test_start", {"test": test_id}), 1064 ("test_end", {"test": test_id, 1065 "status": "PASS"})]) 1066 log_1 = suite_log([("test_start", {"test": test_id}), 1067 ("test_end", {"test": test_id, 1068 "status": "FAIL"})]) 1069 1070 updated = update(tests, log_0, log_1, disable_intermittent="Some message") 1071 new_manifest = updated[0][1] 1072 run_info_1 = default_run_info.copy() 1073 1074 assert new_manifest.get_test(test_id).get( 1075 "disabled", run_info_1) == "Some message" 1076 1077 1078def test_update_stability_conditional_instability(): 1079 tests = [("path/to/test.htm", [test_id], "testharness", b""" 1080[test.htm] 1081 expected: ERROR""")] 1082 1083 log_0 = suite_log([("test_start", {"test": test_id}), 1084 ("test_end", {"test": test_id, 1085 "status": "PASS"})], 1086 run_info={"os": "linux"}) 1087 log_1 = suite_log([("test_start", {"test": test_id}), 1088 ("test_end", {"test": test_id, 1089 "status": "FAIL"})], 1090 run_info={"os": "linux"}) 1091 log_2 = suite_log([("test_start", {"test": test_id}), 1092 ("test_end", {"test": test_id, 1093 "status": "FAIL"})], 1094 run_info={"os": "mac"}) 1095 1096 updated = update(tests, log_0, log_1, log_2, disable_intermittent="Some message") 1097 new_manifest = updated[0][1] 1098 run_info_1 = default_run_info.copy() 1099 run_info_1.update({"os": "linux"}) 1100 run_info_2 = default_run_info.copy() 1101 run_info_2.update({"os": "mac"}) 1102 1103 assert new_manifest.get_test(test_id).get( 1104 "disabled", run_info_1) == "Some message" 1105 with pytest.raises(KeyError): 1106 assert new_manifest.get_test(test_id).get( 1107 "disabled", run_info_2) 1108 assert new_manifest.get_test(test_id).get( 1109 "expected", run_info_2) == "FAIL" 1110 1111 1112def test_update_full(): 1113 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 1114 [test1] 1115 expected: 1116 if debug: TIMEOUT 1117 if not debug and os == "osx": NOTRUN 1118 1119 [test2] 1120 expected: FAIL 1121 1122[test.js] 1123 [test1] 1124 expected: FAIL 1125""")] 1126 1127 log_0 = suite_log([("test_start", {"test": test_id}), 1128 ("test_status", {"test": test_id, 1129 "subtest": "test1", 1130 "status": "FAIL", 1131 "expected": "PASS"}), 1132 ("test_end", {"test": test_id, 1133 "status": "OK"})], 1134 run_info={"debug": False}) 1135 1136 log_1 = suite_log([("test_start", {"test": test_id}), 1137 ("test_status", {"test": test_id, 1138 "subtest": "test1", 1139 "status": "ERROR", 1140 "expected": "PASS"}), 1141 ("test_end", {"test": test_id, 1142 "status": "OK"})], 1143 run_info={"debug": True}) 1144 1145 updated = update(tests, log_0, log_1, full_update=True) 1146 new_manifest = updated[0][1] 1147 1148 run_info_1 = default_run_info.copy() 1149 run_info_1.update({"debug": False, "os": "win"}) 1150 run_info_2 = default_run_info.copy() 1151 run_info_2.update({"debug": True, "os": "osx"}) 1152 1153 assert not new_manifest.is_empty 1154 assert new_manifest.get_test("test.js") is None 1155 assert len(new_manifest.get_test(test_id).children) == 1 1156 assert new_manifest.get_test(test_id).children[0].get( 1157 "expected", run_info_1) == "FAIL" 1158 assert new_manifest.get_test(test_id).children[0].get( 1159 "expected", run_info_2) == "ERROR" 1160 1161 1162def test_update_full_unknown(): 1163 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 1164 [test1] 1165 expected: 1166 if release_or_beta: ERROR 1167 if not debug and os == "osx": NOTRUN 1168""")] 1169 1170 log_0 = suite_log([("test_start", {"test": test_id}), 1171 ("test_status", {"test": test_id, 1172 "subtest": "test1", 1173 "status": "FAIL", 1174 "expected": "PASS"}), 1175 ("test_end", {"test": test_id, 1176 "status": "OK"})], 1177 run_info={"debug": False, "release_or_beta": False}) 1178 1179 log_1 = suite_log([("test_start", {"test": test_id}), 1180 ("test_status", {"test": test_id, 1181 "subtest": "test1", 1182 "status": "FAIL", 1183 "expected": "PASS"}), 1184 ("test_end", {"test": test_id, 1185 "status": "OK"})], 1186 run_info={"debug": True, "release_or_beta": False}) 1187 1188 updated = update(tests, log_0, log_1, full_update=True) 1189 new_manifest = updated[0][1] 1190 1191 run_info_1 = default_run_info.copy() 1192 run_info_1.update({"release_or_beta": False}) 1193 run_info_2 = default_run_info.copy() 1194 run_info_2.update({"release_or_beta": True}) 1195 1196 assert not new_manifest.is_empty 1197 assert new_manifest.get_test(test_id).children[0].get( 1198 "expected", run_info_1) == "FAIL" 1199 assert new_manifest.get_test(test_id).children[0].get( 1200 "expected", run_info_2) == "ERROR" 1201 1202 1203def test_update_full_unknown_missing(): 1204 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 1205 [subtest_deleted] 1206 expected: 1207 if release_or_beta: ERROR 1208 FAIL 1209""")] 1210 1211 log_0 = suite_log([("test_start", {"test": test_id}), 1212 ("test_status", {"test": test_id, 1213 "subtest": "test1", 1214 "status": "PASS", 1215 "expected": "PASS"}), 1216 ("test_end", {"test": test_id, 1217 "status": "OK"})], 1218 run_info={"debug": False, "release_or_beta": False}) 1219 1220 updated = update(tests, log_0, full_update=True) 1221 assert len(updated) == 0 1222 1223 1224def test_update_default(): 1225 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 1226 [test1] 1227 expected: 1228 if os == "mac": FAIL 1229 ERROR""")] 1230 1231 log_0 = suite_log([("test_start", {"test": test_id}), 1232 ("test_status", {"test": test_id, 1233 "subtest": "test1", 1234 "status": "PASS", 1235 "expected": "FAIL"}), 1236 ("test_end", {"test": test_id, 1237 "status": "OK"})], 1238 run_info={"os": "mac"}) 1239 1240 log_1 = suite_log([("test_start", {"test": test_id}), 1241 ("test_status", {"test": test_id, 1242 "subtest": "test1", 1243 "status": "PASS", 1244 "expected": "ERROR"}), 1245 ("test_end", {"test": test_id, 1246 "status": "OK"})], 1247 run_info={"os": "linux"}) 1248 1249 updated = update(tests, log_0, log_1) 1250 new_manifest = updated[0][1] 1251 1252 assert new_manifest.is_empty 1253 1254 1255def test_update_default_1(): 1256 tests = [("path/to/test.htm", [test_id], "testharness", b""" 1257[test.htm] 1258 expected: 1259 if os == "mac": TIMEOUT 1260 ERROR""")] 1261 1262 log_0 = suite_log([("test_start", {"test": test_id}), 1263 ("test_end", {"test": test_id, 1264 "expected": "ERROR", 1265 "status": "FAIL"})], 1266 run_info={"os": "linux"}) 1267 1268 updated = update(tests, log_0) 1269 new_manifest = updated[0][1] 1270 1271 assert not new_manifest.is_empty 1272 1273 run_info_1 = default_run_info.copy() 1274 run_info_1.update({"os": "mac"}) 1275 run_info_2 = default_run_info.copy() 1276 run_info_2.update({"os": "win"}) 1277 1278 assert not new_manifest.is_empty 1279 assert new_manifest.get_test(test_id).get( 1280 "expected", run_info_1) == "TIMEOUT" 1281 assert new_manifest.get_test(test_id).get( 1282 "expected", run_info_2) == "FAIL" 1283 1284 1285def test_update_default_2(): 1286 tests = [("path/to/test.htm", [test_id], "testharness", b""" 1287[test.htm] 1288 expected: 1289 if os == "mac": TIMEOUT 1290 ERROR""")] 1291 1292 log_0 = suite_log([("test_start", {"test": test_id}), 1293 ("test_end", {"test": test_id, 1294 "expected": "ERROR", 1295 "status": "TIMEOUT"})], 1296 run_info={"os": "linux"}) 1297 1298 updated = update(tests, log_0) 1299 new_manifest = updated[0][1] 1300 1301 assert not new_manifest.is_empty 1302 1303 run_info_1 = default_run_info.copy() 1304 run_info_1.update({"os": "mac"}) 1305 run_info_2 = default_run_info.copy() 1306 run_info_2.update({"os": "win"}) 1307 1308 assert not new_manifest.is_empty 1309 assert new_manifest.get_test(test_id).get( 1310 "expected", run_info_1) == "TIMEOUT" 1311 assert new_manifest.get_test(test_id).get( 1312 "expected", run_info_2) == "TIMEOUT" 1313 1314 1315def test_update_assertion_count_0(): 1316 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 1317 max-asserts: 4 1318 min-asserts: 2 1319""")] 1320 1321 log_0 = suite_log([("test_start", {"test": test_id}), 1322 ("assertion_count", {"test": test_id, 1323 "count": 6, 1324 "min_expected": 2, 1325 "max_expected": 4}), 1326 ("test_end", {"test": test_id, 1327 "status": "OK"})]) 1328 1329 updated = update(tests, log_0) 1330 new_manifest = updated[0][1] 1331 1332 assert not new_manifest.is_empty 1333 assert new_manifest.get_test(test_id).get("max-asserts") == "7" 1334 assert new_manifest.get_test(test_id).get("min-asserts") == "2" 1335 1336 1337def test_update_assertion_count_1(): 1338 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 1339 max-asserts: 4 1340 min-asserts: 2 1341""")] 1342 1343 log_0 = suite_log([("test_start", {"test": test_id}), 1344 ("assertion_count", {"test": test_id, 1345 "count": 1, 1346 "min_expected": 2, 1347 "max_expected": 4}), 1348 ("test_end", {"test": test_id, 1349 "status": "OK"})]) 1350 1351 updated = update(tests, log_0) 1352 new_manifest = updated[0][1] 1353 1354 assert not new_manifest.is_empty 1355 assert new_manifest.get_test(test_id).get("max-asserts") == "4" 1356 assert new_manifest.get_test(test_id).has_key("min-asserts") is False 1357 1358 1359def test_update_assertion_count_2(): 1360 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 1361 max-asserts: 4 1362 min-asserts: 2 1363""")] 1364 1365 log_0 = suite_log([("test_start", {"test": test_id}), 1366 ("assertion_count", {"test": test_id, 1367 "count": 3, 1368 "min_expected": 2, 1369 "max_expected": 4}), 1370 ("test_end", {"test": test_id, 1371 "status": "OK"})]) 1372 1373 updated = update(tests, log_0) 1374 assert not updated 1375 1376 1377def test_update_assertion_count_3(): 1378 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm] 1379 max-asserts: 4 1380 min-asserts: 2 1381""")] 1382 1383 log_0 = suite_log([("test_start", {"test": test_id}), 1384 ("assertion_count", {"test": test_id, 1385 "count": 6, 1386 "min_expected": 2, 1387 "max_expected": 4}), 1388 ("test_end", {"test": test_id, 1389 "status": "OK"})], 1390 run_info={"os": "windows"}) 1391 1392 log_1 = suite_log([("test_start", {"test": test_id}), 1393 ("assertion_count", {"test": test_id, 1394 "count": 7, 1395 "min_expected": 2, 1396 "max_expected": 4}), 1397 ("test_end", {"test": test_id, 1398 "status": "OK"})], 1399 run_info={"os": "linux"}) 1400 1401 updated = update(tests, log_0, log_1) 1402 new_manifest = updated[0][1] 1403 1404 assert not new_manifest.is_empty 1405 assert new_manifest.get_test(test_id).get("max-asserts") == "8" 1406 assert new_manifest.get_test(test_id).get("min-asserts") == "2" 1407 1408 1409def test_update_assertion_count_4(): 1410 tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]""")] 1411 1412 log_0 = suite_log([("test_start", {"test": test_id}), 1413 ("assertion_count", {"test": test_id, 1414 "count": 6, 1415 "min_expected": 0, 1416 "max_expected": 0}), 1417 ("test_end", {"test": test_id, 1418 "status": "OK"})], 1419 run_info={"os": "windows"}) 1420 1421 log_1 = suite_log([("test_start", {"test": test_id}), 1422 ("assertion_count", {"test": test_id, 1423 "count": 7, 1424 "min_expected": 0, 1425 "max_expected": 0}), 1426 ("test_end", {"test": test_id, 1427 "status": "OK"})], 1428 run_info={"os": "linux"}) 1429 1430 updated = update(tests, log_0, log_1) 1431 new_manifest = updated[0][1] 1432 1433 assert not new_manifest.is_empty 1434 assert new_manifest.get_test(test_id).get("max-asserts") == "8" 1435 assert new_manifest.get_test(test_id).has_key("min-asserts") is False 1436 1437 1438def test_update_lsan_0(): 1439 tests = [("path/to/test.htm", [test_id], "testharness", b""), 1440 ("path/to/__dir__", [dir_id], None, b"")] 1441 1442 log_0 = suite_log([("lsan_leak", {"scope": "path/to/", 1443 "frames": ["foo", "bar"]})]) 1444 1445 1446 updated = update(tests, log_0) 1447 new_manifest = updated[0][1] 1448 1449 assert not new_manifest.is_empty 1450 assert new_manifest.get("lsan-allowed") == ["foo"] 1451 1452 1453def test_update_lsan_1(): 1454 tests = [("path/to/test.htm", [test_id], "testharness", b""), 1455 ("path/to/__dir__", [dir_id], None, b""" 1456lsan-allowed: [foo]""")] 1457 1458 log_0 = suite_log([("lsan_leak", {"scope": "path/to/", 1459 "frames": ["foo", "bar"]}), 1460 ("lsan_leak", {"scope": "path/to/", 1461 "frames": ["baz", "foobar"]})]) 1462 1463 1464 updated = update(tests, log_0) 1465 new_manifest = updated[0][1] 1466 1467 assert not new_manifest.is_empty 1468 assert new_manifest.get("lsan-allowed") == ["baz", "foo"] 1469 1470 1471def test_update_lsan_2(): 1472 tests = [("path/to/test.htm", [test_id], "testharness", b""), 1473 ("path/__dir__", ["path/__dir__"], None, b""" 1474lsan-allowed: [foo]"""), 1475 ("path/to/__dir__", [dir_id], None, b"")] 1476 1477 log_0 = suite_log([("lsan_leak", {"scope": "path/to/", 1478 "frames": ["foo", "bar"], 1479 "allowed_match": ["foo"]}), 1480 ("lsan_leak", {"scope": "path/to/", 1481 "frames": ["baz", "foobar"]})]) 1482 1483 1484 updated = update(tests, log_0) 1485 new_manifest = updated[0][1] 1486 1487 assert not new_manifest.is_empty 1488 assert new_manifest.get("lsan-allowed") == ["baz"] 1489 1490 1491def test_update_lsan_3(): 1492 tests = [("path/to/test.htm", [test_id], "testharness", b""), 1493 ("path/to/__dir__", [dir_id], None, b"")] 1494 1495 log_0 = suite_log([("lsan_leak", {"scope": "path/to/", 1496 "frames": ["foo", "bar"]})], 1497 run_info={"os": "win"}) 1498 1499 log_1 = suite_log([("lsan_leak", {"scope": "path/to/", 1500 "frames": ["baz", "foobar"]})], 1501 run_info={"os": "linux"}) 1502 1503 1504 updated = update(tests, log_0, log_1) 1505 new_manifest = updated[0][1] 1506 1507 assert not new_manifest.is_empty 1508 assert new_manifest.get("lsan-allowed") == ["baz", "foo"] 1509 1510 1511def test_update_wptreport_0(): 1512 tests = [("path/to/test.htm", [test_id], "testharness", 1513 b"""[test.htm] 1514 [test1] 1515 expected: FAIL""")] 1516 1517 log = {"run_info": default_run_info.copy(), 1518 "results": [ 1519 {"test": "/path/to/test.htm", 1520 "subtests": [{"name": "test1", 1521 "status": "PASS", 1522 "expected": "FAIL"}], 1523 "status": "OK"} 1524 ]} 1525 1526 updated = update(tests, log) 1527 1528 assert len(updated) == 1 1529 assert updated[0][1].is_empty 1530 1531 1532def test_update_wptreport_1(): 1533 tests = [("path/to/test.htm", [test_id], "testharness", b""), 1534 ("path/to/__dir__", [dir_id], None, b"")] 1535 1536 log = {"run_info": default_run_info.copy(), 1537 "results": [], 1538 "lsan_leaks": [{"scope": "path/to/", 1539 "frames": ["baz", "foobar"]}]} 1540 1541 updated = update(tests, log) 1542 1543 assert len(updated) == 1 1544 assert updated[0][1].get("lsan-allowed") == ["baz"] 1545 1546 1547def test_update_leak_total_0(): 1548 tests = [("path/to/test.htm", [test_id], "testharness", b""), 1549 ("path/to/__dir__", [dir_id], None, b"")] 1550 1551 log_0 = suite_log([("mozleak_total", {"scope": "path/to/", 1552 "process": "default", 1553 "bytes": 100, 1554 "threshold": 0, 1555 "objects": []})]) 1556 1557 updated = update(tests, log_0) 1558 new_manifest = updated[0][1] 1559 1560 assert not new_manifest.is_empty 1561 assert new_manifest.get("leak-threshold") == ['default:51200'] 1562 1563 1564def test_update_leak_total_1(): 1565 tests = [("path/to/test.htm", [test_id], "testharness", b""), 1566 ("path/to/__dir__", [dir_id], None, b"")] 1567 1568 log_0 = suite_log([("mozleak_total", {"scope": "path/to/", 1569 "process": "default", 1570 "bytes": 100, 1571 "threshold": 1000, 1572 "objects": []})]) 1573 1574 updated = update(tests, log_0) 1575 assert not updated 1576 1577 1578def test_update_leak_total_2(): 1579 tests = [("path/to/test.htm", [test_id], "testharness", b""), 1580 ("path/to/__dir__", [dir_id], None, b""" 1581leak-total: 110""")] 1582 1583 log_0 = suite_log([("mozleak_total", {"scope": "path/to/", 1584 "process": "default", 1585 "bytes": 100, 1586 "threshold": 110, 1587 "objects": []})]) 1588 1589 updated = update(tests, log_0) 1590 assert not updated 1591 1592 1593def test_update_leak_total_3(): 1594 tests = [("path/to/test.htm", [test_id], "testharness", b""), 1595 ("path/to/__dir__", [dir_id], None, b""" 1596leak-total: 100""")] 1597 1598 log_0 = suite_log([("mozleak_total", {"scope": "path/to/", 1599 "process": "default", 1600 "bytes": 1000, 1601 "threshold": 100, 1602 "objects": []})]) 1603 1604 updated = update(tests, log_0) 1605 new_manifest = updated[0][1] 1606 1607 assert not new_manifest.is_empty 1608 assert new_manifest.get("leak-threshold") == ['default:51200'] 1609 1610 1611def test_update_leak_total_4(): 1612 tests = [("path/to/test.htm", [test_id], "testharness", b""), 1613 ("path/to/__dir__", [dir_id], None, b""" 1614leak-total: 110""")] 1615 1616 log_0 = suite_log([ 1617 ("lsan_leak", {"scope": "path/to/", 1618 "frames": ["foo", "bar"]}), 1619 ("mozleak_total", {"scope": "path/to/", 1620 "process": "default", 1621 "bytes": 100, 1622 "threshold": 110, 1623 "objects": []})]) 1624 1625 updated = update(tests, log_0) 1626 new_manifest = updated[0][1] 1627 1628 assert not new_manifest.is_empty 1629 assert new_manifest.has_key("leak-threshold") is False 1630 1631 1632class TestStep(Step): 1633 def create(self, state): 1634 tests = [("path/to/test.htm", [test_id], "testharness", "")] 1635 state.foo = create_test_manifest(tests) 1636 1637 1638class UpdateRunner(StepRunner): 1639 steps = [TestStep] 1640 1641 1642def test_update_pickle(): 1643 logger = structuredlog.StructuredLogger("expected_test") 1644 args = { 1645 "test_paths": { 1646 "/": {"tests_path": os.path.abspath(os.path.join(here, 1647 os.pardir, 1648 os.pardir, 1649 os.pardir, 1650 os.pardir))}, 1651 }, 1652 "abort": False, 1653 "continue": False, 1654 "sync": False, 1655 } 1656 args2 = args.copy() 1657 args2["abort"] = True 1658 wptupdate = WPTUpdate(logger, **args2) 1659 wptupdate = WPTUpdate(logger, runner_cls=UpdateRunner, **args) 1660 wptupdate.run() 1661