1from io import BytesIO 2from mock import Mock 3 4from manifest import manifest as wptmanifest 5from manifest.item import TestharnessTest, RefTest 6from manifest.utils import to_os_path 7from .. import manifestexpected, wpttest 8 9dir_ini_0 = b"""\ 10prefs: [a:b] 11""" 12 13dir_ini_1 = b"""\ 14prefs: [@Reset, b:c] 15max-asserts: 2 16min-asserts: 1 17tags: [b, c] 18""" 19 20dir_ini_2 = b"""\ 21lsan-max-stack-depth: 42 22""" 23 24test_0 = b"""\ 25[0.html] 26 prefs: [c:d] 27 max-asserts: 3 28 tags: [a, @Reset] 29""" 30 31test_1 = b"""\ 32[1.html] 33 prefs: 34 if os == 'win': [a:b, c:d] 35 expected: 36 if os == 'win': FAIL 37""" 38 39test_2 = b"""\ 40[2.html] 41 lsan-max-stack-depth: 42 42""" 43 44test_3 = b"""\ 45[3.html] 46 [subtest1] 47 expected: [PASS, FAIL] 48 49 [subtest2] 50 disabled: reason 51 52 [subtest3] 53 expected: FAIL 54""" 55 56test_4 = b"""\ 57[4.html] 58 expected: FAIL 59""" 60 61test_5 = b"""\ 62[5.html] 63""" 64 65test_6 = b"""\ 66[6.html] 67 expected: [OK, FAIL] 68""" 69 70test_7 = b"""\ 71[7.html] 72 blink_expect_any_subtest_status: yep 73""" 74 75test_fuzzy = b"""\ 76[fuzzy.html] 77 fuzzy: fuzzy-ref.html:1;200 78""" 79 80 81testharness_test = b"""<script src="/resources/testharness.js"></script> 82<script src="/resources/testharnessreport.js"></script>""" 83 84 85def make_mock_manifest(*items): 86 rv = Mock(tests_root="/foobar") 87 tests = [] 88 rv.__iter__ = lambda self: iter(tests) 89 rv.__getitem__ = lambda self, k: tests[k] 90 for test_type, dir_path, num_tests in items: 91 for i in range(num_tests): 92 filename = dir_path + "/%i.html" % i 93 tests.append((test_type, 94 filename, 95 {TestharnessTest("/foo.bar", filename, "/", filename)})) 96 return rv 97 98def make_test_object(test_name, 99 test_path, 100 index, 101 items, 102 inherit_metadata=None, 103 iterate=False, 104 condition=None): 105 inherit_metadata = inherit_metadata if inherit_metadata is not None else [] 106 condition = condition if condition is not None else {} 107 tests = make_mock_manifest(*items) if isinstance(items, list) else make_mock_manifest(items) 108 109 test_metadata = manifestexpected.static.compile(BytesIO(test_name), 110 condition, 111 data_cls_getter=manifestexpected.data_cls_getter, 112 test_path=test_path, 113 url_base="/") 114 115 test = next(iter(tests[index][2])) if iterate else tests[index][2].pop() 116 return wpttest.from_manifest(tests, test, inherit_metadata, test_metadata.get_test(test.id)) 117 118 119def test_metadata_inherit(): 120 items = [("test", "a", 10), ("test", "a/b", 10), ("test", "c", 10)] 121 inherit_metadata = [ 122 manifestexpected.static.compile( 123 BytesIO(item), 124 {}, 125 data_cls_getter=lambda x,y: manifestexpected.DirectoryManifest) 126 for item in [dir_ini_0, dir_ini_1]] 127 128 test_obj = make_test_object(test_0, "a/0.html", 0, items, inherit_metadata, True) 129 130 assert test_obj.max_assertion_count == 3 131 assert test_obj.min_assertion_count == 1 132 assert test_obj.prefs == {"b": "c", "c": "d"} 133 assert test_obj.tags == {"a", "dir:a"} 134 135 136def test_conditional(): 137 items = [("test", "a", 10), ("test", "a/b", 10), ("test", "c", 10)] 138 139 test_obj = make_test_object(test_1, "a/1.html", 1, items, None, True, {"os": "win"}) 140 141 assert test_obj.prefs == {"a": "b", "c": "d"} 142 assert test_obj.expected() == "FAIL" 143 144 145def test_metadata_lsan_stack_depth(): 146 items = [("test", "a", 10), ("test", "a/b", 10)] 147 148 test_obj = make_test_object(test_2, "a/2.html", 2, items, None, True) 149 150 assert test_obj.lsan_max_stack_depth == 42 151 152 test_obj = make_test_object(test_2, "a/2.html", 1, items, None, True) 153 154 assert test_obj.lsan_max_stack_depth is None 155 156 inherit_metadata = [ 157 manifestexpected.static.compile( 158 BytesIO(dir_ini_2), 159 {}, 160 data_cls_getter=lambda x,y: manifestexpected.DirectoryManifest) 161 ] 162 163 test_obj = make_test_object(test_0, "a/0/html", 0, items, inherit_metadata, False) 164 165 assert test_obj.lsan_max_stack_depth == 42 166 167 168def test_subtests(): 169 test_obj = make_test_object(test_3, "a/3.html", 3, ("test", "a", 4), None, False) 170 assert test_obj.expected("subtest1") == "PASS" 171 assert test_obj.known_intermittent("subtest1") == ["FAIL"] 172 assert test_obj.expected("subtest2") == "PASS" 173 assert test_obj.known_intermittent("subtest2") == [] 174 assert test_obj.expected("subtest3") == "FAIL" 175 assert test_obj.known_intermittent("subtest3") == [] 176 177 178def test_expected_fail(): 179 test_obj = make_test_object(test_4, "a/4.html", 4, ("test", "a", 5), None, False) 180 assert test_obj.expected() == "FAIL" 181 assert test_obj.known_intermittent() == [] 182 183 184def test_no_expected(): 185 test_obj = make_test_object(test_5, "a/5.html", 5, ("test", "a", 6), None, False) 186 assert test_obj.expected() == "OK" 187 assert test_obj.known_intermittent() == [] 188 189 190def test_known_intermittent(): 191 test_obj = make_test_object(test_6, "a/6.html", 6, ("test", "a", 7), None, False) 192 assert test_obj.expected() == "OK" 193 assert test_obj.known_intermittent() == ["FAIL"] 194 195 196def test_expect_any_subtest_status(): 197 test_obj = make_test_object(test_7, "a/7.html", 7, ("test", "a", 8), None, False) 198 assert test_obj.expected() == "OK" 199 assert test_obj.expect_any_subtest_status() is True 200 201 202def test_metadata_fuzzy(): 203 item = RefTest(".", "a/fuzzy.html", "/", "a/fuzzy.html", 204 references=[["/a/fuzzy-ref.html", "=="]], 205 fuzzy=[[["/a/fuzzy.html", '/a/fuzzy-ref.html', '=='], 206 [[2, 3], [10, 15]]]]) 207 s = Mock(rel_path="a/fuzzy.html", rel_path_parts=("a", "fuzzy.html"), hash="0"*40) 208 s.manifest_items = Mock(return_value=(item.item_type, [item])) 209 210 211 manifest = wptmanifest.Manifest() 212 213 assert manifest.update([(s, True)]) is True 214 215 test_metadata = manifestexpected.static.compile(BytesIO(test_fuzzy), 216 {}, 217 data_cls_getter=manifestexpected.data_cls_getter, 218 test_path="a/fuzzy.html", 219 url_base="/") 220 221 test = next(manifest.iterpath(to_os_path("a/fuzzy.html"))) 222 test_obj = wpttest.from_manifest(manifest, test, [], test_metadata.get_test(test.id)) 223 224 assert test_obj.fuzzy == {('/a/fuzzy.html', '/a/fuzzy-ref.html', '=='): [[2, 3], [10, 15]]} 225 assert test_obj.fuzzy_override == {'/a/fuzzy-ref.html': ((1, 1), (200, 200))} 226