1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5import os 6import shutil 7import sys 8import tempfile 9import types 10import uuid 11from collections import defaultdict 12 13from mozlog import reader 14from mozlog import structuredlog 15 16import expected 17import manifestupdate 18import testloader 19import wptmanifest 20import wpttest 21from vcs import git 22manifest = None # Module that will be imported relative to test_root 23 24logger = structuredlog.StructuredLogger("web-platform-tests") 25 26 27def load_test_manifests(serve_root, test_paths): 28 do_delayed_imports(serve_root) 29 manifest_loader = testloader.ManifestLoader(test_paths, False) 30 return manifest_loader.load() 31 32 33def update_expected(test_paths, serve_root, log_file_names, 34 rev_old=None, rev_new="HEAD", ignore_existing=False, 35 sync_root=None, property_order=None, boolean_properties=None): 36 """Update the metadata files for web-platform-tests based on 37 the results obtained in a previous run""" 38 39 manifests = load_test_manifests(serve_root, test_paths) 40 41 change_data = {} 42 43 if sync_root is not None: 44 if rev_old is not None: 45 rev_old = git("rev-parse", rev_old, repo=sync_root).strip() 46 rev_new = git("rev-parse", rev_new, repo=sync_root).strip() 47 48 if rev_old is not None: 49 change_data = load_change_data(rev_old, rev_new, repo=sync_root) 50 51 52 expected_map_by_manifest = update_from_logs(manifests, 53 *log_file_names, 54 ignore_existing=ignore_existing, 55 property_order=property_order, 56 boolean_properties=boolean_properties) 57 58 for test_manifest, expected_map in expected_map_by_manifest.iteritems(): 59 url_base = manifests[test_manifest]["url_base"] 60 metadata_path = test_paths[url_base]["metadata_path"] 61 write_changes(metadata_path, expected_map) 62 63 results_changed = [item.test_path for item in expected_map.itervalues() if item.modified] 64 65 return unexpected_changes(manifests, change_data, results_changed) 66 67 68def do_delayed_imports(serve_root): 69 global manifest 70 from manifest import manifest 71 72 73def files_in_repo(repo_root): 74 return git("ls-tree", "-r", "--name-only", "HEAD").split("\n") 75 76 77def rev_range(rev_old, rev_new, symmetric=False): 78 joiner = ".." if not symmetric else "..." 79 return "".join([rev_old, joiner, rev_new]) 80 81 82def paths_changed(rev_old, rev_new, repo): 83 data = git("diff", "--name-status", rev_range(rev_old, rev_new), repo=repo) 84 lines = [tuple(item.strip() for item in line.strip().split("\t", 1)) 85 for line in data.split("\n") if line.strip()] 86 output = set(lines) 87 return output 88 89 90def load_change_data(rev_old, rev_new, repo): 91 changes = paths_changed(rev_old, rev_new, repo) 92 rv = {} 93 status_keys = {"M": "modified", 94 "A": "new", 95 "D": "deleted"} 96 # TODO: deal with renames 97 for item in changes: 98 rv[item[1]] = status_keys[item[0]] 99 return rv 100 101 102def unexpected_changes(manifests, change_data, files_changed): 103 files_changed = set(files_changed) 104 105 root_manifest = None 106 for manifest, paths in manifests.iteritems(): 107 if paths["url_base"] == "/": 108 root_manifest = manifest 109 break 110 else: 111 return [] 112 113 rv = [] 114 115 return [fn for fn, tests in root_manifest if fn in files_changed and change_data.get(fn) != "M"] 116 117# For each testrun 118# Load all files and scan for the suite_start entry 119# Build a hash of filename: properties 120# For each different set of properties, gather all chunks 121# For each chunk in the set of chunks, go through all tests 122# for each test, make a map of {conditionals: [(platform, new_value)]} 123# Repeat for each platform 124# For each test in the list of tests: 125# for each conditional: 126# If all the new values match (or there aren't any) retain that conditional 127# If any new values mismatch mark the test as needing human attention 128# Check if all the RHS values are the same; if so collapse the conditionals 129 130 131def update_from_logs(manifests, *log_filenames, **kwargs): 132 ignore_existing = kwargs.get("ignore_existing", False) 133 property_order = kwargs.get("property_order") 134 boolean_properties = kwargs.get("boolean_properties") 135 136 expected_map = {} 137 id_test_map = {} 138 139 for test_manifest, paths in manifests.iteritems(): 140 expected_map_manifest, id_path_map_manifest = create_test_tree( 141 paths["metadata_path"], 142 test_manifest, 143 property_order=property_order, 144 boolean_properties=boolean_properties) 145 expected_map[test_manifest] = expected_map_manifest 146 id_test_map.update(id_path_map_manifest) 147 148 updater = ExpectedUpdater(manifests, expected_map, id_test_map, 149 ignore_existing=ignore_existing) 150 for log_filename in log_filenames: 151 with open(log_filename) as f: 152 updater.update_from_log(f) 153 154 for manifest_expected in expected_map.itervalues(): 155 for tree in manifest_expected.itervalues(): 156 for test in tree.iterchildren(): 157 for subtest in test.iterchildren(): 158 subtest.coalesce_expected() 159 test.coalesce_expected() 160 161 return expected_map 162 163def directory_manifests(metadata_path): 164 rv = [] 165 for dirpath, dirname, filenames in os.walk(metadata_path): 166 if "__dir__.ini" in filenames: 167 rel_path = os.path.relpath(dirpath, metadata_path) 168 rv.append(os.path.join(rel_path, "__dir__.ini")) 169 return rv 170 171def write_changes(metadata_path, expected_map): 172 # First write the new manifest files to a temporary directory 173 temp_path = tempfile.mkdtemp(dir=os.path.split(metadata_path)[0]) 174 write_new_expected(temp_path, expected_map) 175 176 # Keep all __dir__.ini files (these are not in expected_map because they 177 # aren't associated with a specific test) 178 keep_files = directory_manifests(metadata_path) 179 180 # Copy all files in the root to the temporary location since 181 # these cannot be ini files 182 keep_files.extend(item for item in os.listdir(metadata_path) if 183 not os.path.isdir(os.path.join(metadata_path, item))) 184 185 for item in keep_files: 186 dest_dir = os.path.dirname(os.path.join(temp_path, item)) 187 if not os.path.exists(dest_dir): 188 os.makedirs(dest_dir) 189 shutil.copyfile(os.path.join(metadata_path, item), 190 os.path.join(temp_path, item)) 191 192 # Then move the old manifest files to a new location 193 temp_path_2 = metadata_path + str(uuid.uuid4()) 194 os.rename(metadata_path, temp_path_2) 195 # Move the new files to the destination location and remove the old files 196 os.rename(temp_path, metadata_path) 197 shutil.rmtree(temp_path_2) 198 199 200def write_new_expected(metadata_path, expected_map): 201 # Serialize the data back to a file 202 for tree in expected_map.itervalues(): 203 if not tree.is_empty: 204 manifest_str = wptmanifest.serialize(tree.node, skip_empty_data=True) 205 assert manifest_str != "" 206 path = expected.expected_path(metadata_path, tree.test_path) 207 dir = os.path.split(path)[0] 208 if not os.path.exists(dir): 209 os.makedirs(dir) 210 with open(path, "w") as f: 211 f.write(manifest_str) 212 213 214class ExpectedUpdater(object): 215 def __init__(self, test_manifests, expected_tree, id_path_map, ignore_existing=False): 216 self.test_manifests = test_manifests 217 self.expected_tree = expected_tree 218 self.id_path_map = id_path_map 219 self.ignore_existing = ignore_existing 220 self.run_info = None 221 self.action_map = {"suite_start": self.suite_start, 222 "test_start": self.test_start, 223 "test_status": self.test_status, 224 "test_end": self.test_end} 225 self.tests_visited = {} 226 227 self.test_cache = {} 228 229 def update_from_log(self, log_file): 230 self.run_info = None 231 log_reader = reader.read(log_file) 232 reader.each_log(log_reader, self.action_map) 233 234 def suite_start(self, data): 235 self.run_info = data["run_info"] 236 237 def test_id(self, id): 238 if type(id) in types.StringTypes: 239 return id 240 else: 241 return tuple(id) 242 243 def test_start(self, data): 244 test_id = self.test_id(data["test"]) 245 try: 246 test_manifest, test = self.id_path_map[test_id] 247 expected_node = self.expected_tree[test_manifest][test].get_test(test_id) 248 except KeyError: 249 print "Test not found %s, skipping" % test_id 250 return 251 self.test_cache[test_id] = expected_node 252 253 if test_id not in self.tests_visited: 254 if self.ignore_existing: 255 expected_node.clear_expected() 256 self.tests_visited[test_id] = set() 257 258 def test_status(self, data): 259 test_id = self.test_id(data["test"]) 260 test = self.test_cache.get(test_id) 261 if test is None: 262 return 263 test_cls = wpttest.manifest_test_cls[test.test_type] 264 265 subtest = test.get_subtest(data["subtest"]) 266 267 self.tests_visited[test.id].add(data["subtest"]) 268 269 result = test_cls.subtest_result_cls( 270 data["subtest"], 271 data["status"], 272 data.get("message")) 273 274 subtest.set_result(self.run_info, result) 275 276 def test_end(self, data): 277 test_id = self.test_id(data["test"]) 278 test = self.test_cache.get(test_id) 279 if test is None: 280 return 281 test_cls = wpttest.manifest_test_cls[test.test_type] 282 283 if data["status"] == "SKIP": 284 return 285 286 result = test_cls.result_cls( 287 data["status"], 288 data.get("message")) 289 290 test.set_result(self.run_info, result) 291 del self.test_cache[test_id] 292 293 294def create_test_tree(metadata_path, test_manifest, property_order=None, 295 boolean_properties=None): 296 expected_map = {} 297 id_test_map = {} 298 exclude_types = frozenset(["stub", "helper", "manual"]) 299 include_types = set(manifest.item_types) - exclude_types 300 for test_path, tests in test_manifest.itertypes(*include_types): 301 expected_data = load_expected(test_manifest, metadata_path, test_path, tests, 302 property_order=property_order, 303 boolean_properties=boolean_properties) 304 if expected_data is None: 305 expected_data = create_expected(test_manifest, 306 test_path, 307 tests, 308 property_order=property_order, 309 boolean_properties=boolean_properties) 310 311 for test in tests: 312 id_test_map[test.id] = (test_manifest, test) 313 expected_map[test] = expected_data 314 315 return expected_map, id_test_map 316 317 318def create_expected(test_manifest, test_path, tests, property_order=None, 319 boolean_properties=None): 320 expected = manifestupdate.ExpectedManifest(None, test_path, test_manifest.url_base, 321 property_order=property_order, 322 boolean_properties=boolean_properties) 323 for test in tests: 324 expected.append(manifestupdate.TestNode.create(test.item_type, test.id)) 325 return expected 326 327 328def load_expected(test_manifest, metadata_path, test_path, tests, property_order=None, 329 boolean_properties=None): 330 expected_manifest = manifestupdate.get_manifest(metadata_path, 331 test_path, 332 test_manifest.url_base, 333 property_order=property_order, 334 boolean_properties=boolean_properties) 335 if expected_manifest is None: 336 return 337 338 tests_by_id = {item.id: item for item in tests} 339 340 # Remove expected data for tests that no longer exist 341 for test in expected_manifest.iterchildren(): 342 if not test.id in tests_by_id: 343 test.remove() 344 345 # Add tests that don't have expected data 346 for test in tests: 347 if not expected_manifest.has_test(test.id): 348 expected_manifest.append(manifestupdate.TestNode.create(test.item_type, test.id)) 349 350 return expected_manifest 351