1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5import os
6import shutil
7import sys
8import tempfile
9import types
10import uuid
11from collections import defaultdict
12
13from mozlog import reader
14from mozlog import structuredlog
15
16import expected
17import manifestupdate
18import testloader
19import wptmanifest
20import wpttest
21from vcs import git
22manifest = None  # Module that will be imported relative to test_root
23
24logger = structuredlog.StructuredLogger("web-platform-tests")
25
26
27def load_test_manifests(serve_root, test_paths):
28    do_delayed_imports(serve_root)
29    manifest_loader = testloader.ManifestLoader(test_paths, False)
30    return manifest_loader.load()
31
32
33def update_expected(test_paths, serve_root, log_file_names,
34                    rev_old=None, rev_new="HEAD", ignore_existing=False,
35                    sync_root=None, property_order=None, boolean_properties=None):
36    """Update the metadata files for web-platform-tests based on
37    the results obtained in a previous run"""
38
39    manifests = load_test_manifests(serve_root, test_paths)
40
41    change_data = {}
42
43    if sync_root is not None:
44        if rev_old is not None:
45            rev_old = git("rev-parse", rev_old, repo=sync_root).strip()
46        rev_new = git("rev-parse", rev_new, repo=sync_root).strip()
47
48        if rev_old is not None:
49            change_data = load_change_data(rev_old, rev_new, repo=sync_root)
50
51
52    expected_map_by_manifest = update_from_logs(manifests,
53                                                *log_file_names,
54                                                ignore_existing=ignore_existing,
55                                                property_order=property_order,
56                                                boolean_properties=boolean_properties)
57
58    for test_manifest, expected_map in expected_map_by_manifest.iteritems():
59        url_base = manifests[test_manifest]["url_base"]
60        metadata_path = test_paths[url_base]["metadata_path"]
61        write_changes(metadata_path, expected_map)
62
63    results_changed = [item.test_path for item in expected_map.itervalues() if item.modified]
64
65    return unexpected_changes(manifests, change_data, results_changed)
66
67
68def do_delayed_imports(serve_root):
69    global manifest
70    from manifest import manifest
71
72
73def files_in_repo(repo_root):
74    return git("ls-tree", "-r", "--name-only", "HEAD").split("\n")
75
76
77def rev_range(rev_old, rev_new, symmetric=False):
78    joiner = ".." if not symmetric else "..."
79    return "".join([rev_old, joiner, rev_new])
80
81
82def paths_changed(rev_old, rev_new, repo):
83    data = git("diff", "--name-status", rev_range(rev_old, rev_new), repo=repo)
84    lines = [tuple(item.strip() for item in line.strip().split("\t", 1))
85             for line in data.split("\n") if line.strip()]
86    output = set(lines)
87    return output
88
89
90def load_change_data(rev_old, rev_new, repo):
91    changes = paths_changed(rev_old, rev_new, repo)
92    rv = {}
93    status_keys = {"M": "modified",
94                   "A": "new",
95                   "D": "deleted"}
96    # TODO: deal with renames
97    for item in changes:
98        rv[item[1]] = status_keys[item[0]]
99    return rv
100
101
102def unexpected_changes(manifests, change_data, files_changed):
103    files_changed = set(files_changed)
104
105    root_manifest = None
106    for manifest, paths in manifests.iteritems():
107        if paths["url_base"] == "/":
108            root_manifest = manifest
109            break
110    else:
111        return []
112
113    rv = []
114
115    return [fn for fn, tests in root_manifest if fn in files_changed and change_data.get(fn) != "M"]
116
117# For each testrun
118# Load all files and scan for the suite_start entry
119# Build a hash of filename: properties
120# For each different set of properties, gather all chunks
121# For each chunk in the set of chunks, go through all tests
122# for each test, make a map of {conditionals: [(platform, new_value)]}
123# Repeat for each platform
124# For each test in the list of tests:
125#   for each conditional:
126#      If all the new values match (or there aren't any) retain that conditional
127#      If any new values mismatch mark the test as needing human attention
128#   Check if all the RHS values are the same; if so collapse the conditionals
129
130
131def update_from_logs(manifests, *log_filenames, **kwargs):
132    ignore_existing = kwargs.get("ignore_existing", False)
133    property_order = kwargs.get("property_order")
134    boolean_properties = kwargs.get("boolean_properties")
135
136    expected_map = {}
137    id_test_map = {}
138
139    for test_manifest, paths in manifests.iteritems():
140        expected_map_manifest, id_path_map_manifest = create_test_tree(
141            paths["metadata_path"],
142            test_manifest,
143            property_order=property_order,
144            boolean_properties=boolean_properties)
145        expected_map[test_manifest] = expected_map_manifest
146        id_test_map.update(id_path_map_manifest)
147
148    updater = ExpectedUpdater(manifests, expected_map, id_test_map,
149                              ignore_existing=ignore_existing)
150    for log_filename in log_filenames:
151        with open(log_filename) as f:
152            updater.update_from_log(f)
153
154    for manifest_expected in expected_map.itervalues():
155        for tree in manifest_expected.itervalues():
156            for test in tree.iterchildren():
157                for subtest in test.iterchildren():
158                    subtest.coalesce_expected()
159                test.coalesce_expected()
160
161    return expected_map
162
163def directory_manifests(metadata_path):
164    rv = []
165    for dirpath, dirname, filenames in os.walk(metadata_path):
166        if "__dir__.ini" in filenames:
167            rel_path = os.path.relpath(dirpath, metadata_path)
168            rv.append(os.path.join(rel_path, "__dir__.ini"))
169    return rv
170
171def write_changes(metadata_path, expected_map):
172    # First write the new manifest files to a temporary directory
173    temp_path = tempfile.mkdtemp(dir=os.path.split(metadata_path)[0])
174    write_new_expected(temp_path, expected_map)
175
176    # Keep all __dir__.ini files (these are not in expected_map because they
177    # aren't associated with a specific test)
178    keep_files = directory_manifests(metadata_path)
179
180    # Copy all files in the root to the temporary location since
181    # these cannot be ini files
182    keep_files.extend(item for item in os.listdir(metadata_path) if
183                      not os.path.isdir(os.path.join(metadata_path, item)))
184
185    for item in keep_files:
186        dest_dir = os.path.dirname(os.path.join(temp_path, item))
187        if not os.path.exists(dest_dir):
188            os.makedirs(dest_dir)
189        shutil.copyfile(os.path.join(metadata_path, item),
190                        os.path.join(temp_path, item))
191
192    # Then move the old manifest files to a new location
193    temp_path_2 = metadata_path + str(uuid.uuid4())
194    os.rename(metadata_path, temp_path_2)
195    # Move the new files to the destination location and remove the old files
196    os.rename(temp_path, metadata_path)
197    shutil.rmtree(temp_path_2)
198
199
200def write_new_expected(metadata_path, expected_map):
201    # Serialize the data back to a file
202    for tree in expected_map.itervalues():
203        if not tree.is_empty:
204            manifest_str = wptmanifest.serialize(tree.node, skip_empty_data=True)
205            assert manifest_str != ""
206            path = expected.expected_path(metadata_path, tree.test_path)
207            dir = os.path.split(path)[0]
208            if not os.path.exists(dir):
209                os.makedirs(dir)
210            with open(path, "w") as f:
211                f.write(manifest_str)
212
213
214class ExpectedUpdater(object):
215    def __init__(self, test_manifests, expected_tree, id_path_map, ignore_existing=False):
216        self.test_manifests = test_manifests
217        self.expected_tree = expected_tree
218        self.id_path_map = id_path_map
219        self.ignore_existing = ignore_existing
220        self.run_info = None
221        self.action_map = {"suite_start": self.suite_start,
222                           "test_start": self.test_start,
223                           "test_status": self.test_status,
224                           "test_end": self.test_end}
225        self.tests_visited = {}
226
227        self.test_cache = {}
228
229    def update_from_log(self, log_file):
230        self.run_info = None
231        log_reader = reader.read(log_file)
232        reader.each_log(log_reader, self.action_map)
233
234    def suite_start(self, data):
235        self.run_info = data["run_info"]
236
237    def test_id(self, id):
238        if type(id) in types.StringTypes:
239            return id
240        else:
241            return tuple(id)
242
243    def test_start(self, data):
244        test_id = self.test_id(data["test"])
245        try:
246            test_manifest, test = self.id_path_map[test_id]
247            expected_node = self.expected_tree[test_manifest][test].get_test(test_id)
248        except KeyError:
249            print "Test not found %s, skipping" % test_id
250            return
251        self.test_cache[test_id] = expected_node
252
253        if test_id not in self.tests_visited:
254            if self.ignore_existing:
255                expected_node.clear_expected()
256            self.tests_visited[test_id] = set()
257
258    def test_status(self, data):
259        test_id = self.test_id(data["test"])
260        test = self.test_cache.get(test_id)
261        if test is None:
262            return
263        test_cls = wpttest.manifest_test_cls[test.test_type]
264
265        subtest = test.get_subtest(data["subtest"])
266
267        self.tests_visited[test.id].add(data["subtest"])
268
269        result = test_cls.subtest_result_cls(
270            data["subtest"],
271            data["status"],
272            data.get("message"))
273
274        subtest.set_result(self.run_info, result)
275
276    def test_end(self, data):
277        test_id = self.test_id(data["test"])
278        test = self.test_cache.get(test_id)
279        if test is None:
280            return
281        test_cls = wpttest.manifest_test_cls[test.test_type]
282
283        if data["status"] == "SKIP":
284            return
285
286        result = test_cls.result_cls(
287            data["status"],
288            data.get("message"))
289
290        test.set_result(self.run_info, result)
291        del self.test_cache[test_id]
292
293
294def create_test_tree(metadata_path, test_manifest, property_order=None,
295                     boolean_properties=None):
296    expected_map = {}
297    id_test_map = {}
298    exclude_types = frozenset(["stub", "helper", "manual"])
299    include_types = set(manifest.item_types) - exclude_types
300    for test_path, tests in test_manifest.itertypes(*include_types):
301        expected_data = load_expected(test_manifest, metadata_path, test_path, tests,
302                                      property_order=property_order,
303                                      boolean_properties=boolean_properties)
304        if expected_data is None:
305            expected_data = create_expected(test_manifest,
306                                            test_path,
307                                            tests,
308                                            property_order=property_order,
309                                            boolean_properties=boolean_properties)
310
311        for test in tests:
312            id_test_map[test.id] = (test_manifest, test)
313            expected_map[test] = expected_data
314
315    return expected_map, id_test_map
316
317
318def create_expected(test_manifest, test_path, tests, property_order=None,
319                    boolean_properties=None):
320    expected = manifestupdate.ExpectedManifest(None, test_path, test_manifest.url_base,
321                                               property_order=property_order,
322                                               boolean_properties=boolean_properties)
323    for test in tests:
324        expected.append(manifestupdate.TestNode.create(test.item_type, test.id))
325    return expected
326
327
328def load_expected(test_manifest, metadata_path, test_path, tests, property_order=None,
329                  boolean_properties=None):
330    expected_manifest = manifestupdate.get_manifest(metadata_path,
331                                                    test_path,
332                                                    test_manifest.url_base,
333                                                    property_order=property_order,
334                                                    boolean_properties=boolean_properties)
335    if expected_manifest is None:
336        return
337
338    tests_by_id = {item.id: item for item in tests}
339
340    # Remove expected data for tests that no longer exist
341    for test in expected_manifest.iterchildren():
342        if not test.id in tests_by_id:
343            test.remove()
344
345    # Add tests that don't have expected data
346    for test in tests:
347        if not expected_manifest.has_test(test.id):
348            expected_manifest.append(manifestupdate.TestNode.create(test.item_type, test.id))
349
350    return expected_manifest
351