1# Copyright (C) 2011 Google Inc.  All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions
5# are met:
6# 1. Redistributions of source code must retain the above copyright
7#    notice, this list of conditions and the following disclaimer.
8# 2. Redistributions in binary form must reproduce the above copyright
9#    notice, this list of conditions and the following disclaimer in the
10#    documentation and/or other materials provided with the distribution.
11#
12# THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
13# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
14# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
15# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
16# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
17# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
18# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
19# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
20# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23#
24
25from contextlib import contextmanager
26import difflib
27import filecmp
28import fnmatch
29import os
30import shutil
31import tempfile
32
33from blinkpy.common.system.executive import Executive
34
35from blinkpy.common import path_finder
36path_finder.add_bindings_scripts_dir_to_sys_path()
37path_finder.add_build_scripts_dir_to_sys_path()
38
39from code_generator_v8 import CodeGeneratorDictionaryImpl
40from code_generator_v8 import CodeGeneratorV8
41from code_generator_v8 import CodeGeneratorUnionType
42from code_generator_v8 import CodeGeneratorCallbackFunction
43from compute_interfaces_info_individual import InterfaceInfoCollector
44from compute_interfaces_info_overall import (compute_interfaces_info_overall,
45                                             interfaces_info)
46from generate_origin_trial_features import generate_origin_trial_features
47from idl_compiler import (generate_bindings,
48                          generate_union_type_containers,
49                          generate_dictionary_impl,
50                          generate_callback_function_impl)
51from json5_generator import Json5File
52from utilities import ComponentInfoProviderCore
53from utilities import ComponentInfoProviderModules
54from utilities import get_file_contents
55from utilities import get_first_interface_name_from_idl
56from utilities import to_snake_case
57
58
59PASS_MESSAGE = 'All tests PASS!'
60FAIL_MESSAGE = """Some tests FAIL!
61To update the reference files, execute:
62    third_party/blink/tools/run_bindings_tests.py --reset-results
63
64If the failures are not due to your changes, test results may be out of sync;
65please rebaseline them in a separate CL, after checking that tests fail in ToT.
66In CL, please set:
67NOTRY=true
68TBR=someone in third_party/blink/renderer/bindings/OWNERS or WATCHLISTS:bindings
69"""
70
71SOURCE_PATH = path_finder.get_source_dir()
72DEPENDENCY_IDL_FILES = frozenset([
73    'test_interface_mixin.idl',
74    'test_interface_mixin_2.idl',
75    'test_interface_mixin_3.idl',
76    'test_interface_partial.idl',
77    'test_interface_partial_2.idl',
78    'test_interface_partial_3.idl',
79    'test_interface_partial_4.idl',
80    'test_interface_partial_secure_context.idl',
81    'test_interface_2_partial.idl',
82    'test_interface_2_partial_2.idl',
83])
84
85COMPONENT_DIRECTORY = frozenset(['core', 'modules'])
86TEST_INPUT_DIRECTORY = os.path.join(SOURCE_PATH, 'bindings', 'tests', 'idls')
87REFERENCE_DIRECTORY = os.path.join(SOURCE_PATH, 'bindings', 'tests', 'results')
88
89# component -> ComponentInfoProvider.
90# Note that this dict contains information about testing idl files, which live
91# in Source/bindings/tests/idls/{core,modules}, not in Source/{core,modules}.
92component_info_providers = {}
93
94
95@contextmanager
96def TemporaryDirectory():
97    """Wrapper for tempfile.mkdtemp() so it's usable with 'with' statement.
98
99    Simple backport of tempfile.TemporaryDirectory from Python 3.2.
100    """
101    name = tempfile.mkdtemp()
102    try:
103        yield name
104    finally:
105        shutil.rmtree(name)
106
107
108def generate_interface_dependencies(runtime_enabled_features):
109    def idl_paths_recursive(directory):
110        # This is slow, especially on Windows, due to os.walk making
111        # excess stat() calls. Faster versions may appear in Python 3.5 or
112        # later:
113        # https://github.com/benhoyt/scandir
114        # http://bugs.python.org/issue11406
115        idl_paths = []
116        for dirpath, _, files in os.walk(directory):
117            idl_paths.extend(os.path.join(dirpath, filename)
118                             for filename in fnmatch.filter(files, '*.idl'))
119        return idl_paths
120
121    def collect_blink_idl_paths():
122        """Returns IDL file paths which blink actually uses."""
123        idl_paths = []
124        for component in COMPONENT_DIRECTORY:
125            directory = os.path.join(SOURCE_PATH, component)
126            idl_paths.extend(idl_paths_recursive(directory))
127        return idl_paths
128
129    def collect_interfaces_info(idl_path_list):
130        info_collector = InterfaceInfoCollector()
131        for idl_path in idl_path_list:
132            info_collector.collect_info(idl_path)
133        info = info_collector.get_info_as_dict()
134        # TestDictionary.{h,cpp} are placed under
135        # Source/bindings/tests/idls/core. However, IdlCompiler generates
136        # TestDictionary.{h,cpp} by using relative_dir.
137        # So the files will be generated under
138        # output_dir/core/bindings/tests/idls/core.
139        # To avoid this issue, we need to clear relative_dir here.
140        for value in info['interfaces_info'].itervalues():
141            value['relative_dir'] = ''
142        component_info = info_collector.get_component_info_as_dict(runtime_enabled_features)
143        return info, component_info
144
145    # We compute interfaces info for *all* IDL files, not just test IDL
146    # files, as code generator output depends on inheritance (both ancestor
147    # chain and inherited extended attributes), and some real interfaces
148    # are special-cased, such as Node.
149    #
150    # For example, when testing the behavior of interfaces that inherit
151    # from Node, we also need to know that these inherit from EventTarget,
152    # since this is also special-cased and Node inherits from EventTarget,
153    # but this inheritance information requires computing dependencies for
154    # the real Node.idl file.
155    non_test_idl_paths = collect_blink_idl_paths()
156    # For bindings test IDL files, we collect interfaces info for each
157    # component so that we can generate union type containers separately.
158    test_idl_paths = {}
159    for component in COMPONENT_DIRECTORY:
160        test_idl_paths[component] = idl_paths_recursive(
161            os.path.join(TEST_INPUT_DIRECTORY, component))
162    # 2nd-stage computation: individual, then overall
163    #
164    # Properly should compute separately by component (currently test
165    # includes are invalid), but that's brittle (would need to update this file
166    # for each new component) and doesn't test the code generator any better
167    # than using a single component.
168    non_test_interfaces_info, non_test_component_info = collect_interfaces_info(non_test_idl_paths)
169    test_interfaces_info = {}
170    test_component_info = {}
171    for component, paths in test_idl_paths.iteritems():
172        test_interfaces_info[component], test_component_info[component] = collect_interfaces_info(paths)
173    # In order to allow test IDL files to override the production IDL files if
174    # they have the same interface name, process the test IDL files after the
175    # non-test IDL files.
176    info_individuals = [non_test_interfaces_info] + test_interfaces_info.values()
177    compute_interfaces_info_overall(info_individuals)
178    # Add typedefs which are specified in the actual IDL files to the testing
179    # component info.
180    test_component_info['core']['typedefs'].update(
181        non_test_component_info['typedefs'])
182    component_info_providers['core'] = ComponentInfoProviderCore(
183        interfaces_info, test_component_info['core'])
184    component_info_providers['modules'] = ComponentInfoProviderModules(
185        interfaces_info, test_component_info['core'],
186        test_component_info['modules'])
187
188
189class IdlCompilerOptions(object):
190    def __init__(self, output_directory, cache_directory, impl_output_directory,
191                 target_component):
192        self.output_directory = output_directory
193        self.cache_directory = cache_directory
194        self.impl_output_directory = impl_output_directory
195        self.target_component = target_component
196
197
198def bindings_tests(output_directory, verbose, suppress_diff):
199    executive = Executive()
200
201    def list_files(directory):
202        if not os.path.isdir(directory):
203            return []
204
205        files = []
206        for component in os.listdir(directory):
207            if component not in COMPONENT_DIRECTORY:
208                continue
209            directory_with_component = os.path.join(directory, component)
210            for filename in os.listdir(directory_with_component):
211                files.append(os.path.join(directory_with_component, filename))
212        return files
213
214    def diff(filename1, filename2):
215        with open(filename1) as file1:
216            file1_lines = file1.readlines()
217        with open(filename2) as file2:
218            file2_lines = file2.readlines()
219
220        # Use Python's difflib module so that diffing works across platforms
221        return ''.join(difflib.context_diff(file1_lines, file2_lines))
222
223    def is_cache_file(filename):
224        return filename.endswith('.cache')
225
226    def delete_cache_files():
227        # FIXME: Instead of deleting cache files, don't generate them.
228        cache_files = [path for path in list_files(output_directory)
229                       if is_cache_file(os.path.basename(path))]
230        for cache_file in cache_files:
231            os.remove(cache_file)
232
233    def identical_file(reference_filename, output_filename):
234        reference_basename = os.path.basename(reference_filename)
235
236        if not os.path.isfile(reference_filename):
237            print 'Missing reference file!'
238            print '(if adding new test, update reference files)'
239            print reference_basename
240            print
241            return False
242
243        if not filecmp.cmp(reference_filename, output_filename):
244            # cmp is much faster than diff, and usual case is "no difference",
245            # so only run diff if cmp detects a difference
246            print 'FAIL: %s' % reference_basename
247            if not suppress_diff:
248                print diff(reference_filename, output_filename)
249            return False
250
251        if verbose:
252            print 'PASS: %s' % reference_basename
253        return True
254
255    def identical_output_files(output_files):
256        reference_files = [os.path.join(REFERENCE_DIRECTORY,
257                                        os.path.relpath(path, output_directory))
258                           for path in output_files]
259        return all([identical_file(reference_filename, output_filename)
260                    for (reference_filename, output_filename) in zip(reference_files, output_files)])
261
262    def no_excess_files(output_files):
263        generated_files = set([os.path.relpath(path, output_directory)
264                               for path in output_files])
265        excess_files = []
266        for path in list_files(REFERENCE_DIRECTORY):
267            relpath = os.path.relpath(path, REFERENCE_DIRECTORY)
268            # Ignore backup files made by a VCS.
269            if os.path.splitext(relpath)[1] == '.orig':
270                continue
271            if relpath not in generated_files:
272                excess_files.append(relpath)
273        if excess_files:
274            print ('Excess reference files! '
275                   '(probably cruft from renaming or deleting):\n' +
276                   '\n'.join(excess_files))
277            return False
278        return True
279
280    def make_runtime_features_dict():
281        input_filename = os.path.join(TEST_INPUT_DIRECTORY, 'runtime_enabled_features.json5')
282        json5_file = Json5File.load_from_files([input_filename])
283        features_map = {}
284        for feature in json5_file.name_dictionaries:
285            features_map[str(feature['name'])] = {
286                'in_origin_trial': feature['in_origin_trial']
287            }
288        return features_map
289
290    try:
291        generate_interface_dependencies(make_runtime_features_dict())
292        for component in COMPONENT_DIRECTORY:
293            output_dir = os.path.join(output_directory, component)
294            if not os.path.exists(output_dir):
295                os.makedirs(output_dir)
296
297            options = IdlCompilerOptions(
298                output_directory=output_dir,
299                impl_output_directory=output_dir,
300                cache_directory=None,
301                target_component=component)
302
303            if component == 'core':
304                partial_interface_output_dir = os.path.join(output_directory,
305                                                            'modules')
306                if not os.path.exists(partial_interface_output_dir):
307                    os.makedirs(partial_interface_output_dir)
308                partial_interface_options = IdlCompilerOptions(
309                    output_directory=partial_interface_output_dir,
310                    impl_output_directory=None,
311                    cache_directory=None,
312                    target_component='modules')
313
314            idl_filenames = []
315            dictionary_impl_filenames = []
316            partial_interface_filenames = []
317            input_directory = os.path.join(TEST_INPUT_DIRECTORY, component)
318            for filename in os.listdir(input_directory):
319                if (filename.endswith('.idl') and
320                        # Dependencies aren't built
321                        # (they are used by the dependent)
322                        filename not in DEPENDENCY_IDL_FILES):
323                    idl_path = os.path.realpath(
324                        os.path.join(input_directory, filename))
325                    idl_filenames.append(idl_path)
326                    idl_basename = os.path.basename(idl_path)
327                    name_from_basename, _ = os.path.splitext(idl_basename)
328                    definition_name = get_first_interface_name_from_idl(get_file_contents(idl_path))
329                    is_partial_interface_idl = to_snake_case(definition_name) != name_from_basename
330                    if not is_partial_interface_idl:
331                        interface_info = interfaces_info[definition_name]
332                        if interface_info['is_dictionary']:
333                            dictionary_impl_filenames.append(idl_path)
334                        if component == 'core' and interface_info[
335                                'dependencies_other_component_full_paths']:
336                            partial_interface_filenames.append(idl_path)
337
338            info_provider = component_info_providers[component]
339            partial_interface_info_provider = component_info_providers['modules']
340
341            generate_union_type_containers(CodeGeneratorUnionType,
342                                           info_provider, options)
343            generate_callback_function_impl(CodeGeneratorCallbackFunction,
344                                            info_provider, options)
345            generate_bindings(
346                CodeGeneratorV8,
347                info_provider,
348                options,
349                idl_filenames)
350            generate_bindings(
351                CodeGeneratorV8,
352                partial_interface_info_provider,
353                partial_interface_options,
354                partial_interface_filenames)
355            generate_dictionary_impl(
356                CodeGeneratorDictionaryImpl,
357                info_provider,
358                options,
359                dictionary_impl_filenames)
360            generate_origin_trial_features(
361                info_provider,
362                options,
363                [filename for filename in idl_filenames
364                 if filename not in dictionary_impl_filenames])
365
366    finally:
367        delete_cache_files()
368
369    # Detect all changes
370    output_files = list_files(output_directory)
371    passed = identical_output_files(output_files)
372    passed &= no_excess_files(output_files)
373
374    if passed:
375        if verbose:
376            print
377            print PASS_MESSAGE
378        return 0
379    print
380    print FAIL_MESSAGE
381    return 1
382
383
384def run_bindings_tests(reset_results, verbose, suppress_diff):
385    # Generate output into the reference directory if resetting results, or
386    # a temp directory if not.
387    if reset_results:
388        print 'Resetting results'
389        return bindings_tests(REFERENCE_DIRECTORY, verbose, suppress_diff)
390    with TemporaryDirectory() as temp_dir:
391        # TODO(peria): Remove this hack.
392        # Some internal algorithms depend on the path of output directory.
393        temp_source_path = os.path.join(temp_dir, 'third_party', 'blink', 'renderer')
394        temp_output_path = os.path.join(temp_source_path, 'bindings', 'tests', 'results')
395        return bindings_tests(temp_output_path, verbose, suppress_diff)
396