1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5# This module contains code for managing WebIDL files and bindings for 6# the build system. 7 8from __future__ import print_function, unicode_literals 9 10import errno 11import hashlib 12import io 13import json 14import logging 15import os 16import six 17 18from copy import deepcopy 19 20from mach.mixin.logging import LoggingMixin 21 22from mozbuild.base import MozbuildObject 23from mozbuild.makeutil import Makefile 24from mozbuild.pythonutil import iter_modules_in_path 25from mozbuild.util import FileAvoidWrite 26 27import mozpack.path as mozpath 28 29# There are various imports in this file in functions to avoid adding 30# dependencies to config.status. See bug 949875. 31 32 33class BuildResult(object): 34 """Represents the result of processing WebIDL files. 35 36 This holds a summary of output file generation during code generation. 37 """ 38 39 def __init__(self): 40 # The .webidl files that had their outputs regenerated. 41 self.inputs = set() 42 43 # The output files that were created. 44 self.created = set() 45 46 # The output files that changed. 47 self.updated = set() 48 49 # The output files that didn't change. 50 self.unchanged = set() 51 52 53class WebIDLCodegenManagerState(dict): 54 """Holds state for the WebIDL code generation manager. 55 56 State is currently just an extended dict. The internal implementation of 57 state should be considered a black box to everyone except 58 WebIDLCodegenManager. But we'll still document it. 59 60 Fields: 61 62 version 63 The integer version of the format. This is to detect incompatible 64 changes between state. It should be bumped whenever the format 65 changes or semantics change. 66 67 webidls 68 A dictionary holding information about every known WebIDL input. 69 Keys are the basenames of input WebIDL files. Values are dicts of 70 metadata. Keys in those dicts are: 71 72 * filename - The full path to the input filename. 73 * inputs - A set of full paths to other webidl files this webidl 74 depends on. 75 * outputs - Set of full output paths that are created/derived from 76 this file. 77 * sha1 - The hexidecimal SHA-1 of the input filename from the last 78 processing time. 79 80 global_inputs 81 A dictionary defining files that influence all processing. Keys 82 are full filenames. Values are hexidecimal SHA-1 from the last 83 processing time. 84 85 dictionaries_convertible_to_js 86 A set of names of dictionaries that are convertible to JS. 87 88 dictionaries_convertible_from_js 89 A set of names of dictionaries that are convertible from JS. 90 """ 91 92 VERSION = 3 93 94 def __init__(self, fh=None): 95 self['version'] = self.VERSION 96 self['webidls'] = {} 97 self['global_depends'] = {} 98 99 if not fh: 100 return 101 102 state = json.load(fh) 103 if state['version'] != self.VERSION: 104 raise Exception('Unknown state version: %s' % state['version']) 105 106 self['version'] = state['version'] 107 self['global_depends'] = state['global_depends'] 108 109 for k, v in state['webidls'].items(): 110 self['webidls'][k] = v 111 112 # Sets are converted to lists for serialization because JSON 113 # doesn't support sets. 114 self['webidls'][k]['inputs'] = set(v['inputs']) 115 self['webidls'][k]['outputs'] = set(v['outputs']) 116 117 self['dictionaries_convertible_to_js'] = set( 118 state['dictionaries_convertible_to_js']) 119 120 self['dictionaries_convertible_from_js'] = set( 121 state['dictionaries_convertible_from_js']) 122 123 def dump(self, fh): 124 """Dump serialized state to a file handle.""" 125 normalized = deepcopy(self) 126 127 for k, v in self['webidls'].items(): 128 # Convert sets to lists because JSON doesn't support sets. 129 normalized['webidls'][k]['outputs'] = sorted(v['outputs']) 130 normalized['webidls'][k]['inputs'] = sorted(v['inputs']) 131 132 normalized['dictionaries_convertible_to_js'] = sorted( 133 self['dictionaries_convertible_to_js']) 134 135 normalized['dictionaries_convertible_from_js'] = sorted( 136 self['dictionaries_convertible_from_js']) 137 138 json.dump(normalized, fh, sort_keys=True) 139 140 141class WebIDLCodegenManager(LoggingMixin): 142 """Manages all code generation around WebIDL. 143 144 To facilitate testing, this object is meant to be generic and reusable. 145 Paths, etc should be parameters and not hardcoded. 146 """ 147 148 # Global parser derived declaration files. 149 GLOBAL_DECLARE_FILES = { 150 'GeneratedAtomList.h', 151 'GeneratedEventList.h', 152 'PrototypeList.h', 153 'RegisterBindings.h', 154 'RegisterWorkerBindings.h', 155 'RegisterWorkerDebuggerBindings.h', 156 'RegisterWorkletBindings.h', 157 'UnionConversions.h', 158 'UnionTypes.h', 159 'WebIDLPrefs.h', 160 'WebIDLSerializable.h', 161 } 162 163 # Global parser derived definition files. 164 GLOBAL_DEFINE_FILES = { 165 'RegisterBindings.cpp', 166 'RegisterWorkerBindings.cpp', 167 'RegisterWorkerDebuggerBindings.cpp', 168 'RegisterWorkletBindings.cpp', 169 'UnionTypes.cpp', 170 'PrototypeList.cpp', 171 'WebIDLPrefs.cpp', 172 'WebIDLSerializable.cpp', 173 } 174 175 def __init__(self, config_path, webidl_root, inputs, exported_header_dir, 176 codegen_dir, state_path, cache_dir=None, make_deps_path=None, 177 make_deps_target=None): 178 """Create an instance that manages WebIDLs in the build system. 179 180 config_path refers to a WebIDL config file (e.g. Bindings.conf). 181 inputs is a 4-tuple describing the input .webidl files and how to 182 process them. Members are: 183 (set(.webidl files), set(basenames of exported files), 184 set(basenames of generated events files), 185 set(example interface names)) 186 187 exported_header_dir and codegen_dir are directories where generated 188 files will be written to. 189 state_path is the path to a file that will receive JSON state from our 190 actions. 191 make_deps_path is the path to a make dependency file that we can 192 optionally write. 193 make_deps_target is the target that receives the make dependencies. It 194 must be defined if using make_deps_path. 195 """ 196 self.populate_logger() 197 198 input_paths, exported_stems, generated_events_stems, example_interfaces = inputs 199 200 self._config_path = config_path 201 self._webidl_root = webidl_root 202 self._input_paths = set(input_paths) 203 self._exported_stems = set(exported_stems) 204 self._generated_events_stems = set(generated_events_stems) 205 self._generated_events_stems_as_array = generated_events_stems 206 self._example_interfaces = set(example_interfaces) 207 self._exported_header_dir = exported_header_dir 208 self._codegen_dir = codegen_dir 209 self._state_path = state_path 210 self._cache_dir = cache_dir 211 self._make_deps_path = make_deps_path 212 self._make_deps_target = make_deps_target 213 214 if ((make_deps_path and not make_deps_target) or 215 (not make_deps_path and make_deps_target)): 216 raise Exception('Must define both make_deps_path and make_deps_target ' 217 'if one is defined.') 218 219 self._parser_results = None 220 self._config = None 221 self._state = WebIDLCodegenManagerState() 222 223 if os.path.exists(state_path): 224 with io.open(state_path, 'r') as fh: 225 try: 226 self._state = WebIDLCodegenManagerState(fh=fh) 227 except Exception as e: 228 self.log(logging.WARN, 'webidl_bad_state', {'msg': str(e)}, 229 'Bad WebIDL state: {msg}') 230 231 @property 232 def config(self): 233 if not self._config: 234 self._parse_webidl() 235 236 return self._config 237 238 def generate_build_files(self): 239 """Generate files required for the build. 240 241 This function is in charge of generating all the .h/.cpp files derived 242 from input .webidl files. Please note that there are build actions 243 required to produce .webidl files and these build actions are 244 explicitly not captured here: this function assumes all .webidl files 245 are present and up to date. 246 247 This routine is called as part of the build to ensure files that need 248 to exist are present and up to date. This routine may not be called if 249 the build dependencies (generated as a result of calling this the first 250 time) say everything is up to date. 251 252 Because reprocessing outputs for every .webidl on every invocation 253 is expensive, we only regenerate the minimal set of files on every 254 invocation. The rules for deciding what needs done are roughly as 255 follows: 256 257 1. If any .webidl changes, reparse all .webidl files and regenerate 258 the global derived files. Only regenerate output files (.h/.cpp) 259 impacted by the modified .webidl files. 260 2. If an non-.webidl dependency (Python files, config file) changes, 261 assume everything is out of date and regenerate the world. This 262 is because changes in those could globally impact every output 263 file. 264 3. If an output file is missing, ensure it is present by performing 265 necessary regeneration. 266 """ 267 # Despite #1 above, we assume the build system is smart enough to not 268 # invoke us if nothing has changed. Therefore, any invocation means 269 # something has changed. And, if anything has changed, we need to 270 # parse the WebIDL. 271 self._parse_webidl() 272 273 result = BuildResult() 274 275 # If we parse, we always update globals - they are cheap and it is 276 # easier that way. 277 created, updated, unchanged = self._write_global_derived() 278 result.created |= created 279 result.updated |= updated 280 result.unchanged |= unchanged 281 282 # If any of the extra dependencies changed, regenerate the world. 283 global_changed, global_hashes = self._global_dependencies_changed() 284 if global_changed: 285 # Make a copy because we may modify. 286 changed_inputs = set(self._input_paths) 287 else: 288 changed_inputs = self._compute_changed_inputs() 289 290 self._state['global_depends'] = global_hashes 291 self._state['dictionaries_convertible_to_js'] = set( 292 d.identifier.name for d in self._config.getDictionariesConvertibleToJS()) 293 self._state['dictionaries_convertible_from_js'] = set( 294 d.identifier.name for d in self._config.getDictionariesConvertibleFromJS()) 295 296 # Generate bindings from .webidl files. 297 for filename in sorted(changed_inputs): 298 basename = mozpath.basename(filename) 299 result.inputs.add(filename) 300 written, deps = self._generate_build_files_for_webidl(filename) 301 result.created |= written[0] 302 result.updated |= written[1] 303 result.unchanged |= written[2] 304 305 self._state['webidls'][basename] = dict( 306 filename=filename, 307 outputs=written[0] | written[1] | written[2], 308 inputs=set(deps), 309 sha1=self._input_hashes[filename], 310 ) 311 312 # Process some special interfaces required for testing. 313 for interface in self._example_interfaces: 314 written = self.generate_example_files(interface) 315 result.created |= written[0] 316 result.updated |= written[1] 317 result.unchanged |= written[2] 318 319 # Generate a make dependency file. 320 if self._make_deps_path: 321 mk = Makefile() 322 codegen_rule = mk.create_rule([self._make_deps_target]) 323 codegen_rule.add_dependencies(six.ensure_text(s) for s in 324 global_hashes.keys()) 325 codegen_rule.add_dependencies(six.ensure_text(p) for p in 326 self._input_paths) 327 328 with FileAvoidWrite(self._make_deps_path) as fh: 329 mk.dump(fh) 330 331 self._save_state() 332 333 return result 334 335 def generate_example_files(self, interface): 336 """Generates example files for a given interface.""" 337 from Codegen import CGExampleRoot 338 339 root = CGExampleRoot(self.config, interface) 340 341 example_paths = self._example_paths(interface) 342 for path in example_paths: 343 print("Generating {}".format(path)) 344 345 return self._maybe_write_codegen(root, *example_paths) 346 347 def _parse_webidl(self): 348 import WebIDL 349 from Configuration import Configuration 350 351 self.log(logging.INFO, 'webidl_parse', 352 {'count': len(self._input_paths)}, 353 'Parsing {count} WebIDL files.') 354 355 hashes = {} 356 parser = WebIDL.Parser(self._cache_dir) 357 358 for path in sorted(self._input_paths): 359 with io.open(path, 'r', encoding='utf-8') as fh: 360 data = fh.read() 361 hashes[path] = hashlib.sha1(six.ensure_binary(data)).hexdigest() 362 parser.parse(data, path) 363 364 # Only these directories may contain WebIDL files with interfaces 365 # which are exposed to the web. WebIDL files in these roots may not 366 # be changed without DOM peer review. 367 # 368 # Other directories may contain WebIDL files as long as they only 369 # contain ChromeOnly interfaces. These are not subject to mandatory 370 # DOM peer review. 371 web_roots = ( 372 # The main WebIDL root. 373 self._webidl_root, 374 # The binding config root, which contains some test-only 375 # interfaces. 376 os.path.dirname(self._config_path), 377 # The objdir sub-directory which contains generated WebIDL files. 378 self._codegen_dir, 379 ) 380 381 self._parser_results = parser.finish() 382 self._config = Configuration(self._config_path, web_roots, 383 self._parser_results, 384 self._generated_events_stems_as_array) 385 self._input_hashes = hashes 386 387 def _write_global_derived(self): 388 from Codegen import GlobalGenRoots 389 390 things = [('declare', f) for f in self.GLOBAL_DECLARE_FILES] 391 things.extend(('define', f) for f in self.GLOBAL_DEFINE_FILES) 392 393 result = (set(), set(), set()) 394 395 for what, filename in things: 396 stem = mozpath.splitext(filename)[0] 397 root = getattr(GlobalGenRoots, stem)(self._config) 398 399 if what == 'declare': 400 code = root.declare() 401 output_root = self._exported_header_dir 402 elif what == 'define': 403 code = root.define() 404 output_root = self._codegen_dir 405 else: 406 raise Exception('Unknown global gen type: %s' % what) 407 408 output_path = mozpath.join(output_root, filename) 409 self._maybe_write_file(output_path, code, result) 410 411 return result 412 413 def _compute_changed_inputs(self): 414 """Compute the set of input files that need to be regenerated.""" 415 changed_inputs = set() 416 expected_outputs = self.expected_build_output_files() 417 418 # Look for missing output files. 419 if any(not os.path.exists(f) for f in expected_outputs): 420 # FUTURE Bug 940469 Only regenerate minimum set. 421 changed_inputs |= self._input_paths 422 423 # That's it for examining output files. We /could/ examine SHA-1's of 424 # output files from a previous run to detect modifications. But that's 425 # a lot of extra work and most build systems don't do that anyway. 426 427 # Now we move on to the input files. 428 old_hashes = {v['filename']: v['sha1'] 429 for v in self._state['webidls'].values()} 430 431 old_filenames = set(old_hashes.keys()) 432 new_filenames = self._input_paths 433 434 # If an old file has disappeared or a new file has arrived, mark 435 # it. 436 changed_inputs |= old_filenames ^ new_filenames 437 438 # For the files in common between runs, compare content. If the file 439 # has changed, mark it. We don't need to perform mtime comparisons 440 # because content is a stronger validator. 441 for filename in old_filenames & new_filenames: 442 if old_hashes[filename] != self._input_hashes[filename]: 443 changed_inputs.add(filename) 444 445 # We've now populated the base set of inputs that have changed. 446 447 # Inherit dependencies from previous run. The full set of dependencies 448 # is associated with each record, so we don't need to perform any fancy 449 # graph traversal. 450 for v in self._state['webidls'].values(): 451 if any(dep for dep in v['inputs'] if dep in changed_inputs): 452 changed_inputs.add(v['filename']) 453 454 # Now check for changes to the set of dictionaries that are convertible to JS 455 oldDictionariesConvertibleToJS = self._state['dictionaries_convertible_to_js'] 456 newDictionariesConvertibleToJS = self._config.getDictionariesConvertibleToJS() 457 newNames = set(d.identifier.name for d in newDictionariesConvertibleToJS) 458 changedDictionaryNames = oldDictionariesConvertibleToJS ^ newNames 459 460 # Now check for changes to the set of dictionaries that are convertible from JS 461 oldDictionariesConvertibleFromJS = self._state['dictionaries_convertible_from_js'] 462 newDictionariesConvertibleFromJS = self._config.getDictionariesConvertibleFromJS() 463 newNames = set(d.identifier.name for d in newDictionariesConvertibleFromJS) 464 changedDictionaryNames |= oldDictionariesConvertibleFromJS ^ newNames 465 466 for name in changedDictionaryNames: 467 d = self._config.getDictionaryIfExists(name) 468 if d: 469 changed_inputs.add(d.filename()) 470 471 # Only use paths that are known to our current state. 472 # This filters out files that were deleted or changed type (e.g. from 473 # static to preprocessed). 474 return changed_inputs & self._input_paths 475 476 def _binding_info(self, p): 477 """Compute binding metadata for an input path. 478 479 Returns a tuple of: 480 481 (stem, binding_stem, is_event, output_files) 482 483 output_files is itself a tuple. The first two items are the binding 484 header and C++ paths, respectively. The 2nd pair are the event header 485 and C++ paths or None if this isn't an event binding. 486 """ 487 basename = mozpath.basename(p) 488 stem = mozpath.splitext(basename)[0] 489 binding_stem = '%sBinding' % stem 490 491 if stem in self._exported_stems: 492 header_dir = self._exported_header_dir 493 else: 494 header_dir = self._codegen_dir 495 496 is_event = stem in self._generated_events_stems 497 498 files = ( 499 mozpath.join(header_dir, '%s.h' % binding_stem), 500 mozpath.join(self._codegen_dir, '%s.cpp' % binding_stem), 501 mozpath.join(header_dir, '%s.h' % stem) if is_event else None, 502 mozpath.join(self._codegen_dir, '%s.cpp' % stem) if is_event else None, 503 ) 504 505 return stem, binding_stem, is_event, header_dir, files 506 507 def _example_paths(self, interface): 508 return ( 509 mozpath.join(self._codegen_dir, '%s-example.h' % interface), 510 mozpath.join(self._codegen_dir, '%s-example.cpp' % interface)) 511 512 def expected_build_output_files(self): 513 """Obtain the set of files generate_build_files() should write.""" 514 paths = set() 515 516 # Account for global generation. 517 for p in self.GLOBAL_DECLARE_FILES: 518 paths.add(mozpath.join(self._exported_header_dir, p)) 519 for p in self.GLOBAL_DEFINE_FILES: 520 paths.add(mozpath.join(self._codegen_dir, p)) 521 522 for p in self._input_paths: 523 stem, binding_stem, is_event, header_dir, files = self._binding_info(p) 524 paths |= {f for f in files if f} 525 526 for interface in self._example_interfaces: 527 for p in self._example_paths(interface): 528 paths.add(p) 529 530 return paths 531 532 def _generate_build_files_for_webidl(self, filename): 533 from Codegen import ( 534 CGBindingRoot, 535 CGEventRoot, 536 ) 537 538 self.log(logging.INFO, 'webidl_generate_build_for_input', 539 {'filename': filename}, 540 'Generating WebIDL files derived from {filename}') 541 542 stem, binding_stem, is_event, header_dir, files = self._binding_info(filename) 543 root = CGBindingRoot(self._config, binding_stem, filename) 544 545 result = self._maybe_write_codegen(root, files[0], files[1]) 546 547 if is_event: 548 generated_event = CGEventRoot(self._config, stem) 549 result = self._maybe_write_codegen(generated_event, files[2], 550 files[3], result) 551 552 return result, root.deps() 553 554 def _global_dependencies_changed(self): 555 """Determine whether the global dependencies have changed.""" 556 current_files = set(iter_modules_in_path(mozpath.dirname(__file__))) 557 558 # We need to catch other .py files from /dom/bindings. We assume these 559 # are in the same directory as the config file. 560 current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path))) 561 562 current_files.add(self._config_path) 563 564 current_hashes = {} 565 for f in current_files: 566 # This will fail if the file doesn't exist. If a current global 567 # dependency doesn't exist, something else is wrong. 568 with io.open(f, 'rb') as fh: 569 current_hashes[f] = hashlib.sha1(fh.read()).hexdigest() 570 571 # The set of files has changed. 572 if current_files ^ set(self._state['global_depends'].keys()): 573 return True, current_hashes 574 575 # Compare hashes. 576 for f, sha1 in current_hashes.items(): 577 if sha1 != self._state['global_depends'][f]: 578 return True, current_hashes 579 580 return False, current_hashes 581 582 def _save_state(self): 583 with io.open(self._state_path, 'w', newline='\n') as fh: 584 self._state.dump(fh) 585 586 def _maybe_write_codegen(self, obj, declare_path, define_path, result=None): 587 assert declare_path and define_path 588 if not result: 589 result = (set(), set(), set()) 590 591 self._maybe_write_file(declare_path, obj.declare(), result) 592 self._maybe_write_file(define_path, obj.define(), result) 593 594 return result 595 596 def _maybe_write_file(self, path, content, result): 597 fh = FileAvoidWrite(path) 598 fh.write(content) 599 existed, updated = fh.close() 600 601 if not existed: 602 result[0].add(path) 603 elif updated: 604 result[1].add(path) 605 else: 606 result[2].add(path) 607 608 609def create_build_system_manager(topsrcdir, topobjdir, dist_dir): 610 """Create a WebIDLCodegenManager for use by the build system.""" 611 src_dir = os.path.join(topsrcdir, 'dom', 'bindings') 612 obj_dir = os.path.join(topobjdir, 'dom', 'bindings') 613 webidl_root = os.path.join(topsrcdir, 'dom', 'webidl') 614 615 with io.open(os.path.join(obj_dir, 'file-lists.json'), 'r') as fh: 616 files = json.load(fh) 617 618 inputs = (files['webidls'], files['exported_stems'], 619 files['generated_events_stems'], files['example_interfaces']) 620 621 cache_dir = os.path.join(obj_dir, '_cache') 622 try: 623 os.makedirs(cache_dir) 624 except OSError as e: 625 if e.errno != errno.EEXIST: 626 raise 627 628 return WebIDLCodegenManager( 629 os.path.join(src_dir, 'Bindings.conf'), 630 webidl_root, 631 inputs, 632 os.path.join(dist_dir, 'include', 'mozilla', 'dom'), 633 obj_dir, 634 os.path.join(obj_dir, 'codegen.json'), 635 cache_dir=cache_dir, 636 # The make rules include a codegen.pp file containing dependencies. 637 make_deps_path=os.path.join(obj_dir, 'codegen.pp'), 638 make_deps_target='webidl.stub', 639 ) 640 641 642class BuildSystemWebIDL(MozbuildObject): 643 @property 644 def manager(self): 645 if not hasattr(self, '_webidl_manager'): 646 self._webidl_manager = create_build_system_manager( 647 self.topsrcdir, self.topobjdir, self.distdir) 648 649 return self._webidl_manager 650