1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5# This module contains code for managing WebIDL files and bindings for 6# the build system. 7 8from __future__ import print_function, unicode_literals 9 10import errno 11import hashlib 12import io 13import json 14import logging 15import os 16import six 17 18from copy import deepcopy 19 20from mach.mixin.logging import LoggingMixin 21 22from mozbuild.base import MozbuildObject 23from mozbuild.makeutil import Makefile 24from mozbuild.pythonutil import iter_modules_in_path 25from mozbuild.util import FileAvoidWrite 26 27import mozpack.path as mozpath 28 29# There are various imports in this file in functions to avoid adding 30# dependencies to config.status. See bug 949875. 31 32 33class BuildResult(object): 34 """Represents the result of processing WebIDL files. 35 36 This holds a summary of output file generation during code generation. 37 """ 38 39 def __init__(self): 40 # The .webidl files that had their outputs regenerated. 41 self.inputs = set() 42 43 # The output files that were created. 44 self.created = set() 45 46 # The output files that changed. 47 self.updated = set() 48 49 # The output files that didn't change. 50 self.unchanged = set() 51 52 53class WebIDLCodegenManagerState(dict): 54 """Holds state for the WebIDL code generation manager. 55 56 State is currently just an extended dict. The internal implementation of 57 state should be considered a black box to everyone except 58 WebIDLCodegenManager. But we'll still document it. 59 60 Fields: 61 62 version 63 The integer version of the format. This is to detect incompatible 64 changes between state. It should be bumped whenever the format 65 changes or semantics change. 66 67 webidls 68 A dictionary holding information about every known WebIDL input. 69 Keys are the basenames of input WebIDL files. Values are dicts of 70 metadata. Keys in those dicts are: 71 72 * filename - The full path to the input filename. 73 * inputs - A set of full paths to other webidl files this webidl 74 depends on. 75 * outputs - Set of full output paths that are created/derived from 76 this file. 77 * sha1 - The hexidecimal SHA-1 of the input filename from the last 78 processing time. 79 80 global_inputs 81 A dictionary defining files that influence all processing. Keys 82 are full filenames. Values are hexidecimal SHA-1 from the last 83 processing time. 84 85 dictionaries_convertible_to_js 86 A set of names of dictionaries that are convertible to JS. 87 88 dictionaries_convertible_from_js 89 A set of names of dictionaries that are convertible from JS. 90 """ 91 92 VERSION = 3 93 94 def __init__(self, fh=None): 95 self["version"] = self.VERSION 96 self["webidls"] = {} 97 self["global_depends"] = {} 98 99 if not fh: 100 return 101 102 state = json.load(fh) 103 if state["version"] != self.VERSION: 104 raise Exception("Unknown state version: %s" % state["version"]) 105 106 self["version"] = state["version"] 107 self["global_depends"] = state["global_depends"] 108 109 for k, v in state["webidls"].items(): 110 self["webidls"][k] = v 111 112 # Sets are converted to lists for serialization because JSON 113 # doesn't support sets. 114 self["webidls"][k]["inputs"] = set(v["inputs"]) 115 self["webidls"][k]["outputs"] = set(v["outputs"]) 116 117 self["dictionaries_convertible_to_js"] = set( 118 state["dictionaries_convertible_to_js"] 119 ) 120 121 self["dictionaries_convertible_from_js"] = set( 122 state["dictionaries_convertible_from_js"] 123 ) 124 125 def dump(self, fh): 126 """Dump serialized state to a file handle.""" 127 normalized = deepcopy(self) 128 129 for k, v in self["webidls"].items(): 130 # Convert sets to lists because JSON doesn't support sets. 131 normalized["webidls"][k]["outputs"] = sorted(v["outputs"]) 132 normalized["webidls"][k]["inputs"] = sorted(v["inputs"]) 133 134 normalized["dictionaries_convertible_to_js"] = sorted( 135 self["dictionaries_convertible_to_js"] 136 ) 137 138 normalized["dictionaries_convertible_from_js"] = sorted( 139 self["dictionaries_convertible_from_js"] 140 ) 141 142 json.dump(normalized, fh, sort_keys=True) 143 144 145class WebIDLCodegenManager(LoggingMixin): 146 """Manages all code generation around WebIDL. 147 148 To facilitate testing, this object is meant to be generic and reusable. 149 Paths, etc should be parameters and not hardcoded. 150 """ 151 152 # Global parser derived declaration files. 153 GLOBAL_DECLARE_FILES = { 154 "GeneratedAtomList.h", 155 "GeneratedEventList.h", 156 "PrototypeList.h", 157 "RegisterBindings.h", 158 "RegisterWorkerBindings.h", 159 "RegisterWorkerDebuggerBindings.h", 160 "RegisterWorkletBindings.h", 161 "UnionConversions.h", 162 "UnionTypes.h", 163 "WebIDLPrefs.h", 164 "WebIDLSerializable.h", 165 } 166 167 # Global parser derived definition files. 168 GLOBAL_DEFINE_FILES = { 169 "RegisterBindings.cpp", 170 "RegisterWorkerBindings.cpp", 171 "RegisterWorkerDebuggerBindings.cpp", 172 "RegisterWorkletBindings.cpp", 173 "UnionTypes.cpp", 174 "PrototypeList.cpp", 175 "WebIDLPrefs.cpp", 176 "WebIDLSerializable.cpp", 177 } 178 179 def __init__( 180 self, 181 config_path, 182 webidl_root, 183 inputs, 184 exported_header_dir, 185 codegen_dir, 186 state_path, 187 cache_dir=None, 188 make_deps_path=None, 189 make_deps_target=None, 190 ): 191 """Create an instance that manages WebIDLs in the build system. 192 193 config_path refers to a WebIDL config file (e.g. Bindings.conf). 194 inputs is a 4-tuple describing the input .webidl files and how to 195 process them. Members are: 196 (set(.webidl files), set(basenames of exported files), 197 set(basenames of generated events files), 198 set(example interface names)) 199 200 exported_header_dir and codegen_dir are directories where generated 201 files will be written to. 202 state_path is the path to a file that will receive JSON state from our 203 actions. 204 make_deps_path is the path to a make dependency file that we can 205 optionally write. 206 make_deps_target is the target that receives the make dependencies. It 207 must be defined if using make_deps_path. 208 """ 209 self.populate_logger() 210 211 input_paths, exported_stems, generated_events_stems, example_interfaces = inputs 212 213 self._config_path = config_path 214 self._webidl_root = webidl_root 215 self._input_paths = set(input_paths) 216 self._exported_stems = set(exported_stems) 217 self._generated_events_stems = set(generated_events_stems) 218 self._generated_events_stems_as_array = generated_events_stems 219 self._example_interfaces = set(example_interfaces) 220 self._exported_header_dir = exported_header_dir 221 self._codegen_dir = codegen_dir 222 self._state_path = state_path 223 self._cache_dir = cache_dir 224 self._make_deps_path = make_deps_path 225 self._make_deps_target = make_deps_target 226 227 if (make_deps_path and not make_deps_target) or ( 228 not make_deps_path and make_deps_target 229 ): 230 raise Exception( 231 "Must define both make_deps_path and make_deps_target " 232 "if one is defined." 233 ) 234 235 self._parser_results = None 236 self._config = None 237 self._state = WebIDLCodegenManagerState() 238 239 if os.path.exists(state_path): 240 with io.open(state_path, "r") as fh: 241 try: 242 self._state = WebIDLCodegenManagerState(fh=fh) 243 except Exception as e: 244 self.log( 245 logging.WARN, 246 "webidl_bad_state", 247 {"msg": str(e)}, 248 "Bad WebIDL state: {msg}", 249 ) 250 251 @property 252 def config(self): 253 if not self._config: 254 self._parse_webidl() 255 256 return self._config 257 258 def generate_build_files(self): 259 """Generate files required for the build. 260 261 This function is in charge of generating all the .h/.cpp files derived 262 from input .webidl files. Please note that there are build actions 263 required to produce .webidl files and these build actions are 264 explicitly not captured here: this function assumes all .webidl files 265 are present and up to date. 266 267 This routine is called as part of the build to ensure files that need 268 to exist are present and up to date. This routine may not be called if 269 the build dependencies (generated as a result of calling this the first 270 time) say everything is up to date. 271 272 Because reprocessing outputs for every .webidl on every invocation 273 is expensive, we only regenerate the minimal set of files on every 274 invocation. The rules for deciding what needs done are roughly as 275 follows: 276 277 1. If any .webidl changes, reparse all .webidl files and regenerate 278 the global derived files. Only regenerate output files (.h/.cpp) 279 impacted by the modified .webidl files. 280 2. If an non-.webidl dependency (Python files, config file) changes, 281 assume everything is out of date and regenerate the world. This 282 is because changes in those could globally impact every output 283 file. 284 3. If an output file is missing, ensure it is present by performing 285 necessary regeneration. 286 """ 287 # Despite #1 above, we assume the build system is smart enough to not 288 # invoke us if nothing has changed. Therefore, any invocation means 289 # something has changed. And, if anything has changed, we need to 290 # parse the WebIDL. 291 self._parse_webidl() 292 293 result = BuildResult() 294 295 # If we parse, we always update globals - they are cheap and it is 296 # easier that way. 297 created, updated, unchanged = self._write_global_derived() 298 result.created |= created 299 result.updated |= updated 300 result.unchanged |= unchanged 301 302 # If any of the extra dependencies changed, regenerate the world. 303 global_changed, global_hashes = self._global_dependencies_changed() 304 if global_changed: 305 # Make a copy because we may modify. 306 changed_inputs = set(self._input_paths) 307 else: 308 changed_inputs = self._compute_changed_inputs() 309 310 self._state["global_depends"] = global_hashes 311 self._state["dictionaries_convertible_to_js"] = set( 312 d.identifier.name for d in self._config.getDictionariesConvertibleToJS() 313 ) 314 self._state["dictionaries_convertible_from_js"] = set( 315 d.identifier.name for d in self._config.getDictionariesConvertibleFromJS() 316 ) 317 318 # Generate bindings from .webidl files. 319 for filename in sorted(changed_inputs): 320 basename = mozpath.basename(filename) 321 result.inputs.add(filename) 322 written, deps = self._generate_build_files_for_webidl(filename) 323 result.created |= written[0] 324 result.updated |= written[1] 325 result.unchanged |= written[2] 326 327 self._state["webidls"][basename] = dict( 328 filename=filename, 329 outputs=written[0] | written[1] | written[2], 330 inputs=set(deps), 331 sha1=self._input_hashes[filename], 332 ) 333 334 # Process some special interfaces required for testing. 335 for interface in self._example_interfaces: 336 written = self.generate_example_files(interface) 337 result.created |= written[0] 338 result.updated |= written[1] 339 result.unchanged |= written[2] 340 341 # Generate a make dependency file. 342 if self._make_deps_path: 343 mk = Makefile() 344 codegen_rule = mk.create_rule([self._make_deps_target]) 345 codegen_rule.add_dependencies( 346 six.ensure_text(s) for s in global_hashes.keys() 347 ) 348 codegen_rule.add_dependencies(six.ensure_text(p) for p in self._input_paths) 349 350 with FileAvoidWrite(self._make_deps_path) as fh: 351 mk.dump(fh) 352 353 self._save_state() 354 355 return result 356 357 def generate_example_files(self, interface): 358 """Generates example files for a given interface.""" 359 from Codegen import CGExampleRoot 360 361 root = CGExampleRoot(self.config, interface) 362 363 example_paths = self._example_paths(interface) 364 for path in example_paths: 365 print("Generating {}".format(path)) 366 367 return self._maybe_write_codegen(root, *example_paths) 368 369 def _parse_webidl(self): 370 import WebIDL 371 from Configuration import Configuration 372 373 self.log( 374 logging.INFO, 375 "webidl_parse", 376 {"count": len(self._input_paths)}, 377 "Parsing {count} WebIDL files.", 378 ) 379 380 hashes = {} 381 parser = WebIDL.Parser(self._cache_dir) 382 383 for path in sorted(self._input_paths): 384 with io.open(path, "r", encoding="utf-8") as fh: 385 data = fh.read() 386 hashes[path] = hashlib.sha1(six.ensure_binary(data)).hexdigest() 387 parser.parse(data, path) 388 389 # Only these directories may contain WebIDL files with interfaces 390 # which are exposed to the web. WebIDL files in these roots may not 391 # be changed without DOM peer review. 392 # 393 # Other directories may contain WebIDL files as long as they only 394 # contain ChromeOnly interfaces. These are not subject to mandatory 395 # DOM peer review. 396 web_roots = ( 397 # The main WebIDL root. 398 self._webidl_root, 399 # The binding config root, which contains some test-only 400 # interfaces. 401 os.path.dirname(self._config_path), 402 # The objdir sub-directory which contains generated WebIDL files. 403 self._codegen_dir, 404 ) 405 406 self._parser_results = parser.finish() 407 self._config = Configuration( 408 self._config_path, 409 web_roots, 410 self._parser_results, 411 self._generated_events_stems_as_array, 412 ) 413 self._input_hashes = hashes 414 415 def _write_global_derived(self): 416 from Codegen import GlobalGenRoots 417 418 things = [("declare", f) for f in self.GLOBAL_DECLARE_FILES] 419 things.extend(("define", f) for f in self.GLOBAL_DEFINE_FILES) 420 421 result = (set(), set(), set()) 422 423 for what, filename in things: 424 stem = mozpath.splitext(filename)[0] 425 root = getattr(GlobalGenRoots, stem)(self._config) 426 427 if what == "declare": 428 code = root.declare() 429 output_root = self._exported_header_dir 430 elif what == "define": 431 code = root.define() 432 output_root = self._codegen_dir 433 else: 434 raise Exception("Unknown global gen type: %s" % what) 435 436 output_path = mozpath.join(output_root, filename) 437 self._maybe_write_file(output_path, code, result) 438 439 return result 440 441 def _compute_changed_inputs(self): 442 """Compute the set of input files that need to be regenerated.""" 443 changed_inputs = set() 444 expected_outputs = self.expected_build_output_files() 445 446 # Look for missing output files. 447 if any(not os.path.exists(f) for f in expected_outputs): 448 # FUTURE Bug 940469 Only regenerate minimum set. 449 changed_inputs |= self._input_paths 450 451 # That's it for examining output files. We /could/ examine SHA-1's of 452 # output files from a previous run to detect modifications. But that's 453 # a lot of extra work and most build systems don't do that anyway. 454 455 # Now we move on to the input files. 456 old_hashes = {v["filename"]: v["sha1"] for v in self._state["webidls"].values()} 457 458 old_filenames = set(old_hashes.keys()) 459 new_filenames = self._input_paths 460 461 # If an old file has disappeared or a new file has arrived, mark 462 # it. 463 changed_inputs |= old_filenames ^ new_filenames 464 465 # For the files in common between runs, compare content. If the file 466 # has changed, mark it. We don't need to perform mtime comparisons 467 # because content is a stronger validator. 468 for filename in old_filenames & new_filenames: 469 if old_hashes[filename] != self._input_hashes[filename]: 470 changed_inputs.add(filename) 471 472 # We've now populated the base set of inputs that have changed. 473 474 # Inherit dependencies from previous run. The full set of dependencies 475 # is associated with each record, so we don't need to perform any fancy 476 # graph traversal. 477 for v in self._state["webidls"].values(): 478 if any(dep for dep in v["inputs"] if dep in changed_inputs): 479 changed_inputs.add(v["filename"]) 480 481 # Now check for changes to the set of dictionaries that are convertible to JS 482 oldDictionariesConvertibleToJS = self._state["dictionaries_convertible_to_js"] 483 newDictionariesConvertibleToJS = self._config.getDictionariesConvertibleToJS() 484 newNames = set(d.identifier.name for d in newDictionariesConvertibleToJS) 485 changedDictionaryNames = oldDictionariesConvertibleToJS ^ newNames 486 487 # Now check for changes to the set of dictionaries that are convertible from JS 488 oldDictionariesConvertibleFromJS = self._state[ 489 "dictionaries_convertible_from_js" 490 ] 491 newDictionariesConvertibleFromJS = ( 492 self._config.getDictionariesConvertibleFromJS() 493 ) 494 newNames = set(d.identifier.name for d in newDictionariesConvertibleFromJS) 495 changedDictionaryNames |= oldDictionariesConvertibleFromJS ^ newNames 496 497 for name in changedDictionaryNames: 498 d = self._config.getDictionaryIfExists(name) 499 if d: 500 changed_inputs.add(d.filename()) 501 502 # Only use paths that are known to our current state. 503 # This filters out files that were deleted or changed type (e.g. from 504 # static to preprocessed). 505 return changed_inputs & self._input_paths 506 507 def _binding_info(self, p): 508 """Compute binding metadata for an input path. 509 510 Returns a tuple of: 511 512 (stem, binding_stem, is_event, output_files) 513 514 output_files is itself a tuple. The first two items are the binding 515 header and C++ paths, respectively. The 2nd pair are the event header 516 and C++ paths or None if this isn't an event binding. 517 """ 518 basename = mozpath.basename(p) 519 stem = mozpath.splitext(basename)[0] 520 binding_stem = "%sBinding" % stem 521 522 if stem in self._exported_stems: 523 header_dir = self._exported_header_dir 524 else: 525 header_dir = self._codegen_dir 526 527 is_event = stem in self._generated_events_stems 528 529 files = ( 530 mozpath.join(header_dir, "%s.h" % binding_stem), 531 mozpath.join(self._codegen_dir, "%s.cpp" % binding_stem), 532 mozpath.join(header_dir, "%s.h" % stem) if is_event else None, 533 mozpath.join(self._codegen_dir, "%s.cpp" % stem) if is_event else None, 534 ) 535 536 return stem, binding_stem, is_event, header_dir, files 537 538 def _example_paths(self, interface): 539 return ( 540 mozpath.join(self._codegen_dir, "%s-example.h" % interface), 541 mozpath.join(self._codegen_dir, "%s-example.cpp" % interface), 542 ) 543 544 def expected_build_output_files(self): 545 """Obtain the set of files generate_build_files() should write.""" 546 paths = set() 547 548 # Account for global generation. 549 for p in self.GLOBAL_DECLARE_FILES: 550 paths.add(mozpath.join(self._exported_header_dir, p)) 551 for p in self.GLOBAL_DEFINE_FILES: 552 paths.add(mozpath.join(self._codegen_dir, p)) 553 554 for p in self._input_paths: 555 stem, binding_stem, is_event, header_dir, files = self._binding_info(p) 556 paths |= {f for f in files if f} 557 558 for interface in self._example_interfaces: 559 for p in self._example_paths(interface): 560 paths.add(p) 561 562 return paths 563 564 def _generate_build_files_for_webidl(self, filename): 565 from Codegen import ( 566 CGBindingRoot, 567 CGEventRoot, 568 ) 569 570 self.log( 571 logging.INFO, 572 "webidl_generate_build_for_input", 573 {"filename": filename}, 574 "Generating WebIDL files derived from {filename}", 575 ) 576 577 stem, binding_stem, is_event, header_dir, files = self._binding_info(filename) 578 root = CGBindingRoot(self._config, binding_stem, filename) 579 580 result = self._maybe_write_codegen(root, files[0], files[1]) 581 582 if is_event: 583 generated_event = CGEventRoot(self._config, stem) 584 result = self._maybe_write_codegen( 585 generated_event, files[2], files[3], result 586 ) 587 588 return result, root.deps() 589 590 def _global_dependencies_changed(self): 591 """Determine whether the global dependencies have changed.""" 592 current_files = set(iter_modules_in_path(mozpath.dirname(__file__))) 593 594 # We need to catch other .py files from /dom/bindings. We assume these 595 # are in the same directory as the config file. 596 current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path))) 597 598 current_files.add(self._config_path) 599 600 current_hashes = {} 601 for f in current_files: 602 # This will fail if the file doesn't exist. If a current global 603 # dependency doesn't exist, something else is wrong. 604 with io.open(f, "rb") as fh: 605 current_hashes[f] = hashlib.sha1(fh.read()).hexdigest() 606 607 # The set of files has changed. 608 if current_files ^ set(self._state["global_depends"].keys()): 609 return True, current_hashes 610 611 # Compare hashes. 612 for f, sha1 in current_hashes.items(): 613 if sha1 != self._state["global_depends"][f]: 614 return True, current_hashes 615 616 return False, current_hashes 617 618 def _save_state(self): 619 with io.open(self._state_path, "w", newline="\n") as fh: 620 self._state.dump(fh) 621 622 def _maybe_write_codegen(self, obj, declare_path, define_path, result=None): 623 assert declare_path and define_path 624 if not result: 625 result = (set(), set(), set()) 626 627 self._maybe_write_file(declare_path, obj.declare(), result) 628 self._maybe_write_file(define_path, obj.define(), result) 629 630 return result 631 632 def _maybe_write_file(self, path, content, result): 633 fh = FileAvoidWrite(path) 634 fh.write(content) 635 existed, updated = fh.close() 636 637 if not existed: 638 result[0].add(path) 639 elif updated: 640 result[1].add(path) 641 else: 642 result[2].add(path) 643 644 645def create_build_system_manager(topsrcdir, topobjdir, dist_dir): 646 """Create a WebIDLCodegenManager for use by the build system.""" 647 src_dir = os.path.join(topsrcdir, "dom", "bindings") 648 obj_dir = os.path.join(topobjdir, "dom", "bindings") 649 webidl_root = os.path.join(topsrcdir, "dom", "webidl") 650 651 with io.open(os.path.join(obj_dir, "file-lists.json"), "r") as fh: 652 files = json.load(fh) 653 654 inputs = ( 655 files["webidls"], 656 files["exported_stems"], 657 files["generated_events_stems"], 658 files["example_interfaces"], 659 ) 660 661 cache_dir = os.path.join(obj_dir, "_cache") 662 try: 663 os.makedirs(cache_dir) 664 except OSError as e: 665 if e.errno != errno.EEXIST: 666 raise 667 668 return WebIDLCodegenManager( 669 os.path.join(src_dir, "Bindings.conf"), 670 webidl_root, 671 inputs, 672 os.path.join(dist_dir, "include", "mozilla", "dom"), 673 obj_dir, 674 os.path.join(obj_dir, "codegen.json"), 675 cache_dir=cache_dir, 676 # The make rules include a codegen.pp file containing dependencies. 677 make_deps_path=os.path.join(obj_dir, "codegen.pp"), 678 make_deps_target="webidl.stub", 679 ) 680 681 682class BuildSystemWebIDL(MozbuildObject): 683 @property 684 def manager(self): 685 if not hasattr(self, "_webidl_manager"): 686 self._webidl_manager = create_build_system_manager( 687 self.topsrcdir, self.topobjdir, self.distdir 688 ) 689 690 return self._webidl_manager 691