1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5# This module contains code for managing WebIDL files and bindings for 6# the build system. 7 8from __future__ import unicode_literals 9 10import errno 11import hashlib 12import json 13import logging 14import os 15 16from copy import deepcopy 17 18from mach.mixin.logging import LoggingMixin 19 20from mozbuild.base import MozbuildObject 21from mozbuild.makeutil import Makefile 22from mozbuild.pythonutil import iter_modules_in_path 23from mozbuild.util import FileAvoidWrite 24 25import mozpack.path as mozpath 26 27# There are various imports in this file in functions to avoid adding 28# dependencies to config.status. See bug 949875. 29 30 31class BuildResult(object): 32 """Represents the result of processing WebIDL files. 33 34 This holds a summary of output file generation during code generation. 35 """ 36 37 def __init__(self): 38 # The .webidl files that had their outputs regenerated. 39 self.inputs = set() 40 41 # The output files that were created. 42 self.created = set() 43 44 # The output files that changed. 45 self.updated = set() 46 47 # The output files that didn't change. 48 self.unchanged = set() 49 50 51class WebIDLCodegenManagerState(dict): 52 """Holds state for the WebIDL code generation manager. 53 54 State is currently just an extended dict. The internal implementation of 55 state should be considered a black box to everyone except 56 WebIDLCodegenManager. But we'll still document it. 57 58 Fields: 59 60 version 61 The integer version of the format. This is to detect incompatible 62 changes between state. It should be bumped whenever the format 63 changes or semantics change. 64 65 webidls 66 A dictionary holding information about every known WebIDL input. 67 Keys are the basenames of input WebIDL files. Values are dicts of 68 metadata. Keys in those dicts are: 69 70 * filename - The full path to the input filename. 71 * inputs - A set of full paths to other webidl files this webidl 72 depends on. 73 * outputs - Set of full output paths that are created/derived from 74 this file. 75 * sha1 - The hexidecimal SHA-1 of the input filename from the last 76 processing time. 77 78 global_inputs 79 A dictionary defining files that influence all processing. Keys 80 are full filenames. Values are hexidecimal SHA-1 from the last 81 processing time. 82 """ 83 84 VERSION = 1 85 86 def __init__(self, fh=None): 87 self['version'] = self.VERSION 88 self['webidls'] = {} 89 self['global_depends'] = {} 90 91 if not fh: 92 return 93 94 state = json.load(fh) 95 if state['version'] != self.VERSION: 96 raise Exception('Unknown state version: %s' % state['version']) 97 98 self['version'] = state['version'] 99 self['global_depends'] = state['global_depends'] 100 101 for k, v in state['webidls'].items(): 102 self['webidls'][k] = v 103 104 # Sets are converted to lists for serialization because JSON 105 # doesn't support sets. 106 self['webidls'][k]['inputs'] = set(v['inputs']) 107 self['webidls'][k]['outputs'] = set(v['outputs']) 108 109 def dump(self, fh): 110 """Dump serialized state to a file handle.""" 111 normalized = deepcopy(self) 112 113 for k, v in self['webidls'].items(): 114 # Convert sets to lists because JSON doesn't support sets. 115 normalized['webidls'][k]['outputs'] = sorted(v['outputs']) 116 normalized['webidls'][k]['inputs'] = sorted(v['inputs']) 117 118 json.dump(normalized, fh, sort_keys=True) 119 120 121class WebIDLCodegenManager(LoggingMixin): 122 """Manages all code generation around WebIDL. 123 124 To facilitate testing, this object is meant to be generic and reusable. 125 Paths, etc should be parameters and not hardcoded. 126 """ 127 128 # Global parser derived declaration files. 129 GLOBAL_DECLARE_FILES = { 130 'GeneratedAtomList.h', 131 'GeneratedEventList.h', 132 'PrototypeList.h', 133 'RegisterBindings.h', 134 'RegisterWorkerBindings.h', 135 'RegisterWorkerDebuggerBindings.h', 136 'RegisterWorkletBindings.h', 137 'ResolveSystemBinding.h', 138 'UnionConversions.h', 139 'UnionTypes.h', 140 } 141 142 # Global parser derived definition files. 143 GLOBAL_DEFINE_FILES = { 144 'RegisterBindings.cpp', 145 'RegisterWorkerBindings.cpp', 146 'RegisterWorkerDebuggerBindings.cpp', 147 'RegisterWorkletBindings.cpp', 148 'ResolveSystemBinding.cpp', 149 'UnionTypes.cpp', 150 'PrototypeList.cpp', 151 } 152 153 def __init__(self, config_path, webidl_root, inputs, exported_header_dir, 154 codegen_dir, state_path, cache_dir=None, make_deps_path=None, 155 make_deps_target=None): 156 """Create an instance that manages WebIDLs in the build system. 157 158 config_path refers to a WebIDL config file (e.g. Bindings.conf). 159 inputs is a 4-tuple describing the input .webidl files and how to 160 process them. Members are: 161 (set(.webidl files), set(basenames of exported files), 162 set(basenames of generated events files), 163 set(example interface names)) 164 165 exported_header_dir and codegen_dir are directories where generated 166 files will be written to. 167 state_path is the path to a file that will receive JSON state from our 168 actions. 169 make_deps_path is the path to a make dependency file that we can 170 optionally write. 171 make_deps_target is the target that receives the make dependencies. It 172 must be defined if using make_deps_path. 173 """ 174 self.populate_logger() 175 176 input_paths, exported_stems, generated_events_stems, example_interfaces = inputs 177 178 self._config_path = config_path 179 self._webidl_root = webidl_root 180 self._input_paths = set(input_paths) 181 self._exported_stems = set(exported_stems) 182 self._generated_events_stems = set(generated_events_stems) 183 self._generated_events_stems_as_array = generated_events_stems 184 self._example_interfaces = set(example_interfaces) 185 self._exported_header_dir = exported_header_dir 186 self._codegen_dir = codegen_dir 187 self._state_path = state_path 188 self._cache_dir = cache_dir 189 self._make_deps_path = make_deps_path 190 self._make_deps_target = make_deps_target 191 192 if ((make_deps_path and not make_deps_target) or 193 (not make_deps_path and make_deps_target)): 194 raise Exception('Must define both make_deps_path and make_deps_target ' 195 'if one is defined.') 196 197 self._parser_results = None 198 self._config = None 199 self._state = WebIDLCodegenManagerState() 200 201 if os.path.exists(state_path): 202 with open(state_path, 'rb') as fh: 203 try: 204 self._state = WebIDLCodegenManagerState(fh=fh) 205 except Exception as e: 206 self.log(logging.WARN, 'webidl_bad_state', {'msg': str(e)}, 207 'Bad WebIDL state: {msg}') 208 209 @property 210 def config(self): 211 if not self._config: 212 self._parse_webidl() 213 214 return self._config 215 216 def generate_build_files(self): 217 """Generate files required for the build. 218 219 This function is in charge of generating all the .h/.cpp files derived 220 from input .webidl files. Please note that there are build actions 221 required to produce .webidl files and these build actions are 222 explicitly not captured here: this function assumes all .webidl files 223 are present and up to date. 224 225 This routine is called as part of the build to ensure files that need 226 to exist are present and up to date. This routine may not be called if 227 the build dependencies (generated as a result of calling this the first 228 time) say everything is up to date. 229 230 Because reprocessing outputs for every .webidl on every invocation 231 is expensive, we only regenerate the minimal set of files on every 232 invocation. The rules for deciding what needs done are roughly as 233 follows: 234 235 1. If any .webidl changes, reparse all .webidl files and regenerate 236 the global derived files. Only regenerate output files (.h/.cpp) 237 impacted by the modified .webidl files. 238 2. If an non-.webidl dependency (Python files, config file) changes, 239 assume everything is out of date and regenerate the world. This 240 is because changes in those could globally impact every output 241 file. 242 3. If an output file is missing, ensure it is present by performing 243 necessary regeneration. 244 """ 245 # Despite #1 above, we assume the build system is smart enough to not 246 # invoke us if nothing has changed. Therefore, any invocation means 247 # something has changed. And, if anything has changed, we need to 248 # parse the WebIDL. 249 self._parse_webidl() 250 251 result = BuildResult() 252 253 # If we parse, we always update globals - they are cheap and it is 254 # easier that way. 255 created, updated, unchanged = self._write_global_derived() 256 result.created |= created 257 result.updated |= updated 258 result.unchanged |= unchanged 259 260 # If any of the extra dependencies changed, regenerate the world. 261 global_changed, global_hashes = self._global_dependencies_changed() 262 if global_changed: 263 # Make a copy because we may modify. 264 changed_inputs = set(self._input_paths) 265 else: 266 changed_inputs = self._compute_changed_inputs() 267 268 self._state['global_depends'] = global_hashes 269 270 # Generate bindings from .webidl files. 271 for filename in sorted(changed_inputs): 272 basename = mozpath.basename(filename) 273 result.inputs.add(filename) 274 written, deps = self._generate_build_files_for_webidl(filename) 275 result.created |= written[0] 276 result.updated |= written[1] 277 result.unchanged |= written[2] 278 279 self._state['webidls'][basename] = dict( 280 filename=filename, 281 outputs=written[0] | written[1] | written[2], 282 inputs=set(deps), 283 sha1=self._input_hashes[filename], 284 ) 285 286 # Process some special interfaces required for testing. 287 for interface in self._example_interfaces: 288 written = self.generate_example_files(interface) 289 result.created |= written[0] 290 result.updated |= written[1] 291 result.unchanged |= written[2] 292 293 # Generate a make dependency file. 294 if self._make_deps_path: 295 mk = Makefile() 296 codegen_rule = mk.create_rule([self._make_deps_target]) 297 codegen_rule.add_dependencies(global_hashes.keys()) 298 codegen_rule.add_dependencies(self._input_paths) 299 300 with FileAvoidWrite(self._make_deps_path) as fh: 301 mk.dump(fh) 302 303 self._save_state() 304 305 return result 306 307 def generate_example_files(self, interface): 308 """Generates example files for a given interface.""" 309 from Codegen import CGExampleRoot 310 311 root = CGExampleRoot(self.config, interface) 312 313 example_paths = self._example_paths(interface) 314 for path in example_paths: 315 print "Generating %s" % path 316 317 return self._maybe_write_codegen(root, *example_paths) 318 319 def _parse_webidl(self): 320 import WebIDL 321 from Configuration import Configuration 322 323 self.log(logging.INFO, 'webidl_parse', 324 {'count': len(self._input_paths)}, 325 'Parsing {count} WebIDL files.') 326 327 hashes = {} 328 parser = WebIDL.Parser(self._cache_dir) 329 330 for path in sorted(self._input_paths): 331 with open(path, 'rb') as fh: 332 data = fh.read() 333 hashes[path] = hashlib.sha1(data).hexdigest() 334 parser.parse(data, path) 335 336 # Only these directories may contain WebIDL files with interfaces 337 # which are exposed to the web. WebIDL files in these roots may not 338 # be changed without DOM peer review. 339 # 340 # Other directories may contain WebIDL files as long as they only 341 # contain ChromeOnly interfaces. These are not subject to mandatory 342 # DOM peer review. 343 web_roots = ( 344 # The main WebIDL root. 345 self._webidl_root, 346 # The binding config root, which contains some test-only 347 # interfaces. 348 os.path.dirname(self._config_path), 349 # The objdir sub-directory which contains generated WebIDL files. 350 self._codegen_dir, 351 ) 352 353 self._parser_results = parser.finish() 354 self._config = Configuration(self._config_path, web_roots, 355 self._parser_results, 356 self._generated_events_stems_as_array) 357 self._input_hashes = hashes 358 359 def _write_global_derived(self): 360 from Codegen import GlobalGenRoots 361 362 things = [('declare', f) for f in self.GLOBAL_DECLARE_FILES] 363 things.extend(('define', f) for f in self.GLOBAL_DEFINE_FILES) 364 365 result = (set(), set(), set()) 366 367 for what, filename in things: 368 stem = mozpath.splitext(filename)[0] 369 root = getattr(GlobalGenRoots, stem)(self._config) 370 371 if what == 'declare': 372 code = root.declare() 373 output_root = self._exported_header_dir 374 elif what == 'define': 375 code = root.define() 376 output_root = self._codegen_dir 377 else: 378 raise Exception('Unknown global gen type: %s' % what) 379 380 output_path = mozpath.join(output_root, filename) 381 self._maybe_write_file(output_path, code, result) 382 383 return result 384 385 def _compute_changed_inputs(self): 386 """Compute the set of input files that need to be regenerated.""" 387 changed_inputs = set() 388 expected_outputs = self.expected_build_output_files() 389 390 # Look for missing output files. 391 if any(not os.path.exists(f) for f in expected_outputs): 392 # FUTURE Bug 940469 Only regenerate minimum set. 393 changed_inputs |= self._input_paths 394 395 # That's it for examining output files. We /could/ examine SHA-1's of 396 # output files from a previous run to detect modifications. But that's 397 # a lot of extra work and most build systems don't do that anyway. 398 399 # Now we move on to the input files. 400 old_hashes = {v['filename']: v['sha1'] 401 for v in self._state['webidls'].values()} 402 403 old_filenames = set(old_hashes.keys()) 404 new_filenames = self._input_paths 405 406 # If an old file has disappeared or a new file has arrived, mark 407 # it. 408 changed_inputs |= old_filenames ^ new_filenames 409 410 # For the files in common between runs, compare content. If the file 411 # has changed, mark it. We don't need to perform mtime comparisons 412 # because content is a stronger validator. 413 for filename in old_filenames & new_filenames: 414 if old_hashes[filename] != self._input_hashes[filename]: 415 changed_inputs.add(filename) 416 417 # We've now populated the base set of inputs that have changed. 418 419 # Inherit dependencies from previous run. The full set of dependencies 420 # is associated with each record, so we don't need to perform any fancy 421 # graph traversal. 422 for v in self._state['webidls'].values(): 423 if any(dep for dep in v['inputs'] if dep in changed_inputs): 424 changed_inputs.add(v['filename']) 425 426 # Only use paths that are known to our current state. 427 # This filters out files that were deleted or changed type (e.g. from 428 # static to preprocessed). 429 return changed_inputs & self._input_paths 430 431 def _binding_info(self, p): 432 """Compute binding metadata for an input path. 433 434 Returns a tuple of: 435 436 (stem, binding_stem, is_event, output_files) 437 438 output_files is itself a tuple. The first two items are the binding 439 header and C++ paths, respectively. The 2nd pair are the event header 440 and C++ paths or None if this isn't an event binding. 441 """ 442 basename = mozpath.basename(p) 443 stem = mozpath.splitext(basename)[0] 444 binding_stem = '%sBinding' % stem 445 446 if stem in self._exported_stems: 447 header_dir = self._exported_header_dir 448 else: 449 header_dir = self._codegen_dir 450 451 is_event = stem in self._generated_events_stems 452 453 files = ( 454 mozpath.join(header_dir, '%s.h' % binding_stem), 455 mozpath.join(self._codegen_dir, '%s.cpp' % binding_stem), 456 mozpath.join(header_dir, '%s.h' % stem) if is_event else None, 457 mozpath.join(self._codegen_dir, '%s.cpp' % stem) if is_event else None, 458 ) 459 460 return stem, binding_stem, is_event, header_dir, files 461 462 def _example_paths(self, interface): 463 return ( 464 mozpath.join(self._codegen_dir, '%s-example.h' % interface), 465 mozpath.join(self._codegen_dir, '%s-example.cpp' % interface)) 466 467 def expected_build_output_files(self): 468 """Obtain the set of files generate_build_files() should write.""" 469 paths = set() 470 471 # Account for global generation. 472 for p in self.GLOBAL_DECLARE_FILES: 473 paths.add(mozpath.join(self._exported_header_dir, p)) 474 for p in self.GLOBAL_DEFINE_FILES: 475 paths.add(mozpath.join(self._codegen_dir, p)) 476 477 for p in self._input_paths: 478 stem, binding_stem, is_event, header_dir, files = self._binding_info(p) 479 paths |= {f for f in files if f} 480 481 for interface in self._example_interfaces: 482 for p in self._example_paths(interface): 483 paths.add(p) 484 485 return paths 486 487 def _generate_build_files_for_webidl(self, filename): 488 from Codegen import ( 489 CGBindingRoot, 490 CGEventRoot, 491 ) 492 493 self.log(logging.INFO, 'webidl_generate_build_for_input', 494 {'filename': filename}, 495 'Generating WebIDL files derived from {filename}') 496 497 stem, binding_stem, is_event, header_dir, files = self._binding_info(filename) 498 root = CGBindingRoot(self._config, binding_stem, filename) 499 500 result = self._maybe_write_codegen(root, files[0], files[1]) 501 502 if is_event: 503 generated_event = CGEventRoot(self._config, stem) 504 result = self._maybe_write_codegen(generated_event, files[2], 505 files[3], result) 506 507 return result, root.deps() 508 509 def _global_dependencies_changed(self): 510 """Determine whether the global dependencies have changed.""" 511 current_files = set(iter_modules_in_path(mozpath.dirname(__file__))) 512 513 # We need to catch other .py files from /dom/bindings. We assume these 514 # are in the same directory as the config file. 515 current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path))) 516 517 current_files.add(self._config_path) 518 519 current_hashes = {} 520 for f in current_files: 521 # This will fail if the file doesn't exist. If a current global 522 # dependency doesn't exist, something else is wrong. 523 with open(f, 'rb') as fh: 524 current_hashes[f] = hashlib.sha1(fh.read()).hexdigest() 525 526 # The set of files has changed. 527 if current_files ^ set(self._state['global_depends'].keys()): 528 return True, current_hashes 529 530 # Compare hashes. 531 for f, sha1 in current_hashes.items(): 532 if sha1 != self._state['global_depends'][f]: 533 return True, current_hashes 534 535 return False, current_hashes 536 537 def _save_state(self): 538 with open(self._state_path, 'wb') as fh: 539 self._state.dump(fh) 540 541 def _maybe_write_codegen(self, obj, declare_path, define_path, result=None): 542 assert declare_path and define_path 543 if not result: 544 result = (set(), set(), set()) 545 546 self._maybe_write_file(declare_path, obj.declare(), result) 547 self._maybe_write_file(define_path, obj.define(), result) 548 549 return result 550 551 def _maybe_write_file(self, path, content, result): 552 fh = FileAvoidWrite(path) 553 fh.write(content) 554 existed, updated = fh.close() 555 556 if not existed: 557 result[0].add(path) 558 elif updated: 559 result[1].add(path) 560 else: 561 result[2].add(path) 562 563 564def create_build_system_manager(topsrcdir, topobjdir, dist_dir): 565 """Create a WebIDLCodegenManager for use by the build system.""" 566 src_dir = os.path.join(topsrcdir, 'dom', 'bindings') 567 obj_dir = os.path.join(topobjdir, 'dom', 'bindings') 568 webidl_root = os.path.join(topsrcdir, 'dom', 'webidl') 569 570 with open(os.path.join(obj_dir, 'file-lists.json'), 'rb') as fh: 571 files = json.load(fh) 572 573 inputs = (files['webidls'], files['exported_stems'], 574 files['generated_events_stems'], files['example_interfaces']) 575 576 cache_dir = os.path.join(obj_dir, '_cache') 577 try: 578 os.makedirs(cache_dir) 579 except OSError as e: 580 if e.errno != errno.EEXIST: 581 raise 582 583 return WebIDLCodegenManager( 584 os.path.join(src_dir, 'Bindings.conf'), 585 webidl_root, 586 inputs, 587 os.path.join(dist_dir, 'include', 'mozilla', 'dom'), 588 obj_dir, 589 os.path.join(obj_dir, 'codegen.json'), 590 cache_dir=cache_dir, 591 # The make rules include a codegen.pp file containing dependencies. 592 make_deps_path=os.path.join(obj_dir, 'codegen.pp'), 593 make_deps_target='codegen.pp', 594 ) 595 596 597class BuildSystemWebIDL(MozbuildObject): 598 @property 599 def manager(self): 600 if not hasattr(self, '_webidl_manager'): 601 self._webidl_manager = create_build_system_manager( 602 self.topsrcdir, self.topobjdir, self.distdir) 603 604 return self._webidl_manager 605