1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5# This module contains code for managing WebIDL files and bindings for 6# the build system. 7 8from __future__ import unicode_literals 9 10import errno 11import hashlib 12import json 13import logging 14import os 15 16from copy import deepcopy 17 18from mach.mixin.logging import LoggingMixin 19 20from mozbuild.base import MozbuildObject 21from mozbuild.makeutil import Makefile 22from mozbuild.pythonutil import iter_modules_in_path 23from mozbuild.util import FileAvoidWrite 24 25import mozpack.path as mozpath 26 27# There are various imports in this file in functions to avoid adding 28# dependencies to config.status. See bug 949875. 29 30 31class BuildResult(object): 32 """Represents the result of processing WebIDL files. 33 34 This holds a summary of output file generation during code generation. 35 """ 36 37 def __init__(self): 38 # The .webidl files that had their outputs regenerated. 39 self.inputs = set() 40 41 # The output files that were created. 42 self.created = set() 43 44 # The output files that changed. 45 self.updated = set() 46 47 # The output files that didn't change. 48 self.unchanged = set() 49 50 51class WebIDLCodegenManagerState(dict): 52 """Holds state for the WebIDL code generation manager. 53 54 State is currently just an extended dict. The internal implementation of 55 state should be considered a black box to everyone except 56 WebIDLCodegenManager. But we'll still document it. 57 58 Fields: 59 60 version 61 The integer version of the format. This is to detect incompatible 62 changes between state. It should be bumped whenever the format 63 changes or semantics change. 64 65 webidls 66 A dictionary holding information about every known WebIDL input. 67 Keys are the basenames of input WebIDL files. Values are dicts of 68 metadata. Keys in those dicts are: 69 70 * filename - The full path to the input filename. 71 * inputs - A set of full paths to other webidl files this webidl 72 depends on. 73 * outputs - Set of full output paths that are created/derived from 74 this file. 75 * sha1 - The hexidecimal SHA-1 of the input filename from the last 76 processing time. 77 78 global_inputs 79 A dictionary defining files that influence all processing. Keys 80 are full filenames. Values are hexidecimal SHA-1 from the last 81 processing time. 82 """ 83 84 VERSION = 1 85 86 def __init__(self, fh=None): 87 self['version'] = self.VERSION 88 self['webidls'] = {} 89 self['global_depends'] = {} 90 91 if not fh: 92 return 93 94 state = json.load(fh) 95 if state['version'] != self.VERSION: 96 raise Exception('Unknown state version: %s' % state['version']) 97 98 self['version'] = state['version'] 99 self['global_depends'] = state['global_depends'] 100 101 for k, v in state['webidls'].items(): 102 self['webidls'][k] = v 103 104 # Sets are converted to lists for serialization because JSON 105 # doesn't support sets. 106 self['webidls'][k]['inputs'] = set(v['inputs']) 107 self['webidls'][k]['outputs'] = set(v['outputs']) 108 109 def dump(self, fh): 110 """Dump serialized state to a file handle.""" 111 normalized = deepcopy(self) 112 113 for k, v in self['webidls'].items(): 114 # Convert sets to lists because JSON doesn't support sets. 115 normalized['webidls'][k]['outputs'] = sorted(v['outputs']) 116 normalized['webidls'][k]['inputs'] = sorted(v['inputs']) 117 118 json.dump(normalized, fh, sort_keys=True) 119 120 121class WebIDLCodegenManager(LoggingMixin): 122 """Manages all code generation around WebIDL. 123 124 To facilitate testing, this object is meant to be generic and reusable. 125 Paths, etc should be parameters and not hardcoded. 126 """ 127 128 # Global parser derived declaration files. 129 GLOBAL_DECLARE_FILES = { 130 'GeneratedAtomList.h', 131 'GeneratedEventList.h', 132 'PrototypeList.h', 133 'RegisterBindings.h', 134 'RegisterWorkerBindings.h', 135 'RegisterWorkerDebuggerBindings.h', 136 'RegisterWorkletBindings.h', 137 'ResolveSystemBinding.h', 138 'UnionConversions.h', 139 'UnionTypes.h', 140 } 141 142 # Global parser derived definition files. 143 GLOBAL_DEFINE_FILES = { 144 'RegisterBindings.cpp', 145 'RegisterWorkerBindings.cpp', 146 'RegisterWorkerDebuggerBindings.cpp', 147 'RegisterWorkletBindings.cpp', 148 'ResolveSystemBinding.cpp', 149 'UnionTypes.cpp', 150 'PrototypeList.cpp', 151 } 152 153 def __init__(self, config_path, inputs, exported_header_dir, 154 codegen_dir, state_path, cache_dir=None, make_deps_path=None, 155 make_deps_target=None): 156 """Create an instance that manages WebIDLs in the build system. 157 158 config_path refers to a WebIDL config file (e.g. Bindings.conf). 159 inputs is a 4-tuple describing the input .webidl files and how to 160 process them. Members are: 161 (set(.webidl files), set(basenames of exported files), 162 set(basenames of generated events files), 163 set(example interface names)) 164 165 exported_header_dir and codegen_dir are directories where generated 166 files will be written to. 167 state_path is the path to a file that will receive JSON state from our 168 actions. 169 make_deps_path is the path to a make dependency file that we can 170 optionally write. 171 make_deps_target is the target that receives the make dependencies. It 172 must be defined if using make_deps_path. 173 """ 174 self.populate_logger() 175 176 input_paths, exported_stems, generated_events_stems, example_interfaces = inputs 177 178 self._config_path = config_path 179 self._input_paths = set(input_paths) 180 self._exported_stems = set(exported_stems) 181 self._generated_events_stems = set(generated_events_stems) 182 self._generated_events_stems_as_array = generated_events_stems 183 self._example_interfaces = set(example_interfaces) 184 self._exported_header_dir = exported_header_dir 185 self._codegen_dir = codegen_dir 186 self._state_path = state_path 187 self._cache_dir = cache_dir 188 self._make_deps_path = make_deps_path 189 self._make_deps_target = make_deps_target 190 191 if ((make_deps_path and not make_deps_target) or 192 (not make_deps_path and make_deps_target)): 193 raise Exception('Must define both make_deps_path and make_deps_target ' 194 'if one is defined.') 195 196 self._parser_results = None 197 self._config = None 198 self._state = WebIDLCodegenManagerState() 199 200 if os.path.exists(state_path): 201 with open(state_path, 'rb') as fh: 202 try: 203 self._state = WebIDLCodegenManagerState(fh=fh) 204 except Exception as e: 205 self.log(logging.WARN, 'webidl_bad_state', {'msg': str(e)}, 206 'Bad WebIDL state: {msg}') 207 208 @property 209 def config(self): 210 if not self._config: 211 self._parse_webidl() 212 213 return self._config 214 215 def generate_build_files(self): 216 """Generate files required for the build. 217 218 This function is in charge of generating all the .h/.cpp files derived 219 from input .webidl files. Please note that there are build actions 220 required to produce .webidl files and these build actions are 221 explicitly not captured here: this function assumes all .webidl files 222 are present and up to date. 223 224 This routine is called as part of the build to ensure files that need 225 to exist are present and up to date. This routine may not be called if 226 the build dependencies (generated as a result of calling this the first 227 time) say everything is up to date. 228 229 Because reprocessing outputs for every .webidl on every invocation 230 is expensive, we only regenerate the minimal set of files on every 231 invocation. The rules for deciding what needs done are roughly as 232 follows: 233 234 1. If any .webidl changes, reparse all .webidl files and regenerate 235 the global derived files. Only regenerate output files (.h/.cpp) 236 impacted by the modified .webidl files. 237 2. If an non-.webidl dependency (Python files, config file) changes, 238 assume everything is out of date and regenerate the world. This 239 is because changes in those could globally impact every output 240 file. 241 3. If an output file is missing, ensure it is present by performing 242 necessary regeneration. 243 """ 244 # Despite #1 above, we assume the build system is smart enough to not 245 # invoke us if nothing has changed. Therefore, any invocation means 246 # something has changed. And, if anything has changed, we need to 247 # parse the WebIDL. 248 self._parse_webidl() 249 250 result = BuildResult() 251 252 # If we parse, we always update globals - they are cheap and it is 253 # easier that way. 254 created, updated, unchanged = self._write_global_derived() 255 result.created |= created 256 result.updated |= updated 257 result.unchanged |= unchanged 258 259 # If any of the extra dependencies changed, regenerate the world. 260 global_changed, global_hashes = self._global_dependencies_changed() 261 if global_changed: 262 # Make a copy because we may modify. 263 changed_inputs = set(self._input_paths) 264 else: 265 changed_inputs = self._compute_changed_inputs() 266 267 self._state['global_depends'] = global_hashes 268 269 # Generate bindings from .webidl files. 270 for filename in sorted(changed_inputs): 271 basename = mozpath.basename(filename) 272 result.inputs.add(filename) 273 written, deps = self._generate_build_files_for_webidl(filename) 274 result.created |= written[0] 275 result.updated |= written[1] 276 result.unchanged |= written[2] 277 278 self._state['webidls'][basename] = dict( 279 filename=filename, 280 outputs=written[0] | written[1] | written[2], 281 inputs=set(deps), 282 sha1=self._input_hashes[filename], 283 ) 284 285 # Process some special interfaces required for testing. 286 for interface in self._example_interfaces: 287 written = self.generate_example_files(interface) 288 result.created |= written[0] 289 result.updated |= written[1] 290 result.unchanged |= written[2] 291 292 # Generate a make dependency file. 293 if self._make_deps_path: 294 mk = Makefile() 295 codegen_rule = mk.create_rule([self._make_deps_target]) 296 codegen_rule.add_dependencies(global_hashes.keys()) 297 codegen_rule.add_dependencies(self._input_paths) 298 299 with FileAvoidWrite(self._make_deps_path) as fh: 300 mk.dump(fh) 301 302 self._save_state() 303 304 return result 305 306 def generate_example_files(self, interface): 307 """Generates example files for a given interface.""" 308 from Codegen import CGExampleRoot 309 310 root = CGExampleRoot(self.config, interface) 311 312 example_paths = self._example_paths(interface) 313 for path in example_paths: 314 print "Generating %s" % path 315 316 return self._maybe_write_codegen(root, *example_paths) 317 318 def _parse_webidl(self): 319 import WebIDL 320 from Configuration import Configuration 321 322 self.log(logging.INFO, 'webidl_parse', 323 {'count': len(self._input_paths)}, 324 'Parsing {count} WebIDL files.') 325 326 hashes = {} 327 parser = WebIDL.Parser(self._cache_dir) 328 329 for path in sorted(self._input_paths): 330 with open(path, 'rb') as fh: 331 data = fh.read() 332 hashes[path] = hashlib.sha1(data).hexdigest() 333 parser.parse(data, path) 334 335 self._parser_results = parser.finish() 336 self._config = Configuration(self._config_path, self._parser_results, 337 self._generated_events_stems_as_array) 338 self._input_hashes = hashes 339 340 def _write_global_derived(self): 341 from Codegen import GlobalGenRoots 342 343 things = [('declare', f) for f in self.GLOBAL_DECLARE_FILES] 344 things.extend(('define', f) for f in self.GLOBAL_DEFINE_FILES) 345 346 result = (set(), set(), set()) 347 348 for what, filename in things: 349 stem = mozpath.splitext(filename)[0] 350 root = getattr(GlobalGenRoots, stem)(self._config) 351 352 if what == 'declare': 353 code = root.declare() 354 output_root = self._exported_header_dir 355 elif what == 'define': 356 code = root.define() 357 output_root = self._codegen_dir 358 else: 359 raise Exception('Unknown global gen type: %s' % what) 360 361 output_path = mozpath.join(output_root, filename) 362 self._maybe_write_file(output_path, code, result) 363 364 return result 365 366 def _compute_changed_inputs(self): 367 """Compute the set of input files that need to be regenerated.""" 368 changed_inputs = set() 369 expected_outputs = self.expected_build_output_files() 370 371 # Look for missing output files. 372 if any(not os.path.exists(f) for f in expected_outputs): 373 # FUTURE Bug 940469 Only regenerate minimum set. 374 changed_inputs |= self._input_paths 375 376 # That's it for examining output files. We /could/ examine SHA-1's of 377 # output files from a previous run to detect modifications. But that's 378 # a lot of extra work and most build systems don't do that anyway. 379 380 # Now we move on to the input files. 381 old_hashes = {v['filename']: v['sha1'] 382 for v in self._state['webidls'].values()} 383 384 old_filenames = set(old_hashes.keys()) 385 new_filenames = self._input_paths 386 387 # If an old file has disappeared or a new file has arrived, mark 388 # it. 389 changed_inputs |= old_filenames ^ new_filenames 390 391 # For the files in common between runs, compare content. If the file 392 # has changed, mark it. We don't need to perform mtime comparisons 393 # because content is a stronger validator. 394 for filename in old_filenames & new_filenames: 395 if old_hashes[filename] != self._input_hashes[filename]: 396 changed_inputs.add(filename) 397 398 # We've now populated the base set of inputs that have changed. 399 400 # Inherit dependencies from previous run. The full set of dependencies 401 # is associated with each record, so we don't need to perform any fancy 402 # graph traversal. 403 for v in self._state['webidls'].values(): 404 if any(dep for dep in v['inputs'] if dep in changed_inputs): 405 changed_inputs.add(v['filename']) 406 407 # Only use paths that are known to our current state. 408 # This filters out files that were deleted or changed type (e.g. from 409 # static to preprocessed). 410 return changed_inputs & self._input_paths 411 412 def _binding_info(self, p): 413 """Compute binding metadata for an input path. 414 415 Returns a tuple of: 416 417 (stem, binding_stem, is_event, output_files) 418 419 output_files is itself a tuple. The first two items are the binding 420 header and C++ paths, respectively. The 2nd pair are the event header 421 and C++ paths or None if this isn't an event binding. 422 """ 423 basename = mozpath.basename(p) 424 stem = mozpath.splitext(basename)[0] 425 binding_stem = '%sBinding' % stem 426 427 if stem in self._exported_stems: 428 header_dir = self._exported_header_dir 429 else: 430 header_dir = self._codegen_dir 431 432 is_event = stem in self._generated_events_stems 433 434 files = ( 435 mozpath.join(header_dir, '%s.h' % binding_stem), 436 mozpath.join(self._codegen_dir, '%s.cpp' % binding_stem), 437 mozpath.join(header_dir, '%s.h' % stem) if is_event else None, 438 mozpath.join(self._codegen_dir, '%s.cpp' % stem) if is_event else None, 439 ) 440 441 return stem, binding_stem, is_event, header_dir, files 442 443 def _example_paths(self, interface): 444 return ( 445 mozpath.join(self._codegen_dir, '%s-example.h' % interface), 446 mozpath.join(self._codegen_dir, '%s-example.cpp' % interface)) 447 448 def expected_build_output_files(self): 449 """Obtain the set of files generate_build_files() should write.""" 450 paths = set() 451 452 # Account for global generation. 453 for p in self.GLOBAL_DECLARE_FILES: 454 paths.add(mozpath.join(self._exported_header_dir, p)) 455 for p in self.GLOBAL_DEFINE_FILES: 456 paths.add(mozpath.join(self._codegen_dir, p)) 457 458 for p in self._input_paths: 459 stem, binding_stem, is_event, header_dir, files = self._binding_info(p) 460 paths |= {f for f in files if f} 461 462 for interface in self._example_interfaces: 463 for p in self._example_paths(interface): 464 paths.add(p) 465 466 return paths 467 468 def _generate_build_files_for_webidl(self, filename): 469 from Codegen import ( 470 CGBindingRoot, 471 CGEventRoot, 472 ) 473 474 self.log(logging.INFO, 'webidl_generate_build_for_input', 475 {'filename': filename}, 476 'Generating WebIDL files derived from {filename}') 477 478 stem, binding_stem, is_event, header_dir, files = self._binding_info(filename) 479 root = CGBindingRoot(self._config, binding_stem, filename) 480 481 result = self._maybe_write_codegen(root, files[0], files[1]) 482 483 if is_event: 484 generated_event = CGEventRoot(self._config, stem) 485 result = self._maybe_write_codegen(generated_event, files[2], 486 files[3], result) 487 488 return result, root.deps() 489 490 def _global_dependencies_changed(self): 491 """Determine whether the global dependencies have changed.""" 492 current_files = set(iter_modules_in_path(mozpath.dirname(__file__))) 493 494 # We need to catch other .py files from /dom/bindings. We assume these 495 # are in the same directory as the config file. 496 current_files |= set(iter_modules_in_path(mozpath.dirname(self._config_path))) 497 498 current_files.add(self._config_path) 499 500 current_hashes = {} 501 for f in current_files: 502 # This will fail if the file doesn't exist. If a current global 503 # dependency doesn't exist, something else is wrong. 504 with open(f, 'rb') as fh: 505 current_hashes[f] = hashlib.sha1(fh.read()).hexdigest() 506 507 # The set of files has changed. 508 if current_files ^ set(self._state['global_depends'].keys()): 509 return True, current_hashes 510 511 # Compare hashes. 512 for f, sha1 in current_hashes.items(): 513 if sha1 != self._state['global_depends'][f]: 514 return True, current_hashes 515 516 return False, current_hashes 517 518 def _save_state(self): 519 with open(self._state_path, 'wb') as fh: 520 self._state.dump(fh) 521 522 def _maybe_write_codegen(self, obj, declare_path, define_path, result=None): 523 assert declare_path and define_path 524 if not result: 525 result = (set(), set(), set()) 526 527 self._maybe_write_file(declare_path, obj.declare(), result) 528 self._maybe_write_file(define_path, obj.define(), result) 529 530 return result 531 532 def _maybe_write_file(self, path, content, result): 533 fh = FileAvoidWrite(path) 534 fh.write(content) 535 existed, updated = fh.close() 536 537 if not existed: 538 result[0].add(path) 539 elif updated: 540 result[1].add(path) 541 else: 542 result[2].add(path) 543 544 545def create_build_system_manager(topsrcdir, topobjdir, dist_dir): 546 """Create a WebIDLCodegenManager for use by the build system.""" 547 src_dir = os.path.join(topsrcdir, 'dom', 'bindings') 548 obj_dir = os.path.join(topobjdir, 'dom', 'bindings') 549 550 with open(os.path.join(obj_dir, 'file-lists.json'), 'rb') as fh: 551 files = json.load(fh) 552 553 inputs = (files['webidls'], files['exported_stems'], 554 files['generated_events_stems'], files['example_interfaces']) 555 556 cache_dir = os.path.join(obj_dir, '_cache') 557 try: 558 os.makedirs(cache_dir) 559 except OSError as e: 560 if e.errno != errno.EEXIST: 561 raise 562 563 return WebIDLCodegenManager( 564 os.path.join(src_dir, 'Bindings.conf'), 565 inputs, 566 os.path.join(dist_dir, 'include', 'mozilla', 'dom'), 567 obj_dir, 568 os.path.join(obj_dir, 'codegen.json'), 569 cache_dir=cache_dir, 570 # The make rules include a codegen.pp file containing dependencies. 571 make_deps_path=os.path.join(obj_dir, 'codegen.pp'), 572 make_deps_target='codegen.pp', 573 ) 574 575 576class BuildSystemWebIDL(MozbuildObject): 577 @property 578 def manager(self): 579 if not hasattr(self, '_webidl_manager'): 580 self._webidl_manager = create_build_system_manager( 581 self.topsrcdir, self.topobjdir, self.distdir) 582 583 return self._webidl_manager 584