1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, unicode_literals 6 7import json 8import os 9 10import mozpack.path as mozpath 11 12from mozbuild.backend.base import BuildBackend 13 14from mozbuild.frontend.context import ( 15 Context, 16 ObjDirPath, 17 Path, 18 RenamedSourcePath, 19 VARIABLES, 20) 21from mozbuild.frontend.data import ( 22 BaseProgram, 23 ChromeManifestEntry, 24 ConfigFileSubstitution, 25 Exports, 26 FinalTargetPreprocessedFiles, 27 FinalTargetFiles, 28 GeneratedSources, 29 GnProjectData, 30 IPDLCollection, 31 SharedLibrary, 32 UnifiedSources, 33 XPIDLFile, 34 WebIDLCollection, 35) 36from mozbuild.jar import ( 37 DeprecatedJarManifest, 38 JarManifestParser, 39) 40from mozbuild.preprocessor import Preprocessor 41from mozpack.chrome.manifest import parse_manifest_line 42 43from mozbuild.util import group_unified_files 44 45class XPIDLManager(object): 46 """Helps manage XPCOM IDLs in the context of the build system.""" 47 def __init__(self, config): 48 self.config = config 49 self.topsrcdir = config.topsrcdir 50 self.topobjdir = config.topobjdir 51 52 self.idls = {} 53 self.modules = {} 54 self.interface_manifests = {} 55 self.chrome_manifests = set() 56 57 def register_idl(self, idl, allow_existing=False): 58 """Registers an IDL file with this instance. 59 60 The IDL file will be built, installed, etc. 61 """ 62 basename = mozpath.basename(idl.source_path) 63 root = mozpath.splitext(basename)[0] 64 xpt = '%s.xpt' % idl.module 65 manifest = mozpath.join(idl.install_target, 'components', 'interfaces.manifest') 66 chrome_manifest = mozpath.join(idl.install_target, 'chrome.manifest') 67 68 entry = { 69 'source': idl.source_path, 70 'module': idl.module, 71 'basename': basename, 72 'root': root, 73 'manifest': manifest, 74 } 75 76 if not allow_existing and entry['basename'] in self.idls: 77 raise Exception('IDL already registered: %s' % entry['basename']) 78 79 self.idls[entry['basename']] = entry 80 t = self.modules.setdefault(entry['module'], (idl.install_target, set())) 81 t[1].add(entry['root']) 82 83 if idl.add_to_manifest: 84 self.interface_manifests.setdefault(manifest, set()).add(xpt) 85 self.chrome_manifests.add(chrome_manifest) 86 87 88class BinariesCollection(object): 89 """Tracks state of binaries produced by the build.""" 90 91 def __init__(self): 92 self.shared_libraries = [] 93 self.programs = [] 94 95class CommonBackend(BuildBackend): 96 """Holds logic common to all build backends.""" 97 98 def _init(self): 99 self._idl_manager = XPIDLManager(self.environment) 100 self._binaries = BinariesCollection() 101 self._configs = set() 102 self._generated_sources = set() 103 104 def consume_object(self, obj): 105 self._configs.add(obj.config) 106 107 if isinstance(obj, XPIDLFile): 108 # TODO bug 1240134 tracks not processing XPIDL files during 109 # artifact builds. 110 self._idl_manager.register_idl(obj) 111 112 elif isinstance(obj, ConfigFileSubstitution): 113 # Do not handle ConfigFileSubstitution for Makefiles. Leave that 114 # to other 115 if mozpath.basename(obj.output_path) == 'Makefile': 116 return False 117 with self._get_preprocessor(obj) as pp: 118 pp.do_include(obj.input_path) 119 self.backend_input_files.add(obj.input_path) 120 121 elif isinstance(obj, WebIDLCollection): 122 self._handle_webidl_collection(obj) 123 124 elif isinstance(obj, IPDLCollection): 125 self._handle_generated_sources(mozpath.join(obj.objdir, f) 126 for f in obj.all_generated_sources()) 127 self._write_unified_files(obj.unified_source_mapping, obj.objdir, 128 poison_windows_h=False) 129 self._handle_ipdl_sources(obj.objdir, 130 list(sorted(obj.all_sources())), 131 list(sorted(obj.all_preprocessed_sources())), 132 list(sorted(obj.all_regular_sources())), 133 obj.unified_source_mapping) 134 135 elif isinstance(obj, UnifiedSources): 136 # Unified sources aren't relevant to artifact builds. 137 if self.environment.is_artifact_build: 138 return True 139 140 if obj.have_unified_mapping: 141 self._write_unified_files(obj.unified_source_mapping, obj.objdir) 142 if hasattr(self, '_process_unified_sources'): 143 self._process_unified_sources(obj) 144 145 elif isinstance(obj, BaseProgram): 146 self._binaries.programs.append(obj) 147 return False 148 149 elif isinstance(obj, SharedLibrary): 150 self._binaries.shared_libraries.append(obj) 151 return False 152 153 elif isinstance(obj, GeneratedSources): 154 self._handle_generated_sources(obj.files) 155 return False 156 157 elif isinstance(obj, Exports): 158 objdir_files = [f.full_path for path, files in obj.files.walk() for f in files if isinstance(f, ObjDirPath)] 159 if objdir_files: 160 self._handle_generated_sources(objdir_files) 161 return False 162 163 elif isinstance(obj, GnProjectData): 164 # These are only handled by special purpose build backends, 165 # ignore them here. 166 return True 167 168 else: 169 return False 170 171 return True 172 173 def consume_finished(self): 174 if len(self._idl_manager.idls): 175 self._write_rust_xpidl_summary(self._idl_manager) 176 self._handle_idl_manager(self._idl_manager) 177 self._handle_generated_sources(mozpath.join(self.environment.topobjdir, 'dist/include/%s.h' % idl['root']) for idl in self._idl_manager.idls.values()) 178 179 180 for config in self._configs: 181 self.backend_input_files.add(config.source) 182 183 # Write out a machine-readable file describing binaries. 184 topobjdir = self.environment.topobjdir 185 with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh: 186 d = { 187 'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries], 188 'programs': [p.to_dict() for p in self._binaries.programs], 189 } 190 json.dump(d, fh, sort_keys=True, indent=4) 191 192 # Write out a file listing generated sources. 193 with self._write_file(mozpath.join(topobjdir, 'generated-sources.json')) as fh: 194 d = { 195 'sources': sorted(self._generated_sources), 196 } 197 json.dump(d, fh, sort_keys=True, indent=4) 198 199 def _handle_generated_sources(self, files): 200 self._generated_sources.update(mozpath.relpath(f, self.environment.topobjdir) for f in files) 201 202 def _handle_webidl_collection(self, webidls): 203 204 bindings_dir = mozpath.join(self.environment.topobjdir, 'dom', 'bindings') 205 206 all_inputs = set(webidls.all_static_sources()) 207 for s in webidls.all_non_static_basenames(): 208 all_inputs.add(mozpath.join(bindings_dir, s)) 209 210 generated_events_stems = webidls.generated_events_stems() 211 exported_stems = webidls.all_regular_stems() 212 213 # The WebIDL manager reads configuration from a JSON file. So, we 214 # need to write this file early. 215 o = dict( 216 webidls=sorted(all_inputs), 217 generated_events_stems=sorted(generated_events_stems), 218 exported_stems=sorted(exported_stems), 219 example_interfaces=sorted(webidls.example_interfaces), 220 ) 221 222 file_lists = mozpath.join(bindings_dir, 'file-lists.json') 223 with self._write_file(file_lists) as fh: 224 json.dump(o, fh, sort_keys=True, indent=2) 225 226 import mozwebidlcodegen 227 228 manager = mozwebidlcodegen.create_build_system_manager( 229 self.environment.topsrcdir, 230 self.environment.topobjdir, 231 mozpath.join(self.environment.topobjdir, 'dist') 232 ) 233 self._handle_generated_sources(manager.expected_build_output_files()) 234 self._write_unified_files(webidls.unified_source_mapping, bindings_dir, 235 poison_windows_h=True) 236 self._handle_webidl_build(bindings_dir, webidls.unified_source_mapping, 237 webidls, 238 manager.expected_build_output_files(), 239 manager.GLOBAL_DEFINE_FILES) 240 241 def _write_unified_file(self, unified_file, source_filenames, 242 output_directory, poison_windows_h=False): 243 with self._write_file(mozpath.join(output_directory, unified_file)) as f: 244 f.write('#define MOZ_UNIFIED_BUILD\n') 245 includeTemplate = '#include "%(cppfile)s"' 246 if poison_windows_h: 247 includeTemplate += ( 248 '\n' 249 '#ifdef _WINDOWS_\n' 250 '#error "%(cppfile)s included windows.h"\n' 251 "#endif") 252 includeTemplate += ( 253 '\n' 254 '#ifdef PL_ARENA_CONST_ALIGN_MASK\n' 255 '#error "%(cppfile)s uses PL_ARENA_CONST_ALIGN_MASK, ' 256 'so it cannot be built in unified mode."\n' 257 '#undef PL_ARENA_CONST_ALIGN_MASK\n' 258 '#endif\n' 259 '#ifdef INITGUID\n' 260 '#error "%(cppfile)s defines INITGUID, ' 261 'so it cannot be built in unified mode."\n' 262 '#undef INITGUID\n' 263 '#endif') 264 f.write('\n'.join(includeTemplate % { "cppfile": s } for 265 s in source_filenames)) 266 267 def _write_unified_files(self, unified_source_mapping, output_directory, 268 poison_windows_h=False): 269 for unified_file, source_filenames in unified_source_mapping: 270 self._write_unified_file(unified_file, source_filenames, 271 output_directory, poison_windows_h) 272 273 def _consume_jar_manifest(self, obj): 274 # Ideally, this would all be handled somehow in the emitter, but 275 # this would require all the magic surrounding l10n and addons in 276 # the recursive make backend to die, which is not going to happen 277 # any time soon enough. 278 # Notably missing: 279 # - DEFINES from config/config.mk 280 # - L10n support 281 # - The equivalent of -e when USE_EXTENSION_MANIFEST is set in 282 # moz.build, but it doesn't matter in dist/bin. 283 pp = Preprocessor() 284 if obj.defines: 285 pp.context.update(obj.defines.defines) 286 pp.context.update(self.environment.defines) 287 pp.context.update( 288 AB_CD='en-US', 289 BUILD_FASTER=1, 290 ) 291 pp.out = JarManifestParser() 292 try: 293 pp.do_include(obj.path.full_path) 294 except DeprecatedJarManifest as e: 295 raise DeprecatedJarManifest('Parsing error while processing %s: %s' 296 % (obj.path.full_path, e.message)) 297 self.backend_input_files |= pp.includes 298 299 for jarinfo in pp.out: 300 jar_context = Context( 301 allowed_variables=VARIABLES, config=obj._context.config) 302 jar_context.push_source(obj._context.main_path) 303 jar_context.push_source(obj.path.full_path) 304 305 install_target = obj.install_target 306 if jarinfo.base: 307 install_target = mozpath.normpath( 308 mozpath.join(install_target, jarinfo.base)) 309 jar_context['FINAL_TARGET'] = install_target 310 if obj.defines: 311 jar_context['DEFINES'] = obj.defines.defines 312 files = jar_context['FINAL_TARGET_FILES'] 313 files_pp = jar_context['FINAL_TARGET_PP_FILES'] 314 315 for e in jarinfo.entries: 316 if e.is_locale: 317 if jarinfo.relativesrcdir: 318 src = '/%s' % jarinfo.relativesrcdir 319 else: 320 src = '' 321 src = mozpath.join(src, 'en-US', e.source) 322 else: 323 src = e.source 324 325 src = Path(jar_context, src) 326 327 if '*' not in e.source and not os.path.exists(src.full_path): 328 if e.is_locale: 329 raise Exception( 330 '%s: Cannot find %s' % (obj.path, e.source)) 331 if e.source.startswith('/'): 332 src = Path(jar_context, '!' + e.source) 333 else: 334 # This actually gets awkward if the jar.mn is not 335 # in the same directory as the moz.build declaring 336 # it, but it's how it works in the recursive make, 337 # not that anything relies on that, but it's simpler. 338 src = Path(obj._context, '!' + e.source) 339 340 output_basename = mozpath.basename(e.output) 341 if output_basename != src.target_basename: 342 src = RenamedSourcePath(jar_context, 343 (src, output_basename)) 344 path = mozpath.dirname(mozpath.join(jarinfo.name, e.output)) 345 346 if e.preprocess: 347 if '*' in e.source: 348 raise Exception('%s: Wildcards are not supported with ' 349 'preprocessing' % obj.path) 350 files_pp[path] += [src] 351 else: 352 files[path] += [src] 353 354 if files: 355 self.consume_object(FinalTargetFiles(jar_context, files)) 356 if files_pp: 357 self.consume_object( 358 FinalTargetPreprocessedFiles(jar_context, files_pp)) 359 360 for m in jarinfo.chrome_manifests: 361 entry = parse_manifest_line( 362 mozpath.dirname(jarinfo.name), 363 m.replace('%', mozpath.basename(jarinfo.name) + '/')) 364 self.consume_object(ChromeManifestEntry( 365 jar_context, '%s.manifest' % jarinfo.name, entry)) 366 367 def _write_rust_xpidl_summary(self, manager): 368 """Write out a rust file which includes the generated xpcom rust modules""" 369 topobjdir = self.environment.topobjdir 370 371 include_tmpl = "include!(concat!(env!(\"MOZ_TOPOBJDIR\"), \"/dist/xpcrs/%s/%s.rs\"))" 372 373 with self._write_file(mozpath.join(topobjdir, 'dist', 'xpcrs', 'rt', 'all.rs')) as fh: 374 fh.write("// THIS FILE IS GENERATED - DO NOT EDIT\n\n") 375 for idl in manager.idls.values(): 376 fh.write(include_tmpl % ("rt", idl['root'])) 377 fh.write(";\n") 378 379 with self._write_file(mozpath.join(topobjdir, 'dist', 'xpcrs', 'bt', 'all.rs')) as fh: 380 fh.write("// THIS FILE IS GENERATED - DO NOT EDIT\n\n") 381 fh.write("&[\n") 382 for idl in manager.idls.values(): 383 fh.write(include_tmpl % ("bt", idl['root'])) 384 fh.write(",\n") 385 fh.write("]\n") 386