1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, unicode_literals 6 7import itertools 8import logging 9import os 10import traceback 11import sys 12import time 13 14from collections import defaultdict, OrderedDict 15from mach.mixin.logging import LoggingMixin 16from mozbuild.util import ( 17 memoize, 18 OrderedDefaultDict, 19) 20 21import mozpack.path as mozpath 22import mozinfo 23import pytoml 24 25from .data import ( 26 BaseRustProgram, 27 BaseSources, 28 ChromeManifestEntry, 29 ComputedFlags, 30 ConfigFileSubstitution, 31 ContextWrapped, 32 Defines, 33 DirectoryTraversal, 34 Exports, 35 FinalTargetFiles, 36 FinalTargetPreprocessedFiles, 37 GeneratedFile, 38 GeneratedSources, 39 GnProjectData, 40 ExternalStaticLibrary, 41 ExternalSharedLibrary, 42 HostDefines, 43 HostLibrary, 44 HostProgram, 45 HostRustProgram, 46 HostSimpleProgram, 47 HostSources, 48 InstallationTarget, 49 IPDLCollection, 50 JARManifest, 51 Library, 52 Linkable, 53 LocalInclude, 54 LocalizedFiles, 55 LocalizedPreprocessedFiles, 56 ObjdirFiles, 57 ObjdirPreprocessedFiles, 58 PerSourceFlag, 59 WebIDLCollection, 60 Program, 61 RustLibrary, 62 HostRustLibrary, 63 RustProgram, 64 RustTest, 65 SharedLibrary, 66 SimpleProgram, 67 Sources, 68 StaticLibrary, 69 TestHarnessFiles, 70 TestManifest, 71 UnifiedSources, 72 VariablePassthru, 73 XPIDLFile, 74) 75from mozpack.chrome.manifest import ( 76 Manifest, 77) 78 79from .reader import SandboxValidationError 80 81from ..testing import ( 82 TEST_MANIFESTS, 83 REFTEST_FLAVORS, 84 WEB_PLATFORM_TESTS_FLAVORS, 85 SupportFilesConverter, 86) 87 88from .context import ( 89 Context, 90 SourcePath, 91 ObjDirPath, 92 Path, 93 SubContext, 94 TemplateContext, 95) 96 97from mozbuild.base import ExecutionSummary 98 99 100class TreeMetadataEmitter(LoggingMixin): 101 """Converts the executed mozbuild files into data structures. 102 103 This is a bridge between reader.py and data.py. It takes what was read by 104 reader.BuildReader and converts it into the classes defined in the data 105 module. 106 """ 107 108 def __init__(self, config): 109 self.populate_logger() 110 111 self.config = config 112 113 mozinfo.find_and_update_from_json(config.topobjdir) 114 115 # Python 2.6 doesn't allow unicode keys to be used for keyword 116 # arguments. This gross hack works around the problem until we 117 # rid ourselves of 2.6. 118 self.info = {} 119 for k, v in mozinfo.info.items(): 120 if isinstance(k, unicode): 121 k = k.encode('ascii') 122 self.info[k] = v 123 124 self._libs = OrderedDefaultDict(list) 125 self._binaries = OrderedDict() 126 self._compile_dirs = set() 127 self._host_compile_dirs = set() 128 self._rust_compile_dirs = set() 129 self._asm_compile_dirs = set() 130 self._compile_flags = dict() 131 self._compile_as_flags = dict() 132 self._linkage = [] 133 self._static_linking_shared = set() 134 self._crate_verified_local = set() 135 self._crate_directories = dict() 136 self._idls = defaultdict(set) 137 138 # Keep track of external paths (third party build systems), starting 139 # from what we run a subconfigure in. We'll eliminate some directories 140 # as we traverse them with moz.build (e.g. js/src). 141 subconfigures = os.path.join(self.config.topobjdir, 'subconfigures') 142 paths = [] 143 if os.path.exists(subconfigures): 144 paths = open(subconfigures).read().splitlines() 145 self._external_paths = set(mozpath.normsep(d) for d in paths) 146 147 self._emitter_time = 0.0 148 self._object_count = 0 149 self._test_files_converter = SupportFilesConverter() 150 151 def summary(self): 152 return ExecutionSummary( 153 'Processed into {object_count:d} build config descriptors in ' 154 '{execution_time:.2f}s', 155 execution_time=self._emitter_time, 156 object_count=self._object_count) 157 158 def emit(self, output, emitfn=None): 159 """Convert the BuildReader output into data structures. 160 161 The return value from BuildReader.read_topsrcdir() (a generator) is 162 typically fed into this function. 163 """ 164 contexts = {} 165 emitfn = emitfn or self.emit_from_context 166 167 def emit_objs(objs): 168 for o in objs: 169 self._object_count += 1 170 yield o 171 172 for out in output: 173 # Nothing in sub-contexts is currently of interest to us. Filter 174 # them all out. 175 if isinstance(out, SubContext): 176 continue 177 178 if isinstance(out, Context): 179 # Keep all contexts around, we will need them later. 180 contexts[out.objdir] = out 181 182 start = time.time() 183 # We need to expand the generator for the timings to work. 184 objs = list(emitfn(out)) 185 self._emitter_time += time.time() - start 186 187 for o in emit_objs(objs): yield o 188 189 else: 190 raise Exception('Unhandled output type: %s' % type(out)) 191 192 # Don't emit Linkable objects when COMPILE_ENVIRONMENT is not set 193 if self.config.substs.get('COMPILE_ENVIRONMENT'): 194 start = time.time() 195 objs = list(self._emit_libs_derived(contexts)) 196 self._emitter_time += time.time() - start 197 198 for o in emit_objs(objs): yield o 199 200 def _emit_libs_derived(self, contexts): 201 202 # First aggregate idl sources. 203 webidl_attrs = [ 204 ('GENERATED_EVENTS_WEBIDL_FILES', lambda c: c.generated_events_sources), 205 ('GENERATED_WEBIDL_FILES', lambda c: c.generated_sources), 206 ('PREPROCESSED_TEST_WEBIDL_FILES', lambda c: c.preprocessed_test_sources), 207 ('PREPROCESSED_WEBIDL_FILES', lambda c: c.preprocessed_sources), 208 ('TEST_WEBIDL_FILES', lambda c: c.test_sources), 209 ('WEBIDL_FILES', lambda c: c.sources), 210 ('WEBIDL_EXAMPLE_INTERFACES', lambda c: c.example_interfaces), 211 ] 212 ipdl_attrs = [ 213 ('IPDL_SOURCES', lambda c: c.sources), 214 ('PREPROCESSED_IPDL_SOURCES', lambda c: c.preprocessed_sources), 215 ] 216 217 idl_sources = {} 218 for root, cls, attrs in ((self.config.substs.get('WEBIDL_ROOT'), 219 WebIDLCollection, webidl_attrs), 220 (self.config.substs.get('IPDL_ROOT'), 221 IPDLCollection, ipdl_attrs)): 222 if root: 223 collection = cls(contexts[root]) 224 for var, src_getter in attrs: 225 src_getter(collection).update(self._idls[var]) 226 227 idl_sources[root] = collection.all_source_files() 228 if isinstance(collection, WebIDLCollection): 229 # Test webidl sources are added here as a somewhat special 230 # case. 231 idl_sources[mozpath.join(root, 'test')] = [s for s in collection.all_test_cpp_basenames()] 232 233 yield collection 234 235 236 # Next do FINAL_LIBRARY linkage. 237 for lib in (l for libs in self._libs.values() for l in libs): 238 if not isinstance(lib, (StaticLibrary, RustLibrary)) or not lib.link_into: 239 continue 240 if lib.link_into not in self._libs: 241 raise SandboxValidationError( 242 'FINAL_LIBRARY ("%s") does not match any LIBRARY_NAME' 243 % lib.link_into, contexts[lib.objdir]) 244 candidates = self._libs[lib.link_into] 245 246 # When there are multiple candidates, but all are in the same 247 # directory and have a different type, we want all of them to 248 # have the library linked. The typical usecase is when building 249 # both a static and a shared library in a directory, and having 250 # that as a FINAL_LIBRARY. 251 if len(set(type(l) for l in candidates)) == len(candidates) and \ 252 len(set(l.objdir for l in candidates)) == 1: 253 for c in candidates: 254 c.link_library(lib) 255 else: 256 raise SandboxValidationError( 257 'FINAL_LIBRARY ("%s") matches a LIBRARY_NAME defined in ' 258 'multiple places:\n %s' % (lib.link_into, 259 '\n '.join(l.objdir for l in candidates)), 260 contexts[lib.objdir]) 261 262 # ...and USE_LIBS linkage. 263 for context, obj, variable in self._linkage: 264 self._link_libraries(context, obj, variable, idl_sources) 265 266 def recurse_refs(lib): 267 for o in lib.refs: 268 yield o 269 if isinstance(o, StaticLibrary): 270 for q in recurse_refs(o): 271 yield q 272 273 # Check that all static libraries refering shared libraries in 274 # USE_LIBS are linked into a shared library or program. 275 for lib in self._static_linking_shared: 276 if all(isinstance(o, StaticLibrary) for o in recurse_refs(lib)): 277 shared_libs = sorted(l.basename for l in lib.linked_libraries 278 if isinstance(l, SharedLibrary)) 279 raise SandboxValidationError( 280 'The static "%s" library is not used in a shared library ' 281 'or a program, but USE_LIBS contains the following shared ' 282 'library names:\n %s\n\nMaybe you can remove the ' 283 'static "%s" library?' % (lib.basename, 284 '\n '.join(shared_libs), lib.basename), 285 contexts[lib.objdir]) 286 287 # Propagate LIBRARY_DEFINES to all child libraries recursively. 288 def propagate_defines(outerlib, defines): 289 outerlib.lib_defines.update(defines) 290 for lib in outerlib.linked_libraries: 291 # Propagate defines only along FINAL_LIBRARY paths, not USE_LIBS 292 # paths. 293 if (isinstance(lib, StaticLibrary) and 294 lib.link_into == outerlib.basename): 295 propagate_defines(lib, defines) 296 297 for lib in (l for libs in self._libs.values() for l in libs): 298 if isinstance(lib, Library): 299 propagate_defines(lib, lib.lib_defines) 300 yield lib 301 302 303 for lib in (l for libs in self._libs.values() for l in libs): 304 lib_defines = list(lib.lib_defines.get_defines()) 305 if lib_defines: 306 objdir_flags = self._compile_flags[lib.objdir] 307 objdir_flags.resolve_flags('LIBRARY_DEFINES', lib_defines) 308 309 objdir_flags = self._compile_as_flags.get(lib.objdir) 310 if objdir_flags: 311 objdir_flags.resolve_flags('LIBRARY_DEFINES', lib_defines) 312 313 for flags_obj in self._compile_flags.values(): 314 yield flags_obj 315 316 for flags_obj in self._compile_as_flags.values(): 317 yield flags_obj 318 319 for obj in self._binaries.values(): 320 yield obj 321 322 323 LIBRARY_NAME_VAR = { 324 'host': 'HOST_LIBRARY_NAME', 325 'target': 'LIBRARY_NAME', 326 } 327 328 LIBSTDCXX_VAR = { 329 'host': 'MOZ_LIBSTDCXX_HOST_VERSION', 330 'target': 'MOZ_LIBSTDCXX_TARGET_VERSION', 331 } 332 333 STDCXXCOMPAT_NAME = { 334 'host': 'host_stdc++compat', 335 'target': 'stdc++compat', 336 } 337 338 def _link_libraries(self, context, obj, variable, extra_sources): 339 """Add linkage declarations to a given object.""" 340 assert isinstance(obj, Linkable) 341 342 if context.objdir in extra_sources: 343 # All "extra sources" are .cpp for the moment, and happen to come 344 # first in order. 345 obj.sources['.cpp'] = extra_sources[context.objdir] + obj.sources['.cpp'] 346 347 for path in context.get(variable, []): 348 self._link_library(context, obj, variable, path) 349 350 # Link system libraries from OS_LIBS/HOST_OS_LIBS. 351 for lib in context.get(variable.replace('USE', 'OS'), []): 352 obj.link_system_library(lib) 353 354 # We have to wait for all the self._link_library calls above to have 355 # happened for obj.cxx_link to be final. 356 if not isinstance(obj, (StaticLibrary, HostLibrary, 357 BaseRustProgram)) and obj.cxx_link: 358 if context.config.substs.get(self.LIBSTDCXX_VAR[obj.KIND]): 359 self._link_library(context, obj, variable, 360 self.STDCXXCOMPAT_NAME[obj.KIND]) 361 if obj.KIND == 'target': 362 for lib in context.config.substs.get('STLPORT_LIBS', []): 363 obj.link_system_library(lib) 364 365 def _link_library(self, context, obj, variable, path): 366 force_static = path.startswith('static:') and obj.KIND == 'target' 367 if force_static: 368 path = path[7:] 369 name = mozpath.basename(path) 370 dir = mozpath.dirname(path) 371 candidates = [l for l in self._libs[name] if l.KIND == obj.KIND] 372 if dir: 373 if dir.startswith('/'): 374 dir = mozpath.normpath( 375 mozpath.join(obj.topobjdir, dir[1:])) 376 else: 377 dir = mozpath.normpath( 378 mozpath.join(obj.objdir, dir)) 379 dir = mozpath.relpath(dir, obj.topobjdir) 380 candidates = [l for l in candidates if l.relobjdir == dir] 381 if not candidates: 382 # If the given directory is under one of the external 383 # (third party) paths, use a fake library reference to 384 # there. 385 for d in self._external_paths: 386 if dir.startswith('%s/' % d): 387 candidates = [self._get_external_library(dir, name, 388 force_static)] 389 break 390 391 if not candidates: 392 raise SandboxValidationError( 393 '%s contains "%s", but there is no "%s" %s in %s.' 394 % (variable, path, name, 395 self.LIBRARY_NAME_VAR[obj.KIND], dir), context) 396 397 if len(candidates) > 1: 398 # If there's more than one remaining candidate, it could be 399 # that there are instances for the same library, in static and 400 # shared form. 401 libs = {} 402 for l in candidates: 403 key = mozpath.join(l.relobjdir, l.basename) 404 if force_static: 405 if isinstance(l, StaticLibrary): 406 libs[key] = l 407 else: 408 if key in libs and isinstance(l, SharedLibrary): 409 libs[key] = l 410 if key not in libs: 411 libs[key] = l 412 candidates = libs.values() 413 if force_static and not candidates: 414 if dir: 415 raise SandboxValidationError( 416 '%s contains "static:%s", but there is no static ' 417 '"%s" %s in %s.' % (variable, path, name, 418 self.LIBRARY_NAME_VAR[obj.KIND], dir), context) 419 raise SandboxValidationError( 420 '%s contains "static:%s", but there is no static "%s" ' 421 '%s in the tree' % (variable, name, name, 422 self.LIBRARY_NAME_VAR[obj.KIND]), context) 423 424 if not candidates: 425 raise SandboxValidationError( 426 '%s contains "%s", which does not match any %s in the tree.' 427 % (variable, path, self.LIBRARY_NAME_VAR[obj.KIND]), 428 context) 429 430 elif len(candidates) > 1: 431 paths = (mozpath.join(l.relsrcdir, 'moz.build') 432 for l in candidates) 433 raise SandboxValidationError( 434 '%s contains "%s", which matches a %s defined in multiple ' 435 'places:\n %s' % (variable, path, 436 self.LIBRARY_NAME_VAR[obj.KIND], 437 '\n '.join(paths)), context) 438 439 elif force_static and not isinstance(candidates[0], StaticLibrary): 440 raise SandboxValidationError( 441 '%s contains "static:%s", but there is only a shared "%s" ' 442 'in %s. You may want to add FORCE_STATIC_LIB=True in ' 443 '%s/moz.build, or remove "static:".' % (variable, path, 444 name, candidates[0].relobjdir, candidates[0].relobjdir), 445 context) 446 447 elif isinstance(obj, StaticLibrary) and isinstance(candidates[0], 448 SharedLibrary): 449 self._static_linking_shared.add(obj) 450 obj.link_library(candidates[0]) 451 452 @memoize 453 def _get_external_library(self, dir, name, force_static): 454 # Create ExternalStaticLibrary or ExternalSharedLibrary object with a 455 # context more or less truthful about where the external library is. 456 context = Context(config=self.config) 457 context.add_source(mozpath.join(self.config.topsrcdir, dir, 'dummy')) 458 if force_static: 459 return ExternalStaticLibrary(context, name) 460 else: 461 return ExternalSharedLibrary(context, name) 462 463 def _parse_cargo_file(self, context): 464 """Parse the Cargo.toml file in context and return a Python object 465 representation of it. Raise a SandboxValidationError if the Cargo.toml 466 file does not exist. Return a tuple of (config, cargo_file).""" 467 cargo_file = mozpath.join(context.srcdir, 'Cargo.toml') 468 if not os.path.exists(cargo_file): 469 raise SandboxValidationError( 470 'No Cargo.toml file found in %s' % cargo_file, context) 471 with open(cargo_file, 'r') as f: 472 return pytoml.load(f), cargo_file 473 474 def _verify_deps(self, context, crate_dir, crate_name, dependencies, description='Dependency'): 475 """Verify that a crate's dependencies all specify local paths.""" 476 for dep_crate_name, values in dependencies.iteritems(): 477 # A simple version number. 478 if isinstance(values, (str, unicode)): 479 raise SandboxValidationError( 480 '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), 481 context) 482 483 dep_path = values.get('path', None) 484 if not dep_path: 485 raise SandboxValidationError( 486 '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), 487 context) 488 489 # Try to catch the case where somebody listed a 490 # local path for development. 491 if os.path.isabs(dep_path): 492 raise SandboxValidationError( 493 '%s %s of crate %s has a non-relative path' % (description, dep_crate_name, crate_name), 494 context) 495 496 if not os.path.exists(mozpath.join(context.config.topsrcdir, crate_dir, dep_path)): 497 raise SandboxValidationError( 498 '%s %s of crate %s refers to a non-existent path' % (description, dep_crate_name, crate_name), 499 context) 500 501 def _rust_library(self, context, libname, static_args, cls=RustLibrary): 502 # We need to note any Rust library for linking purposes. 503 config, cargo_file = self._parse_cargo_file(context) 504 crate_name = config['package']['name'] 505 506 if crate_name != libname: 507 raise SandboxValidationError( 508 'library %s does not match Cargo.toml-defined package %s' % (libname, crate_name), 509 context) 510 511 # Check that the [lib.crate-type] field is correct 512 lib_section = config.get('lib', None) 513 if not lib_section: 514 raise SandboxValidationError( 515 'Cargo.toml for %s has no [lib] section' % libname, 516 context) 517 518 crate_type = lib_section.get('crate-type', None) 519 if not crate_type: 520 raise SandboxValidationError( 521 'Can\'t determine a crate-type for %s from Cargo.toml' % libname, 522 context) 523 524 crate_type = crate_type[0] 525 if crate_type != 'staticlib': 526 raise SandboxValidationError( 527 'crate-type %s is not permitted for %s' % (crate_type, libname), 528 context) 529 530 cargo_target_dir = context.get('RUST_LIBRARY_TARGET_DIR', '.') 531 532 dependencies = set(config.get('dependencies', {}).iterkeys()) 533 534 features = context.get(cls.FEATURES_VAR, []) 535 unique_features = set(features) 536 if len(features) != len(unique_features): 537 raise SandboxValidationError( 538 'features for %s should not contain duplicates: %s' % (libname, features), 539 context) 540 541 return cls(context, libname, cargo_file, crate_type, dependencies, 542 features, cargo_target_dir, **static_args) 543 544 545 def _handle_gn_dirs(self, context): 546 for target_dir in context.get('GN_DIRS', []): 547 context['DIRS'] += [target_dir] 548 gn_dir = context['GN_DIRS'][target_dir] 549 for v in ('variables',): 550 if not getattr(gn_dir, 'variables'): 551 raise SandboxValidationError('Missing value for ' 552 'GN_DIRS["%s"].%s' % (target_dir, v), context) 553 554 non_unified_sources = set() 555 for s in gn_dir.non_unified_sources: 556 source = SourcePath(context, s) 557 if not os.path.exists(source.full_path): 558 raise SandboxValidationError('Cannot find %s.' % source, 559 context) 560 non_unified_sources.add(mozpath.join(context.relsrcdir, s)) 561 562 yield GnProjectData(context, target_dir, gn_dir, non_unified_sources) 563 564 565 def _handle_linkables(self, context, passthru, generated_files): 566 linkables = [] 567 host_linkables = [] 568 def add_program(prog, var): 569 if var.startswith('HOST_'): 570 host_linkables.append(prog) 571 else: 572 linkables.append(prog) 573 574 def check_unique_binary(program, kind): 575 if program in self._binaries: 576 raise SandboxValidationError( 577 'Cannot use "%s" as %s name, ' 578 'because it is already used in %s' % (program, kind, 579 self._binaries[program].relsrcdir), context) 580 for kind, cls in [('PROGRAM', Program), ('HOST_PROGRAM', HostProgram)]: 581 program = context.get(kind) 582 if program: 583 check_unique_binary(program, kind) 584 self._binaries[program] = cls(context, program) 585 self._linkage.append((context, self._binaries[program], 586 kind.replace('PROGRAM', 'USE_LIBS'))) 587 add_program(self._binaries[program], kind) 588 589 all_rust_programs = [] 590 for kind, cls in [('RUST_PROGRAMS', RustProgram), 591 ('HOST_RUST_PROGRAMS', HostRustProgram)]: 592 programs = context[kind] 593 if not programs: 594 continue 595 596 if kind == 'RUST_PROGRAMS': 597 self._rust_compile_dirs.add(context.objdir) 598 all_rust_programs.append((programs, kind, cls)) 599 600 # Verify Rust program definitions. 601 if all_rust_programs: 602 config, cargo_file = self._parse_cargo_file(context); 603 bin_section = config.get('bin', None) 604 if not bin_section: 605 raise SandboxValidationError( 606 'Cargo.toml in %s has no [bin] section' % context.srcdir, 607 context) 608 609 defined_binaries = {b['name'] for b in bin_section} 610 611 for programs, kind, cls in all_rust_programs: 612 for program in programs: 613 if program not in defined_binaries: 614 raise SandboxValidationError( 615 'Cannot find Cargo.toml definition for %s' % program, 616 context) 617 618 check_unique_binary(program, kind) 619 self._binaries[program] = cls(context, program, cargo_file) 620 621 for kind, cls in [ 622 ('SIMPLE_PROGRAMS', SimpleProgram), 623 ('CPP_UNIT_TESTS', SimpleProgram), 624 ('HOST_SIMPLE_PROGRAMS', HostSimpleProgram)]: 625 for program in context[kind]: 626 if program in self._binaries: 627 raise SandboxValidationError( 628 'Cannot use "%s" in %s, ' 629 'because it is already used in %s' % (program, kind, 630 self._binaries[program].relsrcdir), context) 631 self._binaries[program] = cls(context, program, 632 is_unit_test=kind == 'CPP_UNIT_TESTS') 633 self._linkage.append((context, self._binaries[program], 634 'HOST_USE_LIBS' if kind == 'HOST_SIMPLE_PROGRAMS' 635 else 'USE_LIBS')) 636 add_program(self._binaries[program], kind) 637 638 host_libname = context.get('HOST_LIBRARY_NAME') 639 libname = context.get('LIBRARY_NAME') 640 641 if host_libname: 642 if host_libname == libname: 643 raise SandboxValidationError('LIBRARY_NAME and ' 644 'HOST_LIBRARY_NAME must have a different value', context) 645 646 is_rust_library = context.get('IS_RUST_LIBRARY') 647 if is_rust_library: 648 lib = self._rust_library(context, host_libname, {}, cls=HostRustLibrary) 649 else: 650 lib = HostLibrary(context, host_libname) 651 self._libs[host_libname].append(lib) 652 self._linkage.append((context, lib, 'HOST_USE_LIBS')) 653 host_linkables.append(lib) 654 655 final_lib = context.get('FINAL_LIBRARY') 656 if not libname and final_lib: 657 # If no LIBRARY_NAME is given, create one. 658 libname = context.relsrcdir.replace('/', '_') 659 660 static_lib = context.get('FORCE_STATIC_LIB') 661 shared_lib = context.get('FORCE_SHARED_LIB') 662 663 static_name = context.get('STATIC_LIBRARY_NAME') 664 shared_name = context.get('SHARED_LIBRARY_NAME') 665 666 is_framework = context.get('IS_FRAMEWORK') 667 668 soname = context.get('SONAME') 669 670 lib_defines = context.get('LIBRARY_DEFINES') 671 672 shared_args = {} 673 static_args = {} 674 675 if final_lib: 676 if static_lib: 677 raise SandboxValidationError( 678 'FINAL_LIBRARY implies FORCE_STATIC_LIB. ' 679 'Please remove the latter.', context) 680 if shared_lib: 681 raise SandboxValidationError( 682 'FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. ' 683 'Please remove one.', context) 684 if is_framework: 685 raise SandboxValidationError( 686 'FINAL_LIBRARY conflicts with IS_FRAMEWORK. ' 687 'Please remove one.', context) 688 static_args['link_into'] = final_lib 689 static_lib = True 690 691 if libname: 692 if is_framework: 693 if soname: 694 raise SandboxValidationError( 695 'IS_FRAMEWORK conflicts with SONAME. ' 696 'Please remove one.', context) 697 shared_lib = True 698 shared_args['variant'] = SharedLibrary.FRAMEWORK 699 700 if not static_lib and not shared_lib: 701 static_lib = True 702 703 if static_name: 704 if not static_lib: 705 raise SandboxValidationError( 706 'STATIC_LIBRARY_NAME requires FORCE_STATIC_LIB', 707 context) 708 static_args['real_name'] = static_name 709 710 if shared_name: 711 if not shared_lib: 712 raise SandboxValidationError( 713 'SHARED_LIBRARY_NAME requires FORCE_SHARED_LIB', 714 context) 715 shared_args['real_name'] = shared_name 716 717 if soname: 718 if not shared_lib: 719 raise SandboxValidationError( 720 'SONAME requires FORCE_SHARED_LIB', context) 721 shared_args['soname'] = soname 722 723 if context.get('NO_EXPAND_LIBS'): 724 if not static_lib: 725 raise SandboxValidationError( 726 'NO_EXPAND_LIBS can only be set for static libraries.', 727 context) 728 static_args['no_expand_lib'] = True 729 730 if shared_lib and static_lib: 731 if not static_name and not shared_name: 732 raise SandboxValidationError( 733 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' 734 'but neither STATIC_LIBRARY_NAME or ' 735 'SHARED_LIBRARY_NAME is set. At least one is required.', 736 context) 737 if static_name and not shared_name and static_name == libname: 738 raise SandboxValidationError( 739 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' 740 'but STATIC_LIBRARY_NAME is the same as LIBRARY_NAME, ' 741 'and SHARED_LIBRARY_NAME is unset. Please either ' 742 'change STATIC_LIBRARY_NAME or LIBRARY_NAME, or set ' 743 'SHARED_LIBRARY_NAME.', context) 744 if shared_name and not static_name and shared_name == libname: 745 raise SandboxValidationError( 746 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' 747 'but SHARED_LIBRARY_NAME is the same as LIBRARY_NAME, ' 748 'and STATIC_LIBRARY_NAME is unset. Please either ' 749 'change SHARED_LIBRARY_NAME or LIBRARY_NAME, or set ' 750 'STATIC_LIBRARY_NAME.', context) 751 if shared_name and static_name and shared_name == static_name: 752 raise SandboxValidationError( 753 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' 754 'but SHARED_LIBRARY_NAME is the same as ' 755 'STATIC_LIBRARY_NAME. Please change one of them.', 756 context) 757 758 symbols_file = context.get('SYMBOLS_FILE') 759 if symbols_file: 760 if not shared_lib: 761 raise SandboxValidationError( 762 'SYMBOLS_FILE can only be used with a SHARED_LIBRARY.', 763 context) 764 if context.get('DEFFILE'): 765 raise SandboxValidationError( 766 'SYMBOLS_FILE cannot be used along DEFFILE.', 767 context) 768 if isinstance(symbols_file, SourcePath): 769 if not os.path.exists(symbols_file.full_path): 770 raise SandboxValidationError( 771 'Path specified in SYMBOLS_FILE does not exist: %s ' 772 '(resolved to %s)' % (symbols_file, 773 symbols_file.full_path), context) 774 shared_args['symbols_file'] = True 775 else: 776 if symbols_file.target_basename not in generated_files: 777 raise SandboxValidationError( 778 ('Objdir file specified in SYMBOLS_FILE not in ' + 779 'GENERATED_FILES: %s') % (symbols_file,), context) 780 shared_args['symbols_file'] = symbols_file.target_basename 781 782 if shared_lib: 783 lib = SharedLibrary(context, libname, **shared_args) 784 self._libs[libname].append(lib) 785 self._linkage.append((context, lib, 'USE_LIBS')) 786 linkables.append(lib) 787 generated_files.add(lib.lib_name) 788 if symbols_file and isinstance(symbols_file, SourcePath): 789 script = mozpath.join( 790 mozpath.dirname(mozpath.dirname(__file__)), 791 'action', 'generate_symbols_file.py') 792 defines = () 793 if lib.defines: 794 defines = lib.defines.get_defines() 795 yield GeneratedFile(context, script, 796 'generate_symbols_file', lib.symbols_file, 797 [symbols_file], defines) 798 if static_lib: 799 is_rust_library = context.get('IS_RUST_LIBRARY') 800 if is_rust_library: 801 lib = self._rust_library(context, libname, static_args) 802 else: 803 lib = StaticLibrary(context, libname, **static_args) 804 self._libs[libname].append(lib) 805 self._linkage.append((context, lib, 'USE_LIBS')) 806 linkables.append(lib) 807 808 if lib_defines: 809 if not libname: 810 raise SandboxValidationError('LIBRARY_DEFINES needs a ' 811 'LIBRARY_NAME to take effect', context) 812 lib.lib_defines.update(lib_defines) 813 814 # Only emit sources if we have linkables defined in the same context. 815 # Note the linkables are not emitted in this function, but much later, 816 # after aggregation (because of e.g. USE_LIBS processing). 817 if not (linkables or host_linkables): 818 return 819 820 # Avoid emitting compile flags for directories only containing rust 821 # libraries. Emitted compile flags are only relevant to C/C++ sources 822 # for the time being, however ldflags must be emitted for the benefit 823 # of cargo. 824 if not all(isinstance(l, (RustLibrary)) for l in linkables): 825 self._compile_dirs.add(context.objdir) 826 elif linkables: 827 self._rust_compile_dirs.add(context.objdir) 828 829 if host_linkables and not all(isinstance(l, HostRustLibrary) for l in host_linkables): 830 self._host_compile_dirs.add(context.objdir) 831 # TODO: objdirs with only host things in them shouldn't need target 832 # flags, but there's at least one Makefile.in (in 833 # build/unix/elfhack) that relies on the value of LDFLAGS being 834 # passed to one-off rules. 835 self._compile_dirs.add(context.objdir) 836 837 sources = defaultdict(list) 838 gen_sources = defaultdict(list) 839 all_flags = {} 840 for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'): 841 srcs = sources[symbol] 842 gen_srcs = gen_sources[symbol] 843 context_srcs = context.get(symbol, []) 844 for f in context_srcs: 845 full_path = f.full_path 846 if isinstance(f, SourcePath): 847 srcs.append(full_path) 848 else: 849 assert isinstance(f, Path) 850 gen_srcs.append(full_path) 851 if symbol == 'SOURCES': 852 flags = context_srcs[f] 853 if flags: 854 all_flags[full_path] = flags 855 856 if isinstance(f, SourcePath) and not os.path.exists(full_path): 857 raise SandboxValidationError('File listed in %s does not ' 858 'exist: \'%s\'' % (symbol, full_path), context) 859 860 # HOST_SOURCES and UNIFIED_SOURCES only take SourcePaths, so 861 # there should be no generated source in here 862 assert not gen_sources['HOST_SOURCES'] 863 assert not gen_sources['UNIFIED_SOURCES'] 864 865 no_pgo = context.get('NO_PGO') 866 no_pgo_sources = [f for f, flags in all_flags.iteritems() 867 if flags.no_pgo] 868 if no_pgo: 869 if no_pgo_sources: 870 raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo ' 871 'cannot be set at the same time', context) 872 passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo 873 if no_pgo_sources: 874 passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources 875 876 # A map from "canonical suffixes" for a particular source file 877 # language to the range of suffixes associated with that language. 878 # 879 # We deliberately don't list the canonical suffix in the suffix list 880 # in the definition; we'll add it in programmatically after defining 881 # things. 882 suffix_map = { 883 '.s': set(['.asm']), 884 '.c': set(), 885 '.m': set(), 886 '.mm': set(), 887 '.cpp': set(['.cc', '.cxx']), 888 '.S': set(), 889 } 890 891 # The inverse of the above, mapping suffixes to their canonical suffix. 892 canonicalized_suffix_map = {} 893 for suffix, alternatives in suffix_map.iteritems(): 894 alternatives.add(suffix) 895 for a in alternatives: 896 canonicalized_suffix_map[a] = suffix 897 898 def canonical_suffix_for_file(f): 899 return canonicalized_suffix_map[mozpath.splitext(f)[1]] 900 901 # A map from moz.build variables to the canonical suffixes of file 902 # kinds that can be listed therein. 903 all_suffixes = list(suffix_map.keys()) 904 varmap = dict( 905 SOURCES=(Sources, GeneratedSources, all_suffixes), 906 HOST_SOURCES=(HostSources, None, ['.c', '.mm', '.cpp']), 907 UNIFIED_SOURCES=(UnifiedSources, None, ['.c', '.mm', '.cpp']), 908 ) 909 # Track whether there are any C++ source files. 910 # Technically this won't do the right thing for SIMPLE_PROGRAMS in 911 # a directory with mixed C and C++ source, but it's not that important. 912 cxx_sources = defaultdict(bool) 913 914 # Source files to track for linkables associated with this context. 915 ctxt_sources = defaultdict(lambda: defaultdict(list)) 916 917 for variable, (klass, gen_klass, suffixes) in varmap.items(): 918 allowed_suffixes = set().union(*[suffix_map[s] for s in suffixes]) 919 920 # First ensure that we haven't been given filetypes that we don't 921 # recognize. 922 for f in itertools.chain(sources[variable], gen_sources[variable]): 923 ext = mozpath.splitext(f)[1] 924 if ext not in allowed_suffixes: 925 raise SandboxValidationError( 926 '%s has an unknown file type.' % f, context) 927 928 for srcs, cls in ((sources[variable], klass), 929 (gen_sources[variable], gen_klass)): 930 # Now sort the files to let groupby work. 931 sorted_files = sorted(srcs, key=canonical_suffix_for_file) 932 for canonical_suffix, files in itertools.groupby( 933 sorted_files, canonical_suffix_for_file): 934 if canonical_suffix in ('.cpp', '.mm'): 935 cxx_sources[variable] = True 936 elif canonical_suffix in ('.s', '.S'): 937 self._asm_compile_dirs.add(context.objdir) 938 arglist = [context, list(files), canonical_suffix] 939 if variable.startswith('UNIFIED_'): 940 arglist.append(context.get('FILES_PER_UNIFIED_FILE', 16)) 941 obj = cls(*arglist) 942 srcs = obj.files 943 if isinstance(obj, UnifiedSources) and obj.have_unified_mapping: 944 srcs = dict(obj.unified_source_mapping).keys() 945 ctxt_sources[variable][canonical_suffix] += sorted(srcs) 946 yield obj 947 948 if ctxt_sources: 949 for linkable in linkables: 950 for target_var in ('SOURCES', 'UNIFIED_SOURCES'): 951 for suffix, srcs in ctxt_sources[target_var].items(): 952 linkable.sources[suffix] += srcs 953 for host_linkable in host_linkables: 954 for suffix, srcs in ctxt_sources['HOST_SOURCES'].items(): 955 host_linkable.sources[suffix] += srcs 956 957 for f, flags in all_flags.iteritems(): 958 if flags.flags: 959 ext = mozpath.splitext(f)[1] 960 yield PerSourceFlag(context, f, flags.flags) 961 962 # If there are any C++ sources, set all the linkables defined here 963 # to require the C++ linker. 964 for vars, linkable_items in ((('SOURCES', 'UNIFIED_SOURCES'), linkables), 965 (('HOST_SOURCES',), host_linkables)): 966 for var in vars: 967 if cxx_sources[var]: 968 for l in linkable_items: 969 l.cxx_link = True 970 break 971 972 973 def emit_from_context(self, context): 974 """Convert a Context to tree metadata objects. 975 976 This is a generator of mozbuild.frontend.data.ContextDerived instances. 977 """ 978 979 # We only want to emit an InstallationTarget if one of the consulted 980 # variables is defined. Later on, we look up FINAL_TARGET, which has 981 # the side-effect of populating it. So, we need to do this lookup 982 # early. 983 if any(k in context for k in ('FINAL_TARGET', 'XPI_NAME', 'DIST_SUBDIR')): 984 yield InstallationTarget(context) 985 986 for obj in self._handle_gn_dirs(context): 987 yield obj 988 989 # We always emit a directory traversal descriptor. This is needed by 990 # the recursive make backend. 991 for o in self._emit_directory_traversal_from_context(context): yield o 992 993 for obj in self._process_xpidl(context): 994 yield obj 995 996 computed_flags = ComputedFlags(context, context['COMPILE_FLAGS']) 997 computed_link_flags = ComputedFlags(context, context['LINK_FLAGS']) 998 computed_host_flags = ComputedFlags(context, context['HOST_COMPILE_FLAGS']) 999 computed_as_flags = ComputedFlags(context, context['ASM_FLAGS']) 1000 1001 # Proxy some variables as-is until we have richer classes to represent 1002 # them. We should aim to keep this set small because it violates the 1003 # desired abstraction of the build definition away from makefiles. 1004 passthru = VariablePassthru(context) 1005 varlist = [ 1006 'EXTRA_DSO_LDOPTS', 1007 'RCFILE', 1008 'RESFILE', 1009 'RCINCLUDE', 1010 'DEFFILE', 1011 'WIN32_EXE_LDFLAGS', 1012 'USE_EXTENSION_MANIFEST', 1013 'NO_JS_MANIFEST', 1014 'HAS_MISC_RULE', 1015 ] 1016 for v in varlist: 1017 if v in context and context[v]: 1018 passthru.variables[v] = context[v] 1019 1020 if context.config.substs.get('OS_TARGET') == 'WINNT' and \ 1021 context['DELAYLOAD_DLLS']: 1022 context['LDFLAGS'].extend([('-DELAYLOAD:%s' % dll) 1023 for dll in context['DELAYLOAD_DLLS']]) 1024 context['OS_LIBS'].append('delayimp') 1025 1026 for v in ['CMFLAGS', 'CMMFLAGS']: 1027 if v in context and context[v]: 1028 passthru.variables['MOZBUILD_' + v] = context[v] 1029 1030 for v in ['CXXFLAGS', 'CFLAGS']: 1031 if v in context and context[v]: 1032 computed_flags.resolve_flags('MOZBUILD_%s' % v, context[v]) 1033 1034 for v in ['HOST_CXXFLAGS', 'HOST_CFLAGS']: 1035 if v in context and context[v]: 1036 computed_host_flags.resolve_flags('MOZBUILD_%s' % v, context[v]) 1037 1038 if 'LDFLAGS' in context and context['LDFLAGS']: 1039 computed_link_flags.resolve_flags('MOZBUILD', context['LDFLAGS']) 1040 1041 deffile = context['DEFFILE'] 1042 if deffile and context.config.substs.get('OS_ARCH') == 'WINNT': 1043 if context.config.substs.get('GNU_CC'): 1044 computed_link_flags.resolve_flags('DEFFILE', [deffile]) 1045 else: 1046 computed_link_flags.resolve_flags('DEFFILE', 1047 ['-DEF:' + deffile]) 1048 1049 dist_install = context['DIST_INSTALL'] 1050 if dist_install is True: 1051 passthru.variables['DIST_INSTALL'] = True 1052 elif dist_install is False: 1053 passthru.variables['NO_DIST_INSTALL'] = True 1054 1055 # Ideally, this should be done in templates, but this is difficult at 1056 # the moment because USE_STATIC_LIBS can be set after a template 1057 # returns. Eventually, with context-based templates, it will be 1058 # possible. 1059 if (context.config.substs.get('OS_ARCH') == 'WINNT' and 1060 not context.config.substs.get('GNU_CC')): 1061 use_static_lib = (context.get('USE_STATIC_LIBS') and 1062 not context.config.substs.get('MOZ_ASAN')) 1063 rtl_flag = '-MT' if use_static_lib else '-MD' 1064 if (context.config.substs.get('MOZ_DEBUG') and 1065 not context.config.substs.get('MOZ_NO_DEBUG_RTL')): 1066 rtl_flag += 'd' 1067 computed_flags.resolve_flags('RTL', [rtl_flag]) 1068 if not context.config.substs.get('CROSS_COMPILE'): 1069 computed_host_flags.resolve_flags('RTL', [rtl_flag]) 1070 1071 generated_files = set() 1072 localized_generated_files = set() 1073 for obj in self._process_generated_files(context): 1074 for f in obj.outputs: 1075 generated_files.add(f) 1076 if obj.localized: 1077 localized_generated_files.add(f) 1078 yield obj 1079 1080 for path in context['CONFIGURE_SUBST_FILES']: 1081 sub = self._create_substitution(ConfigFileSubstitution, context, 1082 path) 1083 generated_files.add(str(sub.relpath)) 1084 yield sub 1085 1086 for defines_var, cls, backend_flags in (('DEFINES', Defines, (computed_flags, computed_as_flags)), 1087 ('HOST_DEFINES', HostDefines, (computed_host_flags,))): 1088 defines = context.get(defines_var) 1089 if defines: 1090 defines_obj = cls(context, defines) 1091 if isinstance(defines_obj, Defines): 1092 # DEFINES have consumers outside the compile command line, 1093 # HOST_DEFINES do not. 1094 yield defines_obj 1095 else: 1096 # If we don't have explicitly set defines we need to make sure 1097 # initialized values if present end up in computed flags. 1098 defines_obj = cls(context, context[defines_var]) 1099 1100 defines_from_obj = list(defines_obj.get_defines()) 1101 if defines_from_obj: 1102 for flags in backend_flags: 1103 flags.resolve_flags(defines_var, defines_from_obj) 1104 1105 idl_vars = ( 1106 'GENERATED_EVENTS_WEBIDL_FILES', 1107 'GENERATED_WEBIDL_FILES', 1108 'PREPROCESSED_TEST_WEBIDL_FILES', 1109 'PREPROCESSED_WEBIDL_FILES', 1110 'TEST_WEBIDL_FILES', 1111 'WEBIDL_FILES', 1112 'IPDL_SOURCES', 1113 'PREPROCESSED_IPDL_SOURCES', 1114 ) 1115 for context_var in idl_vars: 1116 for name in context.get(context_var, []): 1117 self._idls[context_var].add(mozpath.join(context.srcdir, name)) 1118 # WEBIDL_EXAMPLE_INTERFACES do not correspond to files. 1119 for name in context.get('WEBIDL_EXAMPLE_INTERFACES', []): 1120 self._idls['WEBIDL_EXAMPLE_INTERFACES'].add(name) 1121 1122 local_includes = [] 1123 for local_include in context.get('LOCAL_INCLUDES', []): 1124 full_path = local_include.full_path 1125 if (not isinstance(local_include, ObjDirPath) and 1126 not os.path.exists(full_path)): 1127 raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' 1128 'does not exist: %s (resolved to %s)' % (local_include, 1129 full_path), context) 1130 if (full_path == context.config.topsrcdir or 1131 full_path == context.config.topobjdir): 1132 raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' 1133 'is not allowed: %s (resolved to %s)' % (local_include, 1134 full_path), context) 1135 include_obj = LocalInclude(context, local_include) 1136 local_includes.append(include_obj.path.full_path) 1137 yield include_obj 1138 1139 computed_flags.resolve_flags('LOCAL_INCLUDES', ['-I%s' % p for p in local_includes]) 1140 computed_as_flags.resolve_flags('LOCAL_INCLUDES', ['-I%s' % p for p in local_includes]) 1141 computed_host_flags.resolve_flags('LOCAL_INCLUDES', ['-I%s' % p for p in local_includes]) 1142 1143 for obj in self._handle_linkables(context, passthru, generated_files): 1144 yield obj 1145 1146 generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in self._binaries.keys()]) 1147 1148 components = [] 1149 for var, cls in ( 1150 ('EXPORTS', Exports), 1151 ('FINAL_TARGET_FILES', FinalTargetFiles), 1152 ('FINAL_TARGET_PP_FILES', FinalTargetPreprocessedFiles), 1153 ('LOCALIZED_FILES', LocalizedFiles), 1154 ('LOCALIZED_PP_FILES', LocalizedPreprocessedFiles), 1155 ('OBJDIR_FILES', ObjdirFiles), 1156 ('OBJDIR_PP_FILES', ObjdirPreprocessedFiles), 1157 ('TEST_HARNESS_FILES', TestHarnessFiles), 1158 ): 1159 all_files = context.get(var) 1160 if not all_files: 1161 continue 1162 if dist_install is False and var != 'TEST_HARNESS_FILES': 1163 raise SandboxValidationError( 1164 '%s cannot be used with DIST_INSTALL = False' % var, 1165 context) 1166 has_prefs = False 1167 has_resources = False 1168 for base, files in all_files.walk(): 1169 if var == 'TEST_HARNESS_FILES' and not base: 1170 raise SandboxValidationError( 1171 'Cannot install files to the root of TEST_HARNESS_FILES', context) 1172 if base == 'components': 1173 components.extend(files) 1174 if base == 'defaults/pref': 1175 has_prefs = True 1176 if mozpath.split(base)[0] == 'res': 1177 has_resources = True 1178 for f in files: 1179 if (var in ('FINAL_TARGET_PP_FILES', 1180 'OBJDIR_PP_FILES', 1181 'LOCALIZED_PP_FILES') and 1182 not isinstance(f, SourcePath)): 1183 raise SandboxValidationError( 1184 ('Only source directory paths allowed in ' + 1185 '%s: %s') 1186 % (var, f,), context) 1187 if var.startswith('LOCALIZED_'): 1188 if isinstance(f, SourcePath): 1189 if f.startswith('en-US/'): 1190 pass 1191 elif '/locales/en-US/' in f: 1192 pass 1193 else: 1194 raise SandboxValidationError( 1195 '%s paths must start with `en-US/` or ' 1196 'contain `/locales/en-US/`: %s' 1197 % (var, f,), context) 1198 1199 if not isinstance(f, ObjDirPath): 1200 path = f.full_path 1201 if '*' not in path and not os.path.exists(path): 1202 raise SandboxValidationError( 1203 'File listed in %s does not exist: %s' 1204 % (var, path), context) 1205 else: 1206 # TODO: Bug 1254682 - The '/' check is to allow 1207 # installing files generated from other directories, 1208 # which is done occasionally for tests. However, it 1209 # means we don't fail early if the file isn't actually 1210 # created by the other moz.build file. 1211 if f.target_basename not in generated_files and '/' not in f: 1212 raise SandboxValidationError( 1213 ('Objdir file listed in %s not in ' + 1214 'GENERATED_FILES: %s') % (var, f), context) 1215 1216 if var.startswith('LOCALIZED_'): 1217 # Further require that LOCALIZED_FILES are from 1218 # LOCALIZED_GENERATED_FILES. 1219 if f.target_basename not in localized_generated_files: 1220 raise SandboxValidationError( 1221 ('Objdir file listed in %s not in ' + 1222 'LOCALIZED_GENERATED_FILES: %s') % (var, f), context) 1223 else: 1224 # Additionally, don't allow LOCALIZED_GENERATED_FILES to be used 1225 # in anything *but* LOCALIZED_FILES. 1226 if f.target_basename in localized_generated_files: 1227 raise SandboxValidationError( 1228 ('Outputs of LOCALIZED_GENERATED_FILES cannot be used in %s: ' + 1229 '%s') % (var, f), context) 1230 1231 # Addons (when XPI_NAME is defined) and Applications (when 1232 # DIST_SUBDIR is defined) use a different preferences directory 1233 # (default/preferences) from the one the GRE uses (defaults/pref). 1234 # Hence, we move the files from the latter to the former in that 1235 # case. 1236 if has_prefs and (context.get('XPI_NAME') or 1237 context.get('DIST_SUBDIR')): 1238 all_files.defaults.preferences += all_files.defaults.pref 1239 del all_files.defaults._children['pref'] 1240 1241 if has_resources and (context.get('DIST_SUBDIR') or 1242 context.get('XPI_NAME')): 1243 raise SandboxValidationError( 1244 'RESOURCES_FILES cannot be used with DIST_SUBDIR or ' 1245 'XPI_NAME.', context) 1246 1247 yield cls(context, all_files) 1248 1249 # Check for manifest declarations in EXTRA_{PP_,}COMPONENTS. 1250 if any(e.endswith('.js') for e in components) and \ 1251 not any(e.endswith('.manifest') for e in components) and \ 1252 not context.get('NO_JS_MANIFEST', False): 1253 raise SandboxValidationError('A .js component was specified in EXTRA_COMPONENTS ' 1254 'or EXTRA_PP_COMPONENTS without a matching ' 1255 '.manifest file. See ' 1256 'https://developer.mozilla.org/en/XPCOM/XPCOM_changes_in_Gecko_2.0 .', 1257 context); 1258 1259 for c in components: 1260 if c.endswith('.manifest'): 1261 yield ChromeManifestEntry(context, 'chrome.manifest', 1262 Manifest('components', 1263 mozpath.basename(c))) 1264 1265 if self.config.substs.get('MOZ_RUST_TESTS', None): 1266 rust_test = context.get('RUST_TEST', None) 1267 if rust_test: 1268 # TODO: more sophisticated checking of the declared name vs. 1269 # contents of the Cargo.toml file. 1270 features = context.get('RUST_TEST_FEATURES', []) 1271 1272 yield RustTest(context, rust_test, features) 1273 1274 for obj in self._process_test_manifests(context): 1275 yield obj 1276 1277 for obj in self._process_jar_manifests(context): 1278 yield obj 1279 1280 for name, jar in context.get('JAVA_JAR_TARGETS', {}).items(): 1281 yield ContextWrapped(context, jar) 1282 1283 computed_as_flags.resolve_flags('MOZBUILD', 1284 context.get('ASFLAGS')) 1285 1286 if context.get('USE_YASM') is True: 1287 yasm = context.config.substs.get('YASM') 1288 if not yasm: 1289 raise SandboxValidationError('yasm is not available', context) 1290 passthru.variables['AS'] = yasm 1291 passthru.variables['AS_DASH_C_FLAG'] = '' 1292 computed_as_flags.resolve_flags('OS', 1293 context.config.substs.get('YASM_ASFLAGS', [])) 1294 1295 1296 if passthru.variables: 1297 yield passthru 1298 1299 if context.objdir in self._compile_dirs: 1300 self._compile_flags[context.objdir] = computed_flags 1301 yield computed_link_flags 1302 elif context.objdir in self._rust_compile_dirs: 1303 yield computed_link_flags 1304 1305 if context.objdir in self._asm_compile_dirs: 1306 self._compile_as_flags[context.objdir] = computed_as_flags 1307 1308 if context.objdir in self._host_compile_dirs: 1309 yield computed_host_flags 1310 1311 1312 def _create_substitution(self, cls, context, path): 1313 sub = cls(context) 1314 sub.input_path = '%s.in' % path.full_path 1315 sub.output_path = path.translated 1316 sub.relpath = path 1317 1318 return sub 1319 1320 def _process_xpidl(self, context): 1321 # XPIDL source files get processed and turned into .h and .xpt files. 1322 # If there are multiple XPIDL files in a directory, they get linked 1323 # together into a final .xpt, which has the name defined by 1324 # XPIDL_MODULE. 1325 xpidl_module = context['XPIDL_MODULE'] 1326 1327 if context['XPIDL_SOURCES'] and not xpidl_module: 1328 raise SandboxValidationError('XPIDL_MODULE must be defined if ' 1329 'XPIDL_SOURCES is defined.', context) 1330 1331 if xpidl_module and not context['XPIDL_SOURCES']: 1332 raise SandboxValidationError('XPIDL_MODULE cannot be defined ' 1333 'unless there are XPIDL_SOURCES', context) 1334 1335 if context['XPIDL_SOURCES'] and context['DIST_INSTALL'] is False: 1336 self.log(logging.WARN, 'mozbuild_warning', dict( 1337 path=context.main_path), 1338 '{path}: DIST_INSTALL = False has no effect on XPIDL_SOURCES.') 1339 1340 for idl in context['XPIDL_SOURCES']: 1341 yield XPIDLFile(context, mozpath.join(context.srcdir, idl), 1342 xpidl_module, add_to_manifest=not context['XPIDL_NO_MANIFEST']) 1343 1344 def _process_generated_files(self, context): 1345 for path in context['CONFIGURE_DEFINE_FILES']: 1346 script = mozpath.join(mozpath.dirname(mozpath.dirname(__file__)), 1347 'action', 'process_define_files.py') 1348 yield GeneratedFile(context, script, 'process_define_file', 1349 unicode(path), 1350 [Path(context, path + '.in')]) 1351 1352 generated_files = context.get('GENERATED_FILES') or [] 1353 localized_generated_files = context.get('LOCALIZED_GENERATED_FILES') or [] 1354 if not (generated_files or localized_generated_files): 1355 return 1356 1357 for (localized, gen) in ((False, generated_files), (True, localized_generated_files)): 1358 for f in gen: 1359 flags = gen[f] 1360 outputs = f 1361 inputs = [] 1362 if flags.script: 1363 method = "main" 1364 script = SourcePath(context, flags.script).full_path 1365 1366 # Deal with cases like "C:\\path\\to\\script.py:function". 1367 if '.py:' in script: 1368 script, method = script.rsplit('.py:', 1) 1369 script += '.py' 1370 1371 if not os.path.exists(script): 1372 raise SandboxValidationError( 1373 'Script for generating %s does not exist: %s' 1374 % (f, script), context) 1375 if os.path.splitext(script)[1] != '.py': 1376 raise SandboxValidationError( 1377 'Script for generating %s does not end in .py: %s' 1378 % (f, script), context) 1379 else: 1380 script = None 1381 method = None 1382 1383 for i in flags.inputs: 1384 p = Path(context, i) 1385 if (isinstance(p, SourcePath) and 1386 not os.path.exists(p.full_path)): 1387 raise SandboxValidationError( 1388 'Input for generating %s does not exist: %s' 1389 % (f, p.full_path), context) 1390 inputs.append(p) 1391 1392 yield GeneratedFile(context, script, method, outputs, inputs, 1393 flags.flags, localized=localized) 1394 1395 def _process_test_manifests(self, context): 1396 for prefix, info in TEST_MANIFESTS.items(): 1397 for path, manifest in context.get('%s_MANIFESTS' % prefix, []): 1398 for obj in self._process_test_manifest(context, info, path, manifest): 1399 yield obj 1400 1401 for flavor in REFTEST_FLAVORS: 1402 for path, manifest in context.get('%s_MANIFESTS' % flavor.upper(), []): 1403 for obj in self._process_reftest_manifest(context, flavor, path, manifest): 1404 yield obj 1405 1406 for flavor in WEB_PLATFORM_TESTS_FLAVORS: 1407 for path, manifest in context.get("%s_MANIFESTS" % flavor.upper().replace('-', '_'), []): 1408 for obj in self._process_web_platform_tests_manifest(context, path, manifest): 1409 yield obj 1410 1411 def _process_test_manifest(self, context, info, manifest_path, mpmanifest): 1412 flavor, install_root, install_subdir, package_tests = info 1413 1414 path = manifest_path.full_path 1415 manifest_dir = mozpath.dirname(path) 1416 manifest_reldir = mozpath.dirname(mozpath.relpath(path, 1417 context.config.topsrcdir)) 1418 manifest_sources = [mozpath.relpath(pth, context.config.topsrcdir) 1419 for pth in mpmanifest.source_files] 1420 install_prefix = mozpath.join(install_root, install_subdir) 1421 1422 try: 1423 if not mpmanifest.tests: 1424 raise SandboxValidationError('Empty test manifest: %s' 1425 % path, context) 1426 1427 defaults = mpmanifest.manifest_defaults[os.path.normpath(path)] 1428 obj = TestManifest(context, path, mpmanifest, flavor=flavor, 1429 install_prefix=install_prefix, 1430 relpath=mozpath.join(manifest_reldir, mozpath.basename(path)), 1431 sources=manifest_sources, 1432 dupe_manifest='dupe-manifest' in defaults) 1433 1434 filtered = mpmanifest.tests 1435 1436 missing = [t['name'] for t in filtered if not os.path.exists(t['path'])] 1437 if missing: 1438 raise SandboxValidationError('Test manifest (%s) lists ' 1439 'test that does not exist: %s' % ( 1440 path, ', '.join(missing)), context) 1441 1442 out_dir = mozpath.join(install_prefix, manifest_reldir) 1443 if 'install-to-subdir' in defaults: 1444 # This is terrible, but what are you going to do? 1445 out_dir = mozpath.join(out_dir, defaults['install-to-subdir']) 1446 obj.manifest_obj_relpath = mozpath.join(manifest_reldir, 1447 defaults['install-to-subdir'], 1448 mozpath.basename(path)) 1449 1450 def process_support_files(test): 1451 install_info = self._test_files_converter.convert_support_files( 1452 test, install_root, manifest_dir, out_dir) 1453 1454 obj.pattern_installs.extend(install_info.pattern_installs) 1455 for source, dest in install_info.installs: 1456 obj.installs[source] = (dest, False) 1457 obj.external_installs |= install_info.external_installs 1458 for install_path in install_info.deferred_installs: 1459 if all(['*' not in install_path, 1460 not os.path.isfile(mozpath.join(context.config.topsrcdir, 1461 install_path[2:])), 1462 install_path not in install_info.external_installs]): 1463 raise SandboxValidationError('Error processing test ' 1464 'manifest %s: entry in support-files not present ' 1465 'in the srcdir: %s' % (path, install_path), context) 1466 1467 obj.deferred_installs |= install_info.deferred_installs 1468 1469 for test in filtered: 1470 obj.tests.append(test) 1471 1472 # Some test files are compiled and should not be copied into the 1473 # test package. They function as identifiers rather than files. 1474 if package_tests: 1475 manifest_relpath = mozpath.relpath(test['path'], 1476 mozpath.dirname(test['manifest'])) 1477 obj.installs[mozpath.normpath(test['path'])] = \ 1478 ((mozpath.join(out_dir, manifest_relpath)), True) 1479 1480 process_support_files(test) 1481 1482 for path, m_defaults in mpmanifest.manifest_defaults.items(): 1483 process_support_files(m_defaults) 1484 1485 # We also copy manifests into the output directory, 1486 # including manifests from [include:foo] directives. 1487 for mpath in mpmanifest.manifests(): 1488 mpath = mozpath.normpath(mpath) 1489 out_path = mozpath.join(out_dir, mozpath.basename(mpath)) 1490 obj.installs[mpath] = (out_path, False) 1491 1492 # Some manifests reference files that are auto generated as 1493 # part of the build or shouldn't be installed for some 1494 # reason. Here, we prune those files from the install set. 1495 # FUTURE we should be able to detect autogenerated files from 1496 # other build metadata. Once we do that, we can get rid of this. 1497 for f in defaults.get('generated-files', '').split(): 1498 # We re-raise otherwise the stack trace isn't informative. 1499 try: 1500 del obj.installs[mozpath.join(manifest_dir, f)] 1501 except KeyError: 1502 raise SandboxValidationError('Error processing test ' 1503 'manifest %s: entry in generated-files not present ' 1504 'elsewhere in manifest: %s' % (path, f), context) 1505 1506 yield obj 1507 except (AssertionError, Exception): 1508 raise SandboxValidationError('Error processing test ' 1509 'manifest file %s: %s' % (path, 1510 '\n'.join(traceback.format_exception(*sys.exc_info()))), 1511 context) 1512 1513 def _process_reftest_manifest(self, context, flavor, manifest_path, manifest): 1514 manifest_full_path = manifest_path.full_path 1515 manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, 1516 context.config.topsrcdir)) 1517 1518 # reftest manifests don't come from manifest parser. But they are 1519 # similar enough that we can use the same emitted objects. Note 1520 # that we don't perform any installs for reftests. 1521 obj = TestManifest(context, manifest_full_path, manifest, 1522 flavor=flavor, install_prefix='%s/' % flavor, 1523 relpath=mozpath.join(manifest_reldir, 1524 mozpath.basename(manifest_path))) 1525 1526 for test, source_manifest in sorted(manifest.tests): 1527 obj.tests.append({ 1528 'path': test, 1529 'here': mozpath.dirname(test), 1530 'manifest': source_manifest, 1531 'name': mozpath.basename(test), 1532 'head': '', 1533 'support-files': '', 1534 'subsuite': '', 1535 }) 1536 1537 yield obj 1538 1539 def _process_web_platform_tests_manifest(self, context, paths, manifest): 1540 manifest_path, tests_root = paths 1541 manifest_full_path = mozpath.normpath(mozpath.join( 1542 context.srcdir, manifest_path)) 1543 manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, 1544 context.config.topsrcdir)) 1545 tests_root = mozpath.normpath(mozpath.join(context.srcdir, tests_root)) 1546 1547 # Create a equivalent TestManifest object 1548 obj = TestManifest(context, manifest_full_path, manifest, 1549 flavor="web-platform-tests", 1550 relpath=mozpath.join(manifest_reldir, 1551 mozpath.basename(manifest_path)), 1552 install_prefix="web-platform/") 1553 1554 1555 for test_type, path, tests in manifest: 1556 path = mozpath.join(tests_root, path) 1557 if test_type not in ["testharness", "reftest", "wdspec"]: 1558 continue 1559 1560 for test in tests: 1561 obj.tests.append({ 1562 'path': path, 1563 'here': mozpath.dirname(path), 1564 'manifest': manifest_path, 1565 'name': test.id, 1566 'head': '', 1567 'support-files': '', 1568 'subsuite': '', 1569 }) 1570 1571 yield obj 1572 1573 def _process_jar_manifests(self, context): 1574 jar_manifests = context.get('JAR_MANIFESTS', []) 1575 if len(jar_manifests) > 1: 1576 raise SandboxValidationError('While JAR_MANIFESTS is a list, ' 1577 'it is currently limited to one value.', context) 1578 1579 for path in jar_manifests: 1580 yield JARManifest(context, path) 1581 1582 # Temporary test to look for jar.mn files that creep in without using 1583 # the new declaration. Before, we didn't require jar.mn files to 1584 # declared anywhere (they were discovered). This will detect people 1585 # relying on the old behavior. 1586 if os.path.exists(os.path.join(context.srcdir, 'jar.mn')): 1587 if 'jar.mn' not in jar_manifests: 1588 raise SandboxValidationError('A jar.mn exists but it ' 1589 'is not referenced in the moz.build file. ' 1590 'Please define JAR_MANIFESTS.', context) 1591 1592 def _emit_directory_traversal_from_context(self, context): 1593 o = DirectoryTraversal(context) 1594 o.dirs = context.get('DIRS', []) 1595 1596 # Some paths have a subconfigure, yet also have a moz.build. Those 1597 # shouldn't end up in self._external_paths. 1598 if o.objdir: 1599 self._external_paths -= { o.relobjdir } 1600 1601 yield o 1602