1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7import itertools 8import logging 9import os 10import six 11import sys 12import time 13import traceback 14 15from collections import defaultdict, OrderedDict 16from mach.mixin.logging import LoggingMixin 17from mozbuild.util import memoize, OrderedDefaultDict 18 19import mozpack.path as mozpath 20import mozinfo 21import pytoml 22 23from .data import ( 24 BaseRustProgram, 25 ChromeManifestEntry, 26 ComputedFlags, 27 ConfigFileSubstitution, 28 Defines, 29 DirectoryTraversal, 30 Exports, 31 FinalTargetFiles, 32 FinalTargetPreprocessedFiles, 33 GeneratedFile, 34 GeneratedSources, 35 GnProjectData, 36 ExternalStaticLibrary, 37 ExternalSharedLibrary, 38 HostDefines, 39 HostGeneratedSources, 40 HostLibrary, 41 HostProgram, 42 HostRustProgram, 43 HostSharedLibrary, 44 HostSimpleProgram, 45 HostSources, 46 InstallationTarget, 47 IPDLCollection, 48 JARManifest, 49 Library, 50 Linkable, 51 LocalInclude, 52 LocalizedFiles, 53 LocalizedPreprocessedFiles, 54 ObjdirFiles, 55 ObjdirPreprocessedFiles, 56 PerSourceFlag, 57 WebIDLCollection, 58 Program, 59 RustLibrary, 60 HostRustLibrary, 61 RustProgram, 62 RustTests, 63 SandboxedWasmLibrary, 64 SharedLibrary, 65 SimpleProgram, 66 Sources, 67 StaticLibrary, 68 TestHarnessFiles, 69 TestManifest, 70 UnifiedSources, 71 VariablePassthru, 72 WasmDefines, 73 WasmGeneratedSources, 74 WasmSources, 75 XPCOMComponentManifests, 76 XPIDLModule, 77) 78from mozpack.chrome.manifest import Manifest 79 80from .reader import SandboxValidationError 81 82from ..testing import TEST_MANIFESTS, REFTEST_FLAVORS, SupportFilesConverter 83 84from .context import Context, SourcePath, ObjDirPath, Path, SubContext 85 86from mozbuild.base import ExecutionSummary 87 88 89class TreeMetadataEmitter(LoggingMixin): 90 """Converts the executed mozbuild files into data structures. 91 92 This is a bridge between reader.py and data.py. It takes what was read by 93 reader.BuildReader and converts it into the classes defined in the data 94 module. 95 """ 96 97 def __init__(self, config): 98 self.populate_logger() 99 100 self.config = config 101 102 mozinfo.find_and_update_from_json(config.topobjdir) 103 104 self.info = dict(mozinfo.info) 105 106 self._libs = OrderedDefaultDict(list) 107 self._binaries = OrderedDict() 108 self._compile_dirs = set() 109 self._host_compile_dirs = set() 110 self._wasm_compile_dirs = set() 111 self._asm_compile_dirs = set() 112 self._compile_flags = dict() 113 self._compile_as_flags = dict() 114 self._linkage = [] 115 self._static_linking_shared = set() 116 self._crate_verified_local = set() 117 self._crate_directories = dict() 118 self._idls = defaultdict(set) 119 120 # Keep track of external paths (third party build systems), starting 121 # from what we run a subconfigure in. We'll eliminate some directories 122 # as we traverse them with moz.build (e.g. js/src). 123 subconfigures = os.path.join(self.config.topobjdir, "subconfigures") 124 paths = [] 125 if os.path.exists(subconfigures): 126 paths = open(subconfigures).read().splitlines() 127 self._external_paths = set(mozpath.normsep(d) for d in paths) 128 129 self._emitter_time = 0.0 130 self._object_count = 0 131 self._test_files_converter = SupportFilesConverter() 132 133 def summary(self): 134 return ExecutionSummary( 135 "Processed into {object_count:d} build config descriptors in " 136 "{execution_time:.2f}s", 137 execution_time=self._emitter_time, 138 object_count=self._object_count, 139 ) 140 141 def emit(self, output, emitfn=None): 142 """Convert the BuildReader output into data structures. 143 144 The return value from BuildReader.read_topsrcdir() (a generator) is 145 typically fed into this function. 146 """ 147 contexts = {} 148 emitfn = emitfn or self.emit_from_context 149 150 def emit_objs(objs): 151 for o in objs: 152 self._object_count += 1 153 yield o 154 155 for out in output: 156 # Nothing in sub-contexts is currently of interest to us. Filter 157 # them all out. 158 if isinstance(out, SubContext): 159 continue 160 161 if isinstance(out, Context): 162 # Keep all contexts around, we will need them later. 163 contexts[os.path.normcase(out.objdir)] = out 164 165 start = time.time() 166 # We need to expand the generator for the timings to work. 167 objs = list(emitfn(out)) 168 self._emitter_time += time.time() - start 169 170 for o in emit_objs(objs): 171 yield o 172 173 else: 174 raise Exception("Unhandled output type: %s" % type(out)) 175 176 # Don't emit Linkable objects when COMPILE_ENVIRONMENT is not set 177 if self.config.substs.get("COMPILE_ENVIRONMENT"): 178 start = time.time() 179 objs = list(self._emit_libs_derived(contexts)) 180 self._emitter_time += time.time() - start 181 182 for o in emit_objs(objs): 183 yield o 184 185 def _emit_libs_derived(self, contexts): 186 187 # First aggregate idl sources. 188 webidl_attrs = [ 189 ("GENERATED_EVENTS_WEBIDL_FILES", lambda c: c.generated_events_sources), 190 ("GENERATED_WEBIDL_FILES", lambda c: c.generated_sources), 191 ("PREPROCESSED_TEST_WEBIDL_FILES", lambda c: c.preprocessed_test_sources), 192 ("PREPROCESSED_WEBIDL_FILES", lambda c: c.preprocessed_sources), 193 ("TEST_WEBIDL_FILES", lambda c: c.test_sources), 194 ("WEBIDL_FILES", lambda c: c.sources), 195 ("WEBIDL_EXAMPLE_INTERFACES", lambda c: c.example_interfaces), 196 ] 197 ipdl_attrs = [ 198 ("IPDL_SOURCES", lambda c: c.sources), 199 ("PREPROCESSED_IPDL_SOURCES", lambda c: c.preprocessed_sources), 200 ] 201 xpcom_attrs = [("XPCOM_MANIFESTS", lambda c: c.manifests)] 202 203 idl_sources = {} 204 for root, cls, attrs in ( 205 (self.config.substs.get("WEBIDL_ROOT"), WebIDLCollection, webidl_attrs), 206 (self.config.substs.get("IPDL_ROOT"), IPDLCollection, ipdl_attrs), 207 ( 208 self.config.substs.get("XPCOM_ROOT"), 209 XPCOMComponentManifests, 210 xpcom_attrs, 211 ), 212 ): 213 if root: 214 collection = cls(contexts[os.path.normcase(root)]) 215 for var, src_getter in attrs: 216 src_getter(collection).update(self._idls[var]) 217 218 idl_sources[root] = collection.all_source_files() 219 if isinstance(collection, WebIDLCollection): 220 # Test webidl sources are added here as a somewhat special 221 # case. 222 idl_sources[mozpath.join(root, "test")] = [ 223 s for s in collection.all_test_cpp_basenames() 224 ] 225 226 yield collection 227 228 # Next do FINAL_LIBRARY linkage. 229 for lib in (l for libs in self._libs.values() for l in libs): 230 if not isinstance(lib, (StaticLibrary, RustLibrary)) or not lib.link_into: 231 continue 232 if lib.link_into not in self._libs: 233 raise SandboxValidationError( 234 'FINAL_LIBRARY ("%s") does not match any LIBRARY_NAME' 235 % lib.link_into, 236 contexts[os.path.normcase(lib.objdir)], 237 ) 238 candidates = self._libs[lib.link_into] 239 240 # When there are multiple candidates, but all are in the same 241 # directory and have a different type, we want all of them to 242 # have the library linked. The typical usecase is when building 243 # both a static and a shared library in a directory, and having 244 # that as a FINAL_LIBRARY. 245 if ( 246 len(set(type(l) for l in candidates)) == len(candidates) 247 and len(set(l.objdir for l in candidates)) == 1 248 ): 249 for c in candidates: 250 c.link_library(lib) 251 else: 252 raise SandboxValidationError( 253 'FINAL_LIBRARY ("%s") matches a LIBRARY_NAME defined in ' 254 "multiple places:\n %s" 255 % (lib.link_into, "\n ".join(l.objdir for l in candidates)), 256 contexts[os.path.normcase(lib.objdir)], 257 ) 258 259 # ...and USE_LIBS linkage. 260 for context, obj, variable in self._linkage: 261 self._link_libraries(context, obj, variable, idl_sources) 262 263 def recurse_refs(lib): 264 for o in lib.refs: 265 yield o 266 if isinstance(o, StaticLibrary): 267 for q in recurse_refs(o): 268 yield q 269 270 # Check that all static libraries refering shared libraries in 271 # USE_LIBS are linked into a shared library or program. 272 for lib in self._static_linking_shared: 273 if all(isinstance(o, StaticLibrary) for o in recurse_refs(lib)): 274 shared_libs = sorted( 275 l.basename 276 for l in lib.linked_libraries 277 if isinstance(l, SharedLibrary) 278 ) 279 raise SandboxValidationError( 280 'The static "%s" library is not used in a shared library ' 281 "or a program, but USE_LIBS contains the following shared " 282 "library names:\n %s\n\nMaybe you can remove the " 283 'static "%s" library?' 284 % (lib.basename, "\n ".join(shared_libs), lib.basename), 285 contexts[os.path.normcase(lib.objdir)], 286 ) 287 288 @memoize 289 def rust_libraries(obj): 290 libs = [] 291 for o in obj.linked_libraries: 292 if isinstance(o, (HostRustLibrary, RustLibrary)): 293 libs.append(o) 294 elif isinstance(o, (HostLibrary, StaticLibrary, SandboxedWasmLibrary)): 295 libs.extend(rust_libraries(o)) 296 return libs 297 298 def check_rust_libraries(obj): 299 rust_libs = set(rust_libraries(obj)) 300 if len(rust_libs) <= 1: 301 return 302 if isinstance(obj, (Library, HostLibrary)): 303 what = '"%s" library' % obj.basename 304 else: 305 what = '"%s" program' % obj.name 306 raise SandboxValidationError( 307 "Cannot link the following Rust libraries into the %s:\n" 308 "%s\nOnly one is allowed." 309 % ( 310 what, 311 "\n".join( 312 " - %s" % r.basename 313 for r in sorted(rust_libs, key=lambda r: r.basename) 314 ), 315 ), 316 contexts[os.path.normcase(obj.objdir)], 317 ) 318 319 # Propagate LIBRARY_DEFINES to all child libraries recursively. 320 def propagate_defines(outerlib, defines): 321 outerlib.lib_defines.update(defines) 322 for lib in outerlib.linked_libraries: 323 # Propagate defines only along FINAL_LIBRARY paths, not USE_LIBS 324 # paths. 325 if ( 326 isinstance(lib, StaticLibrary) 327 and lib.link_into == outerlib.basename 328 ): 329 propagate_defines(lib, defines) 330 331 for lib in (l for libs in self._libs.values() for l in libs): 332 if isinstance(lib, Library): 333 propagate_defines(lib, lib.lib_defines) 334 check_rust_libraries(lib) 335 yield lib 336 337 for lib in (l for libs in self._libs.values() for l in libs): 338 lib_defines = list(lib.lib_defines.get_defines()) 339 if lib_defines: 340 objdir_flags = self._compile_flags[lib.objdir] 341 objdir_flags.resolve_flags("LIBRARY_DEFINES", lib_defines) 342 343 objdir_flags = self._compile_as_flags.get(lib.objdir) 344 if objdir_flags: 345 objdir_flags.resolve_flags("LIBRARY_DEFINES", lib_defines) 346 347 for flags_obj in self._compile_flags.values(): 348 yield flags_obj 349 350 for flags_obj in self._compile_as_flags.values(): 351 yield flags_obj 352 353 for obj in self._binaries.values(): 354 if isinstance(obj, Linkable): 355 check_rust_libraries(obj) 356 yield obj 357 358 LIBRARY_NAME_VAR = { 359 "host": "HOST_LIBRARY_NAME", 360 "target": "LIBRARY_NAME", 361 "wasm": "SANDBOXED_WASM_LIBRARY_NAME", 362 } 363 364 LIBSTDCXX_VAR = { 365 "host": "MOZ_LIBSTDCXX_HOST_VERSION", 366 "target": "MOZ_LIBSTDCXX_TARGET_VERSION", 367 "wasm": "MOZ_LIBSTDCXX_TARGET_VERSION", 368 } 369 370 STDCXXCOMPAT_NAME = { 371 "host": "host_stdc++compat", 372 "target": "stdc++compat", 373 "wasm": "stdc++compat", 374 } 375 376 def _link_libraries(self, context, obj, variable, extra_sources): 377 """Add linkage declarations to a given object.""" 378 assert isinstance(obj, Linkable) 379 380 if context.objdir in extra_sources: 381 # All "extra sources" are .cpp for the moment, and happen to come 382 # first in order. 383 obj.sources[".cpp"] = extra_sources[context.objdir] + obj.sources[".cpp"] 384 385 for path in context.get(variable, []): 386 self._link_library(context, obj, variable, path) 387 388 # Link system libraries from OS_LIBS/HOST_OS_LIBS. 389 for lib in context.get(variable.replace("USE", "OS"), []): 390 obj.link_system_library(lib) 391 392 # We have to wait for all the self._link_library calls above to have 393 # happened for obj.cxx_link to be final. 394 # FIXME: Theoretically, HostSharedLibrary shouldn't be here (bug 395 # 1474022). 396 if ( 397 not isinstance( 398 obj, (StaticLibrary, HostLibrary, HostSharedLibrary, BaseRustProgram) 399 ) 400 and obj.cxx_link 401 ): 402 if context.config.substs.get(self.LIBSTDCXX_VAR[obj.KIND]): 403 self._link_library( 404 context, obj, variable, self.STDCXXCOMPAT_NAME[obj.KIND] 405 ) 406 if obj.KIND == "target": 407 for lib in context.config.substs.get("STLPORT_LIBS", []): 408 obj.link_system_library(lib) 409 410 def _link_library(self, context, obj, variable, path): 411 force_static = path.startswith("static:") and obj.KIND == "target" 412 if force_static: 413 path = path[7:] 414 name = mozpath.basename(path) 415 dir = mozpath.dirname(path) 416 candidates = [l for l in self._libs[name] if l.KIND == obj.KIND] 417 if dir: 418 if dir.startswith("/"): 419 dir = mozpath.normpath(mozpath.join(obj.topobjdir, dir[1:])) 420 else: 421 dir = mozpath.normpath(mozpath.join(obj.objdir, dir)) 422 dir = mozpath.relpath(dir, obj.topobjdir) 423 candidates = [l for l in candidates if l.relobjdir == dir] 424 if not candidates: 425 # If the given directory is under one of the external 426 # (third party) paths, use a fake library reference to 427 # there. 428 for d in self._external_paths: 429 if dir.startswith("%s/" % d): 430 candidates = [ 431 self._get_external_library(dir, name, force_static) 432 ] 433 break 434 435 if not candidates: 436 raise SandboxValidationError( 437 '%s contains "%s", but there is no "%s" %s in %s.' 438 % (variable, path, name, self.LIBRARY_NAME_VAR[obj.KIND], dir), 439 context, 440 ) 441 442 if len(candidates) > 1: 443 # If there's more than one remaining candidate, it could be 444 # that there are instances for the same library, in static and 445 # shared form. 446 libs = {} 447 for l in candidates: 448 key = mozpath.join(l.relobjdir, l.basename) 449 if force_static: 450 if isinstance(l, StaticLibrary): 451 libs[key] = l 452 else: 453 if key in libs and isinstance(l, SharedLibrary): 454 libs[key] = l 455 if key not in libs: 456 libs[key] = l 457 candidates = list(libs.values()) 458 if force_static and not candidates: 459 if dir: 460 raise SandboxValidationError( 461 '%s contains "static:%s", but there is no static ' 462 '"%s" %s in %s.' 463 % (variable, path, name, self.LIBRARY_NAME_VAR[obj.KIND], dir), 464 context, 465 ) 466 raise SandboxValidationError( 467 '%s contains "static:%s", but there is no static "%s" ' 468 "%s in the tree" 469 % (variable, name, name, self.LIBRARY_NAME_VAR[obj.KIND]), 470 context, 471 ) 472 473 if not candidates: 474 raise SandboxValidationError( 475 '%s contains "%s", which does not match any %s in the tree.' 476 % (variable, path, self.LIBRARY_NAME_VAR[obj.KIND]), 477 context, 478 ) 479 480 elif len(candidates) > 1: 481 paths = (mozpath.join(l.relsrcdir, "moz.build") for l in candidates) 482 raise SandboxValidationError( 483 '%s contains "%s", which matches a %s defined in multiple ' 484 "places:\n %s" 485 % ( 486 variable, 487 path, 488 self.LIBRARY_NAME_VAR[obj.KIND], 489 "\n ".join(paths), 490 ), 491 context, 492 ) 493 494 elif force_static and not isinstance(candidates[0], StaticLibrary): 495 raise SandboxValidationError( 496 '%s contains "static:%s", but there is only a shared "%s" ' 497 "in %s. You may want to add FORCE_STATIC_LIB=True in " 498 '%s/moz.build, or remove "static:".' 499 % ( 500 variable, 501 path, 502 name, 503 candidates[0].relobjdir, 504 candidates[0].relobjdir, 505 ), 506 context, 507 ) 508 509 elif isinstance(obj, StaticLibrary) and isinstance( 510 candidates[0], SharedLibrary 511 ): 512 self._static_linking_shared.add(obj) 513 obj.link_library(candidates[0]) 514 515 @memoize 516 def _get_external_library(self, dir, name, force_static): 517 # Create ExternalStaticLibrary or ExternalSharedLibrary object with a 518 # context more or less truthful about where the external library is. 519 context = Context(config=self.config) 520 context.add_source(mozpath.join(self.config.topsrcdir, dir, "dummy")) 521 if force_static: 522 return ExternalStaticLibrary(context, name) 523 else: 524 return ExternalSharedLibrary(context, name) 525 526 def _parse_cargo_file(self, context): 527 """Parse the Cargo.toml file in context and return a Python object 528 representation of it. Raise a SandboxValidationError if the Cargo.toml 529 file does not exist. Return a tuple of (config, cargo_file).""" 530 cargo_file = mozpath.join(context.srcdir, "Cargo.toml") 531 if not os.path.exists(cargo_file): 532 raise SandboxValidationError( 533 "No Cargo.toml file found in %s" % cargo_file, context 534 ) 535 with open(cargo_file, "r") as f: 536 return pytoml.load(f), cargo_file 537 538 def _verify_deps( 539 self, context, crate_dir, crate_name, dependencies, description="Dependency" 540 ): 541 """Verify that a crate's dependencies all specify local paths.""" 542 for dep_crate_name, values in six.iteritems(dependencies): 543 # A simple version number. 544 if isinstance(values, (six.binary_type, six.text_type)): 545 raise SandboxValidationError( 546 "%s %s of crate %s does not list a path" 547 % (description, dep_crate_name, crate_name), 548 context, 549 ) 550 551 dep_path = values.get("path", None) 552 if not dep_path: 553 raise SandboxValidationError( 554 "%s %s of crate %s does not list a path" 555 % (description, dep_crate_name, crate_name), 556 context, 557 ) 558 559 # Try to catch the case where somebody listed a 560 # local path for development. 561 if os.path.isabs(dep_path): 562 raise SandboxValidationError( 563 "%s %s of crate %s has a non-relative path" 564 % (description, dep_crate_name, crate_name), 565 context, 566 ) 567 568 if not os.path.exists( 569 mozpath.join(context.config.topsrcdir, crate_dir, dep_path) 570 ): 571 raise SandboxValidationError( 572 "%s %s of crate %s refers to a non-existent path" 573 % (description, dep_crate_name, crate_name), 574 context, 575 ) 576 577 def _rust_library( 578 self, context, libname, static_args, is_gkrust=False, cls=RustLibrary 579 ): 580 # We need to note any Rust library for linking purposes. 581 config, cargo_file = self._parse_cargo_file(context) 582 crate_name = config["package"]["name"] 583 584 if crate_name != libname: 585 raise SandboxValidationError( 586 "library %s does not match Cargo.toml-defined package %s" 587 % (libname, crate_name), 588 context, 589 ) 590 591 # Check that the [lib.crate-type] field is correct 592 lib_section = config.get("lib", None) 593 if not lib_section: 594 raise SandboxValidationError( 595 "Cargo.toml for %s has no [lib] section" % libname, context 596 ) 597 598 crate_type = lib_section.get("crate-type", None) 599 if not crate_type: 600 raise SandboxValidationError( 601 "Can't determine a crate-type for %s from Cargo.toml" % libname, context 602 ) 603 604 crate_type = crate_type[0] 605 if crate_type != "staticlib": 606 raise SandboxValidationError( 607 "crate-type %s is not permitted for %s" % (crate_type, libname), context 608 ) 609 610 dependencies = set(six.iterkeys(config.get("dependencies", {}))) 611 612 features = context.get(cls.FEATURES_VAR, []) 613 unique_features = set(features) 614 if len(features) != len(unique_features): 615 raise SandboxValidationError( 616 "features for %s should not contain duplicates: %s" 617 % (libname, features), 618 context, 619 ) 620 621 return cls( 622 context, 623 libname, 624 cargo_file, 625 crate_type, 626 dependencies, 627 features, 628 is_gkrust, 629 **static_args 630 ) 631 632 def _handle_gn_dirs(self, context): 633 for target_dir in context.get("GN_DIRS", []): 634 context["DIRS"] += [target_dir] 635 gn_dir = context["GN_DIRS"][target_dir] 636 for v in ("variables",): 637 if not getattr(gn_dir, "variables"): 638 raise SandboxValidationError( 639 "Missing value for " 'GN_DIRS["%s"].%s' % (target_dir, v), 640 context, 641 ) 642 643 non_unified_sources = set() 644 for s in gn_dir.non_unified_sources: 645 source = SourcePath(context, s) 646 if not os.path.exists(source.full_path): 647 raise SandboxValidationError("Cannot find %s." % source, context) 648 non_unified_sources.add(mozpath.join(context.relsrcdir, s)) 649 650 yield GnProjectData(context, target_dir, gn_dir, non_unified_sources) 651 652 def _handle_linkables(self, context, passthru, generated_files): 653 linkables = [] 654 host_linkables = [] 655 wasm_linkables = [] 656 657 def add_program(prog, var): 658 if var.startswith("HOST_"): 659 host_linkables.append(prog) 660 else: 661 linkables.append(prog) 662 663 def check_unique_binary(program, kind): 664 if program in self._binaries: 665 raise SandboxValidationError( 666 'Cannot use "%s" as %s name, ' 667 "because it is already used in %s" 668 % (program, kind, self._binaries[program].relsrcdir), 669 context, 670 ) 671 672 for kind, cls in [("PROGRAM", Program), ("HOST_PROGRAM", HostProgram)]: 673 program = context.get(kind) 674 if program: 675 check_unique_binary(program, kind) 676 self._binaries[program] = cls(context, program) 677 self._linkage.append( 678 ( 679 context, 680 self._binaries[program], 681 kind.replace("PROGRAM", "USE_LIBS"), 682 ) 683 ) 684 add_program(self._binaries[program], kind) 685 686 all_rust_programs = [] 687 for kind, cls in [ 688 ("RUST_PROGRAMS", RustProgram), 689 ("HOST_RUST_PROGRAMS", HostRustProgram), 690 ]: 691 programs = context[kind] 692 if not programs: 693 continue 694 695 all_rust_programs.append((programs, kind, cls)) 696 697 # Verify Rust program definitions. 698 if all_rust_programs: 699 config, cargo_file = self._parse_cargo_file(context) 700 bin_section = config.get("bin", None) 701 if not bin_section: 702 raise SandboxValidationError( 703 "Cargo.toml in %s has no [bin] section" % context.srcdir, context 704 ) 705 706 defined_binaries = {b["name"] for b in bin_section} 707 708 for programs, kind, cls in all_rust_programs: 709 for program in programs: 710 if program not in defined_binaries: 711 raise SandboxValidationError( 712 "Cannot find Cargo.toml definition for %s" % program, 713 context, 714 ) 715 716 check_unique_binary(program, kind) 717 self._binaries[program] = cls(context, program, cargo_file) 718 add_program(self._binaries[program], kind) 719 720 for kind, cls in [ 721 ("SIMPLE_PROGRAMS", SimpleProgram), 722 ("CPP_UNIT_TESTS", SimpleProgram), 723 ("HOST_SIMPLE_PROGRAMS", HostSimpleProgram), 724 ]: 725 for program in context[kind]: 726 if program in self._binaries: 727 raise SandboxValidationError( 728 'Cannot use "%s" in %s, ' 729 "because it is already used in %s" 730 % (program, kind, self._binaries[program].relsrcdir), 731 context, 732 ) 733 self._binaries[program] = cls( 734 context, program, is_unit_test=kind == "CPP_UNIT_TESTS" 735 ) 736 self._linkage.append( 737 ( 738 context, 739 self._binaries[program], 740 "HOST_USE_LIBS" 741 if kind == "HOST_SIMPLE_PROGRAMS" 742 else "USE_LIBS", 743 ) 744 ) 745 add_program(self._binaries[program], kind) 746 747 host_libname = context.get("HOST_LIBRARY_NAME") 748 libname = context.get("LIBRARY_NAME") 749 750 if host_libname: 751 if host_libname == libname: 752 raise SandboxValidationError( 753 "LIBRARY_NAME and HOST_LIBRARY_NAME must have a different value", 754 context, 755 ) 756 757 is_rust_library = context.get("IS_RUST_LIBRARY") 758 if is_rust_library: 759 lib = self._rust_library(context, host_libname, {}, cls=HostRustLibrary) 760 elif context.get("FORCE_SHARED_LIB"): 761 lib = HostSharedLibrary(context, host_libname) 762 else: 763 lib = HostLibrary(context, host_libname) 764 self._libs[host_libname].append(lib) 765 self._linkage.append((context, lib, "HOST_USE_LIBS")) 766 host_linkables.append(lib) 767 768 final_lib = context.get("FINAL_LIBRARY") 769 if not libname and final_lib: 770 # If no LIBRARY_NAME is given, create one. 771 libname = context.relsrcdir.replace("/", "_") 772 773 static_lib = context.get("FORCE_STATIC_LIB") 774 shared_lib = context.get("FORCE_SHARED_LIB") 775 776 static_name = context.get("STATIC_LIBRARY_NAME") 777 shared_name = context.get("SHARED_LIBRARY_NAME") 778 779 is_framework = context.get("IS_FRAMEWORK") 780 781 soname = context.get("SONAME") 782 783 lib_defines = context.get("LIBRARY_DEFINES") 784 785 wasm_lib = context.get("SANDBOXED_WASM_LIBRARY_NAME") 786 787 shared_args = {} 788 static_args = {} 789 790 if final_lib: 791 if static_lib: 792 raise SandboxValidationError( 793 "FINAL_LIBRARY implies FORCE_STATIC_LIB. " 794 "Please remove the latter.", 795 context, 796 ) 797 if shared_lib: 798 raise SandboxValidationError( 799 "FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. " 800 "Please remove one.", 801 context, 802 ) 803 if is_framework: 804 raise SandboxValidationError( 805 "FINAL_LIBRARY conflicts with IS_FRAMEWORK. " "Please remove one.", 806 context, 807 ) 808 static_args["link_into"] = final_lib 809 static_lib = True 810 811 if libname: 812 if is_framework: 813 if soname: 814 raise SandboxValidationError( 815 "IS_FRAMEWORK conflicts with SONAME. " "Please remove one.", 816 context, 817 ) 818 shared_lib = True 819 shared_args["variant"] = SharedLibrary.FRAMEWORK 820 821 if not static_lib and not shared_lib: 822 static_lib = True 823 824 if static_name: 825 if not static_lib: 826 raise SandboxValidationError( 827 "STATIC_LIBRARY_NAME requires FORCE_STATIC_LIB", context 828 ) 829 static_args["real_name"] = static_name 830 831 if shared_name: 832 if not shared_lib: 833 raise SandboxValidationError( 834 "SHARED_LIBRARY_NAME requires FORCE_SHARED_LIB", context 835 ) 836 shared_args["real_name"] = shared_name 837 838 if soname: 839 if not shared_lib: 840 raise SandboxValidationError( 841 "SONAME requires FORCE_SHARED_LIB", context 842 ) 843 shared_args["soname"] = soname 844 845 if context.get("NO_EXPAND_LIBS"): 846 if not static_lib: 847 raise SandboxValidationError( 848 "NO_EXPAND_LIBS can only be set for static libraries.", context 849 ) 850 static_args["no_expand_lib"] = True 851 852 if shared_lib and static_lib: 853 if not static_name and not shared_name: 854 raise SandboxValidationError( 855 "Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, " 856 "but neither STATIC_LIBRARY_NAME or " 857 "SHARED_LIBRARY_NAME is set. At least one is required.", 858 context, 859 ) 860 if static_name and not shared_name and static_name == libname: 861 raise SandboxValidationError( 862 "Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, " 863 "but STATIC_LIBRARY_NAME is the same as LIBRARY_NAME, " 864 "and SHARED_LIBRARY_NAME is unset. Please either " 865 "change STATIC_LIBRARY_NAME or LIBRARY_NAME, or set " 866 "SHARED_LIBRARY_NAME.", 867 context, 868 ) 869 if shared_name and not static_name and shared_name == libname: 870 raise SandboxValidationError( 871 "Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, " 872 "but SHARED_LIBRARY_NAME is the same as LIBRARY_NAME, " 873 "and STATIC_LIBRARY_NAME is unset. Please either " 874 "change SHARED_LIBRARY_NAME or LIBRARY_NAME, or set " 875 "STATIC_LIBRARY_NAME.", 876 context, 877 ) 878 if shared_name and static_name and shared_name == static_name: 879 raise SandboxValidationError( 880 "Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, " 881 "but SHARED_LIBRARY_NAME is the same as " 882 "STATIC_LIBRARY_NAME. Please change one of them.", 883 context, 884 ) 885 886 symbols_file = context.get("SYMBOLS_FILE") 887 if symbols_file: 888 if not shared_lib: 889 raise SandboxValidationError( 890 "SYMBOLS_FILE can only be used with a SHARED_LIBRARY.", context 891 ) 892 if context.get("DEFFILE"): 893 raise SandboxValidationError( 894 "SYMBOLS_FILE cannot be used along DEFFILE.", context 895 ) 896 if isinstance(symbols_file, SourcePath): 897 if not os.path.exists(symbols_file.full_path): 898 raise SandboxValidationError( 899 "Path specified in SYMBOLS_FILE does not exist: %s " 900 "(resolved to %s)" % (symbols_file, symbols_file.full_path), 901 context, 902 ) 903 shared_args["symbols_file"] = True 904 else: 905 if symbols_file.target_basename not in generated_files: 906 raise SandboxValidationError( 907 ( 908 "Objdir file specified in SYMBOLS_FILE not in " 909 + "GENERATED_FILES: %s" 910 ) 911 % (symbols_file,), 912 context, 913 ) 914 shared_args["symbols_file"] = symbols_file.target_basename 915 916 if shared_lib: 917 lib = SharedLibrary(context, libname, **shared_args) 918 self._libs[libname].append(lib) 919 self._linkage.append((context, lib, "USE_LIBS")) 920 linkables.append(lib) 921 if not lib.installed: 922 generated_files.add(lib.lib_name) 923 if symbols_file and isinstance(symbols_file, SourcePath): 924 script = mozpath.join( 925 mozpath.dirname(mozpath.dirname(__file__)), 926 "action", 927 "generate_symbols_file.py", 928 ) 929 defines = () 930 if lib.defines: 931 defines = lib.defines.get_defines() 932 yield GeneratedFile( 933 context, 934 script, 935 "generate_symbols_file", 936 lib.symbols_file, 937 [symbols_file], 938 defines, 939 required_during_compile=[lib.symbols_file], 940 ) 941 if static_lib: 942 is_rust_library = context.get("IS_RUST_LIBRARY") 943 if is_rust_library: 944 lib = self._rust_library( 945 context, 946 libname, 947 static_args, 948 is_gkrust=bool(context.get("IS_GKRUST")), 949 ) 950 else: 951 lib = StaticLibrary(context, libname, **static_args) 952 self._libs[libname].append(lib) 953 self._linkage.append((context, lib, "USE_LIBS")) 954 linkables.append(lib) 955 956 if lib_defines: 957 if not libname: 958 raise SandboxValidationError( 959 "LIBRARY_DEFINES needs a " "LIBRARY_NAME to take effect", 960 context, 961 ) 962 lib.lib_defines.update(lib_defines) 963 964 if wasm_lib: 965 if wasm_lib == libname: 966 raise SandboxValidationError( 967 "SANDBOXED_WASM_LIBRARY_NAME and LIBRARY_NAME must have a " 968 "different value.", 969 context, 970 ) 971 if wasm_lib == host_libname: 972 raise SandboxValidationError( 973 "SANDBOXED_WASM_LIBRARY_NAME and HOST_LIBRARY_NAME must " 974 "have a different value.", 975 context, 976 ) 977 if wasm_lib == shared_name: 978 raise SandboxValidationError( 979 "SANDBOXED_WASM_LIBRARY_NAME and SHARED_NAME must have a " 980 "different value.", 981 context, 982 ) 983 if wasm_lib == static_name: 984 raise SandboxValidationError( 985 "SANDBOXED_WASM_LIBRARY_NAME and STATIC_NAME must have a " 986 "different value.", 987 context, 988 ) 989 lib = SandboxedWasmLibrary(context, libname, real_name=wasm_lib) 990 self._libs[libname].append(lib) 991 self._linkage.append((context, lib, "USE_LIBS")) 992 wasm_linkables.append(lib) 993 self._wasm_compile_dirs.add(context.objdir) 994 995 # Only emit sources if we have linkables defined in the same context. 996 # Note the linkables are not emitted in this function, but much later, 997 # after aggregation (because of e.g. USE_LIBS processing). 998 if not (linkables or host_linkables or wasm_linkables): 999 return 1000 1001 self._compile_dirs.add(context.objdir) 1002 1003 if host_linkables and not all( 1004 isinstance(l, HostRustLibrary) for l in host_linkables 1005 ): 1006 self._host_compile_dirs.add(context.objdir) 1007 # TODO: objdirs with only host things in them shouldn't need target 1008 # flags, but there's at least one Makefile.in (in 1009 # build/unix/elfhack) that relies on the value of LDFLAGS being 1010 # passed to one-off rules. 1011 self._compile_dirs.add(context.objdir) 1012 1013 sources = defaultdict(list) 1014 gen_sources = defaultdict(list) 1015 all_flags = {} 1016 for symbol in ("SOURCES", "HOST_SOURCES", "UNIFIED_SOURCES", "WASM_SOURCES"): 1017 srcs = sources[symbol] 1018 gen_srcs = gen_sources[symbol] 1019 context_srcs = context.get(symbol, []) 1020 seen_sources = set() 1021 for f in context_srcs: 1022 if f in seen_sources: 1023 raise SandboxValidationError( 1024 "Source file should only " 1025 "be added to %s once: %s" % (symbol, f), 1026 context, 1027 ) 1028 seen_sources.add(f) 1029 full_path = f.full_path 1030 if isinstance(f, SourcePath): 1031 srcs.append(full_path) 1032 else: 1033 assert isinstance(f, Path) 1034 gen_srcs.append(full_path) 1035 if symbol == "SOURCES": 1036 context_flags = context_srcs[f] 1037 if context_flags: 1038 all_flags[full_path] = context_flags 1039 1040 if isinstance(f, SourcePath) and not os.path.exists(full_path): 1041 raise SandboxValidationError( 1042 "File listed in %s does not " 1043 "exist: '%s'" % (symbol, full_path), 1044 context, 1045 ) 1046 1047 # UNIFIED_SOURCES only take SourcePaths, so there should be no 1048 # generated source in here 1049 assert not gen_sources["UNIFIED_SOURCES"] 1050 1051 no_pgo = context.get("NO_PGO") 1052 no_pgo_sources = [f for f, flags in six.iteritems(all_flags) if flags.no_pgo] 1053 if no_pgo: 1054 if no_pgo_sources: 1055 raise SandboxValidationError( 1056 "NO_PGO and SOURCES[...].no_pgo " "cannot be set at the same time", 1057 context, 1058 ) 1059 passthru.variables["NO_PROFILE_GUIDED_OPTIMIZE"] = no_pgo 1060 if no_pgo_sources: 1061 passthru.variables["NO_PROFILE_GUIDED_OPTIMIZE"] = no_pgo_sources 1062 1063 # A map from "canonical suffixes" for a particular source file 1064 # language to the range of suffixes associated with that language. 1065 # 1066 # We deliberately don't list the canonical suffix in the suffix list 1067 # in the definition; we'll add it in programmatically after defining 1068 # things. 1069 suffix_map = { 1070 ".s": set([".asm"]), 1071 ".c": set(), 1072 ".m": set(), 1073 ".mm": set(), 1074 ".cpp": set([".cc", ".cxx"]), 1075 ".S": set(), 1076 } 1077 1078 # The inverse of the above, mapping suffixes to their canonical suffix. 1079 canonicalized_suffix_map = {} 1080 for suffix, alternatives in six.iteritems(suffix_map): 1081 alternatives.add(suffix) 1082 for a in alternatives: 1083 canonicalized_suffix_map[a] = suffix 1084 1085 def canonical_suffix_for_file(f): 1086 return canonicalized_suffix_map[mozpath.splitext(f)[1]] 1087 1088 # A map from moz.build variables to the canonical suffixes of file 1089 # kinds that can be listed therein. 1090 all_suffixes = list(suffix_map.keys()) 1091 varmap = dict( 1092 SOURCES=(Sources, GeneratedSources, all_suffixes), 1093 HOST_SOURCES=(HostSources, HostGeneratedSources, [".c", ".mm", ".cpp"]), 1094 UNIFIED_SOURCES=(UnifiedSources, None, [".c", ".mm", ".cpp"]), 1095 ) 1096 # Only include a WasmSources or WasmGeneratedSources context if there 1097 # are any WASM_SOURCES. (This is going to matter later because we inject 1098 # an extra .c file to compile with the wasm compiler if, and only if, 1099 # there are any WASM sources.) 1100 if sources["WASM_SOURCES"] or gen_sources["WASM_SOURCES"]: 1101 varmap["WASM_SOURCES"] = (WasmSources, WasmGeneratedSources, [".c", ".cpp"]) 1102 # Track whether there are any C++ source files. 1103 # Technically this won't do the right thing for SIMPLE_PROGRAMS in 1104 # a directory with mixed C and C++ source, but it's not that important. 1105 cxx_sources = defaultdict(bool) 1106 1107 # Source files to track for linkables associated with this context. 1108 ctxt_sources = defaultdict(lambda: defaultdict(list)) 1109 1110 for variable, (klass, gen_klass, suffixes) in varmap.items(): 1111 allowed_suffixes = set().union(*[suffix_map[s] for s in suffixes]) 1112 1113 # First ensure that we haven't been given filetypes that we don't 1114 # recognize. 1115 for f in itertools.chain(sources[variable], gen_sources[variable]): 1116 ext = mozpath.splitext(f)[1] 1117 if ext not in allowed_suffixes: 1118 raise SandboxValidationError( 1119 "%s has an unknown file type." % f, context 1120 ) 1121 1122 for srcs, cls in ( 1123 (sources[variable], klass), 1124 (gen_sources[variable], gen_klass), 1125 ): 1126 # Now sort the files to let groupby work. 1127 sorted_files = sorted(srcs, key=canonical_suffix_for_file) 1128 for canonical_suffix, files in itertools.groupby( 1129 sorted_files, canonical_suffix_for_file 1130 ): 1131 if canonical_suffix in (".cpp", ".mm"): 1132 cxx_sources[variable] = True 1133 elif canonical_suffix in (".s", ".S"): 1134 self._asm_compile_dirs.add(context.objdir) 1135 arglist = [context, list(files), canonical_suffix] 1136 if variable.startswith("UNIFIED_"): 1137 arglist.append(context.get("FILES_PER_UNIFIED_FILE", 16)) 1138 obj = cls(*arglist) 1139 srcs = list(obj.files) 1140 if isinstance(obj, UnifiedSources) and obj.have_unified_mapping: 1141 srcs = dict(obj.unified_source_mapping).keys() 1142 ctxt_sources[variable][canonical_suffix] += sorted(srcs) 1143 yield obj 1144 1145 if ctxt_sources: 1146 for linkable in linkables: 1147 for target_var in ("SOURCES", "UNIFIED_SOURCES"): 1148 for suffix, srcs in ctxt_sources[target_var].items(): 1149 linkable.sources[suffix] += srcs 1150 for host_linkable in host_linkables: 1151 for suffix, srcs in ctxt_sources["HOST_SOURCES"].items(): 1152 host_linkable.sources[suffix] += srcs 1153 for wasm_linkable in wasm_linkables: 1154 for suffix, srcs in ctxt_sources["WASM_SOURCES"].items(): 1155 wasm_linkable.sources[suffix] += srcs 1156 1157 for f, flags in sorted(six.iteritems(all_flags)): 1158 if flags.flags: 1159 ext = mozpath.splitext(f)[1] 1160 yield PerSourceFlag(context, f, flags.flags) 1161 1162 # If there are any C++ sources, set all the linkables defined here 1163 # to require the C++ linker. 1164 for vars, linkable_items in ( 1165 (("SOURCES", "UNIFIED_SOURCES"), linkables), 1166 (("HOST_SOURCES",), host_linkables), 1167 ): 1168 for var in vars: 1169 if cxx_sources[var]: 1170 for l in linkable_items: 1171 l.cxx_link = True 1172 break 1173 1174 def emit_from_context(self, context): 1175 """Convert a Context to tree metadata objects. 1176 1177 This is a generator of mozbuild.frontend.data.ContextDerived instances. 1178 """ 1179 1180 # We only want to emit an InstallationTarget if one of the consulted 1181 # variables is defined. Later on, we look up FINAL_TARGET, which has 1182 # the side-effect of populating it. So, we need to do this lookup 1183 # early. 1184 if any(k in context for k in ("FINAL_TARGET", "XPI_NAME", "DIST_SUBDIR")): 1185 yield InstallationTarget(context) 1186 1187 for obj in self._handle_gn_dirs(context): 1188 yield obj 1189 1190 # We always emit a directory traversal descriptor. This is needed by 1191 # the recursive make backend. 1192 for o in self._emit_directory_traversal_from_context(context): 1193 yield o 1194 1195 for obj in self._process_xpidl(context): 1196 yield obj 1197 1198 computed_flags = ComputedFlags(context, context["COMPILE_FLAGS"]) 1199 computed_link_flags = ComputedFlags(context, context["LINK_FLAGS"]) 1200 computed_host_flags = ComputedFlags(context, context["HOST_COMPILE_FLAGS"]) 1201 computed_as_flags = ComputedFlags(context, context["ASM_FLAGS"]) 1202 computed_wasm_flags = ComputedFlags(context, context["WASM_FLAGS"]) 1203 1204 # Proxy some variables as-is until we have richer classes to represent 1205 # them. We should aim to keep this set small because it violates the 1206 # desired abstraction of the build definition away from makefiles. 1207 passthru = VariablePassthru(context) 1208 varlist = [ 1209 "EXTRA_DSO_LDOPTS", 1210 "RCFILE", 1211 "RCINCLUDE", 1212 "WIN32_EXE_LDFLAGS", 1213 "USE_EXTENSION_MANIFEST", 1214 ] 1215 for v in varlist: 1216 if v in context and context[v]: 1217 passthru.variables[v] = context[v] 1218 1219 if ( 1220 context.config.substs.get("OS_TARGET") == "WINNT" 1221 and context["DELAYLOAD_DLLS"] 1222 ): 1223 if context.config.substs.get("CC_TYPE") != "clang": 1224 context["LDFLAGS"].extend( 1225 [("-DELAYLOAD:%s" % dll) for dll in context["DELAYLOAD_DLLS"]] 1226 ) 1227 else: 1228 context["LDFLAGS"].extend( 1229 [ 1230 ("-Wl,-Xlink=-DELAYLOAD:%s" % dll) 1231 for dll in context["DELAYLOAD_DLLS"] 1232 ] 1233 ) 1234 context["OS_LIBS"].append("delayimp") 1235 1236 for v in ["CMFLAGS", "CMMFLAGS"]: 1237 if v in context and context[v]: 1238 passthru.variables["MOZBUILD_" + v] = context[v] 1239 1240 for v in ["CXXFLAGS", "CFLAGS"]: 1241 if v in context and context[v]: 1242 computed_flags.resolve_flags("MOZBUILD_%s" % v, context[v]) 1243 1244 for v in ["WASM_CFLAGS", "WASM_CXXFLAGS"]: 1245 if v in context and context[v]: 1246 computed_wasm_flags.resolve_flags("MOZBUILD_%s" % v, context[v]) 1247 1248 for v in ["HOST_CXXFLAGS", "HOST_CFLAGS"]: 1249 if v in context and context[v]: 1250 computed_host_flags.resolve_flags("MOZBUILD_%s" % v, context[v]) 1251 1252 if "LDFLAGS" in context and context["LDFLAGS"]: 1253 computed_link_flags.resolve_flags("MOZBUILD", context["LDFLAGS"]) 1254 1255 deffile = context.get("DEFFILE") 1256 if deffile and context.config.substs.get("OS_TARGET") == "WINNT": 1257 if isinstance(deffile, SourcePath): 1258 if not os.path.exists(deffile.full_path): 1259 raise SandboxValidationError( 1260 "Path specified in DEFFILE does not exist: %s " 1261 "(resolved to %s)" % (deffile, deffile.full_path), 1262 context, 1263 ) 1264 path = mozpath.relpath(deffile.full_path, context.objdir) 1265 else: 1266 path = deffile.target_basename 1267 1268 if context.config.substs.get("GNU_CC"): 1269 computed_link_flags.resolve_flags("DEFFILE", [path]) 1270 else: 1271 computed_link_flags.resolve_flags("DEFFILE", ["-DEF:" + path]) 1272 1273 dist_install = context["DIST_INSTALL"] 1274 if dist_install is True: 1275 passthru.variables["DIST_INSTALL"] = True 1276 elif dist_install is False: 1277 passthru.variables["NO_DIST_INSTALL"] = True 1278 1279 # Ideally, this should be done in templates, but this is difficult at 1280 # the moment because USE_STATIC_LIBS can be set after a template 1281 # returns. Eventually, with context-based templates, it will be 1282 # possible. 1283 if context.config.substs.get( 1284 "OS_ARCH" 1285 ) == "WINNT" and not context.config.substs.get("GNU_CC"): 1286 use_static_lib = context.get( 1287 "USE_STATIC_LIBS" 1288 ) and not context.config.substs.get("MOZ_ASAN") 1289 rtl_flag = "-MT" if use_static_lib else "-MD" 1290 if context.config.substs.get("MOZ_DEBUG") and not context.config.substs.get( 1291 "MOZ_NO_DEBUG_RTL" 1292 ): 1293 rtl_flag += "d" 1294 computed_flags.resolve_flags("RTL", [rtl_flag]) 1295 if not context.config.substs.get("CROSS_COMPILE"): 1296 computed_host_flags.resolve_flags("RTL", [rtl_flag]) 1297 computed_wasm_flags.resolve_flags("RTL", [rtl_flag]) 1298 1299 generated_files = set() 1300 localized_generated_files = set() 1301 for obj in self._process_generated_files(context): 1302 for f in obj.outputs: 1303 generated_files.add(f) 1304 if obj.localized: 1305 localized_generated_files.add(f) 1306 yield obj 1307 1308 for path in context["CONFIGURE_SUBST_FILES"]: 1309 sub = self._create_substitution(ConfigFileSubstitution, context, path) 1310 generated_files.add(str(sub.relpath)) 1311 yield sub 1312 1313 for defines_var, cls, backend_flags in ( 1314 ("DEFINES", Defines, (computed_flags, computed_as_flags)), 1315 ("HOST_DEFINES", HostDefines, (computed_host_flags,)), 1316 ("WASM_DEFINES", WasmDefines, (computed_wasm_flags,)), 1317 ): 1318 defines = context.get(defines_var) 1319 if defines: 1320 defines_obj = cls(context, defines) 1321 if isinstance(defines_obj, Defines): 1322 # DEFINES have consumers outside the compile command line, 1323 # HOST_DEFINES do not. 1324 yield defines_obj 1325 else: 1326 # If we don't have explicitly set defines we need to make sure 1327 # initialized values if present end up in computed flags. 1328 defines_obj = cls(context, context[defines_var]) 1329 1330 defines_from_obj = list(defines_obj.get_defines()) 1331 if defines_from_obj: 1332 for flags in backend_flags: 1333 flags.resolve_flags(defines_var, defines_from_obj) 1334 1335 idl_vars = ( 1336 "GENERATED_EVENTS_WEBIDL_FILES", 1337 "GENERATED_WEBIDL_FILES", 1338 "PREPROCESSED_TEST_WEBIDL_FILES", 1339 "PREPROCESSED_WEBIDL_FILES", 1340 "TEST_WEBIDL_FILES", 1341 "WEBIDL_FILES", 1342 "IPDL_SOURCES", 1343 "PREPROCESSED_IPDL_SOURCES", 1344 "XPCOM_MANIFESTS", 1345 ) 1346 for context_var in idl_vars: 1347 for name in context.get(context_var, []): 1348 self._idls[context_var].add(mozpath.join(context.srcdir, name)) 1349 # WEBIDL_EXAMPLE_INTERFACES do not correspond to files. 1350 for name in context.get("WEBIDL_EXAMPLE_INTERFACES", []): 1351 self._idls["WEBIDL_EXAMPLE_INTERFACES"].add(name) 1352 1353 local_includes = [] 1354 for local_include in context.get("LOCAL_INCLUDES", []): 1355 full_path = local_include.full_path 1356 if not isinstance(local_include, ObjDirPath): 1357 if not os.path.exists(full_path): 1358 raise SandboxValidationError( 1359 "Path specified in LOCAL_INCLUDES does not exist: %s (resolved to %s)" 1360 % (local_include, full_path), 1361 context, 1362 ) 1363 if not os.path.isdir(full_path): 1364 raise SandboxValidationError( 1365 "Path specified in LOCAL_INCLUDES " 1366 "is a filename, but a directory is required: %s " 1367 "(resolved to %s)" % (local_include, full_path), 1368 context, 1369 ) 1370 if ( 1371 full_path == context.config.topsrcdir 1372 or full_path == context.config.topobjdir 1373 ): 1374 raise SandboxValidationError( 1375 "Path specified in LOCAL_INCLUDES " 1376 "(%s) resolves to the topsrcdir or topobjdir (%s), which is " 1377 "not allowed" % (local_include, full_path), 1378 context, 1379 ) 1380 include_obj = LocalInclude(context, local_include) 1381 local_includes.append(include_obj.path.full_path) 1382 yield include_obj 1383 1384 computed_flags.resolve_flags( 1385 "LOCAL_INCLUDES", ["-I%s" % p for p in local_includes] 1386 ) 1387 computed_as_flags.resolve_flags( 1388 "LOCAL_INCLUDES", ["-I%s" % p for p in local_includes] 1389 ) 1390 computed_host_flags.resolve_flags( 1391 "LOCAL_INCLUDES", ["-I%s" % p for p in local_includes] 1392 ) 1393 computed_wasm_flags.resolve_flags( 1394 "LOCAL_INCLUDES", ["-I%s" % p for p in local_includes] 1395 ) 1396 1397 for obj in self._handle_linkables(context, passthru, generated_files): 1398 yield obj 1399 1400 generated_files.update( 1401 [ 1402 "%s%s" % (k, self.config.substs.get("BIN_SUFFIX", "")) 1403 for k in self._binaries.keys() 1404 ] 1405 ) 1406 1407 components = [] 1408 for var, cls in ( 1409 ("EXPORTS", Exports), 1410 ("FINAL_TARGET_FILES", FinalTargetFiles), 1411 ("FINAL_TARGET_PP_FILES", FinalTargetPreprocessedFiles), 1412 ("LOCALIZED_FILES", LocalizedFiles), 1413 ("LOCALIZED_PP_FILES", LocalizedPreprocessedFiles), 1414 ("OBJDIR_FILES", ObjdirFiles), 1415 ("OBJDIR_PP_FILES", ObjdirPreprocessedFiles), 1416 ("TEST_HARNESS_FILES", TestHarnessFiles), 1417 ): 1418 all_files = context.get(var) 1419 if not all_files: 1420 continue 1421 if dist_install is False and var != "TEST_HARNESS_FILES": 1422 raise SandboxValidationError( 1423 "%s cannot be used with DIST_INSTALL = False" % var, context 1424 ) 1425 has_prefs = False 1426 has_resources = False 1427 for base, files in all_files.walk(): 1428 if var == "TEST_HARNESS_FILES" and not base: 1429 raise SandboxValidationError( 1430 "Cannot install files to the root of TEST_HARNESS_FILES", 1431 context, 1432 ) 1433 if base == "components": 1434 components.extend(files) 1435 if base == "defaults/pref": 1436 has_prefs = True 1437 if mozpath.split(base)[0] == "res": 1438 has_resources = True 1439 for f in files: 1440 if ( 1441 var 1442 in ( 1443 "FINAL_TARGET_PP_FILES", 1444 "OBJDIR_PP_FILES", 1445 "LOCALIZED_PP_FILES", 1446 ) 1447 and not isinstance(f, SourcePath) 1448 ): 1449 raise SandboxValidationError( 1450 ("Only source directory paths allowed in " + "%s: %s") 1451 % (var, f), 1452 context, 1453 ) 1454 if var.startswith("LOCALIZED_"): 1455 if isinstance(f, SourcePath): 1456 if f.startswith("en-US/"): 1457 pass 1458 elif "locales/en-US/" in f: 1459 pass 1460 else: 1461 raise SandboxValidationError( 1462 "%s paths must start with `en-US/` or " 1463 "contain `locales/en-US/`: %s" % (var, f), 1464 context, 1465 ) 1466 1467 if not isinstance(f, ObjDirPath): 1468 path = f.full_path 1469 if "*" not in path and not os.path.exists(path): 1470 raise SandboxValidationError( 1471 "File listed in %s does not exist: %s" % (var, path), 1472 context, 1473 ) 1474 else: 1475 # TODO: Bug 1254682 - The '/' check is to allow 1476 # installing files generated from other directories, 1477 # which is done occasionally for tests. However, it 1478 # means we don't fail early if the file isn't actually 1479 # created by the other moz.build file. 1480 if f.target_basename not in generated_files and "/" not in f: 1481 raise SandboxValidationError( 1482 ( 1483 "Objdir file listed in %s not in " 1484 + "GENERATED_FILES: %s" 1485 ) 1486 % (var, f), 1487 context, 1488 ) 1489 1490 if var.startswith("LOCALIZED_"): 1491 # Further require that LOCALIZED_FILES are from 1492 # LOCALIZED_GENERATED_FILES. 1493 if f.target_basename not in localized_generated_files: 1494 raise SandboxValidationError( 1495 ( 1496 "Objdir file listed in %s not in " 1497 + "LOCALIZED_GENERATED_FILES: %s" 1498 ) 1499 % (var, f), 1500 context, 1501 ) 1502 else: 1503 # Additionally, don't allow LOCALIZED_GENERATED_FILES to be used 1504 # in anything *but* LOCALIZED_FILES. 1505 if f.target_basename in localized_generated_files: 1506 raise SandboxValidationError( 1507 ( 1508 "Outputs of LOCALIZED_GENERATED_FILES cannot " 1509 "be used in %s: %s" 1510 ) 1511 % (var, f), 1512 context, 1513 ) 1514 1515 # Addons (when XPI_NAME is defined) and Applications (when 1516 # DIST_SUBDIR is defined) use a different preferences directory 1517 # (default/preferences) from the one the GRE uses (defaults/pref). 1518 # Hence, we move the files from the latter to the former in that 1519 # case. 1520 if has_prefs and (context.get("XPI_NAME") or context.get("DIST_SUBDIR")): 1521 all_files.defaults.preferences += all_files.defaults.pref 1522 del all_files.defaults._children["pref"] 1523 1524 if has_resources and ( 1525 context.get("DIST_SUBDIR") or context.get("XPI_NAME") 1526 ): 1527 raise SandboxValidationError( 1528 "RESOURCES_FILES cannot be used with DIST_SUBDIR or " "XPI_NAME.", 1529 context, 1530 ) 1531 1532 yield cls(context, all_files) 1533 1534 for c in components: 1535 if c.endswith(".manifest"): 1536 yield ChromeManifestEntry( 1537 context, 1538 "chrome.manifest", 1539 Manifest("components", mozpath.basename(c)), 1540 ) 1541 1542 rust_tests = context.get("RUST_TESTS", []) 1543 if rust_tests: 1544 # TODO: more sophisticated checking of the declared name vs. 1545 # contents of the Cargo.toml file. 1546 features = context.get("RUST_TEST_FEATURES", []) 1547 1548 yield RustTests(context, rust_tests, features) 1549 1550 for obj in self._process_test_manifests(context): 1551 yield obj 1552 1553 for obj in self._process_jar_manifests(context): 1554 yield obj 1555 1556 computed_as_flags.resolve_flags("MOZBUILD", context.get("ASFLAGS")) 1557 1558 if context.get("USE_NASM") is True: 1559 nasm = context.config.substs.get("NASM") 1560 if not nasm: 1561 raise SandboxValidationError("nasm is not available", context) 1562 passthru.variables["AS"] = nasm 1563 passthru.variables["AS_DASH_C_FLAG"] = "" 1564 passthru.variables["ASOUTOPTION"] = "-o " 1565 computed_as_flags.resolve_flags( 1566 "OS", context.config.substs.get("NASM_ASFLAGS", []) 1567 ) 1568 1569 if context.get("USE_INTEGRATED_CLANGCL_AS") is True: 1570 if context.config.substs.get("CC_TYPE") != "clang-cl": 1571 raise SandboxValidationError("clang-cl is not available", context) 1572 passthru.variables["AS"] = context.config.substs.get("CC") 1573 passthru.variables["AS_DASH_C_FLAG"] = "-c" 1574 passthru.variables["ASOUTOPTION"] = "-o " 1575 1576 if passthru.variables: 1577 yield passthru 1578 1579 if context.objdir in self._compile_dirs: 1580 self._compile_flags[context.objdir] = computed_flags 1581 yield computed_link_flags 1582 1583 if context.objdir in self._asm_compile_dirs: 1584 self._compile_as_flags[context.objdir] = computed_as_flags 1585 1586 if context.objdir in self._host_compile_dirs: 1587 yield computed_host_flags 1588 1589 if context.objdir in self._wasm_compile_dirs: 1590 yield computed_wasm_flags 1591 1592 def _create_substitution(self, cls, context, path): 1593 sub = cls(context) 1594 sub.input_path = "%s.in" % path.full_path 1595 sub.output_path = path.translated 1596 sub.relpath = path 1597 1598 return sub 1599 1600 def _process_xpidl(self, context): 1601 # XPIDL source files get processed and turned into .h and .xpt files. 1602 # If there are multiple XPIDL files in a directory, they get linked 1603 # together into a final .xpt, which has the name defined by 1604 # XPIDL_MODULE. 1605 xpidl_module = context["XPIDL_MODULE"] 1606 1607 if not xpidl_module: 1608 if context["XPIDL_SOURCES"]: 1609 raise SandboxValidationError( 1610 "XPIDL_MODULE must be defined if " "XPIDL_SOURCES is defined.", 1611 context, 1612 ) 1613 return 1614 1615 if not context["XPIDL_SOURCES"]: 1616 raise SandboxValidationError( 1617 "XPIDL_MODULE cannot be defined " "unless there are XPIDL_SOURCES", 1618 context, 1619 ) 1620 1621 if context["DIST_INSTALL"] is False: 1622 self.log( 1623 logging.WARN, 1624 "mozbuild_warning", 1625 dict(path=context.main_path), 1626 "{path}: DIST_INSTALL = False has no effect on XPIDL_SOURCES.", 1627 ) 1628 1629 for idl in context["XPIDL_SOURCES"]: 1630 if not os.path.exists(idl.full_path): 1631 raise SandboxValidationError( 1632 "File %s from XPIDL_SOURCES " "does not exist" % idl.full_path, 1633 context, 1634 ) 1635 1636 yield XPIDLModule(context, xpidl_module, context["XPIDL_SOURCES"]) 1637 1638 def _process_generated_files(self, context): 1639 for path in context["CONFIGURE_DEFINE_FILES"]: 1640 script = mozpath.join( 1641 mozpath.dirname(mozpath.dirname(__file__)), 1642 "action", 1643 "process_define_files.py", 1644 ) 1645 yield GeneratedFile( 1646 context, 1647 script, 1648 "process_define_file", 1649 six.text_type(path), 1650 [Path(context, path + ".in")], 1651 ) 1652 1653 generated_files = context.get("GENERATED_FILES") or [] 1654 localized_generated_files = context.get("LOCALIZED_GENERATED_FILES") or [] 1655 if not (generated_files or localized_generated_files): 1656 return 1657 1658 for (localized, gen) in ( 1659 (False, generated_files), 1660 (True, localized_generated_files), 1661 ): 1662 for f in gen: 1663 flags = gen[f] 1664 outputs = f 1665 inputs = [] 1666 if flags.script: 1667 method = "main" 1668 script = SourcePath(context, flags.script).full_path 1669 1670 # Deal with cases like "C:\\path\\to\\script.py:function". 1671 if ".py:" in script: 1672 script, method = script.rsplit(".py:", 1) 1673 script += ".py" 1674 1675 if not os.path.exists(script): 1676 raise SandboxValidationError( 1677 "Script for generating %s does not exist: %s" % (f, script), 1678 context, 1679 ) 1680 if os.path.splitext(script)[1] != ".py": 1681 raise SandboxValidationError( 1682 "Script for generating %s does not end in .py: %s" 1683 % (f, script), 1684 context, 1685 ) 1686 else: 1687 script = None 1688 method = None 1689 1690 for i in flags.inputs: 1691 p = Path(context, i) 1692 if isinstance(p, SourcePath) and not os.path.exists(p.full_path): 1693 raise SandboxValidationError( 1694 "Input for generating %s does not exist: %s" 1695 % (f, p.full_path), 1696 context, 1697 ) 1698 inputs.append(p) 1699 1700 yield GeneratedFile( 1701 context, 1702 script, 1703 method, 1704 outputs, 1705 inputs, 1706 flags.flags, 1707 localized=localized, 1708 force=flags.force, 1709 ) 1710 1711 def _process_test_manifests(self, context): 1712 for prefix, info in TEST_MANIFESTS.items(): 1713 for path, manifest in context.get("%s_MANIFESTS" % prefix, []): 1714 for obj in self._process_test_manifest(context, info, path, manifest): 1715 yield obj 1716 1717 for flavor in REFTEST_FLAVORS: 1718 for path, manifest in context.get("%s_MANIFESTS" % flavor.upper(), []): 1719 for obj in self._process_reftest_manifest( 1720 context, flavor, path, manifest 1721 ): 1722 yield obj 1723 1724 def _process_test_manifest(self, context, info, manifest_path, mpmanifest): 1725 flavor, install_root, install_subdir, package_tests = info 1726 1727 path = manifest_path.full_path 1728 manifest_dir = mozpath.dirname(path) 1729 manifest_reldir = mozpath.dirname( 1730 mozpath.relpath(path, context.config.topsrcdir) 1731 ) 1732 manifest_sources = [ 1733 mozpath.relpath(pth, context.config.topsrcdir) 1734 for pth in mpmanifest.source_files 1735 ] 1736 install_prefix = mozpath.join(install_root, install_subdir) 1737 1738 try: 1739 if not mpmanifest.tests: 1740 raise SandboxValidationError("Empty test manifest: %s" % path, context) 1741 1742 defaults = mpmanifest.manifest_defaults[os.path.normpath(path)] 1743 obj = TestManifest( 1744 context, 1745 path, 1746 mpmanifest, 1747 flavor=flavor, 1748 install_prefix=install_prefix, 1749 relpath=mozpath.join(manifest_reldir, mozpath.basename(path)), 1750 sources=manifest_sources, 1751 dupe_manifest="dupe-manifest" in defaults, 1752 ) 1753 1754 filtered = mpmanifest.tests 1755 1756 missing = [t["name"] for t in filtered if not os.path.exists(t["path"])] 1757 if missing: 1758 raise SandboxValidationError( 1759 "Test manifest (%s) lists " 1760 "test that does not exist: %s" % (path, ", ".join(missing)), 1761 context, 1762 ) 1763 1764 out_dir = mozpath.join(install_prefix, manifest_reldir) 1765 1766 def process_support_files(test): 1767 install_info = self._test_files_converter.convert_support_files( 1768 test, install_root, manifest_dir, out_dir 1769 ) 1770 1771 obj.pattern_installs.extend(install_info.pattern_installs) 1772 for source, dest in install_info.installs: 1773 obj.installs[source] = (dest, False) 1774 obj.external_installs |= install_info.external_installs 1775 for install_path in install_info.deferred_installs: 1776 if all( 1777 [ 1778 "*" not in install_path, 1779 not os.path.isfile( 1780 mozpath.join(context.config.topsrcdir, install_path[2:]) 1781 ), 1782 install_path not in install_info.external_installs, 1783 ] 1784 ): 1785 raise SandboxValidationError( 1786 "Error processing test " 1787 "manifest %s: entry in support-files not present " 1788 "in the srcdir: %s" % (path, install_path), 1789 context, 1790 ) 1791 1792 obj.deferred_installs |= install_info.deferred_installs 1793 1794 for test in filtered: 1795 obj.tests.append(test) 1796 1797 # Some test files are compiled and should not be copied into the 1798 # test package. They function as identifiers rather than files. 1799 if package_tests: 1800 manifest_relpath = mozpath.relpath( 1801 test["path"], mozpath.dirname(test["manifest"]) 1802 ) 1803 obj.installs[mozpath.normpath(test["path"])] = ( 1804 (mozpath.join(out_dir, manifest_relpath)), 1805 True, 1806 ) 1807 1808 process_support_files(test) 1809 1810 for path, m_defaults in mpmanifest.manifest_defaults.items(): 1811 process_support_files(m_defaults) 1812 1813 # We also copy manifests into the output directory, 1814 # including manifests from [include:foo] directives. 1815 for mpath in mpmanifest.manifests(): 1816 mpath = mozpath.normpath(mpath) 1817 out_path = mozpath.join(out_dir, mozpath.basename(mpath)) 1818 obj.installs[mpath] = (out_path, False) 1819 1820 # Some manifests reference files that are auto generated as 1821 # part of the build or shouldn't be installed for some 1822 # reason. Here, we prune those files from the install set. 1823 # FUTURE we should be able to detect autogenerated files from 1824 # other build metadata. Once we do that, we can get rid of this. 1825 for f in defaults.get("generated-files", "").split(): 1826 # We re-raise otherwise the stack trace isn't informative. 1827 try: 1828 del obj.installs[mozpath.join(manifest_dir, f)] 1829 except KeyError: 1830 raise SandboxValidationError( 1831 "Error processing test " 1832 "manifest %s: entry in generated-files not present " 1833 "elsewhere in manifest: %s" % (path, f), 1834 context, 1835 ) 1836 1837 yield obj 1838 except (AssertionError, Exception): 1839 raise SandboxValidationError( 1840 "Error processing test " 1841 "manifest file %s: %s" 1842 % (path, "\n".join(traceback.format_exception(*sys.exc_info()))), 1843 context, 1844 ) 1845 1846 def _process_reftest_manifest(self, context, flavor, manifest_path, manifest): 1847 manifest_full_path = manifest_path.full_path 1848 manifest_reldir = mozpath.dirname( 1849 mozpath.relpath(manifest_full_path, context.config.topsrcdir) 1850 ) 1851 1852 # reftest manifests don't come from manifest parser. But they are 1853 # similar enough that we can use the same emitted objects. Note 1854 # that we don't perform any installs for reftests. 1855 obj = TestManifest( 1856 context, 1857 manifest_full_path, 1858 manifest, 1859 flavor=flavor, 1860 install_prefix="%s/" % flavor, 1861 relpath=mozpath.join(manifest_reldir, mozpath.basename(manifest_path)), 1862 ) 1863 obj.tests = list(sorted(manifest.tests, key=lambda t: t["path"])) 1864 1865 yield obj 1866 1867 def _process_jar_manifests(self, context): 1868 jar_manifests = context.get("JAR_MANIFESTS", []) 1869 if len(jar_manifests) > 1: 1870 raise SandboxValidationError( 1871 "While JAR_MANIFESTS is a list, " 1872 "it is currently limited to one value.", 1873 context, 1874 ) 1875 1876 for path in jar_manifests: 1877 yield JARManifest(context, path) 1878 1879 # Temporary test to look for jar.mn files that creep in without using 1880 # the new declaration. Before, we didn't require jar.mn files to 1881 # declared anywhere (they were discovered). This will detect people 1882 # relying on the old behavior. 1883 if os.path.exists(os.path.join(context.srcdir, "jar.mn")): 1884 if "jar.mn" not in jar_manifests: 1885 raise SandboxValidationError( 1886 "A jar.mn exists but it " 1887 "is not referenced in the moz.build file. " 1888 "Please define JAR_MANIFESTS.", 1889 context, 1890 ) 1891 1892 def _emit_directory_traversal_from_context(self, context): 1893 o = DirectoryTraversal(context) 1894 o.dirs = context.get("DIRS", []) 1895 1896 # Some paths have a subconfigure, yet also have a moz.build. Those 1897 # shouldn't end up in self._external_paths. 1898 if o.objdir: 1899 self._external_paths -= {o.relobjdir} 1900 1901 yield o 1902