# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, unicode_literals import itertools import logging import os import traceback import sys import time from collections import defaultdict, OrderedDict from mach.mixin.logging import LoggingMixin from mozbuild.util import ( memoize, OrderedDefaultDict, ) import mozpack.path as mozpath import mozinfo import pytoml from .data import ( BaseRustProgram, BaseSources, ChromeManifestEntry, ComputedFlags, ConfigFileSubstitution, ContextWrapped, Defines, DirectoryTraversal, Exports, FinalTargetFiles, FinalTargetPreprocessedFiles, GeneratedFile, GeneratedSources, GnProjectData, ExternalStaticLibrary, ExternalSharedLibrary, HostDefines, HostLibrary, HostProgram, HostRustProgram, HostSimpleProgram, HostSources, InstallationTarget, IPDLCollection, JARManifest, Library, Linkable, LocalInclude, LocalizedFiles, LocalizedPreprocessedFiles, ObjdirFiles, ObjdirPreprocessedFiles, PerSourceFlag, WebIDLCollection, Program, RustLibrary, HostRustLibrary, RustProgram, RustTest, SharedLibrary, SimpleProgram, Sources, StaticLibrary, TestHarnessFiles, TestManifest, UnifiedSources, VariablePassthru, XPIDLFile, ) from mozpack.chrome.manifest import ( Manifest, ) from .reader import SandboxValidationError from ..testing import ( TEST_MANIFESTS, REFTEST_FLAVORS, WEB_PLATFORM_TESTS_FLAVORS, SupportFilesConverter, ) from .context import ( Context, SourcePath, ObjDirPath, Path, SubContext, TemplateContext, ) from mozbuild.base import ExecutionSummary class TreeMetadataEmitter(LoggingMixin): """Converts the executed mozbuild files into data structures. This is a bridge between reader.py and data.py. It takes what was read by reader.BuildReader and converts it into the classes defined in the data module. """ def __init__(self, config): self.populate_logger() self.config = config mozinfo.find_and_update_from_json(config.topobjdir) # Python 2.6 doesn't allow unicode keys to be used for keyword # arguments. This gross hack works around the problem until we # rid ourselves of 2.6. self.info = {} for k, v in mozinfo.info.items(): if isinstance(k, unicode): k = k.encode('ascii') self.info[k] = v self._libs = OrderedDefaultDict(list) self._binaries = OrderedDict() self._compile_dirs = set() self._host_compile_dirs = set() self._rust_compile_dirs = set() self._asm_compile_dirs = set() self._compile_flags = dict() self._compile_as_flags = dict() self._linkage = [] self._static_linking_shared = set() self._crate_verified_local = set() self._crate_directories = dict() self._idls = defaultdict(set) # Keep track of external paths (third party build systems), starting # from what we run a subconfigure in. We'll eliminate some directories # as we traverse them with moz.build (e.g. js/src). subconfigures = os.path.join(self.config.topobjdir, 'subconfigures') paths = [] if os.path.exists(subconfigures): paths = open(subconfigures).read().splitlines() self._external_paths = set(mozpath.normsep(d) for d in paths) self._emitter_time = 0.0 self._object_count = 0 self._test_files_converter = SupportFilesConverter() def summary(self): return ExecutionSummary( 'Processed into {object_count:d} build config descriptors in ' '{execution_time:.2f}s', execution_time=self._emitter_time, object_count=self._object_count) def emit(self, output, emitfn=None): """Convert the BuildReader output into data structures. The return value from BuildReader.read_topsrcdir() (a generator) is typically fed into this function. """ contexts = {} emitfn = emitfn or self.emit_from_context def emit_objs(objs): for o in objs: self._object_count += 1 yield o for out in output: # Nothing in sub-contexts is currently of interest to us. Filter # them all out. if isinstance(out, SubContext): continue if isinstance(out, Context): # Keep all contexts around, we will need them later. contexts[out.objdir] = out start = time.time() # We need to expand the generator for the timings to work. objs = list(emitfn(out)) self._emitter_time += time.time() - start for o in emit_objs(objs): yield o else: raise Exception('Unhandled output type: %s' % type(out)) # Don't emit Linkable objects when COMPILE_ENVIRONMENT is not set if self.config.substs.get('COMPILE_ENVIRONMENT'): start = time.time() objs = list(self._emit_libs_derived(contexts)) self._emitter_time += time.time() - start for o in emit_objs(objs): yield o def _emit_libs_derived(self, contexts): # First aggregate idl sources. webidl_attrs = [ ('GENERATED_EVENTS_WEBIDL_FILES', lambda c: c.generated_events_sources), ('GENERATED_WEBIDL_FILES', lambda c: c.generated_sources), ('PREPROCESSED_TEST_WEBIDL_FILES', lambda c: c.preprocessed_test_sources), ('PREPROCESSED_WEBIDL_FILES', lambda c: c.preprocessed_sources), ('TEST_WEBIDL_FILES', lambda c: c.test_sources), ('WEBIDL_FILES', lambda c: c.sources), ('WEBIDL_EXAMPLE_INTERFACES', lambda c: c.example_interfaces), ] ipdl_attrs = [ ('IPDL_SOURCES', lambda c: c.sources), ('PREPROCESSED_IPDL_SOURCES', lambda c: c.preprocessed_sources), ] idl_sources = {} for root, cls, attrs in ((self.config.substs.get('WEBIDL_ROOT'), WebIDLCollection, webidl_attrs), (self.config.substs.get('IPDL_ROOT'), IPDLCollection, ipdl_attrs)): if root: collection = cls(contexts[root]) for var, src_getter in attrs: src_getter(collection).update(self._idls[var]) idl_sources[root] = collection.all_source_files() if isinstance(collection, WebIDLCollection): # Test webidl sources are added here as a somewhat special # case. idl_sources[mozpath.join(root, 'test')] = [s for s in collection.all_test_cpp_basenames()] yield collection # Next do FINAL_LIBRARY linkage. for lib in (l for libs in self._libs.values() for l in libs): if not isinstance(lib, (StaticLibrary, RustLibrary)) or not lib.link_into: continue if lib.link_into not in self._libs: raise SandboxValidationError( 'FINAL_LIBRARY ("%s") does not match any LIBRARY_NAME' % lib.link_into, contexts[lib.objdir]) candidates = self._libs[lib.link_into] # When there are multiple candidates, but all are in the same # directory and have a different type, we want all of them to # have the library linked. The typical usecase is when building # both a static and a shared library in a directory, and having # that as a FINAL_LIBRARY. if len(set(type(l) for l in candidates)) == len(candidates) and \ len(set(l.objdir for l in candidates)) == 1: for c in candidates: c.link_library(lib) else: raise SandboxValidationError( 'FINAL_LIBRARY ("%s") matches a LIBRARY_NAME defined in ' 'multiple places:\n %s' % (lib.link_into, '\n '.join(l.objdir for l in candidates)), contexts[lib.objdir]) # ...and USE_LIBS linkage. for context, obj, variable in self._linkage: self._link_libraries(context, obj, variable, idl_sources) def recurse_refs(lib): for o in lib.refs: yield o if isinstance(o, StaticLibrary): for q in recurse_refs(o): yield q # Check that all static libraries refering shared libraries in # USE_LIBS are linked into a shared library or program. for lib in self._static_linking_shared: if all(isinstance(o, StaticLibrary) for o in recurse_refs(lib)): shared_libs = sorted(l.basename for l in lib.linked_libraries if isinstance(l, SharedLibrary)) raise SandboxValidationError( 'The static "%s" library is not used in a shared library ' 'or a program, but USE_LIBS contains the following shared ' 'library names:\n %s\n\nMaybe you can remove the ' 'static "%s" library?' % (lib.basename, '\n '.join(shared_libs), lib.basename), contexts[lib.objdir]) # Propagate LIBRARY_DEFINES to all child libraries recursively. def propagate_defines(outerlib, defines): outerlib.lib_defines.update(defines) for lib in outerlib.linked_libraries: # Propagate defines only along FINAL_LIBRARY paths, not USE_LIBS # paths. if (isinstance(lib, StaticLibrary) and lib.link_into == outerlib.basename): propagate_defines(lib, defines) for lib in (l for libs in self._libs.values() for l in libs): if isinstance(lib, Library): propagate_defines(lib, lib.lib_defines) yield lib for lib in (l for libs in self._libs.values() for l in libs): lib_defines = list(lib.lib_defines.get_defines()) if lib_defines: objdir_flags = self._compile_flags[lib.objdir] objdir_flags.resolve_flags('LIBRARY_DEFINES', lib_defines) objdir_flags = self._compile_as_flags.get(lib.objdir) if objdir_flags: objdir_flags.resolve_flags('LIBRARY_DEFINES', lib_defines) for flags_obj in self._compile_flags.values(): yield flags_obj for flags_obj in self._compile_as_flags.values(): yield flags_obj for obj in self._binaries.values(): yield obj LIBRARY_NAME_VAR = { 'host': 'HOST_LIBRARY_NAME', 'target': 'LIBRARY_NAME', } LIBSTDCXX_VAR = { 'host': 'MOZ_LIBSTDCXX_HOST_VERSION', 'target': 'MOZ_LIBSTDCXX_TARGET_VERSION', } STDCXXCOMPAT_NAME = { 'host': 'host_stdc++compat', 'target': 'stdc++compat', } def _link_libraries(self, context, obj, variable, extra_sources): """Add linkage declarations to a given object.""" assert isinstance(obj, Linkable) if context.objdir in extra_sources: # All "extra sources" are .cpp for the moment, and happen to come # first in order. obj.sources['.cpp'] = extra_sources[context.objdir] + obj.sources['.cpp'] for path in context.get(variable, []): self._link_library(context, obj, variable, path) # Link system libraries from OS_LIBS/HOST_OS_LIBS. for lib in context.get(variable.replace('USE', 'OS'), []): obj.link_system_library(lib) # We have to wait for all the self._link_library calls above to have # happened for obj.cxx_link to be final. if not isinstance(obj, (StaticLibrary, HostLibrary, BaseRustProgram)) and obj.cxx_link: if context.config.substs.get(self.LIBSTDCXX_VAR[obj.KIND]): self._link_library(context, obj, variable, self.STDCXXCOMPAT_NAME[obj.KIND]) if obj.KIND == 'target': for lib in context.config.substs.get('STLPORT_LIBS', []): obj.link_system_library(lib) def _link_library(self, context, obj, variable, path): force_static = path.startswith('static:') and obj.KIND == 'target' if force_static: path = path[7:] name = mozpath.basename(path) dir = mozpath.dirname(path) candidates = [l for l in self._libs[name] if l.KIND == obj.KIND] if dir: if dir.startswith('/'): dir = mozpath.normpath( mozpath.join(obj.topobjdir, dir[1:])) else: dir = mozpath.normpath( mozpath.join(obj.objdir, dir)) dir = mozpath.relpath(dir, obj.topobjdir) candidates = [l for l in candidates if l.relobjdir == dir] if not candidates: # If the given directory is under one of the external # (third party) paths, use a fake library reference to # there. for d in self._external_paths: if dir.startswith('%s/' % d): candidates = [self._get_external_library(dir, name, force_static)] break if not candidates: raise SandboxValidationError( '%s contains "%s", but there is no "%s" %s in %s.' % (variable, path, name, self.LIBRARY_NAME_VAR[obj.KIND], dir), context) if len(candidates) > 1: # If there's more than one remaining candidate, it could be # that there are instances for the same library, in static and # shared form. libs = {} for l in candidates: key = mozpath.join(l.relobjdir, l.basename) if force_static: if isinstance(l, StaticLibrary): libs[key] = l else: if key in libs and isinstance(l, SharedLibrary): libs[key] = l if key not in libs: libs[key] = l candidates = libs.values() if force_static and not candidates: if dir: raise SandboxValidationError( '%s contains "static:%s", but there is no static ' '"%s" %s in %s.' % (variable, path, name, self.LIBRARY_NAME_VAR[obj.KIND], dir), context) raise SandboxValidationError( '%s contains "static:%s", but there is no static "%s" ' '%s in the tree' % (variable, name, name, self.LIBRARY_NAME_VAR[obj.KIND]), context) if not candidates: raise SandboxValidationError( '%s contains "%s", which does not match any %s in the tree.' % (variable, path, self.LIBRARY_NAME_VAR[obj.KIND]), context) elif len(candidates) > 1: paths = (mozpath.join(l.relsrcdir, 'moz.build') for l in candidates) raise SandboxValidationError( '%s contains "%s", which matches a %s defined in multiple ' 'places:\n %s' % (variable, path, self.LIBRARY_NAME_VAR[obj.KIND], '\n '.join(paths)), context) elif force_static and not isinstance(candidates[0], StaticLibrary): raise SandboxValidationError( '%s contains "static:%s", but there is only a shared "%s" ' 'in %s. You may want to add FORCE_STATIC_LIB=True in ' '%s/moz.build, or remove "static:".' % (variable, path, name, candidates[0].relobjdir, candidates[0].relobjdir), context) elif isinstance(obj, StaticLibrary) and isinstance(candidates[0], SharedLibrary): self._static_linking_shared.add(obj) obj.link_library(candidates[0]) @memoize def _get_external_library(self, dir, name, force_static): # Create ExternalStaticLibrary or ExternalSharedLibrary object with a # context more or less truthful about where the external library is. context = Context(config=self.config) context.add_source(mozpath.join(self.config.topsrcdir, dir, 'dummy')) if force_static: return ExternalStaticLibrary(context, name) else: return ExternalSharedLibrary(context, name) def _parse_cargo_file(self, context): """Parse the Cargo.toml file in context and return a Python object representation of it. Raise a SandboxValidationError if the Cargo.toml file does not exist. Return a tuple of (config, cargo_file).""" cargo_file = mozpath.join(context.srcdir, 'Cargo.toml') if not os.path.exists(cargo_file): raise SandboxValidationError( 'No Cargo.toml file found in %s' % cargo_file, context) with open(cargo_file, 'r') as f: return pytoml.load(f), cargo_file def _verify_deps(self, context, crate_dir, crate_name, dependencies, description='Dependency'): """Verify that a crate's dependencies all specify local paths.""" for dep_crate_name, values in dependencies.iteritems(): # A simple version number. if isinstance(values, (str, unicode)): raise SandboxValidationError( '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), context) dep_path = values.get('path', None) if not dep_path: raise SandboxValidationError( '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), context) # Try to catch the case where somebody listed a # local path for development. if os.path.isabs(dep_path): raise SandboxValidationError( '%s %s of crate %s has a non-relative path' % (description, dep_crate_name, crate_name), context) if not os.path.exists(mozpath.join(context.config.topsrcdir, crate_dir, dep_path)): raise SandboxValidationError( '%s %s of crate %s refers to a non-existent path' % (description, dep_crate_name, crate_name), context) def _rust_library(self, context, libname, static_args, cls=RustLibrary): # We need to note any Rust library for linking purposes. config, cargo_file = self._parse_cargo_file(context) crate_name = config['package']['name'] if crate_name != libname: raise SandboxValidationError( 'library %s does not match Cargo.toml-defined package %s' % (libname, crate_name), context) # Check that the [lib.crate-type] field is correct lib_section = config.get('lib', None) if not lib_section: raise SandboxValidationError( 'Cargo.toml for %s has no [lib] section' % libname, context) crate_type = lib_section.get('crate-type', None) if not crate_type: raise SandboxValidationError( 'Can\'t determine a crate-type for %s from Cargo.toml' % libname, context) crate_type = crate_type[0] if crate_type != 'staticlib': raise SandboxValidationError( 'crate-type %s is not permitted for %s' % (crate_type, libname), context) cargo_target_dir = context.get('RUST_LIBRARY_TARGET_DIR', '.') dependencies = set(config.get('dependencies', {}).iterkeys()) features = context.get(cls.FEATURES_VAR, []) unique_features = set(features) if len(features) != len(unique_features): raise SandboxValidationError( 'features for %s should not contain duplicates: %s' % (libname, features), context) return cls(context, libname, cargo_file, crate_type, dependencies, features, cargo_target_dir, **static_args) def _handle_gn_dirs(self, context): for target_dir in context.get('GN_DIRS', []): context['DIRS'] += [target_dir] gn_dir = context['GN_DIRS'][target_dir] for v in ('variables',): if not getattr(gn_dir, 'variables'): raise SandboxValidationError('Missing value for ' 'GN_DIRS["%s"].%s' % (target_dir, v), context) non_unified_sources = set() for s in gn_dir.non_unified_sources: source = SourcePath(context, s) if not os.path.exists(source.full_path): raise SandboxValidationError('Cannot find %s.' % source, context) non_unified_sources.add(mozpath.join(context.relsrcdir, s)) yield GnProjectData(context, target_dir, gn_dir, non_unified_sources) def _handle_linkables(self, context, passthru, generated_files): linkables = [] host_linkables = [] def add_program(prog, var): if var.startswith('HOST_'): host_linkables.append(prog) else: linkables.append(prog) def check_unique_binary(program, kind): if program in self._binaries: raise SandboxValidationError( 'Cannot use "%s" as %s name, ' 'because it is already used in %s' % (program, kind, self._binaries[program].relsrcdir), context) for kind, cls in [('PROGRAM', Program), ('HOST_PROGRAM', HostProgram)]: program = context.get(kind) if program: check_unique_binary(program, kind) self._binaries[program] = cls(context, program) self._linkage.append((context, self._binaries[program], kind.replace('PROGRAM', 'USE_LIBS'))) add_program(self._binaries[program], kind) all_rust_programs = [] for kind, cls in [('RUST_PROGRAMS', RustProgram), ('HOST_RUST_PROGRAMS', HostRustProgram)]: programs = context[kind] if not programs: continue if kind == 'RUST_PROGRAMS': self._rust_compile_dirs.add(context.objdir) all_rust_programs.append((programs, kind, cls)) # Verify Rust program definitions. if all_rust_programs: config, cargo_file = self._parse_cargo_file(context); bin_section = config.get('bin', None) if not bin_section: raise SandboxValidationError( 'Cargo.toml in %s has no [bin] section' % context.srcdir, context) defined_binaries = {b['name'] for b in bin_section} for programs, kind, cls in all_rust_programs: for program in programs: if program not in defined_binaries: raise SandboxValidationError( 'Cannot find Cargo.toml definition for %s' % program, context) check_unique_binary(program, kind) self._binaries[program] = cls(context, program, cargo_file) for kind, cls in [ ('SIMPLE_PROGRAMS', SimpleProgram), ('CPP_UNIT_TESTS', SimpleProgram), ('HOST_SIMPLE_PROGRAMS', HostSimpleProgram)]: for program in context[kind]: if program in self._binaries: raise SandboxValidationError( 'Cannot use "%s" in %s, ' 'because it is already used in %s' % (program, kind, self._binaries[program].relsrcdir), context) self._binaries[program] = cls(context, program, is_unit_test=kind == 'CPP_UNIT_TESTS') self._linkage.append((context, self._binaries[program], 'HOST_USE_LIBS' if kind == 'HOST_SIMPLE_PROGRAMS' else 'USE_LIBS')) add_program(self._binaries[program], kind) host_libname = context.get('HOST_LIBRARY_NAME') libname = context.get('LIBRARY_NAME') if host_libname: if host_libname == libname: raise SandboxValidationError('LIBRARY_NAME and ' 'HOST_LIBRARY_NAME must have a different value', context) is_rust_library = context.get('IS_RUST_LIBRARY') if is_rust_library: lib = self._rust_library(context, host_libname, {}, cls=HostRustLibrary) else: lib = HostLibrary(context, host_libname) self._libs[host_libname].append(lib) self._linkage.append((context, lib, 'HOST_USE_LIBS')) host_linkables.append(lib) final_lib = context.get('FINAL_LIBRARY') if not libname and final_lib: # If no LIBRARY_NAME is given, create one. libname = context.relsrcdir.replace('/', '_') static_lib = context.get('FORCE_STATIC_LIB') shared_lib = context.get('FORCE_SHARED_LIB') static_name = context.get('STATIC_LIBRARY_NAME') shared_name = context.get('SHARED_LIBRARY_NAME') is_framework = context.get('IS_FRAMEWORK') soname = context.get('SONAME') lib_defines = context.get('LIBRARY_DEFINES') shared_args = {} static_args = {} if final_lib: if static_lib: raise SandboxValidationError( 'FINAL_LIBRARY implies FORCE_STATIC_LIB. ' 'Please remove the latter.', context) if shared_lib: raise SandboxValidationError( 'FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. ' 'Please remove one.', context) if is_framework: raise SandboxValidationError( 'FINAL_LIBRARY conflicts with IS_FRAMEWORK. ' 'Please remove one.', context) static_args['link_into'] = final_lib static_lib = True if libname: if is_framework: if soname: raise SandboxValidationError( 'IS_FRAMEWORK conflicts with SONAME. ' 'Please remove one.', context) shared_lib = True shared_args['variant'] = SharedLibrary.FRAMEWORK if not static_lib and not shared_lib: static_lib = True if static_name: if not static_lib: raise SandboxValidationError( 'STATIC_LIBRARY_NAME requires FORCE_STATIC_LIB', context) static_args['real_name'] = static_name if shared_name: if not shared_lib: raise SandboxValidationError( 'SHARED_LIBRARY_NAME requires FORCE_SHARED_LIB', context) shared_args['real_name'] = shared_name if soname: if not shared_lib: raise SandboxValidationError( 'SONAME requires FORCE_SHARED_LIB', context) shared_args['soname'] = soname if context.get('NO_EXPAND_LIBS'): if not static_lib: raise SandboxValidationError( 'NO_EXPAND_LIBS can only be set for static libraries.', context) static_args['no_expand_lib'] = True if shared_lib and static_lib: if not static_name and not shared_name: raise SandboxValidationError( 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' 'but neither STATIC_LIBRARY_NAME or ' 'SHARED_LIBRARY_NAME is set. At least one is required.', context) if static_name and not shared_name and static_name == libname: raise SandboxValidationError( 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' 'but STATIC_LIBRARY_NAME is the same as LIBRARY_NAME, ' 'and SHARED_LIBRARY_NAME is unset. Please either ' 'change STATIC_LIBRARY_NAME or LIBRARY_NAME, or set ' 'SHARED_LIBRARY_NAME.', context) if shared_name and not static_name and shared_name == libname: raise SandboxValidationError( 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' 'but SHARED_LIBRARY_NAME is the same as LIBRARY_NAME, ' 'and STATIC_LIBRARY_NAME is unset. Please either ' 'change SHARED_LIBRARY_NAME or LIBRARY_NAME, or set ' 'STATIC_LIBRARY_NAME.', context) if shared_name and static_name and shared_name == static_name: raise SandboxValidationError( 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' 'but SHARED_LIBRARY_NAME is the same as ' 'STATIC_LIBRARY_NAME. Please change one of them.', context) symbols_file = context.get('SYMBOLS_FILE') if symbols_file: if not shared_lib: raise SandboxValidationError( 'SYMBOLS_FILE can only be used with a SHARED_LIBRARY.', context) if context.get('DEFFILE'): raise SandboxValidationError( 'SYMBOLS_FILE cannot be used along DEFFILE.', context) if isinstance(symbols_file, SourcePath): if not os.path.exists(symbols_file.full_path): raise SandboxValidationError( 'Path specified in SYMBOLS_FILE does not exist: %s ' '(resolved to %s)' % (symbols_file, symbols_file.full_path), context) shared_args['symbols_file'] = True else: if symbols_file.target_basename not in generated_files: raise SandboxValidationError( ('Objdir file specified in SYMBOLS_FILE not in ' + 'GENERATED_FILES: %s') % (symbols_file,), context) shared_args['symbols_file'] = symbols_file.target_basename if shared_lib: lib = SharedLibrary(context, libname, **shared_args) self._libs[libname].append(lib) self._linkage.append((context, lib, 'USE_LIBS')) linkables.append(lib) generated_files.add(lib.lib_name) if symbols_file and isinstance(symbols_file, SourcePath): script = mozpath.join( mozpath.dirname(mozpath.dirname(__file__)), 'action', 'generate_symbols_file.py') defines = () if lib.defines: defines = lib.defines.get_defines() yield GeneratedFile(context, script, 'generate_symbols_file', lib.symbols_file, [symbols_file], defines) if static_lib: is_rust_library = context.get('IS_RUST_LIBRARY') if is_rust_library: lib = self._rust_library(context, libname, static_args) else: lib = StaticLibrary(context, libname, **static_args) self._libs[libname].append(lib) self._linkage.append((context, lib, 'USE_LIBS')) linkables.append(lib) if lib_defines: if not libname: raise SandboxValidationError('LIBRARY_DEFINES needs a ' 'LIBRARY_NAME to take effect', context) lib.lib_defines.update(lib_defines) # Only emit sources if we have linkables defined in the same context. # Note the linkables are not emitted in this function, but much later, # after aggregation (because of e.g. USE_LIBS processing). if not (linkables or host_linkables): return # Avoid emitting compile flags for directories only containing rust # libraries. Emitted compile flags are only relevant to C/C++ sources # for the time being, however ldflags must be emitted for the benefit # of cargo. if not all(isinstance(l, (RustLibrary)) for l in linkables): self._compile_dirs.add(context.objdir) elif linkables: self._rust_compile_dirs.add(context.objdir) if host_linkables and not all(isinstance(l, HostRustLibrary) for l in host_linkables): self._host_compile_dirs.add(context.objdir) # TODO: objdirs with only host things in them shouldn't need target # flags, but there's at least one Makefile.in (in # build/unix/elfhack) that relies on the value of LDFLAGS being # passed to one-off rules. self._compile_dirs.add(context.objdir) sources = defaultdict(list) gen_sources = defaultdict(list) all_flags = {} for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'): srcs = sources[symbol] gen_srcs = gen_sources[symbol] context_srcs = context.get(symbol, []) for f in context_srcs: full_path = f.full_path if isinstance(f, SourcePath): srcs.append(full_path) else: assert isinstance(f, Path) gen_srcs.append(full_path) if symbol == 'SOURCES': flags = context_srcs[f] if flags: all_flags[full_path] = flags if isinstance(f, SourcePath) and not os.path.exists(full_path): raise SandboxValidationError('File listed in %s does not ' 'exist: \'%s\'' % (symbol, full_path), context) # HOST_SOURCES and UNIFIED_SOURCES only take SourcePaths, so # there should be no generated source in here assert not gen_sources['HOST_SOURCES'] assert not gen_sources['UNIFIED_SOURCES'] no_pgo = context.get('NO_PGO') no_pgo_sources = [f for f, flags in all_flags.iteritems() if flags.no_pgo] if no_pgo: if no_pgo_sources: raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo ' 'cannot be set at the same time', context) passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo if no_pgo_sources: passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources # A map from "canonical suffixes" for a particular source file # language to the range of suffixes associated with that language. # # We deliberately don't list the canonical suffix in the suffix list # in the definition; we'll add it in programmatically after defining # things. suffix_map = { '.s': set(['.asm']), '.c': set(), '.m': set(), '.mm': set(), '.cpp': set(['.cc', '.cxx']), '.S': set(), } # The inverse of the above, mapping suffixes to their canonical suffix. canonicalized_suffix_map = {} for suffix, alternatives in suffix_map.iteritems(): alternatives.add(suffix) for a in alternatives: canonicalized_suffix_map[a] = suffix def canonical_suffix_for_file(f): return canonicalized_suffix_map[mozpath.splitext(f)[1]] # A map from moz.build variables to the canonical suffixes of file # kinds that can be listed therein. all_suffixes = list(suffix_map.keys()) varmap = dict( SOURCES=(Sources, GeneratedSources, all_suffixes), HOST_SOURCES=(HostSources, None, ['.c', '.mm', '.cpp']), UNIFIED_SOURCES=(UnifiedSources, None, ['.c', '.mm', '.cpp']), ) # Track whether there are any C++ source files. # Technically this won't do the right thing for SIMPLE_PROGRAMS in # a directory with mixed C and C++ source, but it's not that important. cxx_sources = defaultdict(bool) # Source files to track for linkables associated with this context. ctxt_sources = defaultdict(lambda: defaultdict(list)) for variable, (klass, gen_klass, suffixes) in varmap.items(): allowed_suffixes = set().union(*[suffix_map[s] for s in suffixes]) # First ensure that we haven't been given filetypes that we don't # recognize. for f in itertools.chain(sources[variable], gen_sources[variable]): ext = mozpath.splitext(f)[1] if ext not in allowed_suffixes: raise SandboxValidationError( '%s has an unknown file type.' % f, context) for srcs, cls in ((sources[variable], klass), (gen_sources[variable], gen_klass)): # Now sort the files to let groupby work. sorted_files = sorted(srcs, key=canonical_suffix_for_file) for canonical_suffix, files in itertools.groupby( sorted_files, canonical_suffix_for_file): if canonical_suffix in ('.cpp', '.mm'): cxx_sources[variable] = True elif canonical_suffix in ('.s', '.S'): self._asm_compile_dirs.add(context.objdir) arglist = [context, list(files), canonical_suffix] if variable.startswith('UNIFIED_'): arglist.append(context.get('FILES_PER_UNIFIED_FILE', 16)) obj = cls(*arglist) srcs = obj.files if isinstance(obj, UnifiedSources) and obj.have_unified_mapping: srcs = dict(obj.unified_source_mapping).keys() ctxt_sources[variable][canonical_suffix] += sorted(srcs) yield obj if ctxt_sources: for linkable in linkables: for target_var in ('SOURCES', 'UNIFIED_SOURCES'): for suffix, srcs in ctxt_sources[target_var].items(): linkable.sources[suffix] += srcs for host_linkable in host_linkables: for suffix, srcs in ctxt_sources['HOST_SOURCES'].items(): host_linkable.sources[suffix] += srcs for f, flags in all_flags.iteritems(): if flags.flags: ext = mozpath.splitext(f)[1] yield PerSourceFlag(context, f, flags.flags) # If there are any C++ sources, set all the linkables defined here # to require the C++ linker. for vars, linkable_items in ((('SOURCES', 'UNIFIED_SOURCES'), linkables), (('HOST_SOURCES',), host_linkables)): for var in vars: if cxx_sources[var]: for l in linkable_items: l.cxx_link = True break def emit_from_context(self, context): """Convert a Context to tree metadata objects. This is a generator of mozbuild.frontend.data.ContextDerived instances. """ # We only want to emit an InstallationTarget if one of the consulted # variables is defined. Later on, we look up FINAL_TARGET, which has # the side-effect of populating it. So, we need to do this lookup # early. if any(k in context for k in ('FINAL_TARGET', 'XPI_NAME', 'DIST_SUBDIR')): yield InstallationTarget(context) for obj in self._handle_gn_dirs(context): yield obj # We always emit a directory traversal descriptor. This is needed by # the recursive make backend. for o in self._emit_directory_traversal_from_context(context): yield o for obj in self._process_xpidl(context): yield obj computed_flags = ComputedFlags(context, context['COMPILE_FLAGS']) computed_link_flags = ComputedFlags(context, context['LINK_FLAGS']) computed_host_flags = ComputedFlags(context, context['HOST_COMPILE_FLAGS']) computed_as_flags = ComputedFlags(context, context['ASM_FLAGS']) # Proxy some variables as-is until we have richer classes to represent # them. We should aim to keep this set small because it violates the # desired abstraction of the build definition away from makefiles. passthru = VariablePassthru(context) varlist = [ 'EXTRA_DSO_LDOPTS', 'RCFILE', 'RESFILE', 'RCINCLUDE', 'DEFFILE', 'WIN32_EXE_LDFLAGS', 'USE_EXTENSION_MANIFEST', 'NO_JS_MANIFEST', 'HAS_MISC_RULE', ] for v in varlist: if v in context and context[v]: passthru.variables[v] = context[v] if context.config.substs.get('OS_TARGET') == 'WINNT' and \ context['DELAYLOAD_DLLS']: context['LDFLAGS'].extend([('-DELAYLOAD:%s' % dll) for dll in context['DELAYLOAD_DLLS']]) context['OS_LIBS'].append('delayimp') for v in ['CMFLAGS', 'CMMFLAGS']: if v in context and context[v]: passthru.variables['MOZBUILD_' + v] = context[v] for v in ['CXXFLAGS', 'CFLAGS']: if v in context and context[v]: computed_flags.resolve_flags('MOZBUILD_%s' % v, context[v]) for v in ['HOST_CXXFLAGS', 'HOST_CFLAGS']: if v in context and context[v]: computed_host_flags.resolve_flags('MOZBUILD_%s' % v, context[v]) if 'LDFLAGS' in context and context['LDFLAGS']: computed_link_flags.resolve_flags('MOZBUILD', context['LDFLAGS']) deffile = context['DEFFILE'] if deffile and context.config.substs.get('OS_ARCH') == 'WINNT': if context.config.substs.get('GNU_CC'): computed_link_flags.resolve_flags('DEFFILE', [deffile]) else: computed_link_flags.resolve_flags('DEFFILE', ['-DEF:' + deffile]) dist_install = context['DIST_INSTALL'] if dist_install is True: passthru.variables['DIST_INSTALL'] = True elif dist_install is False: passthru.variables['NO_DIST_INSTALL'] = True # Ideally, this should be done in templates, but this is difficult at # the moment because USE_STATIC_LIBS can be set after a template # returns. Eventually, with context-based templates, it will be # possible. if (context.config.substs.get('OS_ARCH') == 'WINNT' and not context.config.substs.get('GNU_CC')): use_static_lib = (context.get('USE_STATIC_LIBS') and not context.config.substs.get('MOZ_ASAN')) rtl_flag = '-MT' if use_static_lib else '-MD' if (context.config.substs.get('MOZ_DEBUG') and not context.config.substs.get('MOZ_NO_DEBUG_RTL')): rtl_flag += 'd' computed_flags.resolve_flags('RTL', [rtl_flag]) if not context.config.substs.get('CROSS_COMPILE'): computed_host_flags.resolve_flags('RTL', [rtl_flag]) generated_files = set() localized_generated_files = set() for obj in self._process_generated_files(context): for f in obj.outputs: generated_files.add(f) if obj.localized: localized_generated_files.add(f) yield obj for path in context['CONFIGURE_SUBST_FILES']: sub = self._create_substitution(ConfigFileSubstitution, context, path) generated_files.add(str(sub.relpath)) yield sub for defines_var, cls, backend_flags in (('DEFINES', Defines, (computed_flags, computed_as_flags)), ('HOST_DEFINES', HostDefines, (computed_host_flags,))): defines = context.get(defines_var) if defines: defines_obj = cls(context, defines) if isinstance(defines_obj, Defines): # DEFINES have consumers outside the compile command line, # HOST_DEFINES do not. yield defines_obj else: # If we don't have explicitly set defines we need to make sure # initialized values if present end up in computed flags. defines_obj = cls(context, context[defines_var]) defines_from_obj = list(defines_obj.get_defines()) if defines_from_obj: for flags in backend_flags: flags.resolve_flags(defines_var, defines_from_obj) idl_vars = ( 'GENERATED_EVENTS_WEBIDL_FILES', 'GENERATED_WEBIDL_FILES', 'PREPROCESSED_TEST_WEBIDL_FILES', 'PREPROCESSED_WEBIDL_FILES', 'TEST_WEBIDL_FILES', 'WEBIDL_FILES', 'IPDL_SOURCES', 'PREPROCESSED_IPDL_SOURCES', ) for context_var in idl_vars: for name in context.get(context_var, []): self._idls[context_var].add(mozpath.join(context.srcdir, name)) # WEBIDL_EXAMPLE_INTERFACES do not correspond to files. for name in context.get('WEBIDL_EXAMPLE_INTERFACES', []): self._idls['WEBIDL_EXAMPLE_INTERFACES'].add(name) local_includes = [] for local_include in context.get('LOCAL_INCLUDES', []): full_path = local_include.full_path if (not isinstance(local_include, ObjDirPath) and not os.path.exists(full_path)): raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' 'does not exist: %s (resolved to %s)' % (local_include, full_path), context) if (full_path == context.config.topsrcdir or full_path == context.config.topobjdir): raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' 'is not allowed: %s (resolved to %s)' % (local_include, full_path), context) include_obj = LocalInclude(context, local_include) local_includes.append(include_obj.path.full_path) yield include_obj computed_flags.resolve_flags('LOCAL_INCLUDES', ['-I%s' % p for p in local_includes]) computed_as_flags.resolve_flags('LOCAL_INCLUDES', ['-I%s' % p for p in local_includes]) computed_host_flags.resolve_flags('LOCAL_INCLUDES', ['-I%s' % p for p in local_includes]) for obj in self._handle_linkables(context, passthru, generated_files): yield obj generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in self._binaries.keys()]) components = [] for var, cls in ( ('EXPORTS', Exports), ('FINAL_TARGET_FILES', FinalTargetFiles), ('FINAL_TARGET_PP_FILES', FinalTargetPreprocessedFiles), ('LOCALIZED_FILES', LocalizedFiles), ('LOCALIZED_PP_FILES', LocalizedPreprocessedFiles), ('OBJDIR_FILES', ObjdirFiles), ('OBJDIR_PP_FILES', ObjdirPreprocessedFiles), ('TEST_HARNESS_FILES', TestHarnessFiles), ): all_files = context.get(var) if not all_files: continue if dist_install is False and var != 'TEST_HARNESS_FILES': raise SandboxValidationError( '%s cannot be used with DIST_INSTALL = False' % var, context) has_prefs = False has_resources = False for base, files in all_files.walk(): if var == 'TEST_HARNESS_FILES' and not base: raise SandboxValidationError( 'Cannot install files to the root of TEST_HARNESS_FILES', context) if base == 'components': components.extend(files) if base == 'defaults/pref': has_prefs = True if mozpath.split(base)[0] == 'res': has_resources = True for f in files: if (var in ('FINAL_TARGET_PP_FILES', 'OBJDIR_PP_FILES', 'LOCALIZED_PP_FILES') and not isinstance(f, SourcePath)): raise SandboxValidationError( ('Only source directory paths allowed in ' + '%s: %s') % (var, f,), context) if var.startswith('LOCALIZED_'): if isinstance(f, SourcePath): if f.startswith('en-US/'): pass elif '/locales/en-US/' in f: pass else: raise SandboxValidationError( '%s paths must start with `en-US/` or ' 'contain `/locales/en-US/`: %s' % (var, f,), context) if not isinstance(f, ObjDirPath): path = f.full_path if '*' not in path and not os.path.exists(path): raise SandboxValidationError( 'File listed in %s does not exist: %s' % (var, path), context) else: # TODO: Bug 1254682 - The '/' check is to allow # installing files generated from other directories, # which is done occasionally for tests. However, it # means we don't fail early if the file isn't actually # created by the other moz.build file. if f.target_basename not in generated_files and '/' not in f: raise SandboxValidationError( ('Objdir file listed in %s not in ' + 'GENERATED_FILES: %s') % (var, f), context) if var.startswith('LOCALIZED_'): # Further require that LOCALIZED_FILES are from # LOCALIZED_GENERATED_FILES. if f.target_basename not in localized_generated_files: raise SandboxValidationError( ('Objdir file listed in %s not in ' + 'LOCALIZED_GENERATED_FILES: %s') % (var, f), context) else: # Additionally, don't allow LOCALIZED_GENERATED_FILES to be used # in anything *but* LOCALIZED_FILES. if f.target_basename in localized_generated_files: raise SandboxValidationError( ('Outputs of LOCALIZED_GENERATED_FILES cannot be used in %s: ' + '%s') % (var, f), context) # Addons (when XPI_NAME is defined) and Applications (when # DIST_SUBDIR is defined) use a different preferences directory # (default/preferences) from the one the GRE uses (defaults/pref). # Hence, we move the files from the latter to the former in that # case. if has_prefs and (context.get('XPI_NAME') or context.get('DIST_SUBDIR')): all_files.defaults.preferences += all_files.defaults.pref del all_files.defaults._children['pref'] if has_resources and (context.get('DIST_SUBDIR') or context.get('XPI_NAME')): raise SandboxValidationError( 'RESOURCES_FILES cannot be used with DIST_SUBDIR or ' 'XPI_NAME.', context) yield cls(context, all_files) # Check for manifest declarations in EXTRA_{PP_,}COMPONENTS. if any(e.endswith('.js') for e in components) and \ not any(e.endswith('.manifest') for e in components) and \ not context.get('NO_JS_MANIFEST', False): raise SandboxValidationError('A .js component was specified in EXTRA_COMPONENTS ' 'or EXTRA_PP_COMPONENTS without a matching ' '.manifest file. See ' 'https://developer.mozilla.org/en/XPCOM/XPCOM_changes_in_Gecko_2.0 .', context); for c in components: if c.endswith('.manifest'): yield ChromeManifestEntry(context, 'chrome.manifest', Manifest('components', mozpath.basename(c))) if self.config.substs.get('MOZ_RUST_TESTS', None): rust_test = context.get('RUST_TEST', None) if rust_test: # TODO: more sophisticated checking of the declared name vs. # contents of the Cargo.toml file. features = context.get('RUST_TEST_FEATURES', []) yield RustTest(context, rust_test, features) for obj in self._process_test_manifests(context): yield obj for obj in self._process_jar_manifests(context): yield obj for name, jar in context.get('JAVA_JAR_TARGETS', {}).items(): yield ContextWrapped(context, jar) computed_as_flags.resolve_flags('MOZBUILD', context.get('ASFLAGS')) if context.get('USE_YASM') is True: yasm = context.config.substs.get('YASM') if not yasm: raise SandboxValidationError('yasm is not available', context) passthru.variables['AS'] = yasm passthru.variables['AS_DASH_C_FLAG'] = '' computed_as_flags.resolve_flags('OS', context.config.substs.get('YASM_ASFLAGS', [])) if passthru.variables: yield passthru if context.objdir in self._compile_dirs: self._compile_flags[context.objdir] = computed_flags yield computed_link_flags elif context.objdir in self._rust_compile_dirs: yield computed_link_flags if context.objdir in self._asm_compile_dirs: self._compile_as_flags[context.objdir] = computed_as_flags if context.objdir in self._host_compile_dirs: yield computed_host_flags def _create_substitution(self, cls, context, path): sub = cls(context) sub.input_path = '%s.in' % path.full_path sub.output_path = path.translated sub.relpath = path return sub def _process_xpidl(self, context): # XPIDL source files get processed and turned into .h and .xpt files. # If there are multiple XPIDL files in a directory, they get linked # together into a final .xpt, which has the name defined by # XPIDL_MODULE. xpidl_module = context['XPIDL_MODULE'] if context['XPIDL_SOURCES'] and not xpidl_module: raise SandboxValidationError('XPIDL_MODULE must be defined if ' 'XPIDL_SOURCES is defined.', context) if xpidl_module and not context['XPIDL_SOURCES']: raise SandboxValidationError('XPIDL_MODULE cannot be defined ' 'unless there are XPIDL_SOURCES', context) if context['XPIDL_SOURCES'] and context['DIST_INSTALL'] is False: self.log(logging.WARN, 'mozbuild_warning', dict( path=context.main_path), '{path}: DIST_INSTALL = False has no effect on XPIDL_SOURCES.') for idl in context['XPIDL_SOURCES']: yield XPIDLFile(context, mozpath.join(context.srcdir, idl), xpidl_module, add_to_manifest=not context['XPIDL_NO_MANIFEST']) def _process_generated_files(self, context): for path in context['CONFIGURE_DEFINE_FILES']: script = mozpath.join(mozpath.dirname(mozpath.dirname(__file__)), 'action', 'process_define_files.py') yield GeneratedFile(context, script, 'process_define_file', unicode(path), [Path(context, path + '.in')]) generated_files = context.get('GENERATED_FILES') or [] localized_generated_files = context.get('LOCALIZED_GENERATED_FILES') or [] if not (generated_files or localized_generated_files): return for (localized, gen) in ((False, generated_files), (True, localized_generated_files)): for f in gen: flags = gen[f] outputs = f inputs = [] if flags.script: method = "main" script = SourcePath(context, flags.script).full_path # Deal with cases like "C:\\path\\to\\script.py:function". if '.py:' in script: script, method = script.rsplit('.py:', 1) script += '.py' if not os.path.exists(script): raise SandboxValidationError( 'Script for generating %s does not exist: %s' % (f, script), context) if os.path.splitext(script)[1] != '.py': raise SandboxValidationError( 'Script for generating %s does not end in .py: %s' % (f, script), context) else: script = None method = None for i in flags.inputs: p = Path(context, i) if (isinstance(p, SourcePath) and not os.path.exists(p.full_path)): raise SandboxValidationError( 'Input for generating %s does not exist: %s' % (f, p.full_path), context) inputs.append(p) yield GeneratedFile(context, script, method, outputs, inputs, flags.flags, localized=localized) def _process_test_manifests(self, context): for prefix, info in TEST_MANIFESTS.items(): for path, manifest in context.get('%s_MANIFESTS' % prefix, []): for obj in self._process_test_manifest(context, info, path, manifest): yield obj for flavor in REFTEST_FLAVORS: for path, manifest in context.get('%s_MANIFESTS' % flavor.upper(), []): for obj in self._process_reftest_manifest(context, flavor, path, manifest): yield obj for flavor in WEB_PLATFORM_TESTS_FLAVORS: for path, manifest in context.get("%s_MANIFESTS" % flavor.upper().replace('-', '_'), []): for obj in self._process_web_platform_tests_manifest(context, path, manifest): yield obj def _process_test_manifest(self, context, info, manifest_path, mpmanifest): flavor, install_root, install_subdir, package_tests = info path = manifest_path.full_path manifest_dir = mozpath.dirname(path) manifest_reldir = mozpath.dirname(mozpath.relpath(path, context.config.topsrcdir)) manifest_sources = [mozpath.relpath(pth, context.config.topsrcdir) for pth in mpmanifest.source_files] install_prefix = mozpath.join(install_root, install_subdir) try: if not mpmanifest.tests: raise SandboxValidationError('Empty test manifest: %s' % path, context) defaults = mpmanifest.manifest_defaults[os.path.normpath(path)] obj = TestManifest(context, path, mpmanifest, flavor=flavor, install_prefix=install_prefix, relpath=mozpath.join(manifest_reldir, mozpath.basename(path)), sources=manifest_sources, dupe_manifest='dupe-manifest' in defaults) filtered = mpmanifest.tests missing = [t['name'] for t in filtered if not os.path.exists(t['path'])] if missing: raise SandboxValidationError('Test manifest (%s) lists ' 'test that does not exist: %s' % ( path, ', '.join(missing)), context) out_dir = mozpath.join(install_prefix, manifest_reldir) if 'install-to-subdir' in defaults: # This is terrible, but what are you going to do? out_dir = mozpath.join(out_dir, defaults['install-to-subdir']) obj.manifest_obj_relpath = mozpath.join(manifest_reldir, defaults['install-to-subdir'], mozpath.basename(path)) def process_support_files(test): install_info = self._test_files_converter.convert_support_files( test, install_root, manifest_dir, out_dir) obj.pattern_installs.extend(install_info.pattern_installs) for source, dest in install_info.installs: obj.installs[source] = (dest, False) obj.external_installs |= install_info.external_installs for install_path in install_info.deferred_installs: if all(['*' not in install_path, not os.path.isfile(mozpath.join(context.config.topsrcdir, install_path[2:])), install_path not in install_info.external_installs]): raise SandboxValidationError('Error processing test ' 'manifest %s: entry in support-files not present ' 'in the srcdir: %s' % (path, install_path), context) obj.deferred_installs |= install_info.deferred_installs for test in filtered: obj.tests.append(test) # Some test files are compiled and should not be copied into the # test package. They function as identifiers rather than files. if package_tests: manifest_relpath = mozpath.relpath(test['path'], mozpath.dirname(test['manifest'])) obj.installs[mozpath.normpath(test['path'])] = \ ((mozpath.join(out_dir, manifest_relpath)), True) process_support_files(test) for path, m_defaults in mpmanifest.manifest_defaults.items(): process_support_files(m_defaults) # We also copy manifests into the output directory, # including manifests from [include:foo] directives. for mpath in mpmanifest.manifests(): mpath = mozpath.normpath(mpath) out_path = mozpath.join(out_dir, mozpath.basename(mpath)) obj.installs[mpath] = (out_path, False) # Some manifests reference files that are auto generated as # part of the build or shouldn't be installed for some # reason. Here, we prune those files from the install set. # FUTURE we should be able to detect autogenerated files from # other build metadata. Once we do that, we can get rid of this. for f in defaults.get('generated-files', '').split(): # We re-raise otherwise the stack trace isn't informative. try: del obj.installs[mozpath.join(manifest_dir, f)] except KeyError: raise SandboxValidationError('Error processing test ' 'manifest %s: entry in generated-files not present ' 'elsewhere in manifest: %s' % (path, f), context) yield obj except (AssertionError, Exception): raise SandboxValidationError('Error processing test ' 'manifest file %s: %s' % (path, '\n'.join(traceback.format_exception(*sys.exc_info()))), context) def _process_reftest_manifest(self, context, flavor, manifest_path, manifest): manifest_full_path = manifest_path.full_path manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, context.config.topsrcdir)) # reftest manifests don't come from manifest parser. But they are # similar enough that we can use the same emitted objects. Note # that we don't perform any installs for reftests. obj = TestManifest(context, manifest_full_path, manifest, flavor=flavor, install_prefix='%s/' % flavor, relpath=mozpath.join(manifest_reldir, mozpath.basename(manifest_path))) for test, source_manifest in sorted(manifest.tests): obj.tests.append({ 'path': test, 'here': mozpath.dirname(test), 'manifest': source_manifest, 'name': mozpath.basename(test), 'head': '', 'support-files': '', 'subsuite': '', }) yield obj def _process_web_platform_tests_manifest(self, context, paths, manifest): manifest_path, tests_root = paths manifest_full_path = mozpath.normpath(mozpath.join( context.srcdir, manifest_path)) manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, context.config.topsrcdir)) tests_root = mozpath.normpath(mozpath.join(context.srcdir, tests_root)) # Create a equivalent TestManifest object obj = TestManifest(context, manifest_full_path, manifest, flavor="web-platform-tests", relpath=mozpath.join(manifest_reldir, mozpath.basename(manifest_path)), install_prefix="web-platform/") for test_type, path, tests in manifest: path = mozpath.join(tests_root, path) if test_type not in ["testharness", "reftest", "wdspec"]: continue for test in tests: obj.tests.append({ 'path': path, 'here': mozpath.dirname(path), 'manifest': manifest_path, 'name': test.id, 'head': '', 'support-files': '', 'subsuite': '', }) yield obj def _process_jar_manifests(self, context): jar_manifests = context.get('JAR_MANIFESTS', []) if len(jar_manifests) > 1: raise SandboxValidationError('While JAR_MANIFESTS is a list, ' 'it is currently limited to one value.', context) for path in jar_manifests: yield JARManifest(context, path) # Temporary test to look for jar.mn files that creep in without using # the new declaration. Before, we didn't require jar.mn files to # declared anywhere (they were discovered). This will detect people # relying on the old behavior. if os.path.exists(os.path.join(context.srcdir, 'jar.mn')): if 'jar.mn' not in jar_manifests: raise SandboxValidationError('A jar.mn exists but it ' 'is not referenced in the moz.build file. ' 'Please define JAR_MANIFESTS.', context) def _emit_directory_traversal_from_context(self, context): o = DirectoryTraversal(context) o.dirs = context.get('DIRS', []) # Some paths have a subconfigure, yet also have a moz.build. Those # shouldn't end up in self._external_paths. if o.objdir: self._external_paths -= { o.relobjdir } yield o