1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, unicode_literals
6
7import json
8import os
9
10import mozpack.path as mozpath
11
12from mozbuild.backend.base import BuildBackend
13
14from mozbuild.frontend.context import (
15    Context,
16    ObjDirPath,
17    Path,
18    RenamedSourcePath,
19    VARIABLES,
20)
21from mozbuild.frontend.data import (
22    BaseProgram,
23    ChromeManifestEntry,
24    ConfigFileSubstitution,
25    Exports,
26    FinalTargetPreprocessedFiles,
27    FinalTargetFiles,
28    GeneratedSources,
29    GnProjectData,
30    IPDLCollection,
31    SharedLibrary,
32    UnifiedSources,
33    XPIDLFile,
34    WebIDLCollection,
35)
36from mozbuild.jar import (
37    DeprecatedJarManifest,
38    JarManifestParser,
39)
40from mozbuild.preprocessor import Preprocessor
41from mozpack.chrome.manifest import parse_manifest_line
42
43from mozbuild.util import group_unified_files
44
45class XPIDLManager(object):
46    """Helps manage XPCOM IDLs in the context of the build system."""
47    def __init__(self, config):
48        self.config = config
49        self.topsrcdir = config.topsrcdir
50        self.topobjdir = config.topobjdir
51
52        self.idls = {}
53        self.modules = {}
54        self.interface_manifests = {}
55        self.chrome_manifests = set()
56
57    def register_idl(self, idl, allow_existing=False):
58        """Registers an IDL file with this instance.
59
60        The IDL file will be built, installed, etc.
61        """
62        basename = mozpath.basename(idl.source_path)
63        root = mozpath.splitext(basename)[0]
64        xpt = '%s.xpt' % idl.module
65        manifest = mozpath.join(idl.install_target, 'components', 'interfaces.manifest')
66        chrome_manifest = mozpath.join(idl.install_target, 'chrome.manifest')
67
68        entry = {
69            'source': idl.source_path,
70            'module': idl.module,
71            'basename': basename,
72            'root': root,
73            'manifest': manifest,
74        }
75
76        if not allow_existing and entry['basename'] in self.idls:
77            raise Exception('IDL already registered: %s' % entry['basename'])
78
79        self.idls[entry['basename']] = entry
80        t = self.modules.setdefault(entry['module'], (idl.install_target, set()))
81        t[1].add(entry['root'])
82
83        if idl.add_to_manifest:
84            self.interface_manifests.setdefault(manifest, set()).add(xpt)
85            self.chrome_manifests.add(chrome_manifest)
86
87
88class BinariesCollection(object):
89    """Tracks state of binaries produced by the build."""
90
91    def __init__(self):
92        self.shared_libraries = []
93        self.programs = []
94
95class CommonBackend(BuildBackend):
96    """Holds logic common to all build backends."""
97
98    def _init(self):
99        self._idl_manager = XPIDLManager(self.environment)
100        self._binaries = BinariesCollection()
101        self._configs = set()
102        self._generated_sources = set()
103
104    def consume_object(self, obj):
105        self._configs.add(obj.config)
106
107        if isinstance(obj, XPIDLFile):
108            # TODO bug 1240134 tracks not processing XPIDL files during
109            # artifact builds.
110            self._idl_manager.register_idl(obj)
111
112        elif isinstance(obj, ConfigFileSubstitution):
113            # Do not handle ConfigFileSubstitution for Makefiles. Leave that
114            # to other
115            if mozpath.basename(obj.output_path) == 'Makefile':
116                return False
117            with self._get_preprocessor(obj) as pp:
118                pp.do_include(obj.input_path)
119            self.backend_input_files.add(obj.input_path)
120
121        elif isinstance(obj, WebIDLCollection):
122            self._handle_webidl_collection(obj)
123
124        elif isinstance(obj, IPDLCollection):
125            self._handle_generated_sources(mozpath.join(obj.objdir, f)
126                                           for f in obj.all_generated_sources())
127            self._write_unified_files(obj.unified_source_mapping, obj.objdir,
128                                      poison_windows_h=False)
129            self._handle_ipdl_sources(obj.objdir,
130                                      list(sorted(obj.all_sources())),
131                                      list(sorted(obj.all_preprocessed_sources())),
132                                      list(sorted(obj.all_regular_sources())),
133                                      obj.unified_source_mapping)
134
135        elif isinstance(obj, UnifiedSources):
136            # Unified sources aren't relevant to artifact builds.
137            if self.environment.is_artifact_build:
138                return True
139
140            if obj.have_unified_mapping:
141                self._write_unified_files(obj.unified_source_mapping, obj.objdir)
142            if hasattr(self, '_process_unified_sources'):
143                self._process_unified_sources(obj)
144
145        elif isinstance(obj, BaseProgram):
146            self._binaries.programs.append(obj)
147            return False
148
149        elif isinstance(obj, SharedLibrary):
150            self._binaries.shared_libraries.append(obj)
151            return False
152
153        elif isinstance(obj, GeneratedSources):
154            self._handle_generated_sources(obj.files)
155            return False
156
157        elif isinstance(obj, Exports):
158            objdir_files = [f.full_path for path, files in obj.files.walk() for f in files if isinstance(f, ObjDirPath)]
159            if objdir_files:
160                self._handle_generated_sources(objdir_files)
161            return False
162
163        elif isinstance(obj, GnProjectData):
164            # These are only handled by special purpose build backends,
165            # ignore them here.
166            return True
167
168        else:
169            return False
170
171        return True
172
173    def consume_finished(self):
174        if len(self._idl_manager.idls):
175            self._write_rust_xpidl_summary(self._idl_manager)
176            self._handle_idl_manager(self._idl_manager)
177            self._handle_generated_sources(mozpath.join(self.environment.topobjdir, 'dist/include/%s.h' % idl['root']) for idl in self._idl_manager.idls.values())
178
179
180        for config in self._configs:
181            self.backend_input_files.add(config.source)
182
183        # Write out a machine-readable file describing binaries.
184        topobjdir = self.environment.topobjdir
185        with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
186            d = {
187                'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
188                'programs': [p.to_dict() for p in self._binaries.programs],
189            }
190            json.dump(d, fh, sort_keys=True, indent=4)
191
192        # Write out a file listing generated sources.
193        with self._write_file(mozpath.join(topobjdir, 'generated-sources.json')) as fh:
194            d = {
195                'sources': sorted(self._generated_sources),
196            }
197            json.dump(d, fh, sort_keys=True, indent=4)
198
199    def _handle_generated_sources(self, files):
200        self._generated_sources.update(mozpath.relpath(f, self.environment.topobjdir) for f in files)
201
202    def _handle_webidl_collection(self, webidls):
203
204        bindings_dir = mozpath.join(self.environment.topobjdir, 'dom', 'bindings')
205
206        all_inputs = set(webidls.all_static_sources())
207        for s in webidls.all_non_static_basenames():
208            all_inputs.add(mozpath.join(bindings_dir, s))
209
210        generated_events_stems = webidls.generated_events_stems()
211        exported_stems = webidls.all_regular_stems()
212
213        # The WebIDL manager reads configuration from a JSON file. So, we
214        # need to write this file early.
215        o = dict(
216            webidls=sorted(all_inputs),
217            generated_events_stems=sorted(generated_events_stems),
218            exported_stems=sorted(exported_stems),
219            example_interfaces=sorted(webidls.example_interfaces),
220        )
221
222        file_lists = mozpath.join(bindings_dir, 'file-lists.json')
223        with self._write_file(file_lists) as fh:
224            json.dump(o, fh, sort_keys=True, indent=2)
225
226        import mozwebidlcodegen
227
228        manager = mozwebidlcodegen.create_build_system_manager(
229            self.environment.topsrcdir,
230            self.environment.topobjdir,
231            mozpath.join(self.environment.topobjdir, 'dist')
232        )
233        self._handle_generated_sources(manager.expected_build_output_files())
234        self._write_unified_files(webidls.unified_source_mapping, bindings_dir,
235                                  poison_windows_h=True)
236        self._handle_webidl_build(bindings_dir, webidls.unified_source_mapping,
237                                  webidls,
238                                  manager.expected_build_output_files(),
239                                  manager.GLOBAL_DEFINE_FILES)
240
241    def _write_unified_file(self, unified_file, source_filenames,
242                            output_directory, poison_windows_h=False):
243        with self._write_file(mozpath.join(output_directory, unified_file)) as f:
244            f.write('#define MOZ_UNIFIED_BUILD\n')
245            includeTemplate = '#include "%(cppfile)s"'
246            if poison_windows_h:
247                includeTemplate += (
248                    '\n'
249                    '#ifdef _WINDOWS_\n'
250                    '#error "%(cppfile)s included windows.h"\n'
251                    "#endif")
252            includeTemplate += (
253                '\n'
254                '#ifdef PL_ARENA_CONST_ALIGN_MASK\n'
255                '#error "%(cppfile)s uses PL_ARENA_CONST_ALIGN_MASK, '
256                'so it cannot be built in unified mode."\n'
257                '#undef PL_ARENA_CONST_ALIGN_MASK\n'
258                '#endif\n'
259                '#ifdef INITGUID\n'
260                '#error "%(cppfile)s defines INITGUID, '
261                'so it cannot be built in unified mode."\n'
262                '#undef INITGUID\n'
263                '#endif')
264            f.write('\n'.join(includeTemplate % { "cppfile": s } for
265                              s in source_filenames))
266
267    def _write_unified_files(self, unified_source_mapping, output_directory,
268                             poison_windows_h=False):
269        for unified_file, source_filenames in unified_source_mapping:
270            self._write_unified_file(unified_file, source_filenames,
271                                     output_directory, poison_windows_h)
272
273    def _consume_jar_manifest(self, obj):
274        # Ideally, this would all be handled somehow in the emitter, but
275        # this would require all the magic surrounding l10n and addons in
276        # the recursive make backend to die, which is not going to happen
277        # any time soon enough.
278        # Notably missing:
279        # - DEFINES from config/config.mk
280        # - L10n support
281        # - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
282        #   moz.build, but it doesn't matter in dist/bin.
283        pp = Preprocessor()
284        if obj.defines:
285            pp.context.update(obj.defines.defines)
286        pp.context.update(self.environment.defines)
287        pp.context.update(
288            AB_CD='en-US',
289            BUILD_FASTER=1,
290        )
291        pp.out = JarManifestParser()
292        try:
293            pp.do_include(obj.path.full_path)
294        except DeprecatedJarManifest as e:
295            raise DeprecatedJarManifest('Parsing error while processing %s: %s'
296                                        % (obj.path.full_path, e.message))
297        self.backend_input_files |= pp.includes
298
299        for jarinfo in pp.out:
300            jar_context = Context(
301                allowed_variables=VARIABLES, config=obj._context.config)
302            jar_context.push_source(obj._context.main_path)
303            jar_context.push_source(obj.path.full_path)
304
305            install_target = obj.install_target
306            if jarinfo.base:
307                install_target = mozpath.normpath(
308                    mozpath.join(install_target, jarinfo.base))
309            jar_context['FINAL_TARGET'] = install_target
310            if obj.defines:
311                jar_context['DEFINES'] = obj.defines.defines
312            files = jar_context['FINAL_TARGET_FILES']
313            files_pp = jar_context['FINAL_TARGET_PP_FILES']
314
315            for e in jarinfo.entries:
316                if e.is_locale:
317                    if jarinfo.relativesrcdir:
318                        src = '/%s' % jarinfo.relativesrcdir
319                    else:
320                        src = ''
321                    src = mozpath.join(src, 'en-US', e.source)
322                else:
323                    src = e.source
324
325                src = Path(jar_context, src)
326
327                if '*' not in e.source and not os.path.exists(src.full_path):
328                    if e.is_locale:
329                        raise Exception(
330                            '%s: Cannot find %s' % (obj.path, e.source))
331                    if e.source.startswith('/'):
332                        src = Path(jar_context, '!' + e.source)
333                    else:
334                        # This actually gets awkward if the jar.mn is not
335                        # in the same directory as the moz.build declaring
336                        # it, but it's how it works in the recursive make,
337                        # not that anything relies on that, but it's simpler.
338                        src = Path(obj._context, '!' + e.source)
339
340                output_basename = mozpath.basename(e.output)
341                if output_basename != src.target_basename:
342                    src = RenamedSourcePath(jar_context,
343                                            (src, output_basename))
344                path = mozpath.dirname(mozpath.join(jarinfo.name, e.output))
345
346                if e.preprocess:
347                    if '*' in e.source:
348                        raise Exception('%s: Wildcards are not supported with '
349                                        'preprocessing' % obj.path)
350                    files_pp[path] += [src]
351                else:
352                    files[path] += [src]
353
354            if files:
355                self.consume_object(FinalTargetFiles(jar_context, files))
356            if files_pp:
357                self.consume_object(
358                    FinalTargetPreprocessedFiles(jar_context, files_pp))
359
360            for m in jarinfo.chrome_manifests:
361                entry = parse_manifest_line(
362                    mozpath.dirname(jarinfo.name),
363                    m.replace('%', mozpath.basename(jarinfo.name) + '/'))
364                self.consume_object(ChromeManifestEntry(
365                    jar_context, '%s.manifest' % jarinfo.name, entry))
366
367    def _write_rust_xpidl_summary(self, manager):
368        """Write out a rust file which includes the generated xpcom rust modules"""
369        topobjdir = self.environment.topobjdir
370
371        include_tmpl = "include!(concat!(env!(\"MOZ_TOPOBJDIR\"), \"/dist/xpcrs/%s/%s.rs\"))"
372
373        with self._write_file(mozpath.join(topobjdir, 'dist', 'xpcrs', 'rt', 'all.rs')) as fh:
374            fh.write("// THIS FILE IS GENERATED - DO NOT EDIT\n\n")
375            for idl in manager.idls.values():
376                fh.write(include_tmpl % ("rt", idl['root']))
377                fh.write(";\n")
378
379        with self._write_file(mozpath.join(topobjdir, 'dist', 'xpcrs', 'bt', 'all.rs')) as fh:
380            fh.write("// THIS FILE IS GENERATED - DO NOT EDIT\n\n")
381            fh.write("&[\n")
382            for idl in manager.idls.values():
383                fh.write(include_tmpl % ("bt", idl['root']))
384                fh.write(",\n")
385            fh.write("]\n")
386