1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import
6
7import os
8import sys
9import json
10
11from collections import Iterable, OrderedDict
12from types import StringTypes, ModuleType
13
14import mozpack.path as mozpath
15
16from mozbuild.util import (
17    FileAvoidWrite,
18    memoized_property,
19    ReadOnlyDict,
20)
21from mozbuild.shellutil import quote as shell_quote
22
23
24if sys.version_info.major == 2:
25    text_type = unicode
26else:
27    text_type = str
28
29
30class BuildConfig(object):
31    """Represents the output of configure."""
32
33    _CODE_CACHE = {}
34
35    def __init__(self):
36        self.topsrcdir = None
37        self.topobjdir = None
38        self.defines = {}
39        self.non_global_defines = []
40        self.substs = {}
41        self.files = []
42        self.mozconfig = None
43
44    @classmethod
45    def from_config_status(cls, path):
46        """Create an instance from a config.status file."""
47        code_cache = cls._CODE_CACHE
48        mtime = os.path.getmtime(path)
49
50        # cache the compiled code as it can be reused
51        # we cache it the first time, or if the file changed
52        if not path in code_cache or code_cache[path][0] != mtime:
53            # Add config.status manually to sys.modules so it gets picked up by
54            # iter_modules_in_path() for automatic dependencies.
55            mod = ModuleType('config.status')
56            mod.__file__ = path
57            sys.modules['config.status'] = mod
58
59            with open(path, 'rt') as fh:
60                source = fh.read()
61                code_cache[path] = (
62                    mtime,
63                    compile(source, path, 'exec', dont_inherit=1)
64                )
65
66        g = {
67            '__builtins__': __builtins__,
68            '__file__': path,
69        }
70        l = {}
71        exec(code_cache[path][1], g, l)
72
73        config = BuildConfig()
74
75        for name in l['__all__']:
76            setattr(config, name, l[name])
77
78        return config
79
80
81class ConfigEnvironment(object):
82    """Perform actions associated with a configured but bare objdir.
83
84    The purpose of this class is to preprocess files from the source directory
85    and output results in the object directory.
86
87    There are two types of files: config files and config headers,
88    each treated through a different member function.
89
90    Creating a ConfigEnvironment requires a few arguments:
91      - topsrcdir and topobjdir are, respectively, the top source and
92        the top object directory.
93      - defines is a dict filled from AC_DEFINE and AC_DEFINE_UNQUOTED in
94        autoconf.
95      - non_global_defines are a list of names appearing in defines above
96        that are not meant to be exported in ACDEFINES (see below)
97      - substs is a dict filled from AC_SUBST in autoconf.
98
99    ConfigEnvironment automatically defines one additional substs variable
100    from all the defines not appearing in non_global_defines:
101      - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
102        preprocessor command lines. The order in which defines were given
103        when creating the ConfigEnvironment is preserved.
104    and two other additional subst variables from all the other substs:
105      - ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted
106        order, for use in autoconf.mk. It includes ACDEFINES
107        Only substs with a VALUE are included, such that the resulting file
108        doesn't change when new empty substs are added.
109        This results in less invalidation of build dependencies in the case
110        of autoconf.mk..
111      - ALLEMPTYSUBSTS contains the substs with an empty value, in the form
112        NAME =.
113
114    ConfigEnvironment expects a "top_srcdir" subst to be set with the top
115    source directory, in msys format on windows. It is used to derive a
116    "srcdir" subst when treating config files. It can either be an absolute
117    path or a path relative to the topobjdir.
118    """
119
120    def __init__(self, topsrcdir, topobjdir, defines=None,
121        non_global_defines=None, substs=None, source=None, mozconfig=None):
122
123        if not source:
124            source = mozpath.join(topobjdir, 'config.status')
125        self.source = source
126        self.defines = ReadOnlyDict(defines or {})
127        self.non_global_defines = non_global_defines or []
128        self.substs = dict(substs or {})
129        self.topsrcdir = mozpath.abspath(topsrcdir)
130        self.topobjdir = mozpath.abspath(topobjdir)
131        self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None
132        self.lib_prefix = self.substs.get('LIB_PREFIX', '')
133        self.rust_lib_prefix = self.substs.get('RUST_LIB_PREFIX', '')
134        if 'LIB_SUFFIX' in self.substs:
135            self.lib_suffix = '.%s' % self.substs['LIB_SUFFIX']
136        if 'RUST_LIB_SUFFIX' in self.substs:
137            self.rust_lib_suffix = '.%s' % self.substs['RUST_LIB_SUFFIX']
138        self.dll_prefix = self.substs.get('DLL_PREFIX', '')
139        self.dll_suffix = self.substs.get('DLL_SUFFIX', '')
140        if self.substs.get('IMPORT_LIB_SUFFIX'):
141            self.import_prefix = self.lib_prefix
142            self.import_suffix = '.%s' % self.substs['IMPORT_LIB_SUFFIX']
143        else:
144            self.import_prefix = self.dll_prefix
145            self.import_suffix = self.dll_suffix
146        self.bin_suffix = self.substs.get('BIN_SUFFIX', '')
147
148        global_defines = [name for name in self.defines
149            if not name in self.non_global_defines]
150        self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name,
151            shell_quote(self.defines[name]).replace('$', '$$'))
152            for name in sorted(global_defines)])
153        def serialize(name, obj):
154            if isinstance(obj, StringTypes):
155                return obj
156            if isinstance(obj, Iterable):
157                return ' '.join(obj)
158            raise Exception('Unhandled type %s for %s', type(obj), str(name))
159        self.substs['ALLSUBSTS'] = '\n'.join(sorted(['%s = %s' % (name,
160            serialize(name, self.substs[name])) for name in self.substs if self.substs[name]]))
161        self.substs['ALLEMPTYSUBSTS'] = '\n'.join(sorted(['%s =' % name
162            for name in self.substs if not self.substs[name]]))
163
164        self.substs = ReadOnlyDict(self.substs)
165
166        self.external_source_dir = None
167        external = self.substs.get('EXTERNAL_SOURCE_DIR', '')
168        if external:
169            external = mozpath.normpath(external)
170            if not os.path.isabs(external):
171                external = mozpath.join(self.topsrcdir, external)
172            self.external_source_dir = mozpath.normpath(external)
173
174        # Populate a Unicode version of substs. This is an optimization to make
175        # moz.build reading faster, since each sandbox needs a Unicode version
176        # of these variables and doing it over a thousand times is a hotspot
177        # during sandbox execution!
178        # Bug 844509 tracks moving everything to Unicode.
179        self.substs_unicode = {}
180
181        def decode(v):
182            if not isinstance(v, text_type):
183                try:
184                    return v.decode('utf-8')
185                except UnicodeDecodeError:
186                    return v.decode('utf-8', 'replace')
187
188        for k, v in self.substs.items():
189            if not isinstance(v, StringTypes):
190                if isinstance(v, Iterable):
191                    type(v)(decode(i) for i in v)
192            elif not isinstance(v, text_type):
193                v = decode(v)
194
195            self.substs_unicode[k] = v
196
197        self.substs_unicode = ReadOnlyDict(self.substs_unicode)
198
199    @property
200    def is_artifact_build(self):
201        return self.substs.get('MOZ_ARTIFACT_BUILDS', False)
202
203    @memoized_property
204    def acdefines(self):
205        acdefines = dict((name, self.defines[name])
206                         for name in self.defines
207                         if name not in self.non_global_defines)
208        return ReadOnlyDict(acdefines)
209
210    @staticmethod
211    def from_config_status(path):
212        config = BuildConfig.from_config_status(path)
213
214        return ConfigEnvironment(config.topsrcdir, config.topobjdir,
215            config.defines, config.non_global_defines, config.substs, path)
216
217
218class PartialConfigDict(object):
219    """Facilitates mapping the config.statusd defines & substs with dict-like access.
220
221    This allows a buildconfig client to use buildconfig.defines['FOO'] (and
222    similar for substs), where the value of FOO is delay-loaded until it is
223    needed.
224    """
225    def __init__(self, config_statusd, typ, environ_override=False):
226        self._dict = {}
227        self._datadir = mozpath.join(config_statusd, typ)
228        self._config_track = mozpath.join(self._datadir, 'config.track')
229        self._files = set()
230        self._environ_override = environ_override
231
232    def _load_config_track(self):
233        existing_files = set()
234        try:
235            with open(self._config_track) as fh:
236                existing_files.update(fh.read().splitlines())
237        except IOError:
238            pass
239        return existing_files
240
241    def _write_file(self, key, value):
242        filename = mozpath.join(self._datadir, key)
243        with FileAvoidWrite(filename) as fh:
244            json.dump(value, fh, indent=4)
245        return filename
246
247    def _fill_group(self, values):
248        # Clear out any cached values. This is mostly for tests that will check
249        # the environment, write out a new set of variables, and then check the
250        # environment again. Normally only configure ends up calling this
251        # function, and other consumers create their own
252        # PartialConfigEnvironments in new python processes.
253        self._dict = {}
254
255        existing_files = self._load_config_track()
256
257        new_files = set()
258        for k, v in values.iteritems():
259            new_files.add(self._write_file(k, v))
260
261        for filename in existing_files - new_files:
262            # We can't actually os.remove() here, since make would not see that the
263            # file has been removed and that the target needs to be updated. Instead
264            # we just overwrite the file with a value of None, which is equivalent
265            # to a non-existing file.
266            with FileAvoidWrite(filename) as fh:
267                json.dump(None, fh)
268
269        with FileAvoidWrite(self._config_track) as fh:
270            for f in sorted(new_files):
271                fh.write('%s\n' % f)
272
273    def __getitem__(self, key):
274        if self._environ_override:
275            if (key not in ('CPP', 'CXXCPP', 'SHELL')) and (key in os.environ):
276                return os.environ[key]
277
278        if key not in self._dict:
279            data = None
280            try:
281                filename = mozpath.join(self._datadir, key)
282                self._files.add(filename)
283                with open(filename) as f:
284                    data = json.load(f)
285            except IOError:
286                pass
287            self._dict[key] = data
288
289        if self._dict[key] is None:
290            raise KeyError("'%s'" % key)
291        return self._dict[key]
292
293    def __setitem__(self, key, value):
294        self._dict[key] = value
295
296    def get(self, key, default=None):
297        return self[key] if key in self else default
298
299    def __contains__(self, key):
300        try:
301            return self[key] is not None
302        except KeyError:
303            return False
304
305    def iteritems(self):
306        existing_files = self._load_config_track()
307        for f in existing_files:
308            # The track file contains filenames, and the basename is the
309            # variable name.
310            var = mozpath.basename(f)
311            yield var, self[var]
312
313
314class PartialConfigEnvironment(object):
315    """Allows access to individual config.status items via config.statusd/* files.
316
317    This class is similar to the full ConfigEnvironment, which uses
318    config.status, except this allows access and tracks dependencies to
319    individual configure values. It is intended to be used during the build
320    process to handle things like GENERATED_FILES, CONFIGURE_DEFINE_FILES, and
321    anything else that may need to access specific substs or defines.
322
323    Creating a PartialConfigEnvironment requires only the topobjdir, which is
324    needed to distinguish between the top-level environment and the js/src
325    environment.
326
327    The PartialConfigEnvironment automatically defines one additional subst variable
328    from all the defines not appearing in non_global_defines:
329      - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
330        preprocessor command lines. The order in which defines were given
331        when creating the ConfigEnvironment is preserved.
332
333    and one additional define from all the defines as a dictionary:
334      - ALLDEFINES contains all of the global defines as a dictionary. This is
335      intended to be used instead of the defines structure from config.status so
336      that scripts can depend directly on its value.
337    """
338    def __init__(self, topobjdir):
339        config_statusd = mozpath.join(topobjdir, 'config.statusd')
340        self.substs = PartialConfigDict(config_statusd, 'substs', environ_override=True)
341        self.defines = PartialConfigDict(config_statusd, 'defines')
342        self.topobjdir = topobjdir
343
344    def write_vars(self, config):
345        substs = config['substs'].copy()
346        defines = config['defines'].copy()
347
348        global_defines = [
349            name for name in config['defines']
350            if name not in config['non_global_defines']
351        ]
352        acdefines = ' '.join(['-D%s=%s' % (name,
353            shell_quote(config['defines'][name]).replace('$', '$$'))
354            for name in sorted(global_defines)])
355        substs['ACDEFINES'] = acdefines
356
357        all_defines = OrderedDict()
358        for k in global_defines:
359            all_defines[k] = config['defines'][k]
360        defines['ALLDEFINES'] = all_defines
361
362        self.substs._fill_group(substs)
363        self.defines._fill_group(defines)
364
365    def get_dependencies(self):
366        return ['$(wildcard %s)' % f for f in self.substs._files | self.defines._files]
367