1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, unicode_literals
6
7import filecmp
8import os
9import re
10import sys
11import subprocess
12import traceback
13
14from collections import defaultdict
15from mozpack import path as mozpath
16
17
18MOZ_MYCONFIG_ERROR = '''
19The MOZ_MYCONFIG environment variable to define the location of mozconfigs
20is deprecated. If you wish to define the mozconfig path via an environment
21variable, use MOZCONFIG instead.
22'''.strip()
23
24MOZCONFIG_LEGACY_PATH = '''
25You currently have a mozconfig at %s. This implicit location is no longer
26supported. Please move it to %s/.mozconfig or set an explicit path
27via the $MOZCONFIG environment variable.
28'''.strip()
29
30MOZCONFIG_BAD_EXIT_CODE = '''
31Evaluation of your mozconfig exited with an error. This could be triggered
32by a command inside your mozconfig failing. Please change your mozconfig
33to not error and/or to catch errors in executed commands.
34'''.strip()
35
36MOZCONFIG_BAD_OUTPUT = '''
37Evaluation of your mozconfig produced unexpected output.  This could be
38triggered by a command inside your mozconfig failing or producing some warnings
39or error messages. Please change your mozconfig to not error and/or to catch
40errors in executed commands.
41'''.strip()
42
43
44class MozconfigFindException(Exception):
45    """Raised when a mozconfig location is not defined properly."""
46
47
48class MozconfigLoadException(Exception):
49    """Raised when a mozconfig could not be loaded properly.
50
51    This typically indicates a malformed or misbehaving mozconfig file.
52    """
53
54    def __init__(self, path, message, output=None):
55        self.path = path
56        self.output = output
57        Exception.__init__(self, message)
58
59
60class MozconfigLoader(object):
61    """Handles loading and parsing of mozconfig files."""
62
63    RE_MAKE_VARIABLE = re.compile('''
64        ^\s*                    # Leading whitespace
65        (?P<var>[a-zA-Z_0-9]+)  # Variable name
66        \s* [?:]?= \s*          # Assignment operator surrounded by optional
67                                # spaces
68        (?P<value>.*$)''',      # Everything else (likely the value)
69        re.VERBOSE)
70
71    # Default mozconfig files in the topsrcdir.
72    DEFAULT_TOPSRCDIR_PATHS = ('.mozconfig', 'mozconfig')
73
74    DEPRECATED_TOPSRCDIR_PATHS = ('mozconfig.sh', 'myconfig.sh')
75    DEPRECATED_HOME_PATHS = ('.mozconfig', '.mozconfig.sh', '.mozmyconfig.sh')
76
77    IGNORE_SHELL_VARIABLES = {'_'}
78
79    ENVIRONMENT_VARIABLES = {
80        'CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'MOZ_OBJDIR',
81    }
82
83    AUTODETECT = object()
84
85    def __init__(self, topsrcdir):
86        self.topsrcdir = topsrcdir
87
88    @property
89    def _loader_script(self):
90        our_dir = os.path.abspath(os.path.dirname(__file__))
91
92        return os.path.join(our_dir, 'mozconfig_loader')
93
94    def find_mozconfig(self, env=os.environ):
95        """Find the active mozconfig file for the current environment.
96
97        This emulates the logic in mozconfig-find.
98
99        1) If ENV[MOZCONFIG] is set, use that
100        2) If $TOPSRCDIR/mozconfig or $TOPSRCDIR/.mozconfig exists, use it.
101        3) If both exist or if there are legacy locations detected, error out.
102
103        The absolute path to the found mozconfig will be returned on success.
104        None will be returned if no mozconfig could be found. A
105        MozconfigFindException will be raised if there is a bad state,
106        including conditions from #3 above.
107        """
108        # Check for legacy methods first.
109
110        if 'MOZ_MYCONFIG' in env:
111            raise MozconfigFindException(MOZ_MYCONFIG_ERROR)
112
113        env_path = env.get('MOZCONFIG', None) or None
114        if env_path is not None:
115            if not os.path.isabs(env_path):
116                potential_roots = [self.topsrcdir, os.getcwd()]
117                # Attempt to eliminate duplicates for e.g.
118                # self.topsrcdir == os.curdir.
119                potential_roots = set(os.path.abspath(p) for p in potential_roots)
120                existing = [root for root in potential_roots
121                            if os.path.exists(os.path.join(root, env_path))]
122                if len(existing) > 1:
123                    # There are multiple files, but we might have a setup like:
124                    #
125                    # somedirectory/
126                    #   srcdir/
127                    #   objdir/
128                    #
129                    # MOZCONFIG=../srcdir/some/path/to/mozconfig
130                    #
131                    # and be configuring from the objdir.  So even though we
132                    # have multiple existing files, they are actually the same
133                    # file.
134                    mozconfigs = [os.path.join(root, env_path)
135                                  for root in existing]
136                    if not all(map(lambda p1, p2: filecmp.cmp(p1, p2, shallow=False),
137                                   mozconfigs[:-1], mozconfigs[1:])):
138                        raise MozconfigFindException(
139                            'MOZCONFIG environment variable refers to a path that ' +
140                            'exists in more than one of ' + ', '.join(potential_roots) +
141                            '. Remove all but one.')
142                elif not existing:
143                    raise MozconfigFindException(
144                        'MOZCONFIG environment variable refers to a path that ' +
145                        'does not exist in any of ' + ', '.join(potential_roots))
146
147                env_path = os.path.join(existing[0], env_path)
148            elif not os.path.exists(env_path): # non-relative path
149                raise MozconfigFindException(
150                    'MOZCONFIG environment variable refers to a path that '
151                    'does not exist: ' + env_path)
152
153            if not os.path.isfile(env_path):
154                raise MozconfigFindException(
155                    'MOZCONFIG environment variable refers to a '
156                    'non-file: ' + env_path)
157
158        srcdir_paths = [os.path.join(self.topsrcdir, p) for p in
159            self.DEFAULT_TOPSRCDIR_PATHS]
160        existing = [p for p in srcdir_paths if os.path.isfile(p)]
161
162        if env_path is None and len(existing) > 1:
163            raise MozconfigFindException('Multiple default mozconfig files '
164                'present. Remove all but one. ' + ', '.join(existing))
165
166        path = None
167
168        if env_path is not None:
169            path = env_path
170        elif len(existing):
171            assert len(existing) == 1
172            path = existing[0]
173
174        if path is not None:
175            return os.path.abspath(path)
176
177        deprecated_paths = [os.path.join(self.topsrcdir, s) for s in
178            self.DEPRECATED_TOPSRCDIR_PATHS]
179
180        home = env.get('HOME', None)
181        if home is not None:
182            deprecated_paths.extend([os.path.join(home, s) for s in
183            self.DEPRECATED_HOME_PATHS])
184
185        for path in deprecated_paths:
186            if os.path.exists(path):
187                raise MozconfigFindException(
188                    MOZCONFIG_LEGACY_PATH % (path, self.topsrcdir))
189
190        return None
191
192    def read_mozconfig(self, path=None):
193        """Read the contents of a mozconfig into a data structure.
194
195        This takes the path to a mozconfig to load. If the given path is
196        AUTODETECT, will try to find a mozconfig from the environment using
197        find_mozconfig().
198
199        mozconfig files are shell scripts. So, we can't just parse them.
200        Instead, we run the shell script in a wrapper which allows us to record
201        state from execution. Thus, the output from a mozconfig is a friendly
202        static data structure.
203        """
204        if path is self.AUTODETECT:
205            path = self.find_mozconfig()
206
207        result = {
208            'path': path,
209            'topobjdir': None,
210            'configure_args': None,
211            'make_flags': None,
212            'make_extra': None,
213            'env': None,
214            'vars': None,
215        }
216
217        if path is None:
218            return result
219
220        path = mozpath.normsep(path)
221
222        result['configure_args'] = []
223        result['make_extra'] = []
224        result['make_flags'] = []
225
226        env = dict(os.environ)
227
228        # Since mozconfig_loader is a shell script, running it "normally"
229        # actually leads to two shell executions on Windows. Avoid this by
230        # directly calling sh mozconfig_loader.
231        shell = 'sh'
232        if 'MOZILLABUILD' in os.environ:
233            shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh'
234        if sys.platform == 'win32':
235            shell = shell + '.exe'
236
237        command = [shell, mozpath.normsep(self._loader_script),
238                   mozpath.normsep(self.topsrcdir), path, sys.executable,
239                   mozpath.join(mozpath.dirname(self._loader_script),
240                                'action', 'dump_env.py')]
241
242        try:
243            # We need to capture stderr because that's where the shell sends
244            # errors if execution fails.
245            output = subprocess.check_output(command, stderr=subprocess.STDOUT,
246                cwd=self.topsrcdir, env=env)
247        except subprocess.CalledProcessError as e:
248            lines = e.output.splitlines()
249
250            # Output before actual execution shouldn't be relevant.
251            try:
252                index = lines.index('------END_BEFORE_SOURCE')
253                lines = lines[index + 1:]
254            except ValueError:
255                pass
256
257            raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines)
258
259        try:
260            parsed = self._parse_loader_output(output)
261        except AssertionError:
262            # _parse_loader_output uses assertions to verify the
263            # well-formedness of the shell output; when these fail, it
264            # generally means there was a problem with the output, but we
265            # include the assertion traceback just to be sure.
266            print('Assertion failed in _parse_loader_output:')
267            traceback.print_exc()
268            raise MozconfigLoadException(path, MOZCONFIG_BAD_OUTPUT,
269                                         output.splitlines())
270
271        def diff_vars(vars_before, vars_after):
272            set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES
273            set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES
274            added = set2 - set1
275            removed = set1 - set2
276            maybe_modified = set1 & set2
277            changed = {
278                'added': {},
279                'removed': {},
280                'modified': {},
281                'unmodified': {},
282            }
283
284            for key in added:
285                changed['added'][key] = vars_after[key]
286
287            for key in removed:
288                changed['removed'][key] = vars_before[key]
289
290            for key in maybe_modified:
291                if vars_before[key] != vars_after[key]:
292                    changed['modified'][key] = (
293                        vars_before[key], vars_after[key])
294                elif key in self.ENVIRONMENT_VARIABLES:
295                    # In order for irrelevant environment variable changes not
296                    # to incur in re-running configure, only a set of
297                    # environment variables are stored when they are
298                    # unmodified. Otherwise, changes such as using a different
299                    # terminal window, or even rebooting, would trigger
300                    # reconfigures.
301                    changed['unmodified'][key] = vars_after[key]
302
303            return changed
304
305        result['env'] = diff_vars(parsed['env_before'], parsed['env_after'])
306
307        # Environment variables also appear as shell variables, but that's
308        # uninteresting duplication of information. Filter them out.
309        filt = lambda x, y: {k: v for k, v in x.items() if k not in y}
310        result['vars'] = diff_vars(
311            filt(parsed['vars_before'], parsed['env_before']),
312            filt(parsed['vars_after'], parsed['env_after'])
313        )
314
315        result['configure_args'] = [self._expand(o) for o in parsed['ac']]
316
317        if 'MOZ_OBJDIR' in parsed['env_before']:
318            result['topobjdir'] = parsed['env_before']['MOZ_OBJDIR']
319
320        mk = [self._expand(o) for o in parsed['mk']]
321
322        for o in mk:
323            match = self.RE_MAKE_VARIABLE.match(o)
324
325            if match is None:
326                result['make_extra'].append(o)
327                continue
328
329            name, value = match.group('var'), match.group('value')
330
331            if name == 'MOZ_MAKE_FLAGS':
332                result['make_flags'] = value.split()
333                continue
334
335            if name == 'MOZ_OBJDIR':
336                result['topobjdir'] = value
337                continue
338
339            result['make_extra'].append(o)
340
341        return result
342
343    def _parse_loader_output(self, output):
344        mk_options = []
345        ac_options = []
346        before_source = {}
347        after_source = {}
348        env_before_source = {}
349        env_after_source = {}
350
351        current = None
352        current_type = None
353        in_variable = None
354
355        for line in output.splitlines():
356
357            # XXX This is an ugly hack. Data may be lost from things
358            # like environment variable values.
359            # See https://bugzilla.mozilla.org/show_bug.cgi?id=831381
360            line = line.decode('mbcs' if sys.platform == 'win32' else 'utf-8',
361                               'ignore')
362
363            if not line:
364                continue
365
366            if line.startswith('------BEGIN_'):
367                assert current_type is None
368                assert current is None
369                assert not in_variable
370                current_type = line[len('------BEGIN_'):]
371                current = []
372                continue
373
374            if line.startswith('------END_'):
375                assert not in_variable
376                section = line[len('------END_'):]
377                assert current_type == section
378
379                if current_type == 'AC_OPTION':
380                    ac_options.append('\n'.join(current))
381                elif current_type == 'MK_OPTION':
382                    mk_options.append('\n'.join(current))
383
384                current = None
385                current_type = None
386                continue
387
388            assert current_type is not None
389
390            vars_mapping = {
391                'BEFORE_SOURCE': before_source,
392                'AFTER_SOURCE': after_source,
393                'ENV_BEFORE_SOURCE': env_before_source,
394                'ENV_AFTER_SOURCE': env_after_source,
395            }
396
397            if current_type in vars_mapping:
398                # mozconfigs are sourced using the Bourne shell (or at least
399                # in Bourne shell mode). This means |set| simply lists
400                # variables from the current shell (not functions). (Note that
401                # if Bash is installed in /bin/sh it acts like regular Bourne
402                # and doesn't print functions.) So, lines should have the
403                # form:
404                #
405                #  key='value'
406                #  key=value
407                #
408                # The only complication is multi-line variables. Those have the
409                # form:
410                #
411                #  key='first
412                #  second'
413
414                # TODO Bug 818377 Properly handle multi-line variables of form:
415                # $ foo="a='b'
416                # c='d'"
417                # $ set
418                # foo='a='"'"'b'"'"'
419                # c='"'"'d'"'"
420
421                name = in_variable
422                value = None
423                if in_variable:
424                    # Reached the end of a multi-line variable.
425                    if line.endswith("'") and not line.endswith("\\'"):
426                        current.append(line[:-1])
427                        value = '\n'.join(current)
428                        in_variable = None
429                    else:
430                        current.append(line)
431                        continue
432                else:
433                    equal_pos = line.find('=')
434
435                    if equal_pos < 1:
436                        # TODO log warning?
437                        continue
438
439                    name = line[0:equal_pos]
440                    value = line[equal_pos + 1:]
441
442                    if len(value):
443                        has_quote = value[0] == "'"
444
445                        if has_quote:
446                            value = value[1:]
447
448                        # Lines with a quote not ending in a quote are multi-line.
449                        if has_quote and not value.endswith("'"):
450                            in_variable = name
451                            current.append(value)
452                            continue
453                        else:
454                            value = value[:-1] if has_quote else value
455
456                assert name is not None
457
458                vars_mapping[current_type][name] = value
459
460                current = []
461
462                continue
463
464            current.append(line)
465
466        return {
467            'mk': mk_options,
468            'ac': ac_options,
469            'vars_before': before_source,
470            'vars_after': after_source,
471            'env_before': env_before_source,
472            'env_after': env_after_source,
473        }
474
475    def _expand(self, s):
476        return s.replace('@TOPSRCDIR@', self.topsrcdir)
477