1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, print_function, unicode_literals
6
7import copy
8import re
9import types
10import unittest
11
12from fnmatch import fnmatch
13from StringIO import StringIO
14from textwrap import dedent
15
16from mozunit import (
17    main,
18    MockedOpen,
19)
20
21from mozbuild.preprocessor import Preprocessor
22from mozbuild.util import ReadOnlyNamespace
23from mozpack import path as mozpath
24
25
26class CompilerPreprocessor(Preprocessor):
27    # The C preprocessor only expands macros when they are not in C strings.
28    # For now, we don't look very hard for C strings because they don't matter
29    # that much for our unit tests, but we at least avoid expanding in the
30    # simple "FOO" case.
31    VARSUBST = re.compile('(?<!")(?P<VAR>\w+)(?!")', re.U)
32    NON_WHITESPACE = re.compile('\S')
33    HAS_FEATURE = re.compile('(__has_feature)\(([^\)]*)\)')
34
35    def __init__(self, *args, **kwargs):
36        Preprocessor.__init__(self, *args, **kwargs)
37        self.do_filter('c_substitution')
38        self.setMarker('#\s*')
39
40    def do_if(self, expression, **kwargs):
41        # The C preprocessor handles numbers following C rules, which is a
42        # different handling than what our Preprocessor does out of the box.
43        # Hack around it enough that the configure tests work properly.
44        context = self.context
45        def normalize_numbers(value):
46            if isinstance(value, types.StringTypes):
47                if value[-1:] == 'L' and value[:-1].isdigit():
48                    value = int(value[:-1])
49            return value
50        # Our Preprocessor doesn't handle macros with parameters, so we hack
51        # around that for __has_feature()-like things.
52        def normalize_has_feature(expr):
53            return self.HAS_FEATURE.sub(r'\1\2', expr)
54        self.context = self.Context(
55            (normalize_has_feature(k), normalize_numbers(v))
56            for k, v in context.iteritems()
57        )
58        try:
59            return Preprocessor.do_if(self, normalize_has_feature(expression),
60                                      **kwargs)
61        finally:
62            self.context = context
63
64    class Context(dict):
65        def __missing__(self, key):
66            return None
67
68    def filter_c_substitution(self, line):
69        def repl(matchobj):
70            varname = matchobj.group('VAR')
71            if varname in self.context:
72                result = str(self.context[varname])
73                # The C preprocessor inserts whitespaces around expanded
74                # symbols.
75                start, end = matchobj.span('VAR')
76                if self.NON_WHITESPACE.match(line[start-1:start]):
77                    result = ' ' + result
78                if self.NON_WHITESPACE.match(line[end:end+1]):
79                    result = result + ' '
80                return result
81            return matchobj.group(0)
82        return self.VARSUBST.sub(repl, line)
83
84
85class TestCompilerPreprocessor(unittest.TestCase):
86    def test_expansion(self):
87        pp = CompilerPreprocessor({
88            'A': 1,
89            'B': '2',
90            'C': 'c',
91            'D': 'd'
92        })
93        pp.out = StringIO()
94        input = StringIO('A.B.C "D"')
95        input.name = 'foo'
96        pp.do_include(input)
97
98        self.assertEquals(pp.out.getvalue(), '1 . 2 . c "D"')
99
100    def test_condition(self):
101        pp = CompilerPreprocessor({
102            'A': 1,
103            'B': '2',
104            'C': '0L',
105        })
106        pp.out = StringIO()
107        input = StringIO(dedent('''\
108            #ifdef A
109            IFDEF_A
110            #endif
111            #if A
112            IF_A
113            #endif
114            #  if B
115            IF_B
116            #  else
117            IF_NOT_B
118            #  endif
119            #if !C
120            IF_NOT_C
121            #else
122            IF_C
123            #endif
124        '''))
125        input.name = 'foo'
126        pp.do_include(input)
127
128        self.assertEquals('IFDEF_A\nIF_A\nIF_B\nIF_NOT_C\n', pp.out.getvalue())
129
130
131class FakeCompiler(dict):
132    '''Defines a fake compiler for use in toolchain tests below.
133
134    The definitions given when creating an instance can have one of two
135    forms:
136    - a dict giving preprocessor symbols and their respective value, e.g.
137        { '__GNUC__': 4, '__STDC__': 1 }
138    - a dict associating flags to preprocessor symbols. An entry for `None`
139      is required in this case. Those are the baseline preprocessor symbols.
140      Additional entries describe additional flags to set or existing flags
141      to unset (with a value of `False`).
142        {
143          None: { '__GNUC__': 4, '__STDC__': 1, '__STRICT_ANSI__': 1 },
144          '-std=gnu99': { '__STDC_VERSION__': '199901L',
145                          '__STRICT_ANSI__': False },
146        }
147      With the dict above, invoking the preprocessor with no additional flags
148      would define __GNUC__, __STDC__ and __STRICT_ANSI__, and with -std=gnu99,
149      __GNUC__, __STDC__, and __STDC_VERSION__ (__STRICT_ANSI__ would be
150      unset).
151      It is also possible to have different symbols depending on the source
152      file extension. In this case, the key is '*.ext'. e.g.
153        {
154          '*.c': { '__STDC__': 1 },
155          '*.cpp': { '__cplusplus': '199711L' },
156        }
157
158    All the given definitions are merged together.
159
160    A FakeCompiler instance itself can be used as a definition to create
161    another FakeCompiler.
162
163    For convenience, FakeCompiler instances can be added (+) to one another.
164    '''
165    def __init__(self, *definitions):
166        for definition in definitions:
167            if all(not isinstance(d, dict) for d in definition.itervalues()):
168                definition = {None: definition}
169            for key, value in definition.iteritems():
170                self.setdefault(key, {}).update(value)
171
172    def __call__(self, stdin, args):
173        files = [arg for arg in args if not arg.startswith('-')]
174        flags = [arg for arg in args if arg.startswith('-')]
175        if '-E' in flags:
176            assert len(files) == 1
177            file = files[0]
178            pp = CompilerPreprocessor(self[None])
179
180            def apply_defn(defn):
181                for k, v in defn.iteritems():
182                    if v is False:
183                        if k in pp.context:
184                            del pp.context[k]
185                    else:
186                        pp.context[k] = v
187
188            for glob, defn in self.iteritems():
189                if glob and not glob.startswith('-') and fnmatch(file, glob):
190                    apply_defn(defn)
191
192            for flag in flags:
193                apply_defn(self.get(flag, {}))
194
195            pp.out = StringIO()
196            pp.do_include(file)
197            return 0, pp.out.getvalue(), ''
198        elif '-c' in flags:
199            if '-funknown-flag' in flags:
200                return 1, '', ''
201            return 0, '', ''
202
203        return 1, '', ''
204
205    def __add__(self, other):
206        return FakeCompiler(self, other)
207
208
209class TestFakeCompiler(unittest.TestCase):
210    def test_fake_compiler(self):
211        with MockedOpen({
212            'file': 'A B C',
213            'file.c': 'A B C',
214        }):
215            compiler = FakeCompiler({
216                'A': '1',
217                'B': '2',
218            })
219            self.assertEquals(compiler(None, ['-E', 'file']),
220                              (0, '1 2 C', ''))
221
222            compiler = FakeCompiler({
223                None: {
224                    'A': '1',
225                    'B': '2',
226                },
227                '-foo': {
228                    'C': 'foo',
229                },
230                '-bar': {
231                    'B': 'bar',
232                    'C': 'bar',
233                },
234                '-qux': {
235                    'B': False,
236                },
237                '*.c': {
238                    'B': '42',
239                },
240            })
241            self.assertEquals(compiler(None, ['-E', 'file']),
242                              (0, '1 2 C', ''))
243            self.assertEquals(compiler(None, ['-E', '-foo', 'file']),
244                              (0, '1 2 foo', ''))
245            self.assertEquals(compiler(None, ['-E', '-bar', 'file']),
246                              (0, '1 bar bar', ''))
247            self.assertEquals(compiler(None, ['-E', '-qux', 'file']),
248                              (0, '1 B C', ''))
249            self.assertEquals(compiler(None, ['-E', '-foo', '-bar', 'file']),
250                              (0, '1 bar bar', ''))
251            self.assertEquals(compiler(None, ['-E', '-bar', '-foo', 'file']),
252                              (0, '1 bar foo', ''))
253            self.assertEquals(compiler(None, ['-E', '-bar', '-qux', 'file']),
254                              (0, '1 B bar', ''))
255            self.assertEquals(compiler(None, ['-E', '-qux', '-bar', 'file']),
256                              (0, '1 bar bar', ''))
257            self.assertEquals(compiler(None, ['-E', 'file.c']),
258                              (0, '1 42 C', ''))
259            self.assertEquals(compiler(None, ['-E', '-bar', 'file.c']),
260                              (0, '1 bar bar', ''))
261
262    def test_multiple_definitions(self):
263        compiler = FakeCompiler({
264            'A': 1,
265            'B': 2,
266        }, {
267            'C': 3,
268        })
269
270        self.assertEquals(compiler, {
271            None: {
272                'A': 1,
273                'B': 2,
274                'C': 3,
275            },
276        })
277        compiler = FakeCompiler({
278            'A': 1,
279            'B': 2,
280        }, {
281            'B': 4,
282            'C': 3,
283        })
284
285        self.assertEquals(compiler, {
286            None: {
287                'A': 1,
288                'B': 4,
289                'C': 3,
290            },
291        })
292        compiler = FakeCompiler({
293            'A': 1,
294            'B': 2,
295        }, {
296            None: {
297                'B': 4,
298                'C': 3,
299            },
300            '-foo': {
301                'D': 5,
302            },
303        })
304
305        self.assertEquals(compiler, {
306            None: {
307                'A': 1,
308                'B': 4,
309                'C': 3,
310            },
311            '-foo': {
312                'D': 5,
313            },
314        })
315
316        compiler = FakeCompiler({
317            None: {
318                'A': 1,
319                'B': 2,
320            },
321            '-foo': {
322                'D': 5,
323            },
324        }, {
325            '-foo': {
326                'D': 5,
327            },
328            '-bar': {
329                'E': 6,
330            },
331        })
332
333        self.assertEquals(compiler, {
334            None: {
335                'A': 1,
336                'B': 2,
337            },
338            '-foo': {
339                'D': 5,
340            },
341            '-bar': {
342                'E': 6,
343            },
344        })
345
346
347class CompilerResult(ReadOnlyNamespace):
348    '''Helper of convenience to manipulate toolchain results in unit tests
349
350    When adding a dict, the result is a new CompilerResult with the values
351    from the dict replacing those from the CompilerResult, except for `flags`,
352    where the value from the dict extends the `flags` in `self`.
353    '''
354
355    def __init__(self, wrapper=None, compiler='', version='', type='',
356                 language='', flags=None):
357        if flags is None:
358            flags = []
359        if wrapper is None:
360            wrapper = []
361        super(CompilerResult, self).__init__(
362            flags=flags,
363            version=version,
364            type=type,
365            compiler=mozpath.abspath(compiler),
366            wrapper=wrapper,
367            language=language,
368        )
369
370    def __add__(self, other):
371        assert isinstance(other, dict)
372        result = copy.deepcopy(self.__dict__)
373        for k, v in other.iteritems():
374            if k == 'flags':
375                result.setdefault(k, []).extend(v)
376            else:
377                result[k] = v
378        return CompilerResult(**result)
379
380
381class TestCompilerResult(unittest.TestCase):
382    def test_compiler_result(self):
383        result = CompilerResult()
384        self.assertEquals(result.__dict__, {
385            'wrapper': [],
386            'compiler': mozpath.abspath(''),
387            'version': '',
388            'type': '',
389            'language': '',
390            'flags': [],
391        })
392
393        result = CompilerResult(
394            compiler='/usr/bin/gcc',
395            version='4.2.1',
396            type='gcc',
397            language='C',
398            flags=['-std=gnu99'],
399        )
400        self.assertEquals(result.__dict__, {
401            'wrapper': [],
402            'compiler': mozpath.abspath('/usr/bin/gcc'),
403            'version': '4.2.1',
404            'type': 'gcc',
405            'language': 'C',
406            'flags': ['-std=gnu99'],
407        })
408
409        result2 = result + {'flags': ['-m32']}
410        self.assertEquals(result2.__dict__, {
411            'wrapper': [],
412            'compiler': mozpath.abspath('/usr/bin/gcc'),
413            'version': '4.2.1',
414            'type': 'gcc',
415            'language': 'C',
416            'flags': ['-std=gnu99', '-m32'],
417        })
418        # Original flags are untouched.
419        self.assertEquals(result.flags, ['-std=gnu99'])
420
421        result3 = result + {
422            'compiler': '/usr/bin/gcc-4.7',
423            'version': '4.7.3',
424            'flags': ['-m32'],
425        }
426        self.assertEquals(result3.__dict__, {
427            'wrapper': [],
428            'compiler': mozpath.abspath('/usr/bin/gcc-4.7'),
429            'version': '4.7.3',
430            'type': 'gcc',
431            'language': 'C',
432            'flags': ['-std=gnu99', '-m32'],
433        })
434
435
436if __name__ == '__main__':
437    main()
438