1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7import copy 8import re 9import types 10import unittest 11 12from fnmatch import fnmatch 13from StringIO import StringIO 14from textwrap import dedent 15 16from mozunit import ( 17 main, 18 MockedOpen, 19) 20 21from mozbuild.preprocessor import Preprocessor 22from mozbuild.util import ReadOnlyNamespace 23from mozpack import path as mozpath 24 25 26class CompilerPreprocessor(Preprocessor): 27 # The C preprocessor only expands macros when they are not in C strings. 28 # For now, we don't look very hard for C strings because they don't matter 29 # that much for our unit tests, but we at least avoid expanding in the 30 # simple "FOO" case. 31 VARSUBST = re.compile('(?<!")(?P<VAR>\w+)(?!")', re.U) 32 NON_WHITESPACE = re.compile('\S') 33 HAS_FEATURE = re.compile('(__has_feature)\(([^\)]*)\)') 34 35 def __init__(self, *args, **kwargs): 36 Preprocessor.__init__(self, *args, **kwargs) 37 self.do_filter('c_substitution') 38 self.setMarker('#\s*') 39 40 def do_if(self, expression, **kwargs): 41 # The C preprocessor handles numbers following C rules, which is a 42 # different handling than what our Preprocessor does out of the box. 43 # Hack around it enough that the configure tests work properly. 44 context = self.context 45 def normalize_numbers(value): 46 if isinstance(value, types.StringTypes): 47 if value[-1:] == 'L' and value[:-1].isdigit(): 48 value = int(value[:-1]) 49 return value 50 # Our Preprocessor doesn't handle macros with parameters, so we hack 51 # around that for __has_feature()-like things. 52 def normalize_has_feature(expr): 53 return self.HAS_FEATURE.sub(r'\1\2', expr) 54 self.context = self.Context( 55 (normalize_has_feature(k), normalize_numbers(v)) 56 for k, v in context.iteritems() 57 ) 58 try: 59 return Preprocessor.do_if(self, normalize_has_feature(expression), 60 **kwargs) 61 finally: 62 self.context = context 63 64 class Context(dict): 65 def __missing__(self, key): 66 return None 67 68 def filter_c_substitution(self, line): 69 def repl(matchobj): 70 varname = matchobj.group('VAR') 71 if varname in self.context: 72 result = str(self.context[varname]) 73 # The C preprocessor inserts whitespaces around expanded 74 # symbols. 75 start, end = matchobj.span('VAR') 76 if self.NON_WHITESPACE.match(line[start-1:start]): 77 result = ' ' + result 78 if self.NON_WHITESPACE.match(line[end:end+1]): 79 result = result + ' ' 80 return result 81 return matchobj.group(0) 82 return self.VARSUBST.sub(repl, line) 83 84 85class TestCompilerPreprocessor(unittest.TestCase): 86 def test_expansion(self): 87 pp = CompilerPreprocessor({ 88 'A': 1, 89 'B': '2', 90 'C': 'c', 91 'D': 'd' 92 }) 93 pp.out = StringIO() 94 input = StringIO('A.B.C "D"') 95 input.name = 'foo' 96 pp.do_include(input) 97 98 self.assertEquals(pp.out.getvalue(), '1 . 2 . c "D"') 99 100 def test_condition(self): 101 pp = CompilerPreprocessor({ 102 'A': 1, 103 'B': '2', 104 'C': '0L', 105 }) 106 pp.out = StringIO() 107 input = StringIO(dedent('''\ 108 #ifdef A 109 IFDEF_A 110 #endif 111 #if A 112 IF_A 113 #endif 114 # if B 115 IF_B 116 # else 117 IF_NOT_B 118 # endif 119 #if !C 120 IF_NOT_C 121 #else 122 IF_C 123 #endif 124 ''')) 125 input.name = 'foo' 126 pp.do_include(input) 127 128 self.assertEquals('IFDEF_A\nIF_A\nIF_B\nIF_NOT_C\n', pp.out.getvalue()) 129 130 131class FakeCompiler(dict): 132 '''Defines a fake compiler for use in toolchain tests below. 133 134 The definitions given when creating an instance can have one of two 135 forms: 136 - a dict giving preprocessor symbols and their respective value, e.g. 137 { '__GNUC__': 4, '__STDC__': 1 } 138 - a dict associating flags to preprocessor symbols. An entry for `None` 139 is required in this case. Those are the baseline preprocessor symbols. 140 Additional entries describe additional flags to set or existing flags 141 to unset (with a value of `False`). 142 { 143 None: { '__GNUC__': 4, '__STDC__': 1, '__STRICT_ANSI__': 1 }, 144 '-std=gnu99': { '__STDC_VERSION__': '199901L', 145 '__STRICT_ANSI__': False }, 146 } 147 With the dict above, invoking the preprocessor with no additional flags 148 would define __GNUC__, __STDC__ and __STRICT_ANSI__, and with -std=gnu99, 149 __GNUC__, __STDC__, and __STDC_VERSION__ (__STRICT_ANSI__ would be 150 unset). 151 It is also possible to have different symbols depending on the source 152 file extension. In this case, the key is '*.ext'. e.g. 153 { 154 '*.c': { '__STDC__': 1 }, 155 '*.cpp': { '__cplusplus': '199711L' }, 156 } 157 158 All the given definitions are merged together. 159 160 A FakeCompiler instance itself can be used as a definition to create 161 another FakeCompiler. 162 163 For convenience, FakeCompiler instances can be added (+) to one another. 164 ''' 165 def __init__(self, *definitions): 166 for definition in definitions: 167 if all(not isinstance(d, dict) for d in definition.itervalues()): 168 definition = {None: definition} 169 for key, value in definition.iteritems(): 170 self.setdefault(key, {}).update(value) 171 172 def __call__(self, stdin, args): 173 files = [arg for arg in args if not arg.startswith('-')] 174 flags = [arg for arg in args if arg.startswith('-')] 175 if '-E' in flags: 176 assert len(files) == 1 177 file = files[0] 178 pp = CompilerPreprocessor(self[None]) 179 180 def apply_defn(defn): 181 for k, v in defn.iteritems(): 182 if v is False: 183 if k in pp.context: 184 del pp.context[k] 185 else: 186 pp.context[k] = v 187 188 for glob, defn in self.iteritems(): 189 if glob and not glob.startswith('-') and fnmatch(file, glob): 190 apply_defn(defn) 191 192 for flag in flags: 193 apply_defn(self.get(flag, {})) 194 195 pp.out = StringIO() 196 pp.do_include(file) 197 return 0, pp.out.getvalue(), '' 198 elif '-c' in flags: 199 if '-funknown-flag' in flags: 200 return 1, '', '' 201 return 0, '', '' 202 203 return 1, '', '' 204 205 def __add__(self, other): 206 return FakeCompiler(self, other) 207 208 209class TestFakeCompiler(unittest.TestCase): 210 def test_fake_compiler(self): 211 with MockedOpen({ 212 'file': 'A B C', 213 'file.c': 'A B C', 214 }): 215 compiler = FakeCompiler({ 216 'A': '1', 217 'B': '2', 218 }) 219 self.assertEquals(compiler(None, ['-E', 'file']), 220 (0, '1 2 C', '')) 221 222 compiler = FakeCompiler({ 223 None: { 224 'A': '1', 225 'B': '2', 226 }, 227 '-foo': { 228 'C': 'foo', 229 }, 230 '-bar': { 231 'B': 'bar', 232 'C': 'bar', 233 }, 234 '-qux': { 235 'B': False, 236 }, 237 '*.c': { 238 'B': '42', 239 }, 240 }) 241 self.assertEquals(compiler(None, ['-E', 'file']), 242 (0, '1 2 C', '')) 243 self.assertEquals(compiler(None, ['-E', '-foo', 'file']), 244 (0, '1 2 foo', '')) 245 self.assertEquals(compiler(None, ['-E', '-bar', 'file']), 246 (0, '1 bar bar', '')) 247 self.assertEquals(compiler(None, ['-E', '-qux', 'file']), 248 (0, '1 B C', '')) 249 self.assertEquals(compiler(None, ['-E', '-foo', '-bar', 'file']), 250 (0, '1 bar bar', '')) 251 self.assertEquals(compiler(None, ['-E', '-bar', '-foo', 'file']), 252 (0, '1 bar foo', '')) 253 self.assertEquals(compiler(None, ['-E', '-bar', '-qux', 'file']), 254 (0, '1 B bar', '')) 255 self.assertEquals(compiler(None, ['-E', '-qux', '-bar', 'file']), 256 (0, '1 bar bar', '')) 257 self.assertEquals(compiler(None, ['-E', 'file.c']), 258 (0, '1 42 C', '')) 259 self.assertEquals(compiler(None, ['-E', '-bar', 'file.c']), 260 (0, '1 bar bar', '')) 261 262 def test_multiple_definitions(self): 263 compiler = FakeCompiler({ 264 'A': 1, 265 'B': 2, 266 }, { 267 'C': 3, 268 }) 269 270 self.assertEquals(compiler, { 271 None: { 272 'A': 1, 273 'B': 2, 274 'C': 3, 275 }, 276 }) 277 compiler = FakeCompiler({ 278 'A': 1, 279 'B': 2, 280 }, { 281 'B': 4, 282 'C': 3, 283 }) 284 285 self.assertEquals(compiler, { 286 None: { 287 'A': 1, 288 'B': 4, 289 'C': 3, 290 }, 291 }) 292 compiler = FakeCompiler({ 293 'A': 1, 294 'B': 2, 295 }, { 296 None: { 297 'B': 4, 298 'C': 3, 299 }, 300 '-foo': { 301 'D': 5, 302 }, 303 }) 304 305 self.assertEquals(compiler, { 306 None: { 307 'A': 1, 308 'B': 4, 309 'C': 3, 310 }, 311 '-foo': { 312 'D': 5, 313 }, 314 }) 315 316 compiler = FakeCompiler({ 317 None: { 318 'A': 1, 319 'B': 2, 320 }, 321 '-foo': { 322 'D': 5, 323 }, 324 }, { 325 '-foo': { 326 'D': 5, 327 }, 328 '-bar': { 329 'E': 6, 330 }, 331 }) 332 333 self.assertEquals(compiler, { 334 None: { 335 'A': 1, 336 'B': 2, 337 }, 338 '-foo': { 339 'D': 5, 340 }, 341 '-bar': { 342 'E': 6, 343 }, 344 }) 345 346 347class CompilerResult(ReadOnlyNamespace): 348 '''Helper of convenience to manipulate toolchain results in unit tests 349 350 When adding a dict, the result is a new CompilerResult with the values 351 from the dict replacing those from the CompilerResult, except for `flags`, 352 where the value from the dict extends the `flags` in `self`. 353 ''' 354 355 def __init__(self, wrapper=None, compiler='', version='', type='', 356 language='', flags=None): 357 if flags is None: 358 flags = [] 359 if wrapper is None: 360 wrapper = [] 361 super(CompilerResult, self).__init__( 362 flags=flags, 363 version=version, 364 type=type, 365 compiler=mozpath.abspath(compiler), 366 wrapper=wrapper, 367 language=language, 368 ) 369 370 def __add__(self, other): 371 assert isinstance(other, dict) 372 result = copy.deepcopy(self.__dict__) 373 for k, v in other.iteritems(): 374 if k == 'flags': 375 result.setdefault(k, []).extend(v) 376 else: 377 result[k] = v 378 return CompilerResult(**result) 379 380 381class TestCompilerResult(unittest.TestCase): 382 def test_compiler_result(self): 383 result = CompilerResult() 384 self.assertEquals(result.__dict__, { 385 'wrapper': [], 386 'compiler': mozpath.abspath(''), 387 'version': '', 388 'type': '', 389 'language': '', 390 'flags': [], 391 }) 392 393 result = CompilerResult( 394 compiler='/usr/bin/gcc', 395 version='4.2.1', 396 type='gcc', 397 language='C', 398 flags=['-std=gnu99'], 399 ) 400 self.assertEquals(result.__dict__, { 401 'wrapper': [], 402 'compiler': mozpath.abspath('/usr/bin/gcc'), 403 'version': '4.2.1', 404 'type': 'gcc', 405 'language': 'C', 406 'flags': ['-std=gnu99'], 407 }) 408 409 result2 = result + {'flags': ['-m32']} 410 self.assertEquals(result2.__dict__, { 411 'wrapper': [], 412 'compiler': mozpath.abspath('/usr/bin/gcc'), 413 'version': '4.2.1', 414 'type': 'gcc', 415 'language': 'C', 416 'flags': ['-std=gnu99', '-m32'], 417 }) 418 # Original flags are untouched. 419 self.assertEquals(result.flags, ['-std=gnu99']) 420 421 result3 = result + { 422 'compiler': '/usr/bin/gcc-4.7', 423 'version': '4.7.3', 424 'flags': ['-m32'], 425 } 426 self.assertEquals(result3.__dict__, { 427 'wrapper': [], 428 'compiler': mozpath.abspath('/usr/bin/gcc-4.7'), 429 'version': '4.7.3', 430 'type': 'gcc', 431 'language': 'C', 432 'flags': ['-std=gnu99', '-m32'], 433 }) 434 435 436if __name__ == '__main__': 437 main() 438