1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7import codecs 8import inspect 9import logging 10import os 11import re 12import six 13from six.moves import builtins as __builtin__ 14import sys 15import types 16from collections import OrderedDict 17from contextlib import contextmanager 18from functools import wraps 19from mozbuild.configure.options import ( 20 CommandLineHelper, 21 ConflictingOptionError, 22 HELP_OPTIONS_CATEGORY, 23 InvalidOptionError, 24 Option, 25 OptionValue, 26) 27from mozbuild.configure.help import HelpFormatter 28from mozbuild.configure.util import ( 29 ConfigureOutputHandler, 30 getpreferredencoding, 31 LineIO, 32) 33from mozbuild.util import ( 34 ensure_subprocess_env, 35 exec_, 36 memoize, 37 memoized_property, 38 ReadOnlyDict, 39 ReadOnlyNamespace, 40 system_encoding, 41) 42 43import mozpack.path as mozpath 44 45 46# TRACE logging level, below (thus more verbose than) DEBUG 47TRACE = 5 48 49 50class ConfigureError(Exception): 51 pass 52 53 54class SandboxDependsFunction(object): 55 '''Sandbox-visible representation of @depends functions.''' 56 57 def __init__(self, unsandboxed): 58 self._or = unsandboxed.__or__ 59 self._and = unsandboxed.__and__ 60 self._getattr = unsandboxed.__getattr__ 61 62 def __call__(self, *arg, **kwargs): 63 raise ConfigureError('The `%s` function may not be called' 64 % self.__name__) 65 66 def __or__(self, other): 67 if not isinstance(other, SandboxDependsFunction): 68 raise ConfigureError('Can only do binary arithmetic operations ' 69 'with another @depends function.') 70 return self._or(other).sandboxed 71 72 def __and__(self, other): 73 if not isinstance(other, SandboxDependsFunction): 74 raise ConfigureError('Can only do binary arithmetic operations ' 75 'with another @depends function.') 76 return self._and(other).sandboxed 77 78 def __cmp__(self, other): 79 raise ConfigureError('Cannot compare @depends functions.') 80 81 def __eq__(self, other): 82 raise ConfigureError('Cannot compare @depends functions.') 83 84 def __hash__(self): 85 return object.__hash__(self) 86 87 def __ne__(self, other): 88 raise ConfigureError('Cannot compare @depends functions.') 89 90 def __lt__(self, other): 91 raise ConfigureError('Cannot compare @depends functions.') 92 93 def __le__(self, other): 94 raise ConfigureError('Cannot compare @depends functions.') 95 96 def __gt__(self, other): 97 raise ConfigureError('Cannot compare @depends functions.') 98 99 def __ge__(self, other): 100 raise ConfigureError('Cannot compare @depends functions.') 101 102 def __getattr__(self, key): 103 return self._getattr(key).sandboxed 104 105 def __nonzero__(self): 106 raise ConfigureError( 107 'Cannot do boolean operations on @depends functions.') 108 109 110class DependsFunction(object): 111 __slots__ = ( 112 '_func', '_name', 'dependencies', 'when', 'sandboxed', 'sandbox', 113 '_result') 114 115 def __init__(self, sandbox, func, dependencies, when=None): 116 assert isinstance(sandbox, ConfigureSandbox) 117 assert not inspect.isgeneratorfunction(func) 118 self._func = func 119 self._name = func.__name__ 120 self.dependencies = dependencies 121 self.sandboxed = wraps(func)(SandboxDependsFunction(self)) 122 self.sandbox = sandbox 123 self.when = when 124 sandbox._depends[self.sandboxed] = self 125 126 # Only @depends functions with a dependency on '--help' are executed 127 # immediately. Everything else is queued for later execution. 128 if sandbox._help_option in dependencies: 129 sandbox._value_for(self) 130 elif not sandbox._help: 131 sandbox._execution_queue.append((sandbox._value_for, (self,))) 132 133 @property 134 def name(self): 135 return self._name 136 137 @name.setter 138 def name(self, value): 139 self._name = value 140 141 @property 142 def sandboxed_dependencies(self): 143 return [ 144 d.sandboxed if isinstance(d, DependsFunction) else d 145 for d in self.dependencies 146 ] 147 148 @memoize 149 def result(self): 150 if self.when and not self.sandbox._value_for(self.when): 151 return None 152 153 resolved_args = [self.sandbox._value_for(d) 154 for d in self.dependencies] 155 return self._func(*resolved_args) 156 157 def __repr__(self): 158 return '<%s %s(%s)>' % ( 159 self.__class__.__name__, 160 self.name, 161 ', '.join(repr(d) for d in self.dependencies), 162 ) 163 164 def __or__(self, other): 165 if isinstance(other, SandboxDependsFunction): 166 other = self.sandbox._depends.get(other) 167 assert isinstance(other, DependsFunction) 168 assert self.sandbox is other.sandbox 169 return CombinedDependsFunction(self.sandbox, self.or_impl, 170 (self, other)) 171 172 @staticmethod 173 def or_impl(iterable): 174 # Applies "or" to all the items of iterable. 175 # e.g. if iterable contains a, b and c, returns `a or b or c`. 176 for i in iterable: 177 if i: 178 return i 179 return i 180 181 def __and__(self, other): 182 if isinstance(other, SandboxDependsFunction): 183 other = self.sandbox._depends.get(other) 184 assert isinstance(other, DependsFunction) 185 assert self.sandbox is other.sandbox 186 return CombinedDependsFunction(self.sandbox, self.and_impl, 187 (self, other)) 188 189 @staticmethod 190 def and_impl(iterable): 191 # Applies "and" to all the items of iterable. 192 # e.g. if iterable contains a, b and c, returns `a and b and c`. 193 for i in iterable: 194 if not i: 195 return i 196 return i 197 198 def __getattr__(self, key): 199 if key.startswith('_'): 200 return super(DependsFunction, self).__getattr__(key) 201 # Our function may return None or an object that simply doesn't have 202 # the wanted key. In that case, just return None. 203 return TrivialDependsFunction( 204 self.sandbox, lambda x: getattr(x, key, None), [self], self.when) 205 206 207class TrivialDependsFunction(DependsFunction): 208 '''Like a DependsFunction, but the linter won't expect it to have a 209 dependency on --help ever.''' 210 211 212class CombinedDependsFunction(DependsFunction): 213 def __init__(self, sandbox, func, dependencies): 214 flatten_deps = [] 215 for d in dependencies: 216 if isinstance(d, CombinedDependsFunction) and d._func is func: 217 for d2 in d.dependencies: 218 if d2 not in flatten_deps: 219 flatten_deps.append(d2) 220 elif d not in flatten_deps: 221 flatten_deps.append(d) 222 223 super(CombinedDependsFunction, self).__init__( 224 sandbox, func, flatten_deps) 225 226 @memoize 227 def result(self): 228 resolved_args = (self.sandbox._value_for(d) 229 for d in self.dependencies) 230 return self._func(resolved_args) 231 232 def __eq__(self, other): 233 return (isinstance(other, self.__class__) and 234 self._func is other._func and 235 set(self.dependencies) == set(other.dependencies)) 236 237 def __hash__(self): 238 return object.__hash__(self) 239 240 def __ne__(self, other): 241 return not self == other 242 243 244class SandboxedGlobal(dict): 245 '''Identifiable dict type for use as function global''' 246 247 248def forbidden_import(*args, **kwargs): 249 raise ImportError('Importing modules is forbidden') 250 251 252class ConfigureSandbox(dict): 253 """Represents a sandbox for executing Python code for build configuration. 254 This is a different kind of sandboxing than the one used for moz.build 255 processing. 256 257 The sandbox has 9 primitives: 258 - option 259 - depends 260 - template 261 - imports 262 - include 263 - set_config 264 - set_define 265 - imply_option 266 - only_when 267 268 `option`, `include`, `set_config`, `set_define` and `imply_option` are 269 functions. `depends`, `template`, and `imports` are decorators. `only_when` 270 is a context_manager. 271 272 These primitives are declared as name_impl methods to this class and 273 the mapping name -> name_impl is done automatically in __getitem__. 274 275 Additional primitives should be frowned upon to keep the sandbox itself as 276 simple as possible. Instead, helpers should be created within the sandbox 277 with the existing primitives. 278 279 The sandbox is given, at creation, a dict where the yielded configuration 280 will be stored. 281 282 config = {} 283 sandbox = ConfigureSandbox(config) 284 sandbox.run(path) 285 do_stuff(config) 286 """ 287 288 # The default set of builtins. We expose unicode as str to make sandboxed 289 # files more python3-ready. 290 BUILTINS = ReadOnlyDict({ 291 b: getattr(__builtin__, b, None) 292 for b in ('None', 'False', 'True', 'int', 'bool', 'any', 'all', 'len', 293 'list', 'tuple', 'set', 'dict', 'isinstance', 'getattr', 294 'hasattr', 'enumerate', 'range', 'zip', 'AssertionError', 295 '__build_class__', # will be None on py2 296 ) 297 }, __import__=forbidden_import, str=six.text_type) 298 299 # Expose a limited set of functions from os.path 300 OS = ReadOnlyNamespace(path=ReadOnlyNamespace(**{ 301 k: getattr(mozpath, k, getattr(os.path, k)) 302 for k in ('abspath', 'basename', 'dirname', 'isabs', 'join', 303 'normcase', 'normpath', 'realpath', 'relpath') 304 })) 305 306 def __init__(self, config, environ=os.environ, argv=sys.argv, 307 stdout=sys.stdout, stderr=sys.stderr, logger=None): 308 dict.__setitem__(self, '__builtins__', self.BUILTINS) 309 310 self._environ = dict(environ) 311 312 self._paths = [] 313 self._all_paths = set() 314 self._templates = set() 315 # Associate SandboxDependsFunctions to DependsFunctions. 316 self._depends = OrderedDict() 317 self._seen = set() 318 # Store the @imports added to a given function. 319 self._imports = {} 320 321 self._options = OrderedDict() 322 # Store raw option (as per command line or environment) for each Option 323 self._raw_options = OrderedDict() 324 325 # Store options added with `imply_option`, and the reason they were 326 # added (which can either have been given to `imply_option`, or 327 # inferred. Their order matters, so use a list. 328 self._implied_options = [] 329 330 # Store all results from _prepare_function 331 self._prepared_functions = set() 332 333 # Queue of functions to execute, with their arguments 334 self._execution_queue = [] 335 336 # Store the `when`s associated to some options. 337 self._conditions = {} 338 339 # A list of conditions to apply as a default `when` for every *_impl() 340 self._default_conditions = [] 341 342 self._helper = CommandLineHelper(environ, argv) 343 344 assert isinstance(config, dict) 345 self._config = config 346 347 # Tracks how many templates "deep" we are in the stack. 348 self._template_depth = 0 349 350 logging.addLevelName(TRACE, 'TRACE') 351 if logger is None: 352 logger = moz_logger = logging.getLogger('moz.configure') 353 logger.setLevel(logging.DEBUG) 354 formatter = logging.Formatter('%(levelname)s: %(message)s') 355 handler = ConfigureOutputHandler(stdout, stderr) 356 handler.setFormatter(formatter) 357 queue_debug = handler.queue_debug 358 logger.addHandler(handler) 359 360 else: 361 assert isinstance(logger, logging.Logger) 362 moz_logger = None 363 @contextmanager 364 def queue_debug(): 365 yield 366 367 self._logger = logger 368 369 # Some callers will manage to log a bytestring with characters in it 370 # that can't be converted to ascii. Make our log methods robust to this 371 # by detecting the encoding that a producer is likely to have used. 372 encoding = getpreferredencoding() 373 374 def wrapped_log_method(logger, key): 375 method = getattr(logger, key) 376 377 def wrapped(*args, **kwargs): 378 out_args = [ 379 six.ensure_text(arg, encoding=encoding or 'utf-8') 380 if isinstance(arg, six.binary_type) else arg for arg in args 381 ] 382 return method(*out_args, **kwargs) 383 return wrapped 384 385 log_namespace = { 386 k: wrapped_log_method(logger, k) 387 for k in ('debug', 'info', 'warning', 'error') 388 } 389 log_namespace['queue_debug'] = queue_debug 390 self.log_impl = ReadOnlyNamespace(**log_namespace) 391 392 self._help = None 393 self._help_option = self.option_impl( 394 '--help', help='print this message', category=HELP_OPTIONS_CATEGORY) 395 self._seen.add(self._help_option) 396 397 self._always = DependsFunction(self, lambda: True, []) 398 self._never = DependsFunction(self, lambda: False, []) 399 400 if self._value_for(self._help_option): 401 self._help = HelpFormatter(argv[0]) 402 self._help.add(self._help_option) 403 elif moz_logger: 404 handler = logging.FileHandler('config.log', mode='w', delay=True, 405 encoding='utf-8') 406 handler.setFormatter(formatter) 407 logger.addHandler(handler) 408 409 def include_file(self, path): 410 '''Include one file in the sandbox. Users of this class probably want 411 to use `run` instead. 412 413 Note: this will execute all template invocations, as well as @depends 414 functions that depend on '--help', but nothing else. 415 ''' 416 417 if self._paths: 418 path = mozpath.join(mozpath.dirname(self._paths[-1]), path) 419 path = mozpath.normpath(path) 420 if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)): 421 raise ConfigureError( 422 'Cannot include `%s` because it is not in a subdirectory ' 423 'of `%s`' % (path, mozpath.dirname(self._paths[0]))) 424 else: 425 path = mozpath.realpath(mozpath.abspath(path)) 426 if path in self._all_paths: 427 raise ConfigureError( 428 'Cannot include `%s` because it was included already.' % path) 429 self._paths.append(path) 430 self._all_paths.add(path) 431 432 source = open(path, 'rb').read() 433 434 code = compile(source, path, 'exec') 435 436 exec_(code, self) 437 438 self._paths.pop(-1) 439 440 def run(self, path=None): 441 '''Executes the given file within the sandbox, as well as everything 442 pending from any other included file, and ensure the overall 443 consistency of the executed script(s).''' 444 if path: 445 self.include_file(path) 446 447 for option in six.itervalues(self._options): 448 # All options must be referenced by some @depends function 449 if option not in self._seen: 450 raise ConfigureError( 451 'Option `%s` is not handled ; reference it with a @depends' 452 % option.option 453 ) 454 455 self._value_for(option) 456 457 # All implied options should exist. 458 for implied_option in self._implied_options: 459 value = self._resolve(implied_option.value) 460 if value is not None: 461 # There are two ways to end up here: either the implied option 462 # is unknown, or it's known but there was a dependency loop 463 # that prevented the implication from being applied. 464 option = self._options.get(implied_option.name) 465 if not option: 466 raise ConfigureError( 467 '`%s`, emitted from `%s` line %d, is unknown.' 468 % (implied_option.option, implied_option.caller[1], 469 implied_option.caller[2])) 470 # If the option is known, check that the implied value doesn't 471 # conflict with what value was attributed to the option. 472 if (implied_option.when and 473 not self._value_for(implied_option.when)): 474 continue 475 option_value = self._value_for_option(option) 476 if value != option_value: 477 reason = implied_option.reason 478 if isinstance(reason, Option): 479 reason = self._raw_options.get(reason) or reason.option 480 reason = reason.split('=', 1)[0] 481 value = OptionValue.from_(value) 482 raise InvalidOptionError( 483 "'%s' implied by '%s' conflicts with '%s' from the %s" 484 % (value.format(option.option), reason, 485 option_value.format(option.option), option_value.origin)) 486 487 # All options should have been removed (handled) by now. 488 for arg in self._helper: 489 without_value = arg.split('=', 1)[0] 490 msg = 'Unknown option: %s' % without_value 491 if self._help: 492 self._logger.warning(msg) 493 else: 494 raise InvalidOptionError(msg) 495 496 # Run the execution queue 497 for func, args in self._execution_queue: 498 func(*args) 499 500 if self._help: 501 with LineIO(self.log_impl.info) as out: 502 self._help.usage(out) 503 504 def __getitem__(self, key): 505 impl = '%s_impl' % key 506 func = getattr(self, impl, None) 507 if func: 508 return func 509 510 return super(ConfigureSandbox, self).__getitem__(key) 511 512 def __setitem__(self, key, value): 513 if (key in self.BUILTINS or key == '__builtins__' or 514 hasattr(self, '%s_impl' % key)): 515 raise KeyError('Cannot reassign builtins') 516 517 if inspect.isfunction(value) and value not in self._templates: 518 value = self._prepare_function(value) 519 520 elif (not isinstance(value, SandboxDependsFunction) and 521 value not in self._templates and 522 not (inspect.isclass(value) and issubclass(value, Exception))): 523 raise KeyError('Cannot assign `%s` because it is neither a ' 524 '@depends nor a @template' % key) 525 526 if isinstance(value, SandboxDependsFunction): 527 self._depends[value].name = key 528 529 return super(ConfigureSandbox, self).__setitem__(key, value) 530 531 def _resolve(self, arg): 532 if isinstance(arg, SandboxDependsFunction): 533 return self._value_for_depends(self._depends[arg]) 534 return arg 535 536 def _value_for(self, obj): 537 if isinstance(obj, SandboxDependsFunction): 538 assert obj in self._depends 539 return self._value_for_depends(self._depends[obj]) 540 541 elif isinstance(obj, DependsFunction): 542 return self._value_for_depends(obj) 543 544 elif isinstance(obj, Option): 545 return self._value_for_option(obj) 546 547 assert False 548 549 @memoize 550 def _value_for_depends(self, obj): 551 value = obj.result() 552 self._logger.log(TRACE, '%r = %r', obj, value) 553 return value 554 555 @memoize 556 def _value_for_option(self, option): 557 implied = {} 558 matching_implied_options = [ 559 o for o in self._implied_options if o.name in (option.name, option.env) 560 ] 561 # Update self._implied_options before going into the loop with the non-matching 562 # options. 563 self._implied_options = [ 564 o for o in self._implied_options if o.name not in (option.name, option.env) 565 ] 566 567 for implied_option in matching_implied_options: 568 if (implied_option.when and 569 not self._value_for(implied_option.when)): 570 continue 571 572 value = self._resolve(implied_option.value) 573 574 if value is not None: 575 value = OptionValue.from_(value) 576 opt = value.format(implied_option.option) 577 self._helper.add(opt, 'implied') 578 implied[opt] = implied_option 579 580 try: 581 value, option_string = self._helper.handle(option) 582 except ConflictingOptionError as e: 583 reason = implied[e.arg].reason 584 if isinstance(reason, Option): 585 reason = self._raw_options.get(reason) or reason.option 586 reason = reason.split('=', 1)[0] 587 raise InvalidOptionError( 588 "'%s' implied by '%s' conflicts with '%s' from the %s" 589 % (e.arg, reason, e.old_arg, e.old_origin)) 590 591 if value.origin == 'implied': 592 recursed_value = getattr(self, '__value_for_option').get((option,)) 593 if recursed_value is not None: 594 _, filename, line, _, _, _ = implied[value.format(option.option)].caller 595 raise ConfigureError( 596 "'%s' appears somewhere in the direct or indirect dependencies when " 597 "resolving imply_option at %s:%d" % (option.option, filename, line)) 598 599 if option_string: 600 self._raw_options[option] = option_string 601 602 when = self._conditions.get(option) 603 # If `when` resolves to a false-ish value, we always return None. 604 # This makes option(..., when='--foo') equivalent to 605 # option(..., when=depends('--foo')(lambda x: x)). 606 if when and not self._value_for(when) and value is not None: 607 # If the option was passed explicitly, we throw an error that 608 # the option is not available. Except when the option was passed 609 # from the environment, because that would be too cumbersome. 610 if value.origin not in ('default', 'environment'): 611 raise InvalidOptionError( 612 '%s is not available in this configuration' 613 % option_string.split('=', 1)[0]) 614 self._logger.log(TRACE, '%r = None', option) 615 return None 616 617 self._logger.log(TRACE, '%r = %r', option, value) 618 return value 619 620 def _dependency(self, arg, callee_name, arg_name=None): 621 if isinstance(arg, six.string_types): 622 prefix, name, values = Option.split_option(arg) 623 if values != (): 624 raise ConfigureError("Option must not contain an '='") 625 if name not in self._options: 626 raise ConfigureError("'%s' is not a known option. " 627 "Maybe it's declared too late?" 628 % arg) 629 arg = self._options[name] 630 self._seen.add(arg) 631 elif isinstance(arg, SandboxDependsFunction): 632 assert arg in self._depends 633 arg = self._depends[arg] 634 else: 635 raise TypeError( 636 "Cannot use object of type '%s' as %sargument to %s" 637 % (type(arg).__name__, '`%s` ' % arg_name if arg_name else '', 638 callee_name)) 639 return arg 640 641 def _normalize_when(self, when, callee_name): 642 if when is True: 643 when = self._always 644 elif when is False: 645 when = self._never 646 elif when is not None: 647 when = self._dependency(when, callee_name, 'when') 648 649 if self._default_conditions: 650 # Create a pseudo @depends function for the combination of all 651 # default conditions and `when`. 652 dependencies = [when] if when else [] 653 dependencies.extend(self._default_conditions) 654 if len(dependencies) == 1: 655 return dependencies[0] 656 return CombinedDependsFunction(self, all, dependencies) 657 return when 658 659 @contextmanager 660 def only_when_impl(self, when): 661 '''Implementation of only_when() 662 663 `only_when` is a context manager that essentially makes calls to 664 other sandbox functions within the context block ignored. 665 ''' 666 when = self._normalize_when(when, 'only_when') 667 if when and self._default_conditions[-1:] != [when]: 668 self._default_conditions.append(when) 669 yield 670 self._default_conditions.pop() 671 else: 672 yield 673 674 def option_impl(self, *args, **kwargs): 675 '''Implementation of option() 676 This function creates and returns an Option() object, passing it the 677 resolved arguments (uses the result of functions when functions are 678 passed). In most cases, the result of this function is not expected to 679 be used. 680 Command line argument/environment variable parsing for this Option is 681 handled here. 682 ''' 683 when = self._normalize_when(kwargs.get('when'), 'option') 684 args = [self._resolve(arg) for arg in args] 685 kwargs = {k: self._resolve(v) for k, v in six.iteritems(kwargs) 686 if k != 'when'} 687 # The Option constructor needs to look up the stack to infer a category 688 # for the Option, since the category is based on the filename where the 689 # Option is defined. However, if the Option is defined in a template, we 690 # want the category to reference the caller of the template rather than 691 # the caller of the option() function. 692 kwargs['define_depth'] = self._template_depth * 3 693 option = Option(*args, **kwargs) 694 if when: 695 self._conditions[option] = when 696 if option.name in self._options: 697 raise ConfigureError('Option `%s` already defined' % option.option) 698 if option.env in self._options: 699 raise ConfigureError('Option `%s` already defined' % option.env) 700 if option.name: 701 self._options[option.name] = option 702 if option.env: 703 self._options[option.env] = option 704 705 if self._help and (when is None or self._value_for(when)): 706 self._help.add(option) 707 708 return option 709 710 def depends_impl(self, *args, **kwargs): 711 '''Implementation of @depends() 712 This function is a decorator. It returns a function that subsequently 713 takes a function and returns a dummy function. The dummy function 714 identifies the actual function for the sandbox, while preventing 715 further function calls from within the sandbox. 716 717 @depends() takes a variable number of option strings or dummy function 718 references. The decorated function is called as soon as the decorator 719 is called, and the arguments it receives are the OptionValue or 720 function results corresponding to each of the arguments to @depends. 721 As an exception, when a HelpFormatter is attached, only functions that 722 have '--help' in their @depends argument list are called. 723 724 The decorated function is altered to use a different global namespace 725 for its execution. This different global namespace exposes a limited 726 set of functions from os.path. 727 ''' 728 for k in kwargs: 729 if k != 'when': 730 raise TypeError( 731 "depends_impl() got an unexpected keyword argument '%s'" 732 % k) 733 734 when = self._normalize_when(kwargs.get('when'), '@depends') 735 736 if not when and not args: 737 raise ConfigureError('@depends needs at least one argument') 738 739 dependencies = tuple(self._dependency(arg, '@depends') for arg in args) 740 741 conditions = [ 742 self._conditions[d] 743 for d in dependencies 744 if d in self._conditions and isinstance(d, Option) 745 ] 746 for c in conditions: 747 if c != when: 748 raise ConfigureError('@depends function needs the same `when` ' 749 'as options it depends on') 750 751 def decorator(func): 752 if inspect.isgeneratorfunction(func): 753 raise ConfigureError( 754 'Cannot decorate generator functions with @depends') 755 func = self._prepare_function(func) 756 depends = DependsFunction(self, func, dependencies, when=when) 757 return depends.sandboxed 758 759 return decorator 760 761 def include_impl(self, what, when=None): 762 '''Implementation of include(). 763 Allows to include external files for execution in the sandbox. 764 It is possible to use a @depends function as argument, in which case 765 the result of the function is the file name to include. This latter 766 feature is only really meant for --enable-application/--enable-project. 767 ''' 768 with self.only_when_impl(when): 769 what = self._resolve(what) 770 if what: 771 if not isinstance(what, six.string_types): 772 raise TypeError("Unexpected type: '%s'" % type(what).__name__) 773 self.include_file(what) 774 775 def template_impl(self, func): 776 '''Implementation of @template. 777 This function is a decorator. Template functions are called 778 immediately. They are altered so that their global namespace exposes 779 a limited set of functions from os.path, as well as `depends` and 780 `option`. 781 Templates allow to simplify repetitive constructs, or to implement 782 helper decorators and somesuch. 783 ''' 784 def update_globals(glob): 785 glob.update( 786 (k[:-len('_impl')], getattr(self, k)) 787 for k in dir(self) if k.endswith('_impl') and k != 'template_impl' 788 ) 789 glob.update((k, v) for k, v in six.iteritems(self) if k not in glob) 790 791 template = self._prepare_function(func, update_globals) 792 793 # Any function argument to the template must be prepared to be sandboxed. 794 # If the template itself returns a function (in which case, it's very 795 # likely a decorator), that function must be prepared to be sandboxed as 796 # well. 797 def wrap_template(template): 798 isfunction = inspect.isfunction 799 800 def maybe_prepare_function(obj): 801 if isfunction(obj): 802 return self._prepare_function(obj) 803 return obj 804 805 # The following function may end up being prepared to be sandboxed, 806 # so it mustn't depend on anything from the global scope in this 807 # file. It can however depend on variables from the closure, thus 808 # maybe_prepare_function and isfunction are declared above to be 809 # available there. 810 @self.wraps(template) 811 def wrapper(*args, **kwargs): 812 args = [maybe_prepare_function(arg) for arg in args] 813 kwargs = {k: maybe_prepare_function(v) 814 for k, v in kwargs.items()} 815 self._template_depth += 1 816 ret = template(*args, **kwargs) 817 self._template_depth -= 1 818 if isfunction(ret): 819 # We can't expect the sandboxed code to think about all the 820 # details of implementing decorators, so do some of the 821 # work for them. If the function takes exactly one function 822 # as argument and returns a function, it must be a 823 # decorator, so mark the returned function as wrapping the 824 # function passed in. 825 if len(args) == 1 and not kwargs and isfunction(args[0]): 826 ret = self.wraps(args[0])(ret) 827 return wrap_template(ret) 828 return ret 829 return wrapper 830 831 wrapper = wrap_template(template) 832 self._templates.add(wrapper) 833 return wrapper 834 835 def wraps(self, func): 836 return wraps(func) 837 838 RE_MODULE = re.compile('^[a-zA-Z0-9_\.]+$') 839 840 def imports_impl(self, _import, _from=None, _as=None): 841 '''Implementation of @imports. 842 This decorator imports the given _import from the given _from module 843 optionally under a different _as name. 844 The options correspond to the various forms for the import builtin. 845 846 @imports('sys') 847 @imports(_from='mozpack', _import='path', _as='mozpath') 848 ''' 849 for value, required in ( 850 (_import, True), (_from, False), (_as, False)): 851 852 if not isinstance(value, six.string_types) and ( 853 required or value is not None): 854 raise TypeError("Unexpected type: '%s'" % type(value).__name__) 855 if value is not None and not self.RE_MODULE.match(value): 856 raise ValueError("Invalid argument to @imports: '%s'" % value) 857 if _as and '.' in _as: 858 raise ValueError("Invalid argument to @imports: '%s'" % _as) 859 860 def decorator(func): 861 if func in self._templates: 862 raise ConfigureError( 863 '@imports must appear after @template') 864 if func in self._depends: 865 raise ConfigureError( 866 '@imports must appear after @depends') 867 # For the imports to apply in the order they appear in the 868 # .configure file, we accumulate them in reverse order and apply 869 # them later. 870 imports = self._imports.setdefault(func, []) 871 imports.insert(0, (_from, _import, _as)) 872 return func 873 874 return decorator 875 876 def _apply_imports(self, func, glob): 877 for _from, _import, _as in self._imports.pop(func, ()): 878 self._get_one_import(_from, _import, _as, glob) 879 880 def _handle_wrapped_import(self, _from, _import, _as, glob): 881 """Given the name of a module, "import" a mocked package into the glob 882 iff the module is one that we wrap (either for the sandbox or for the 883 purpose of testing). Applies if the wrapped module is exposed by an 884 attribute of `self`. 885 886 For example, if the import statement is `from os import environ`, then 887 this function will set 888 glob['environ'] = self._wrapped_os.environ. 889 890 Iff this function handles the given import, return True. 891 """ 892 module = (_from or _import).split('.')[0] 893 attr = '_wrapped_' + module 894 wrapped = getattr(self, attr, None) 895 if wrapped: 896 if _as or _from: 897 obj = self._recursively_get_property( 898 module, (_from + '.' if _from else '') + _import, wrapped) 899 glob[_as or _import] = obj 900 else: 901 glob[module] = wrapped 902 return True 903 else: 904 return False 905 906 def _recursively_get_property(self, module, what, wrapped): 907 """Traverse the wrapper object `wrapped` (which represents the module 908 `module`) and return the property represented by `what`, which may be a 909 series of nested attributes. 910 911 For example, if `module` is 'os' and `what` is 'os.path.join', 912 return `wrapped.path.join`. 913 """ 914 if what == module: 915 return wrapped 916 assert what.startswith(module + '.') 917 attrs = what[len(module + '.'):].split('.') 918 for attr in attrs: 919 wrapped = getattr(wrapped, attr) 920 return wrapped 921 922 @memoized_property 923 def _wrapped_os(self): 924 wrapped_os = {} 925 exec_('from os import *', {}, wrapped_os) 926 # Special case os and os.environ so that os.environ is our copy of 927 # the environment. 928 wrapped_os['environ'] = self._environ 929 return ReadOnlyNamespace(**wrapped_os) 930 931 @memoized_property 932 def _wrapped_subprocess(self): 933 wrapped_subprocess = {} 934 exec_('from subprocess import *', {}, wrapped_subprocess) 935 936 def wrap(function): 937 def wrapper(*args, **kwargs): 938 if 'env' not in kwargs: 939 kwargs['env'] = dict(self._environ) 940 # Subprocess on older Pythons can't handle unicode keys or 941 # values in environment dicts while subprocess on newer Pythons 942 # needs text in the env. Normalize automagically so callers 943 # don't have to deal with this. 944 kwargs['env'] = ensure_subprocess_env(kwargs['env'], encoding=system_encoding) 945 return function(*args, **kwargs) 946 return wrapper 947 948 for f in ('call', 'check_call', 'check_output', 'Popen'): 949 wrapped_subprocess[f] = wrap(wrapped_subprocess[f]) 950 951 return ReadOnlyNamespace(**wrapped_subprocess) 952 953 @memoized_property 954 def _wrapped_six(self): 955 if six.PY3: 956 return six 957 wrapped_six = {} 958 exec_('from six import *', {}, wrapped_six) 959 wrapped_six_moves = {} 960 exec_('from six.moves import *', {}, wrapped_six_moves) 961 wrapped_six_moves_builtins = {} 962 exec_('from six.moves.builtins import *', {}, 963 wrapped_six_moves_builtins) 964 965 # Special case for the open() builtin, because otherwise, using it 966 # fails with "IOError: file() constructor not accessible in 967 # restricted mode". We also make open() look more like python 3's, 968 # decoding to unicode strings unless the mode says otherwise. 969 def wrapped_open(name, mode=None, buffering=None): 970 args = (name,) 971 kwargs = {} 972 if buffering is not None: 973 kwargs['buffering'] = buffering 974 if mode is not None: 975 args += (mode,) 976 if 'b' in mode: 977 return open(*args, **kwargs) 978 kwargs['encoding'] = system_encoding 979 return codecs.open(*args, **kwargs) 980 981 wrapped_six_moves_builtins['open'] = wrapped_open 982 wrapped_six_moves['builtins'] = ReadOnlyNamespace( 983 **wrapped_six_moves_builtins) 984 wrapped_six['moves'] = ReadOnlyNamespace(**wrapped_six_moves) 985 986 return ReadOnlyNamespace(**wrapped_six) 987 988 def _get_one_import(self, _from, _import, _as, glob): 989 """Perform the given import, placing the result into the dict glob.""" 990 if not _from and _import == '__builtin__': 991 glob[_as or '__builtin__'] = __builtin__ 992 return 993 if _from == '__builtin__': 994 _from = 'six.moves.builtins' 995 # The special `__sandbox__` module gives access to the sandbox 996 # instance. 997 if not _from and _import == '__sandbox__': 998 glob[_as or _import] = self 999 return 1000 if self._handle_wrapped_import(_from, _import, _as, glob): 1001 return 1002 # If we've gotten this far, we should just do a normal import. 1003 # Until this proves to be a performance problem, just construct an 1004 # import statement and execute it. 1005 import_line = '%simport %s%s' % ( 1006 ('from %s ' % _from) if _from else '', _import, 1007 (' as %s' % _as) if _as else '') 1008 exec_(import_line, {}, glob) 1009 1010 def _resolve_and_set(self, data, name, value, when=None): 1011 # Don't set anything when --help was on the command line 1012 if self._help: 1013 return 1014 if when and not self._value_for(when): 1015 return 1016 name = self._resolve(name) 1017 if name is None: 1018 return 1019 if not isinstance(name, six.string_types): 1020 raise TypeError("Unexpected type: '%s'" % type(name).__name__) 1021 if name in data: 1022 raise ConfigureError( 1023 "Cannot add '%s' to configuration: Key already " 1024 "exists" % name) 1025 value = self._resolve(value) 1026 if value is not None: 1027 if self._logger.isEnabledFor(TRACE): 1028 if data is self._config: 1029 self._logger.log(TRACE, 'set_config(%s, %r)', name, value) 1030 elif data is self._config.get('DEFINES'): 1031 self._logger.log(TRACE, 'set_define(%s, %r)', name, value) 1032 data[name] = value 1033 1034 def set_config_impl(self, name, value, when=None): 1035 '''Implementation of set_config(). 1036 Set the configuration items with the given name to the given value. 1037 Both `name` and `value` can be references to @depends functions, 1038 in which case the result from these functions is used. If the result 1039 of either function is None, the configuration item is not set. 1040 ''' 1041 when = self._normalize_when(when, 'set_config') 1042 1043 self._execution_queue.append(( 1044 self._resolve_and_set, (self._config, name, value, when))) 1045 1046 def set_define_impl(self, name, value, when=None): 1047 '''Implementation of set_define(). 1048 Set the define with the given name to the given value. Both `name` and 1049 `value` can be references to @depends functions, in which case the 1050 result from these functions is used. If the result of either function 1051 is None, the define is not set. If the result is False, the define is 1052 explicitly undefined (-U). 1053 ''' 1054 when = self._normalize_when(when, 'set_define') 1055 1056 defines = self._config.setdefault('DEFINES', {}) 1057 self._execution_queue.append(( 1058 self._resolve_and_set, (defines, name, value, when))) 1059 1060 def imply_option_impl(self, option, value, reason=None, when=None): 1061 '''Implementation of imply_option(). 1062 Injects additional options as if they had been passed on the command 1063 line. The `option` argument is a string as in option()'s `name` or 1064 `env`. The option must be declared after `imply_option` references it. 1065 The `value` argument indicates the value to pass to the option. 1066 It can be: 1067 - True. In this case `imply_option` injects the positive option 1068 1069 (--enable-foo/--with-foo). 1070 imply_option('--enable-foo', True) 1071 imply_option('--disable-foo', True) 1072 1073 are both equivalent to `--enable-foo` on the command line. 1074 1075 - False. In this case `imply_option` injects the negative option 1076 1077 (--disable-foo/--without-foo). 1078 imply_option('--enable-foo', False) 1079 imply_option('--disable-foo', False) 1080 1081 are both equivalent to `--disable-foo` on the command line. 1082 1083 - None. In this case `imply_option` does nothing. 1084 imply_option('--enable-foo', None) 1085 imply_option('--disable-foo', None) 1086 1087 are both equivalent to not passing any flag on the command line. 1088 1089 - a string or a tuple. In this case `imply_option` injects the positive 1090 option with the given value(s). 1091 1092 imply_option('--enable-foo', 'a') 1093 imply_option('--disable-foo', 'a') 1094 1095 are both equivalent to `--enable-foo=a` on the command line. 1096 imply_option('--enable-foo', ('a', 'b')) 1097 imply_option('--disable-foo', ('a', 'b')) 1098 1099 are both equivalent to `--enable-foo=a,b` on the command line. 1100 1101 Because imply_option('--disable-foo', ...) can be misleading, it is 1102 recommended to use the positive form ('--enable' or '--with') for 1103 `option`. 1104 1105 The `value` argument can also be (and usually is) a reference to a 1106 @depends function, in which case the result of that function will be 1107 used as per the descripted mapping above. 1108 1109 The `reason` argument indicates what caused the option to be implied. 1110 It is necessary when it cannot be inferred from the `value`. 1111 ''' 1112 1113 when = self._normalize_when(when, 'imply_option') 1114 1115 # Don't do anything when --help was on the command line 1116 if self._help: 1117 return 1118 if not reason and isinstance(value, SandboxDependsFunction): 1119 deps = self._depends[value].dependencies 1120 possible_reasons = [d for d in deps if d != self._help_option] 1121 if len(possible_reasons) == 1: 1122 if isinstance(possible_reasons[0], Option): 1123 reason = possible_reasons[0] 1124 if not reason and (isinstance(value, (bool, tuple)) or 1125 isinstance(value, six.string_types)): 1126 # A reason can be provided automatically when imply_option 1127 # is called with an immediate value. 1128 _, filename, line, _, _, _ = inspect.stack()[1] 1129 reason = "imply_option at %s:%s" % (filename, line) 1130 1131 if not reason: 1132 raise ConfigureError( 1133 "Cannot infer what implies '%s'. Please add a `reason` to " 1134 "the `imply_option` call." 1135 % option) 1136 1137 prefix, name, values = Option.split_option(option) 1138 if values != (): 1139 raise ConfigureError("Implied option must not contain an '='") 1140 1141 self._implied_options.append(ReadOnlyNamespace( 1142 option=option, 1143 prefix=prefix, 1144 name=name, 1145 value=value, 1146 caller=inspect.stack()[1], 1147 reason=reason, 1148 when=when, 1149 )) 1150 1151 def _prepare_function(self, func, update_globals=None): 1152 '''Alter the given function global namespace with the common ground 1153 for @depends, and @template. 1154 ''' 1155 if not inspect.isfunction(func): 1156 raise TypeError("Unexpected type: '%s'" % type(func).__name__) 1157 if func in self._prepared_functions: 1158 return func 1159 1160 glob = SandboxedGlobal( 1161 (k, v) 1162 for k, v in six.iteritems(func.__globals__) 1163 if (inspect.isfunction(v) and v not in self._templates) or ( 1164 inspect.isclass(v) and issubclass(v, Exception)) 1165 ) 1166 glob.update( 1167 __builtins__=self.BUILTINS, 1168 __file__=self._paths[-1] if self._paths else '', 1169 __name__=self._paths[-1] if self._paths else '', 1170 os=self.OS, 1171 log=self.log_impl, 1172 ) 1173 if update_globals: 1174 update_globals(glob) 1175 1176 # The execution model in the sandbox doesn't guarantee the execution 1177 # order will always be the same for a given function, and if it uses 1178 # variables from a closure that are changed after the function is 1179 # declared, depending when the function is executed, the value of the 1180 # variable can differ. For consistency, we force the function to use 1181 # the value from the earliest it can be run, which is at declaration. 1182 # Note this is not entirely bullet proof (if the value is e.g. a list, 1183 # the list contents could have changed), but covers the bases. 1184 closure = None 1185 if func.__closure__: 1186 def makecell(content): 1187 def f(): 1188 content 1189 return f.__closure__[0] 1190 1191 closure = tuple(makecell(cell.cell_contents) 1192 for cell in func.__closure__) 1193 1194 new_func = self.wraps(func)(types.FunctionType( 1195 func.__code__, 1196 glob, 1197 func.__name__, 1198 func.__defaults__, 1199 closure 1200 )) 1201 @self.wraps(new_func) 1202 def wrapped(*args, **kwargs): 1203 if func in self._imports: 1204 self._apply_imports(func, glob) 1205 return new_func(*args, **kwargs) 1206 1207 self._prepared_functions.add(wrapped) 1208 return wrapped 1209