1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7import codecs 8import inspect 9import logging 10import os 11import re 12import six 13from six.moves import builtins as __builtin__ 14import sys 15import types 16from collections import OrderedDict 17from contextlib import contextmanager 18from functools import wraps 19from mozbuild.configure.options import ( 20 CommandLineHelper, 21 ConflictingOptionError, 22 HELP_OPTIONS_CATEGORY, 23 InvalidOptionError, 24 Option, 25 OptionValue, 26) 27from mozbuild.configure.help import HelpFormatter 28from mozbuild.configure.util import ConfigureOutputHandler, getpreferredencoding, LineIO 29from mozbuild.util import ( 30 exec_, 31 memoize, 32 memoized_property, 33 ReadOnlyDict, 34 ReadOnlyNamespace, 35 system_encoding, 36) 37 38import mozpack.path as mozpath 39 40 41# TRACE logging level, below (thus more verbose than) DEBUG 42TRACE = 5 43 44 45class ConfigureError(Exception): 46 pass 47 48 49class SandboxDependsFunction(object): 50 """Sandbox-visible representation of @depends functions.""" 51 52 def __init__(self, unsandboxed): 53 self._or = unsandboxed.__or__ 54 self._and = unsandboxed.__and__ 55 self._getattr = unsandboxed.__getattr__ 56 57 def __call__(self, *arg, **kwargs): 58 raise ConfigureError("The `%s` function may not be called" % self.__name__) 59 60 def __or__(self, other): 61 if not isinstance(other, SandboxDependsFunction): 62 raise ConfigureError( 63 "Can only do binary arithmetic operations " 64 "with another @depends function." 65 ) 66 return self._or(other).sandboxed 67 68 def __and__(self, other): 69 if not isinstance(other, SandboxDependsFunction): 70 raise ConfigureError( 71 "Can only do binary arithmetic operations " 72 "with another @depends function." 73 ) 74 return self._and(other).sandboxed 75 76 def __cmp__(self, other): 77 raise ConfigureError("Cannot compare @depends functions.") 78 79 def __eq__(self, other): 80 raise ConfigureError("Cannot compare @depends functions.") 81 82 def __hash__(self): 83 return object.__hash__(self) 84 85 def __ne__(self, other): 86 raise ConfigureError("Cannot compare @depends functions.") 87 88 def __lt__(self, other): 89 raise ConfigureError("Cannot compare @depends functions.") 90 91 def __le__(self, other): 92 raise ConfigureError("Cannot compare @depends functions.") 93 94 def __gt__(self, other): 95 raise ConfigureError("Cannot compare @depends functions.") 96 97 def __ge__(self, other): 98 raise ConfigureError("Cannot compare @depends functions.") 99 100 def __getattr__(self, key): 101 return self._getattr(key).sandboxed 102 103 def __nonzero__(self): 104 raise ConfigureError("Cannot do boolean operations on @depends functions.") 105 106 107class DependsFunction(object): 108 __slots__ = ( 109 "_func", 110 "_name", 111 "dependencies", 112 "when", 113 "sandboxed", 114 "sandbox", 115 "_result", 116 ) 117 118 def __init__(self, sandbox, func, dependencies, when=None): 119 assert isinstance(sandbox, ConfigureSandbox) 120 assert not inspect.isgeneratorfunction(func) 121 self._func = func 122 self._name = func.__name__ 123 self.dependencies = dependencies 124 self.sandboxed = wraps(func)(SandboxDependsFunction(self)) 125 self.sandbox = sandbox 126 self.when = when 127 sandbox._depends[self.sandboxed] = self 128 129 # Only @depends functions with a dependency on '--help' are executed 130 # immediately. Everything else is queued for later execution. 131 if sandbox._help_option in dependencies: 132 sandbox._value_for(self) 133 elif not sandbox._help: 134 sandbox._execution_queue.append((sandbox._value_for, (self,))) 135 136 @property 137 def name(self): 138 return self._name 139 140 @name.setter 141 def name(self, value): 142 self._name = value 143 144 @property 145 def sandboxed_dependencies(self): 146 return [ 147 d.sandboxed if isinstance(d, DependsFunction) else d 148 for d in self.dependencies 149 ] 150 151 @memoize 152 def result(self): 153 if self.when and not self.sandbox._value_for(self.when): 154 return None 155 156 resolved_args = [self.sandbox._value_for(d) for d in self.dependencies] 157 return self._func(*resolved_args) 158 159 def __repr__(self): 160 return "<%s %s(%s)>" % ( 161 self.__class__.__name__, 162 self.name, 163 ", ".join(repr(d) for d in self.dependencies), 164 ) 165 166 def __or__(self, other): 167 if isinstance(other, SandboxDependsFunction): 168 other = self.sandbox._depends.get(other) 169 assert isinstance(other, DependsFunction) 170 assert self.sandbox is other.sandbox 171 return CombinedDependsFunction(self.sandbox, self.or_impl, (self, other)) 172 173 @staticmethod 174 def or_impl(iterable): 175 # Applies "or" to all the items of iterable. 176 # e.g. if iterable contains a, b and c, returns `a or b or c`. 177 for i in iterable: 178 if i: 179 return i 180 return i 181 182 def __and__(self, other): 183 if isinstance(other, SandboxDependsFunction): 184 other = self.sandbox._depends.get(other) 185 assert isinstance(other, DependsFunction) 186 assert self.sandbox is other.sandbox 187 return CombinedDependsFunction(self.sandbox, self.and_impl, (self, other)) 188 189 @staticmethod 190 def and_impl(iterable): 191 # Applies "and" to all the items of iterable. 192 # e.g. if iterable contains a, b and c, returns `a and b and c`. 193 for i in iterable: 194 if not i: 195 return i 196 return i 197 198 def __getattr__(self, key): 199 if key.startswith("_"): 200 return super(DependsFunction, self).__getattr__(key) 201 # Our function may return None or an object that simply doesn't have 202 # the wanted key. In that case, just return None. 203 return TrivialDependsFunction( 204 self.sandbox, lambda x: getattr(x, key, None), [self], self.when 205 ) 206 207 208class TrivialDependsFunction(DependsFunction): 209 """Like a DependsFunction, but the linter won't expect it to have a 210 dependency on --help ever.""" 211 212 213class CombinedDependsFunction(DependsFunction): 214 def __init__(self, sandbox, func, dependencies): 215 flatten_deps = [] 216 for d in dependencies: 217 if isinstance(d, CombinedDependsFunction) and d._func is func: 218 for d2 in d.dependencies: 219 if d2 not in flatten_deps: 220 flatten_deps.append(d2) 221 elif d not in flatten_deps: 222 flatten_deps.append(d) 223 224 super(CombinedDependsFunction, self).__init__(sandbox, func, flatten_deps) 225 226 @memoize 227 def result(self): 228 resolved_args = (self.sandbox._value_for(d) for d in self.dependencies) 229 return self._func(resolved_args) 230 231 def __eq__(self, other): 232 return ( 233 isinstance(other, self.__class__) 234 and self._func is other._func 235 and set(self.dependencies) == set(other.dependencies) 236 ) 237 238 def __hash__(self): 239 return object.__hash__(self) 240 241 def __ne__(self, other): 242 return not self == other 243 244 245class SandboxedGlobal(dict): 246 """Identifiable dict type for use as function global""" 247 248 249def forbidden_import(*args, **kwargs): 250 raise ImportError("Importing modules is forbidden") 251 252 253class ConfigureSandbox(dict): 254 """Represents a sandbox for executing Python code for build configuration. 255 This is a different kind of sandboxing than the one used for moz.build 256 processing. 257 258 The sandbox has 9 primitives: 259 - option 260 - depends 261 - template 262 - imports 263 - include 264 - set_config 265 - set_define 266 - imply_option 267 - only_when 268 269 `option`, `include`, `set_config`, `set_define` and `imply_option` are 270 functions. `depends`, `template`, and `imports` are decorators. `only_when` 271 is a context_manager. 272 273 These primitives are declared as name_impl methods to this class and 274 the mapping name -> name_impl is done automatically in __getitem__. 275 276 Additional primitives should be frowned upon to keep the sandbox itself as 277 simple as possible. Instead, helpers should be created within the sandbox 278 with the existing primitives. 279 280 The sandbox is given, at creation, a dict where the yielded configuration 281 will be stored. 282 283 config = {} 284 sandbox = ConfigureSandbox(config) 285 sandbox.run(path) 286 do_stuff(config) 287 """ 288 289 # The default set of builtins. We expose unicode as str to make sandboxed 290 # files more python3-ready. 291 BUILTINS = ReadOnlyDict( 292 { 293 b: getattr(__builtin__, b, None) 294 for b in ( 295 "AssertionError", 296 "False", 297 "None", 298 "True", 299 "__build_class__", # will be None on py2 300 "all", 301 "any", 302 "bool", 303 "dict", 304 "enumerate", 305 "getattr", 306 "hasattr", 307 "int", 308 "isinstance", 309 "len", 310 "list", 311 "range", 312 "set", 313 "sorted", 314 "tuple", 315 "zip", 316 ) 317 }, 318 __import__=forbidden_import, 319 str=six.text_type, 320 ) 321 322 # Expose a limited set of functions from os.path 323 OS = ReadOnlyNamespace( 324 path=ReadOnlyNamespace( 325 **{ 326 k: getattr(mozpath, k, getattr(os.path, k)) 327 for k in ( 328 "abspath", 329 "basename", 330 "dirname", 331 "isabs", 332 "join", 333 "normcase", 334 "normpath", 335 "realpath", 336 "relpath", 337 ) 338 } 339 ) 340 ) 341 342 def __init__( 343 self, 344 config, 345 environ=os.environ, 346 argv=sys.argv, 347 stdout=sys.stdout, 348 stderr=sys.stderr, 349 logger=None, 350 ): 351 dict.__setitem__(self, "__builtins__", self.BUILTINS) 352 353 self._environ = dict(environ) 354 355 self._paths = [] 356 self._all_paths = set() 357 self._templates = set() 358 # Associate SandboxDependsFunctions to DependsFunctions. 359 self._depends = OrderedDict() 360 self._seen = set() 361 # Store the @imports added to a given function. 362 self._imports = {} 363 364 self._options = OrderedDict() 365 # Store raw option (as per command line or environment) for each Option 366 self._raw_options = OrderedDict() 367 368 # Store options added with `imply_option`, and the reason they were 369 # added (which can either have been given to `imply_option`, or 370 # inferred. Their order matters, so use a list. 371 self._implied_options = [] 372 373 # Store all results from _prepare_function 374 self._prepared_functions = set() 375 376 # Queue of functions to execute, with their arguments 377 self._execution_queue = [] 378 379 # Store the `when`s associated to some options. 380 self._conditions = {} 381 382 # A list of conditions to apply as a default `when` for every *_impl() 383 self._default_conditions = [] 384 385 self._helper = CommandLineHelper(environ, argv) 386 387 assert isinstance(config, dict) 388 self._config = config 389 390 # Tracks how many templates "deep" we are in the stack. 391 self._template_depth = 0 392 393 logging.addLevelName(TRACE, "TRACE") 394 if logger is None: 395 logger = moz_logger = logging.getLogger("moz.configure") 396 logger.setLevel(logging.DEBUG) 397 formatter = logging.Formatter("%(levelname)s: %(message)s") 398 handler = ConfigureOutputHandler(stdout, stderr) 399 handler.setFormatter(formatter) 400 queue_debug = handler.queue_debug 401 logger.addHandler(handler) 402 403 else: 404 assert isinstance(logger, logging.Logger) 405 moz_logger = None 406 407 @contextmanager 408 def queue_debug(): 409 yield 410 411 self._logger = logger 412 413 # Some callers will manage to log a bytestring with characters in it 414 # that can't be converted to ascii. Make our log methods robust to this 415 # by detecting the encoding that a producer is likely to have used. 416 encoding = getpreferredencoding() 417 418 def wrapped_log_method(logger, key): 419 method = getattr(logger, key) 420 421 def wrapped(*args, **kwargs): 422 out_args = [ 423 six.ensure_text(arg, encoding=encoding or "utf-8") 424 if isinstance(arg, six.binary_type) 425 else arg 426 for arg in args 427 ] 428 return method(*out_args, **kwargs) 429 430 return wrapped 431 432 log_namespace = { 433 k: wrapped_log_method(logger, k) 434 for k in ("debug", "info", "warning", "error") 435 } 436 log_namespace["queue_debug"] = queue_debug 437 self.log_impl = ReadOnlyNamespace(**log_namespace) 438 439 self._help = None 440 self._help_option = self.option_impl( 441 "--help", help="print this message", category=HELP_OPTIONS_CATEGORY 442 ) 443 self._seen.add(self._help_option) 444 445 self._always = DependsFunction(self, lambda: True, []) 446 self._never = DependsFunction(self, lambda: False, []) 447 448 if self._value_for(self._help_option): 449 self._help = HelpFormatter(argv[0]) 450 self._help.add(self._help_option) 451 elif moz_logger: 452 handler = logging.FileHandler( 453 "config.log", mode="w", delay=True, encoding="utf-8" 454 ) 455 handler.setFormatter(formatter) 456 logger.addHandler(handler) 457 458 def include_file(self, path): 459 """Include one file in the sandbox. Users of this class probably want 460 to use `run` instead. 461 462 Note: this will execute all template invocations, as well as @depends 463 functions that depend on '--help', but nothing else. 464 """ 465 466 if self._paths: 467 path = mozpath.join(mozpath.dirname(self._paths[-1]), path) 468 path = mozpath.normpath(path) 469 if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)): 470 raise ConfigureError( 471 "Cannot include `%s` because it is not in a subdirectory " 472 "of `%s`" % (path, mozpath.dirname(self._paths[0])) 473 ) 474 else: 475 path = mozpath.realpath(mozpath.abspath(path)) 476 if path in self._all_paths: 477 raise ConfigureError( 478 "Cannot include `%s` because it was included already." % path 479 ) 480 self._paths.append(path) 481 self._all_paths.add(path) 482 483 with open(path, "rb") as fh: 484 source = fh.read() 485 486 code = compile(source, path, "exec") 487 488 exec_(code, self) 489 490 self._paths.pop(-1) 491 492 def run(self, path=None): 493 """Executes the given file within the sandbox, as well as everything 494 pending from any other included file, and ensure the overall 495 consistency of the executed script(s).""" 496 if path: 497 self.include_file(path) 498 499 for option in six.itervalues(self._options): 500 # All options must be referenced by some @depends function 501 if option not in self._seen: 502 raise ConfigureError( 503 "Option `%s` is not handled ; reference it with a @depends" 504 % option.option 505 ) 506 507 self._value_for(option) 508 509 # All implied options should exist. 510 for implied_option in self._implied_options: 511 value = self._resolve(implied_option.value) 512 if value is not None: 513 # There are two ways to end up here: either the implied option 514 # is unknown, or it's known but there was a dependency loop 515 # that prevented the implication from being applied. 516 option = self._options.get(implied_option.name) 517 if not option: 518 raise ConfigureError( 519 "`%s`, emitted from `%s` line %d, is unknown." 520 % ( 521 implied_option.option, 522 implied_option.caller[1], 523 implied_option.caller[2], 524 ) 525 ) 526 # If the option is known, check that the implied value doesn't 527 # conflict with what value was attributed to the option. 528 if implied_option.when and not self._value_for(implied_option.when): 529 continue 530 option_value = self._value_for_option(option) 531 if value != option_value: 532 reason = implied_option.reason 533 if isinstance(reason, Option): 534 reason = self._raw_options.get(reason) or reason.option 535 reason = reason.split("=", 1)[0] 536 value = OptionValue.from_(value) 537 raise InvalidOptionError( 538 "'%s' implied by '%s' conflicts with '%s' from the %s" 539 % ( 540 value.format(option.option), 541 reason, 542 option_value.format(option.option), 543 option_value.origin, 544 ) 545 ) 546 547 # All options should have been removed (handled) by now. 548 for arg in self._helper: 549 without_value = arg.split("=", 1)[0] 550 msg = "Unknown option: %s" % without_value 551 if self._help: 552 self._logger.warning(msg) 553 else: 554 raise InvalidOptionError(msg) 555 556 # Run the execution queue 557 for func, args in self._execution_queue: 558 func(*args) 559 560 if self._help: 561 with LineIO(self.log_impl.info) as out: 562 self._help.usage(out) 563 564 def __getitem__(self, key): 565 impl = "%s_impl" % key 566 func = getattr(self, impl, None) 567 if func: 568 return func 569 570 return super(ConfigureSandbox, self).__getitem__(key) 571 572 def __setitem__(self, key, value): 573 if ( 574 key in self.BUILTINS 575 or key == "__builtins__" 576 or hasattr(self, "%s_impl" % key) 577 ): 578 raise KeyError("Cannot reassign builtins") 579 580 if inspect.isfunction(value) and value not in self._templates: 581 value = self._prepare_function(value) 582 583 elif ( 584 not isinstance(value, SandboxDependsFunction) 585 and value not in self._templates 586 and not (inspect.isclass(value) and issubclass(value, Exception)) 587 ): 588 raise KeyError( 589 "Cannot assign `%s` because it is neither a " 590 "@depends nor a @template" % key 591 ) 592 593 if isinstance(value, SandboxDependsFunction): 594 self._depends[value].name = key 595 596 return super(ConfigureSandbox, self).__setitem__(key, value) 597 598 def _resolve(self, arg): 599 if isinstance(arg, SandboxDependsFunction): 600 return self._value_for_depends(self._depends[arg]) 601 return arg 602 603 def _value_for(self, obj): 604 if isinstance(obj, SandboxDependsFunction): 605 assert obj in self._depends 606 return self._value_for_depends(self._depends[obj]) 607 608 elif isinstance(obj, DependsFunction): 609 return self._value_for_depends(obj) 610 611 elif isinstance(obj, Option): 612 return self._value_for_option(obj) 613 614 assert False 615 616 @memoize 617 def _value_for_depends(self, obj): 618 value = obj.result() 619 self._logger.log(TRACE, "%r = %r", obj, value) 620 return value 621 622 @memoize 623 def _value_for_option(self, option): 624 implied = {} 625 matching_implied_options = [ 626 o for o in self._implied_options if o.name in (option.name, option.env) 627 ] 628 # Update self._implied_options before going into the loop with the non-matching 629 # options. 630 self._implied_options = [ 631 o for o in self._implied_options if o.name not in (option.name, option.env) 632 ] 633 634 for implied_option in matching_implied_options: 635 if implied_option.when and not self._value_for(implied_option.when): 636 continue 637 638 value = self._resolve(implied_option.value) 639 640 if value is not None: 641 value = OptionValue.from_(value) 642 opt = value.format(implied_option.option) 643 self._helper.add(opt, "implied") 644 implied[opt] = implied_option 645 646 try: 647 value, option_string = self._helper.handle(option) 648 except ConflictingOptionError as e: 649 reason = implied[e.arg].reason 650 if isinstance(reason, Option): 651 reason = self._raw_options.get(reason) or reason.option 652 reason = reason.split("=", 1)[0] 653 raise InvalidOptionError( 654 "'%s' implied by '%s' conflicts with '%s' from the %s" 655 % (e.arg, reason, e.old_arg, e.old_origin) 656 ) 657 658 if value.origin == "implied": 659 recursed_value = getattr(self, "__value_for_option").get((option,)) 660 if recursed_value is not None: 661 _, filename, line, _, _, _ = implied[value.format(option.option)].caller 662 raise ConfigureError( 663 "'%s' appears somewhere in the direct or indirect dependencies when " 664 "resolving imply_option at %s:%d" % (option.option, filename, line) 665 ) 666 667 if option_string: 668 self._raw_options[option] = option_string 669 670 when = self._conditions.get(option) 671 # If `when` resolves to a false-ish value, we always return None. 672 # This makes option(..., when='--foo') equivalent to 673 # option(..., when=depends('--foo')(lambda x: x)). 674 if when and not self._value_for(when) and value is not None: 675 # If the option was passed explicitly, we throw an error that 676 # the option is not available. Except when the option was passed 677 # from the environment, because that would be too cumbersome. 678 if value.origin not in ("default", "environment"): 679 raise InvalidOptionError( 680 "%s is not available in this configuration" 681 % option_string.split("=", 1)[0] 682 ) 683 self._logger.log(TRACE, "%r = None", option) 684 return None 685 686 self._logger.log(TRACE, "%r = %r", option, value) 687 return value 688 689 def _dependency(self, arg, callee_name, arg_name=None): 690 if isinstance(arg, six.string_types): 691 prefix, name, values = Option.split_option(arg) 692 if values != (): 693 raise ConfigureError("Option must not contain an '='") 694 if name not in self._options: 695 raise ConfigureError( 696 "'%s' is not a known option. " "Maybe it's declared too late?" % arg 697 ) 698 arg = self._options[name] 699 self._seen.add(arg) 700 elif isinstance(arg, SandboxDependsFunction): 701 assert arg in self._depends 702 arg = self._depends[arg] 703 else: 704 raise TypeError( 705 "Cannot use object of type '%s' as %sargument to %s" 706 % ( 707 type(arg).__name__, 708 "`%s` " % arg_name if arg_name else "", 709 callee_name, 710 ) 711 ) 712 return arg 713 714 def _normalize_when(self, when, callee_name): 715 if when is True: 716 when = self._always 717 elif when is False: 718 when = self._never 719 elif when is not None: 720 when = self._dependency(when, callee_name, "when") 721 722 if self._default_conditions: 723 # Create a pseudo @depends function for the combination of all 724 # default conditions and `when`. 725 dependencies = [when] if when else [] 726 dependencies.extend(self._default_conditions) 727 if len(dependencies) == 1: 728 return dependencies[0] 729 return CombinedDependsFunction(self, all, dependencies) 730 return when 731 732 @contextmanager 733 def only_when_impl(self, when): 734 """Implementation of only_when() 735 736 `only_when` is a context manager that essentially makes calls to 737 other sandbox functions within the context block ignored. 738 """ 739 when = self._normalize_when(when, "only_when") 740 if when and self._default_conditions[-1:] != [when]: 741 self._default_conditions.append(when) 742 yield 743 self._default_conditions.pop() 744 else: 745 yield 746 747 def option_impl(self, *args, **kwargs): 748 """Implementation of option() 749 This function creates and returns an Option() object, passing it the 750 resolved arguments (uses the result of functions when functions are 751 passed). In most cases, the result of this function is not expected to 752 be used. 753 Command line argument/environment variable parsing for this Option is 754 handled here. 755 """ 756 when = self._normalize_when(kwargs.get("when"), "option") 757 args = [self._resolve(arg) for arg in args] 758 kwargs = {k: self._resolve(v) for k, v in six.iteritems(kwargs) if k != "when"} 759 # The Option constructor needs to look up the stack to infer a category 760 # for the Option, since the category is based on the filename where the 761 # Option is defined. However, if the Option is defined in a template, we 762 # want the category to reference the caller of the template rather than 763 # the caller of the option() function. 764 kwargs["define_depth"] = self._template_depth * 3 765 option = Option(*args, **kwargs) 766 if when: 767 self._conditions[option] = when 768 if option.name in self._options: 769 raise ConfigureError("Option `%s` already defined" % option.option) 770 if option.env in self._options: 771 raise ConfigureError("Option `%s` already defined" % option.env) 772 if option.name: 773 self._options[option.name] = option 774 if option.env: 775 self._options[option.env] = option 776 777 if self._help and (when is None or self._value_for(when)): 778 self._help.add(option) 779 780 return option 781 782 def depends_impl(self, *args, **kwargs): 783 """Implementation of @depends() 784 This function is a decorator. It returns a function that subsequently 785 takes a function and returns a dummy function. The dummy function 786 identifies the actual function for the sandbox, while preventing 787 further function calls from within the sandbox. 788 789 @depends() takes a variable number of option strings or dummy function 790 references. The decorated function is called as soon as the decorator 791 is called, and the arguments it receives are the OptionValue or 792 function results corresponding to each of the arguments to @depends. 793 As an exception, when a HelpFormatter is attached, only functions that 794 have '--help' in their @depends argument list are called. 795 796 The decorated function is altered to use a different global namespace 797 for its execution. This different global namespace exposes a limited 798 set of functions from os.path. 799 """ 800 for k in kwargs: 801 if k != "when": 802 raise TypeError( 803 "depends_impl() got an unexpected keyword argument '%s'" % k 804 ) 805 806 when = self._normalize_when(kwargs.get("when"), "@depends") 807 808 if not when and not args: 809 raise ConfigureError("@depends needs at least one argument") 810 811 dependencies = tuple(self._dependency(arg, "@depends") for arg in args) 812 813 conditions = [ 814 self._conditions[d] 815 for d in dependencies 816 if d in self._conditions and isinstance(d, Option) 817 ] 818 for c in conditions: 819 if c != when: 820 raise ConfigureError( 821 "@depends function needs the same `when` " 822 "as options it depends on" 823 ) 824 825 def decorator(func): 826 if inspect.isgeneratorfunction(func): 827 raise ConfigureError( 828 "Cannot decorate generator functions with @depends" 829 ) 830 func = self._prepare_function(func) 831 depends = DependsFunction(self, func, dependencies, when=when) 832 return depends.sandboxed 833 834 return decorator 835 836 def include_impl(self, what, when=None): 837 """Implementation of include(). 838 Allows to include external files for execution in the sandbox. 839 It is possible to use a @depends function as argument, in which case 840 the result of the function is the file name to include. This latter 841 feature is only really meant for --enable-application/--enable-project. 842 """ 843 with self.only_when_impl(when): 844 what = self._resolve(what) 845 if what: 846 if not isinstance(what, six.string_types): 847 raise TypeError("Unexpected type: '%s'" % type(what).__name__) 848 self.include_file(what) 849 850 def template_impl(self, func): 851 """Implementation of @template. 852 This function is a decorator. Template functions are called 853 immediately. They are altered so that their global namespace exposes 854 a limited set of functions from os.path, as well as `depends` and 855 `option`. 856 Templates allow to simplify repetitive constructs, or to implement 857 helper decorators and somesuch. 858 """ 859 860 def update_globals(glob): 861 glob.update( 862 (k[: -len("_impl")], getattr(self, k)) 863 for k in dir(self) 864 if k.endswith("_impl") and k != "template_impl" 865 ) 866 glob.update((k, v) for k, v in six.iteritems(self) if k not in glob) 867 868 template = self._prepare_function(func, update_globals) 869 870 # Any function argument to the template must be prepared to be sandboxed. 871 # If the template itself returns a function (in which case, it's very 872 # likely a decorator), that function must be prepared to be sandboxed as 873 # well. 874 def wrap_template(template): 875 isfunction = inspect.isfunction 876 877 def maybe_prepare_function(obj): 878 if isfunction(obj): 879 return self._prepare_function(obj) 880 return obj 881 882 # The following function may end up being prepared to be sandboxed, 883 # so it mustn't depend on anything from the global scope in this 884 # file. It can however depend on variables from the closure, thus 885 # maybe_prepare_function and isfunction are declared above to be 886 # available there. 887 @self.wraps(template) 888 def wrapper(*args, **kwargs): 889 args = [maybe_prepare_function(arg) for arg in args] 890 kwargs = {k: maybe_prepare_function(v) for k, v in kwargs.items()} 891 self._template_depth += 1 892 ret = template(*args, **kwargs) 893 self._template_depth -= 1 894 if isfunction(ret): 895 # We can't expect the sandboxed code to think about all the 896 # details of implementing decorators, so do some of the 897 # work for them. If the function takes exactly one function 898 # as argument and returns a function, it must be a 899 # decorator, so mark the returned function as wrapping the 900 # function passed in. 901 if len(args) == 1 and not kwargs and isfunction(args[0]): 902 ret = self.wraps(args[0])(ret) 903 return wrap_template(ret) 904 return ret 905 906 return wrapper 907 908 wrapper = wrap_template(template) 909 self._templates.add(wrapper) 910 return wrapper 911 912 def wraps(self, func): 913 return wraps(func) 914 915 RE_MODULE = re.compile("^[a-zA-Z0-9_\.]+$") 916 917 def imports_impl(self, _import, _from=None, _as=None): 918 """Implementation of @imports. 919 This decorator imports the given _import from the given _from module 920 optionally under a different _as name. 921 The options correspond to the various forms for the import builtin. 922 923 @imports('sys') 924 @imports(_from='mozpack', _import='path', _as='mozpath') 925 """ 926 for value, required in ((_import, True), (_from, False), (_as, False)): 927 928 if not isinstance(value, six.string_types) and ( 929 required or value is not None 930 ): 931 raise TypeError("Unexpected type: '%s'" % type(value).__name__) 932 if value is not None and not self.RE_MODULE.match(value): 933 raise ValueError("Invalid argument to @imports: '%s'" % value) 934 if _as and "." in _as: 935 raise ValueError("Invalid argument to @imports: '%s'" % _as) 936 937 def decorator(func): 938 if func in self._templates: 939 raise ConfigureError("@imports must appear after @template") 940 if func in self._depends: 941 raise ConfigureError("@imports must appear after @depends") 942 # For the imports to apply in the order they appear in the 943 # .configure file, we accumulate them in reverse order and apply 944 # them later. 945 imports = self._imports.setdefault(func, []) 946 imports.insert(0, (_from, _import, _as)) 947 return func 948 949 return decorator 950 951 def _apply_imports(self, func, glob): 952 for _from, _import, _as in self._imports.pop(func, ()): 953 self._get_one_import(_from, _import, _as, glob) 954 955 def _handle_wrapped_import(self, _from, _import, _as, glob): 956 """Given the name of a module, "import" a mocked package into the glob 957 iff the module is one that we wrap (either for the sandbox or for the 958 purpose of testing). Applies if the wrapped module is exposed by an 959 attribute of `self`. 960 961 For example, if the import statement is `from os import environ`, then 962 this function will set 963 glob['environ'] = self._wrapped_os.environ. 964 965 Iff this function handles the given import, return True. 966 """ 967 module = (_from or _import).split(".")[0] 968 attr = "_wrapped_" + module 969 wrapped = getattr(self, attr, None) 970 if wrapped: 971 if _as or _from: 972 obj = self._recursively_get_property( 973 module, (_from + "." if _from else "") + _import, wrapped 974 ) 975 glob[_as or _import] = obj 976 else: 977 glob[module] = wrapped 978 return True 979 else: 980 return False 981 982 def _recursively_get_property(self, module, what, wrapped): 983 """Traverse the wrapper object `wrapped` (which represents the module 984 `module`) and return the property represented by `what`, which may be a 985 series of nested attributes. 986 987 For example, if `module` is 'os' and `what` is 'os.path.join', 988 return `wrapped.path.join`. 989 """ 990 if what == module: 991 return wrapped 992 assert what.startswith(module + ".") 993 attrs = what[len(module + ".") :].split(".") 994 for attr in attrs: 995 wrapped = getattr(wrapped, attr) 996 return wrapped 997 998 @memoized_property 999 def _wrapped_os(self): 1000 wrapped_os = {} 1001 exec_("from os import *", {}, wrapped_os) 1002 # Special case os and os.environ so that os.environ is our copy of 1003 # the environment. 1004 wrapped_os["environ"] = self._environ 1005 return ReadOnlyNamespace(**wrapped_os) 1006 1007 @memoized_property 1008 def _wrapped_subprocess(self): 1009 wrapped_subprocess = {} 1010 exec_("from subprocess import *", {}, wrapped_subprocess) 1011 1012 def wrap(function): 1013 def wrapper(*args, **kwargs): 1014 if kwargs.get("env") is None and self._environ: 1015 kwargs["env"] = dict(self._environ) 1016 1017 return function(*args, **kwargs) 1018 1019 return wrapper 1020 1021 for f in ("call", "check_call", "check_output", "Popen", "run"): 1022 # `run` is new to python 3.5. In case this still runs from python2 1023 # code, avoid failing here. 1024 if f in wrapped_subprocess: 1025 wrapped_subprocess[f] = wrap(wrapped_subprocess[f]) 1026 1027 return ReadOnlyNamespace(**wrapped_subprocess) 1028 1029 @memoized_property 1030 def _wrapped_six(self): 1031 if six.PY3: 1032 return six 1033 wrapped_six = {} 1034 exec_("from six import *", {}, wrapped_six) 1035 wrapped_six_moves = {} 1036 exec_("from six.moves import *", {}, wrapped_six_moves) 1037 wrapped_six_moves_builtins = {} 1038 exec_("from six.moves.builtins import *", {}, wrapped_six_moves_builtins) 1039 1040 # Special case for the open() builtin, because otherwise, using it 1041 # fails with "IOError: file() constructor not accessible in 1042 # restricted mode". We also make open() look more like python 3's, 1043 # decoding to unicode strings unless the mode says otherwise. 1044 def wrapped_open(name, mode=None, buffering=None): 1045 args = (name,) 1046 kwargs = {} 1047 if buffering is not None: 1048 kwargs["buffering"] = buffering 1049 if mode is not None: 1050 args += (mode,) 1051 if "b" in mode: 1052 return open(*args, **kwargs) 1053 kwargs["encoding"] = system_encoding 1054 return codecs.open(*args, **kwargs) 1055 1056 wrapped_six_moves_builtins["open"] = wrapped_open 1057 wrapped_six_moves["builtins"] = ReadOnlyNamespace(**wrapped_six_moves_builtins) 1058 wrapped_six["moves"] = ReadOnlyNamespace(**wrapped_six_moves) 1059 1060 return ReadOnlyNamespace(**wrapped_six) 1061 1062 def _get_one_import(self, _from, _import, _as, glob): 1063 """Perform the given import, placing the result into the dict glob.""" 1064 if not _from and _import == "__builtin__": 1065 glob[_as or "__builtin__"] = __builtin__ 1066 return 1067 if _from == "__builtin__": 1068 _from = "six.moves.builtins" 1069 # The special `__sandbox__` module gives access to the sandbox 1070 # instance. 1071 if not _from and _import == "__sandbox__": 1072 glob[_as or _import] = self 1073 return 1074 if self._handle_wrapped_import(_from, _import, _as, glob): 1075 return 1076 # If we've gotten this far, we should just do a normal import. 1077 # Until this proves to be a performance problem, just construct an 1078 # import statement and execute it. 1079 import_line = "%simport %s%s" % ( 1080 ("from %s " % _from) if _from else "", 1081 _import, 1082 (" as %s" % _as) if _as else "", 1083 ) 1084 exec_(import_line, {}, glob) 1085 1086 def _resolve_and_set(self, data, name, value, when=None): 1087 # Don't set anything when --help was on the command line 1088 if self._help: 1089 return 1090 if when and not self._value_for(when): 1091 return 1092 name = self._resolve(name) 1093 if name is None: 1094 return 1095 if not isinstance(name, six.string_types): 1096 raise TypeError("Unexpected type: '%s'" % type(name).__name__) 1097 if name in data: 1098 raise ConfigureError( 1099 "Cannot add '%s' to configuration: Key already " "exists" % name 1100 ) 1101 value = self._resolve(value) 1102 if value is not None: 1103 if self._logger.isEnabledFor(TRACE): 1104 if data is self._config: 1105 self._logger.log(TRACE, "set_config(%s, %r)", name, value) 1106 elif data is self._config.get("DEFINES"): 1107 self._logger.log(TRACE, "set_define(%s, %r)", name, value) 1108 data[name] = value 1109 1110 def set_config_impl(self, name, value, when=None): 1111 """Implementation of set_config(). 1112 Set the configuration items with the given name to the given value. 1113 Both `name` and `value` can be references to @depends functions, 1114 in which case the result from these functions is used. If the result 1115 of either function is None, the configuration item is not set. 1116 """ 1117 when = self._normalize_when(when, "set_config") 1118 1119 self._execution_queue.append( 1120 (self._resolve_and_set, (self._config, name, value, when)) 1121 ) 1122 1123 def set_define_impl(self, name, value, when=None): 1124 """Implementation of set_define(). 1125 Set the define with the given name to the given value. Both `name` and 1126 `value` can be references to @depends functions, in which case the 1127 result from these functions is used. If the result of either function 1128 is None, the define is not set. If the result is False, the define is 1129 explicitly undefined (-U). 1130 """ 1131 when = self._normalize_when(when, "set_define") 1132 1133 defines = self._config.setdefault("DEFINES", {}) 1134 self._execution_queue.append( 1135 (self._resolve_and_set, (defines, name, value, when)) 1136 ) 1137 1138 def imply_option_impl(self, option, value, reason=None, when=None): 1139 """Implementation of imply_option(). 1140 Injects additional options as if they had been passed on the command 1141 line. The `option` argument is a string as in option()'s `name` or 1142 `env`. The option must be declared after `imply_option` references it. 1143 The `value` argument indicates the value to pass to the option. 1144 It can be: 1145 - True. In this case `imply_option` injects the positive option 1146 1147 (--enable-foo/--with-foo). 1148 imply_option('--enable-foo', True) 1149 imply_option('--disable-foo', True) 1150 1151 are both equivalent to `--enable-foo` on the command line. 1152 1153 - False. In this case `imply_option` injects the negative option 1154 1155 (--disable-foo/--without-foo). 1156 imply_option('--enable-foo', False) 1157 imply_option('--disable-foo', False) 1158 1159 are both equivalent to `--disable-foo` on the command line. 1160 1161 - None. In this case `imply_option` does nothing. 1162 imply_option('--enable-foo', None) 1163 imply_option('--disable-foo', None) 1164 1165 are both equivalent to not passing any flag on the command line. 1166 1167 - a string or a tuple. In this case `imply_option` injects the positive 1168 option with the given value(s). 1169 1170 imply_option('--enable-foo', 'a') 1171 imply_option('--disable-foo', 'a') 1172 1173 are both equivalent to `--enable-foo=a` on the command line. 1174 imply_option('--enable-foo', ('a', 'b')) 1175 imply_option('--disable-foo', ('a', 'b')) 1176 1177 are both equivalent to `--enable-foo=a,b` on the command line. 1178 1179 Because imply_option('--disable-foo', ...) can be misleading, it is 1180 recommended to use the positive form ('--enable' or '--with') for 1181 `option`. 1182 1183 The `value` argument can also be (and usually is) a reference to a 1184 @depends function, in which case the result of that function will be 1185 used as per the descripted mapping above. 1186 1187 The `reason` argument indicates what caused the option to be implied. 1188 It is necessary when it cannot be inferred from the `value`. 1189 """ 1190 1191 when = self._normalize_when(when, "imply_option") 1192 1193 # Don't do anything when --help was on the command line 1194 if self._help: 1195 return 1196 if not reason and isinstance(value, SandboxDependsFunction): 1197 deps = self._depends[value].dependencies 1198 possible_reasons = [d for d in deps if d != self._help_option] 1199 if len(possible_reasons) == 1: 1200 if isinstance(possible_reasons[0], Option): 1201 reason = possible_reasons[0] 1202 if not reason and ( 1203 isinstance(value, (bool, tuple)) or isinstance(value, six.string_types) 1204 ): 1205 # A reason can be provided automatically when imply_option 1206 # is called with an immediate value. 1207 _, filename, line, _, _, _ = inspect.stack()[1] 1208 reason = "imply_option at %s:%s" % (filename, line) 1209 1210 if not reason: 1211 raise ConfigureError( 1212 "Cannot infer what implies '%s'. Please add a `reason` to " 1213 "the `imply_option` call." % option 1214 ) 1215 1216 prefix, name, values = Option.split_option(option) 1217 if values != (): 1218 raise ConfigureError("Implied option must not contain an '='") 1219 1220 self._implied_options.append( 1221 ReadOnlyNamespace( 1222 option=option, 1223 prefix=prefix, 1224 name=name, 1225 value=value, 1226 caller=inspect.stack()[1], 1227 reason=reason, 1228 when=when, 1229 ) 1230 ) 1231 1232 def _prepare_function(self, func, update_globals=None): 1233 """Alter the given function global namespace with the common ground 1234 for @depends, and @template. 1235 """ 1236 if not inspect.isfunction(func): 1237 raise TypeError("Unexpected type: '%s'" % type(func).__name__) 1238 if func in self._prepared_functions: 1239 return func 1240 1241 glob = SandboxedGlobal( 1242 (k, v) 1243 for k, v in six.iteritems(func.__globals__) 1244 if (inspect.isfunction(v) and v not in self._templates) 1245 or (inspect.isclass(v) and issubclass(v, Exception)) 1246 ) 1247 glob.update( 1248 __builtins__=self.BUILTINS, 1249 __file__=self._paths[-1] if self._paths else "", 1250 __name__=self._paths[-1] if self._paths else "", 1251 os=self.OS, 1252 log=self.log_impl, 1253 namespace=ReadOnlyNamespace, 1254 ) 1255 if update_globals: 1256 update_globals(glob) 1257 1258 # The execution model in the sandbox doesn't guarantee the execution 1259 # order will always be the same for a given function, and if it uses 1260 # variables from a closure that are changed after the function is 1261 # declared, depending when the function is executed, the value of the 1262 # variable can differ. For consistency, we force the function to use 1263 # the value from the earliest it can be run, which is at declaration. 1264 # Note this is not entirely bullet proof (if the value is e.g. a list, 1265 # the list contents could have changed), but covers the bases. 1266 closure = None 1267 if func.__closure__: 1268 1269 def makecell(content): 1270 def f(): 1271 content 1272 1273 return f.__closure__[0] 1274 1275 closure = tuple(makecell(cell.cell_contents) for cell in func.__closure__) 1276 1277 new_func = self.wraps(func)( 1278 types.FunctionType( 1279 func.__code__, glob, func.__name__, func.__defaults__, closure 1280 ) 1281 ) 1282 1283 @self.wraps(new_func) 1284 def wrapped(*args, **kwargs): 1285 if func in self._imports: 1286 self._apply_imports(func, glob) 1287 return new_func(*args, **kwargs) 1288 1289 self._prepared_functions.add(wrapped) 1290 return wrapped 1291