1from __future__ import print_function 2 3import argparse 4import itertools 5import os 6import random 7import re 8import shlex 9import string 10import sys 11import traceback 12import warnings 13from collections import OrderedDict 14from fnmatch import fnmatchcase 15from subprocess import list2cmdline 16from threading import Thread 17 18import pkg_resources 19import pluggy 20import py 21import toml 22 23import tox 24from tox.constants import INFO 25from tox.interpreters import Interpreters, NoInterpreterInfo 26from tox.reporter import ( 27 REPORTER_TIMESTAMP_ON_ENV, 28 error, 29 update_default_reporter, 30 using, 31 verbosity1, 32) 33from tox.util.path import ensure_empty_dir 34 35from .parallel import ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY 36from .parallel import add_parallel_config, add_parallel_flags 37from .reporter import add_verbosity_commands 38 39hookimpl = tox.hookimpl 40"""DEPRECATED - REMOVE - this is left for compatibility with plugins importing this from here. 41 42Instead create a hookimpl in your code with: 43 44 import pluggy 45 hookimpl = pluggy.HookimplMarker("tox") 46""" 47 48default_factors = tox.PYTHON.DEFAULT_FACTORS 49"""DEPRECATED MOVE - please update to new location.""" 50 51WITHIN_PROVISION = os.environ.get(str("TOX_PROVISION")) == "1" 52 53 54def get_plugin_manager(plugins=()): 55 # initialize plugin manager 56 import tox.venv 57 58 pm = pluggy.PluginManager("tox") 59 pm.add_hookspecs(tox.hookspecs) 60 pm.register(tox.config) 61 pm.register(tox.interpreters) 62 pm.register(tox.venv) 63 pm.register(tox.session) 64 from tox import package 65 66 pm.register(package) 67 pm.load_setuptools_entrypoints("tox") 68 for plugin in plugins: 69 pm.register(plugin) 70 pm.check_pending() 71 return pm 72 73 74class Parser: 75 """Command line and ini-parser control object.""" 76 77 def __init__(self): 78 class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter): 79 def __init__(self, prog): 80 super(HelpFormatter, self).__init__(prog, max_help_position=35, width=190) 81 82 self.argparser = argparse.ArgumentParser( 83 description="tox options", add_help=False, prog="tox", formatter_class=HelpFormatter 84 ) 85 self._testenv_attr = [] 86 87 def add_argument(self, *args, **kwargs): 88 """ add argument to command line parser. This takes the 89 same arguments that ``argparse.ArgumentParser.add_argument``. 90 """ 91 return self.argparser.add_argument(*args, **kwargs) 92 93 def add_testenv_attribute(self, name, type, help, default=None, postprocess=None): 94 """ add an ini-file variable for "testenv" section. 95 96 Types are specified as strings like "bool", "line-list", "string", "argv", "path", 97 "argvlist". 98 99 The ``postprocess`` function will be called for each testenv 100 like ``postprocess(testenv_config=testenv_config, value=value)`` 101 where ``value`` is the value as read from the ini (or the default value) 102 and ``testenv_config`` is a :py:class:`tox.config.TestenvConfig` instance 103 which will receive all ini-variables as object attributes. 104 105 Any postprocess function must return a value which will then be set 106 as the final value in the testenv section. 107 """ 108 self._testenv_attr.append(VenvAttribute(name, type, default, help, postprocess)) 109 110 def add_testenv_attribute_obj(self, obj): 111 """ add an ini-file variable as an object. 112 113 This works as the ``add_testenv_attribute`` function but expects 114 "name", "type", "help", and "postprocess" attributes on the object. 115 """ 116 assert hasattr(obj, "name") 117 assert hasattr(obj, "type") 118 assert hasattr(obj, "help") 119 assert hasattr(obj, "postprocess") 120 self._testenv_attr.append(obj) 121 122 def parse_cli(self, args, strict=False): 123 args, argv = self.argparser.parse_known_args(args) 124 if argv and (strict or WITHIN_PROVISION): 125 self.argparser.error("unrecognized arguments: %s".format(" ".join(argv))) 126 return args 127 128 def _format_help(self): 129 return self.argparser.format_help() 130 131 132class VenvAttribute: 133 def __init__(self, name, type, default, help, postprocess): 134 self.name = name 135 self.type = type 136 self.default = default 137 self.help = help 138 self.postprocess = postprocess 139 140 141class DepOption: 142 name = "deps" 143 type = "line-list" 144 help = "each line specifies a dependency in pip/setuptools format." 145 default = () 146 147 def postprocess(self, testenv_config, value): 148 deps = [] 149 config = testenv_config.config 150 for depline in value: 151 m = re.match(r":(\w+):\s*(\S+)", depline) 152 if m: 153 iname, name = m.groups() 154 ixserver = config.indexserver[iname] 155 else: 156 name = depline.strip() 157 ixserver = None 158 # we need to process options, in case they contain a space, 159 # as the subprocess call to pip install will otherwise fail. 160 # in case of a short option, we remove the space 161 for option in tox.PIP.INSTALL_SHORT_OPTIONS_ARGUMENT: 162 if name.startswith(option): 163 name = "{}{}".format(option, name[len(option) :].strip()) 164 # in case of a long option, we add an equal sign 165 for option in tox.PIP.INSTALL_LONG_OPTIONS_ARGUMENT: 166 name_start = "{} ".format(option) 167 if name.startswith(name_start): 168 name = "{}={}".format(option, name[len(option) :].strip()) 169 name = self._cut_off_dep_comment(name) 170 name = self._replace_forced_dep(name, config) 171 deps.append(DepConfig(name, ixserver)) 172 return deps 173 174 def _replace_forced_dep(self, name, config): 175 """Override given dependency config name. Take ``--force-dep-version`` option into account. 176 177 :param name: dep config, for example ["pkg==1.0", "other==2.0"]. 178 :param config: ``Config`` instance 179 :return: the new dependency that should be used for virtual environments 180 """ 181 if not config.option.force_dep: 182 return name 183 for forced_dep in config.option.force_dep: 184 if self._is_same_dep(forced_dep, name): 185 return forced_dep 186 return name 187 188 @staticmethod 189 def _cut_off_dep_comment(name): 190 return re.sub(r"\s+#.*", "", name).strip() 191 192 @classmethod 193 def _is_same_dep(cls, dep1, dep2): 194 """Definitions are the same if they refer to the same package, even if versions differ.""" 195 dep1_name = pkg_resources.Requirement.parse(dep1).project_name 196 try: 197 dep2_name = pkg_resources.Requirement.parse(dep2).project_name 198 except pkg_resources.RequirementParseError: 199 # we couldn't parse a version, probably a URL 200 return False 201 return dep1_name == dep2_name 202 203 204class PosargsOption: 205 name = "args_are_paths" 206 type = "bool" 207 default = True 208 help = "treat positional args in commands as paths" 209 210 def postprocess(self, testenv_config, value): 211 config = testenv_config.config 212 args = config.option.args 213 if args: 214 if value: 215 args = [] 216 for arg in config.option.args: 217 if arg and not os.path.isabs(arg): 218 origpath = os.path.join(config.invocationcwd.strpath, arg) 219 if os.path.exists(origpath): 220 arg = os.path.relpath(origpath, testenv_config.changedir.strpath) 221 args.append(arg) 222 testenv_config._reader.addsubstitutions(args) 223 return value 224 225 226class InstallcmdOption: 227 name = "install_command" 228 type = "argv" 229 default = "python -m pip install {opts} {packages}" 230 help = "install command for dependencies and package under test." 231 232 def postprocess(self, testenv_config, value): 233 if "{packages}" not in value: 234 raise tox.exception.ConfigError( 235 "'install_command' must contain '{packages}' substitution" 236 ) 237 return value 238 239 240def parseconfig(args, plugins=()): 241 """Parse the configuration file and create a Config object. 242 243 :param plugins: 244 :param list[str] args: list of arguments. 245 :rtype: :class:`Config` 246 :raise SystemExit: toxinit file is not found 247 """ 248 pm = get_plugin_manager(plugins) 249 config, option = parse_cli(args, pm) 250 update_default_reporter(config.option.quiet_level, config.option.verbose_level) 251 252 for config_file in propose_configs(option.configfile): 253 config_type = config_file.basename 254 255 content = None 256 if config_type == "pyproject.toml": 257 toml_content = get_py_project_toml(config_file) 258 try: 259 content = toml_content["tool"]["tox"]["legacy_tox_ini"] 260 except KeyError: 261 continue 262 ParseIni(config, config_file, content) 263 pm.hook.tox_configure(config=config) # post process config object 264 break 265 else: 266 msg = "tox config file (either {}) not found" 267 candidates = ", ".join(INFO.CONFIG_CANDIDATES) 268 feedback(msg.format(candidates), sysexit=not (option.help or option.helpini)) 269 return config 270 271 272def get_py_project_toml(path): 273 with open(str(path)) as file_handler: 274 config_data = toml.load(file_handler) 275 return config_data 276 277 278def propose_configs(cli_config_file): 279 from_folder = py.path.local() 280 if cli_config_file is not None: 281 if os.path.isfile(cli_config_file): 282 yield py.path.local(cli_config_file) 283 return 284 if os.path.isdir(cli_config_file): 285 from_folder = py.path.local(cli_config_file) 286 else: 287 print( 288 "ERROR: {} is neither file or directory".format(cli_config_file), file=sys.stderr 289 ) 290 return 291 for basename in INFO.CONFIG_CANDIDATES: 292 if from_folder.join(basename).isfile(): 293 yield from_folder.join(basename) 294 for path in from_folder.parts(reverse=True): 295 ini_path = path.join(basename) 296 if ini_path.check(): 297 yield ini_path 298 299 300def parse_cli(args, pm): 301 parser = Parser() 302 pm.hook.tox_addoption(parser=parser) 303 option = parser.parse_cli(args) 304 if option.version: 305 print(get_version_info(pm)) 306 raise SystemExit(0) 307 interpreters = Interpreters(hook=pm.hook) 308 config = Config( 309 pluginmanager=pm, option=option, interpreters=interpreters, parser=parser, args=args 310 ) 311 return config, option 312 313 314def feedback(msg, sysexit=False): 315 print("ERROR: {}".format(msg), file=sys.stderr) 316 if sysexit: 317 raise SystemExit(1) 318 319 320def get_version_info(pm): 321 out = ["{} imported from {}".format(tox.__version__, tox.__file__)] 322 plugin_dist_info = pm.list_plugin_distinfo() 323 if plugin_dist_info: 324 out.append("registered plugins:") 325 for mod, egg_info in plugin_dist_info: 326 source = getattr(mod, "__file__", repr(mod)) 327 out.append(" {}-{} at {}".format(egg_info.project_name, egg_info.version, source)) 328 return "\n".join(out) 329 330 331class SetenvDict(object): 332 _DUMMY = object() 333 334 def __init__(self, definitions, reader): 335 self.definitions = definitions 336 self.reader = reader 337 self.resolved = {} 338 self._lookupstack = [] 339 340 def __repr__(self): 341 return "{}: {}".format(self.__class__.__name__, self.definitions) 342 343 def __contains__(self, name): 344 return name in self.definitions 345 346 def get(self, name, default=None): 347 try: 348 return self.resolved[name] 349 except KeyError: 350 try: 351 if name in self._lookupstack: 352 raise KeyError(name) 353 val = self.definitions[name] 354 except KeyError: 355 return os.environ.get(name, default) 356 self._lookupstack.append(name) 357 try: 358 self.resolved[name] = res = self.reader._replace(val) 359 finally: 360 self._lookupstack.pop() 361 return res 362 363 def __getitem__(self, name): 364 x = self.get(name, self._DUMMY) 365 if x is self._DUMMY: 366 raise KeyError(name) 367 return x 368 369 def keys(self): 370 return self.definitions.keys() 371 372 def __setitem__(self, name, value): 373 self.definitions[name] = value 374 self.resolved[name] = value 375 376 377@tox.hookimpl 378def tox_addoption(parser): 379 parser.add_argument( 380 "--version", 381 action="store_true", 382 dest="version", 383 help="report version information to stdout.", 384 ) 385 parser.add_argument( 386 "-h", "--help", action="store_true", dest="help", help="show help about options" 387 ) 388 parser.add_argument( 389 "--help-ini", "--hi", action="store_true", dest="helpini", help="show help about ini-names" 390 ) 391 add_verbosity_commands(parser) 392 parser.add_argument( 393 "--showconfig", 394 action="store_true", 395 help="show live configuration (by default all env, with -l only default targets," 396 " specific via TOXENV/-e)", 397 ) 398 parser.add_argument( 399 "-l", 400 "--listenvs", 401 action="store_true", 402 dest="listenvs", 403 help="show list of test environments (with description if verbose)", 404 ) 405 parser.add_argument( 406 "-a", 407 "--listenvs-all", 408 action="store_true", 409 dest="listenvs_all", 410 help="show list of all defined environments (with description if verbose)", 411 ) 412 parser.add_argument( 413 "-c", 414 action="store", 415 default=None, 416 dest="configfile", 417 help="config file name or directory with 'tox.ini' file.", 418 ) 419 parser.add_argument( 420 "-e", 421 action="append", 422 dest="env", 423 metavar="envlist", 424 help="work against specified environments (ALL selects all).", 425 ) 426 parser.add_argument( 427 "--notest", action="store_true", dest="notest", help="skip invoking test commands." 428 ) 429 parser.add_argument( 430 "--sdistonly", 431 action="store_true", 432 dest="sdistonly", 433 help="only perform the sdist packaging activity.", 434 ) 435 add_parallel_flags(parser) 436 parser.add_argument( 437 "--parallel--safe-build", 438 action="store_true", 439 dest="parallel_safe_build", 440 help="(deprecated) ensure two tox builds can run in parallel " 441 "(uses a lock file in the tox workdir with .lock extension)", 442 ) 443 parser.add_argument( 444 "--installpkg", 445 action="store", 446 default=None, 447 metavar="PATH", 448 help="use specified package for installation into venv, instead of creating an sdist.", 449 ) 450 parser.add_argument( 451 "--develop", 452 action="store_true", 453 dest="develop", 454 help="install package in the venv using 'setup.py develop' via 'pip -e .'", 455 ) 456 parser.add_argument( 457 "-i", 458 "--index-url", 459 action="append", 460 dest="indexurl", 461 metavar="URL", 462 help="set indexserver url (if URL is of form name=url set the " 463 "url for the 'name' indexserver, specifically)", 464 ) 465 parser.add_argument( 466 "--pre", 467 action="store_true", 468 dest="pre", 469 help="install pre-releases and development versions of dependencies. " 470 "This will pass the --pre option to install_command " 471 "(pip by default).", 472 ) 473 parser.add_argument( 474 "-r", 475 "--recreate", 476 action="store_true", 477 dest="recreate", 478 help="force recreation of virtual environments", 479 ) 480 parser.add_argument( 481 "--result-json", 482 action="store", 483 dest="resultjson", 484 metavar="PATH", 485 help="write a json file with detailed information " 486 "about all commands and results involved.", 487 ) 488 489 # We choose 1 to 4294967295 because it is the range of PYTHONHASHSEED. 490 parser.add_argument( 491 "--hashseed", 492 action="store", 493 metavar="SEED", 494 default=None, 495 help="set PYTHONHASHSEED to SEED before running commands. " 496 "Defaults to a random integer in the range [1, 4294967295] " 497 "([1, 1024] on Windows). " 498 "Passing 'noset' suppresses this behavior.", 499 ) 500 parser.add_argument( 501 "--force-dep", 502 action="append", 503 metavar="REQ", 504 default=None, 505 help="Forces a certain version of one of the dependencies " 506 "when configuring the virtual environment. REQ Examples " 507 "'pytest<2.7' or 'django>=1.6'.", 508 ) 509 parser.add_argument( 510 "--sitepackages", 511 action="store_true", 512 help="override sitepackages setting to True in all envs", 513 ) 514 parser.add_argument( 515 "--alwayscopy", action="store_true", help="override alwayscopy setting to True in all envs" 516 ) 517 518 cli_skip_missing_interpreter(parser) 519 parser.add_argument( 520 "--workdir", 521 action="store", 522 dest="workdir", 523 metavar="PATH", 524 default=None, 525 help="tox working directory", 526 ) 527 528 parser.add_argument( 529 "args", nargs="*", help="additional arguments available to command positional substitution" 530 ) 531 532 parser.add_testenv_attribute( 533 name="envdir", 534 type="path", 535 default="{toxworkdir}/{envname}", 536 help="set venv directory -- be very careful when changing this as tox " 537 "will remove this directory when recreating an environment", 538 ) 539 540 # add various core venv interpreter attributes 541 def setenv(testenv_config, value): 542 setenv = value 543 config = testenv_config.config 544 if "PYTHONHASHSEED" not in setenv and config.hashseed is not None: 545 setenv["PYTHONHASHSEED"] = config.hashseed 546 547 setenv["TOX_ENV_NAME"] = str(testenv_config.envname) 548 setenv["TOX_ENV_DIR"] = str(testenv_config.envdir) 549 return setenv 550 551 parser.add_testenv_attribute( 552 name="setenv", 553 type="dict_setenv", 554 postprocess=setenv, 555 help="list of X=Y lines with environment variable settings", 556 ) 557 558 def basepython_default(testenv_config, value): 559 """either user set or proposed from the factor name 560 561 in both cases we check that the factor name implied python version and the resolved 562 python interpreter version match up; if they don't we warn, unless ignore base 563 python conflict is set in which case the factor name implied version if forced 564 """ 565 for factor in testenv_config.factors: 566 if factor in tox.PYTHON.DEFAULT_FACTORS: 567 implied_python = tox.PYTHON.DEFAULT_FACTORS[factor] 568 break 569 else: 570 implied_python, factor = None, None 571 572 if testenv_config.config.ignore_basepython_conflict and implied_python is not None: 573 return implied_python 574 575 proposed_python = (implied_python or sys.executable) if value is None else str(value) 576 if implied_python is not None and implied_python != proposed_python: 577 testenv_config.basepython = proposed_python 578 match = tox.PYTHON.PY_FACTORS_RE.match(factor) 579 implied_version = match.group(2) if match else None 580 if implied_version is not None: 581 python_info_for_proposed = testenv_config.python_info 582 if not isinstance(python_info_for_proposed, NoInterpreterInfo): 583 proposed_version = "".join( 584 str(i) for i in python_info_for_proposed.version_info[0:2] 585 ) 586 # '27'.startswith('2') or '27'.startswith('27') 587 if not proposed_version.startswith(implied_version): 588 # TODO(stephenfin): Raise an exception here in tox 4.0 589 warnings.warn( 590 "conflicting basepython version (set {}, should be {}) for env '{}';" 591 "resolve conflict or set ignore_basepython_conflict".format( 592 proposed_version, implied_version, testenv_config.envname 593 ) 594 ) 595 return proposed_python 596 597 parser.add_testenv_attribute( 598 name="basepython", 599 type="basepython", 600 default=None, 601 postprocess=basepython_default, 602 help="executable name or path of interpreter used to create a virtual test environment.", 603 ) 604 605 def merge_description(testenv_config, value): 606 """the reader by default joins generated description with new line, 607 replace new line with space""" 608 return value.replace("\n", " ") 609 610 parser.add_testenv_attribute( 611 name="description", 612 type="string", 613 default="", 614 postprocess=merge_description, 615 help="short description of this environment", 616 ) 617 618 parser.add_testenv_attribute( 619 name="envtmpdir", type="path", default="{envdir}/tmp", help="venv temporary directory" 620 ) 621 622 parser.add_testenv_attribute( 623 name="envlogdir", type="path", default="{envdir}/log", help="venv log directory" 624 ) 625 626 parser.add_testenv_attribute( 627 name="downloadcache", 628 type="string", 629 default=None, 630 help="(ignored) has no effect anymore, pip-8 uses local caching by default", 631 ) 632 633 parser.add_testenv_attribute( 634 name="changedir", 635 type="path", 636 default="{toxinidir}", 637 help="directory to change to when running commands", 638 ) 639 640 parser.add_testenv_attribute_obj(PosargsOption()) 641 642 parser.add_testenv_attribute( 643 name="skip_install", 644 type="bool", 645 default=False, 646 help="Do not install the current package. This can be used when you need the virtualenv " 647 "management but do not want to install the current package", 648 ) 649 650 parser.add_testenv_attribute( 651 name="ignore_errors", 652 type="bool", 653 default=False, 654 help="if set to True all commands will be executed irrespective of their result error " 655 "status.", 656 ) 657 658 def recreate(testenv_config, value): 659 if testenv_config.config.option.recreate: 660 return True 661 return value 662 663 parser.add_testenv_attribute( 664 name="recreate", 665 type="bool", 666 default=False, 667 postprocess=recreate, 668 help="always recreate this test environment.", 669 ) 670 671 def passenv(testenv_config, value): 672 # Flatten the list to deal with space-separated values. 673 value = list(itertools.chain.from_iterable([x.split(" ") for x in value])) 674 675 passenv = { 676 "PATH", 677 "PIP_INDEX_URL", 678 "LANG", 679 "LANGUAGE", 680 "LD_LIBRARY_PATH", 681 "TOX_WORK_DIR", 682 str(REPORTER_TIMESTAMP_ON_ENV), 683 str(PARALLEL_ENV_VAR_KEY), 684 } 685 686 # read in global passenv settings 687 p = os.environ.get("TOX_TESTENV_PASSENV", None) 688 if p is not None: 689 env_values = [x for x in p.split() if x] 690 value.extend(env_values) 691 692 # we ensure that tmp directory settings are passed on 693 # we could also set it to the per-venv "envtmpdir" 694 # but this leads to very long paths when run with jenkins 695 # so we just pass it on by default for now. 696 if tox.INFO.IS_WIN: 697 passenv.add("SYSTEMDRIVE") # needed for pip6 698 passenv.add("SYSTEMROOT") # needed for python's crypto module 699 passenv.add("PATHEXT") # needed for discovering executables 700 passenv.add("COMSPEC") # needed for distutils cygwincompiler 701 passenv.add("TEMP") 702 passenv.add("TMP") 703 # for `multiprocessing.cpu_count()` on Windows (prior to Python 3.4). 704 passenv.add("NUMBER_OF_PROCESSORS") 705 passenv.add("PROCESSOR_ARCHITECTURE") # platform.machine() 706 passenv.add("USERPROFILE") # needed for `os.path.expanduser()` 707 passenv.add("MSYSTEM") # fixes #429 708 else: 709 passenv.add("TMPDIR") 710 for spec in value: 711 for name in os.environ: 712 if fnmatchcase(name.upper(), spec.upper()): 713 passenv.add(name) 714 return passenv 715 716 parser.add_testenv_attribute( 717 name="passenv", 718 type="line-list", 719 postprocess=passenv, 720 help="environment variables needed during executing test commands (taken from invocation " 721 "environment). Note that tox always passes through some basic environment variables " 722 "which are needed for basic functioning of the Python system. See --showconfig for the " 723 "eventual passenv setting.", 724 ) 725 726 parser.add_testenv_attribute( 727 name="whitelist_externals", 728 type="line-list", 729 help="each lines specifies a path or basename for which tox will not warn " 730 "about it coming from outside the test environment.", 731 ) 732 733 parser.add_testenv_attribute( 734 name="platform", 735 type="string", 736 default=".*", 737 help="regular expression which must match against ``sys.platform``. " 738 "otherwise testenv will be skipped.", 739 ) 740 741 def sitepackages(testenv_config, value): 742 return testenv_config.config.option.sitepackages or value 743 744 def alwayscopy(testenv_config, value): 745 return testenv_config.config.option.alwayscopy or value 746 747 parser.add_testenv_attribute( 748 name="sitepackages", 749 type="bool", 750 default=False, 751 postprocess=sitepackages, 752 help="Set to ``True`` if you want to create virtual environments that also " 753 "have access to globally installed packages.", 754 ) 755 756 parser.add_testenv_attribute( 757 "download", 758 type="bool", 759 default=False, 760 help="download the latest pip, setuptools and wheel when creating the virtual" 761 "environment (default is to use the one bundled in virtualenv)", 762 ) 763 764 parser.add_testenv_attribute( 765 name="alwayscopy", 766 type="bool", 767 default=False, 768 postprocess=alwayscopy, 769 help="Set to ``True`` if you want virtualenv to always copy files rather " 770 "than symlinking.", 771 ) 772 773 def pip_pre(testenv_config, value): 774 return testenv_config.config.option.pre or value 775 776 parser.add_testenv_attribute( 777 name="pip_pre", 778 type="bool", 779 default=False, 780 postprocess=pip_pre, 781 help="If ``True``, adds ``--pre`` to the ``opts`` passed to the install command. ", 782 ) 783 784 def develop(testenv_config, value): 785 option = testenv_config.config.option 786 return not option.installpkg and (value or option.develop) 787 788 parser.add_testenv_attribute( 789 name="usedevelop", 790 type="bool", 791 postprocess=develop, 792 default=False, 793 help="install package in develop/editable mode", 794 ) 795 796 parser.add_testenv_attribute_obj(InstallcmdOption()) 797 798 parser.add_testenv_attribute( 799 name="list_dependencies_command", 800 type="argv", 801 default="python -m pip freeze", 802 help="list dependencies for a virtual environment", 803 ) 804 805 parser.add_testenv_attribute_obj(DepOption()) 806 807 parser.add_testenv_attribute( 808 name="commands", 809 type="argvlist", 810 default="", 811 help="each line specifies a test command and can use substitution.", 812 ) 813 814 parser.add_testenv_attribute( 815 name="commands_pre", 816 type="argvlist", 817 default="", 818 help="each line specifies a setup command action and can use substitution.", 819 ) 820 821 parser.add_testenv_attribute( 822 name="commands_post", 823 type="argvlist", 824 default="", 825 help="each line specifies a teardown command and can use substitution.", 826 ) 827 828 parser.add_testenv_attribute( 829 "ignore_outcome", 830 type="bool", 831 default=False, 832 help="if set to True a failing result of this testenv will not make " 833 "tox fail, only a warning will be produced", 834 ) 835 836 parser.add_testenv_attribute( 837 "extras", 838 type="line-list", 839 help="list of extras to install with the source distribution or develop install", 840 ) 841 842 add_parallel_config(parser) 843 844 845def cli_skip_missing_interpreter(parser): 846 class SkipMissingInterpreterAction(argparse.Action): 847 def __call__(self, parser, namespace, values, option_string=None): 848 value = "true" if values is None else values 849 if value not in ("config", "true", "false"): 850 raise argparse.ArgumentTypeError("value must be config, true or false") 851 setattr(namespace, self.dest, value) 852 853 parser.add_argument( 854 "-s", 855 "--skip-missing-interpreters", 856 default="config", 857 metavar="val", 858 nargs="?", 859 action=SkipMissingInterpreterAction, 860 help="don't fail tests for missing interpreters: {config,true,false} choice", 861 ) 862 863 864class Config(object): 865 """Global Tox config object.""" 866 867 def __init__(self, pluginmanager, option, interpreters, parser, args): 868 self.envconfigs = OrderedDict() 869 """Mapping envname -> envconfig""" 870 self.invocationcwd = py.path.local() 871 self.interpreters = interpreters 872 self.pluginmanager = pluginmanager 873 self.option = option 874 self._parser = parser 875 self._testenv_attr = parser._testenv_attr 876 self.args = args 877 878 """option namespace containing all parsed command line options""" 879 880 @property 881 def homedir(self): 882 homedir = get_homedir() 883 if homedir is None: 884 homedir = self.toxinidir # FIXME XXX good idea? 885 return homedir 886 887 888class TestenvConfig: 889 """Testenv Configuration object. 890 891 In addition to some core attributes/properties this config object holds all 892 per-testenv ini attributes as attributes, see "tox --help-ini" for an overview. 893 """ 894 895 def __init__(self, envname, config, factors, reader): 896 #: test environment name 897 self.envname = envname 898 #: global tox config object 899 self.config = config 900 #: set of factors 901 self.factors = factors 902 self._reader = reader 903 self.missing_subs = [] 904 """Holds substitutions that could not be resolved. 905 906 Pre 2.8.1 missing substitutions crashed with a ConfigError although this would not be a 907 problem if the env is not part of the current testrun. So we need to remember this and 908 check later when the testenv is actually run and crash only then. 909 """ 910 911 def get_envbindir(self): 912 """Path to directory where scripts/binaries reside.""" 913 if tox.INFO.IS_WIN and "jython" not in self.basepython and "pypy" not in self.basepython: 914 return self.envdir.join("Scripts") 915 else: 916 return self.envdir.join("bin") 917 918 @property 919 def envbindir(self): 920 return self.get_envbindir() 921 922 @property 923 def envpython(self): 924 """Path to python executable.""" 925 return self.get_envpython() 926 927 def get_envpython(self): 928 """ path to python/jython executable. """ 929 if "jython" in str(self.basepython): 930 name = "jython" 931 else: 932 name = "python" 933 return self.envbindir.join(name) 934 935 def get_envsitepackagesdir(self): 936 """Return sitepackagesdir of the virtualenv environment. 937 938 NOTE: Only available during execution, not during parsing. 939 """ 940 x = self.config.interpreters.get_sitepackagesdir(info=self.python_info, envdir=self.envdir) 941 return x 942 943 @property 944 def python_info(self): 945 """Return sitepackagesdir of the virtualenv environment.""" 946 return self.config.interpreters.get_info(envconfig=self) 947 948 def getsupportedinterpreter(self): 949 if tox.INFO.IS_WIN and self.basepython and "jython" in self.basepython: 950 raise tox.exception.UnsupportedInterpreter( 951 "Jython/Windows does not support installing scripts" 952 ) 953 info = self.config.interpreters.get_info(envconfig=self) 954 if not info.executable: 955 raise tox.exception.InterpreterNotFound(self.basepython) 956 if not info.version_info: 957 raise tox.exception.InvocationError( 958 "Failed to get version_info for {}: {}".format(info.name, info.err) 959 ) 960 return info.executable 961 962 963testenvprefix = "testenv:" 964 965 966def get_homedir(): 967 try: 968 return py.path.local._gethomedir() 969 except Exception: 970 return None 971 972 973def make_hashseed(): 974 max_seed = 4294967295 975 if tox.INFO.IS_WIN: 976 max_seed = 1024 977 return str(random.randint(1, max_seed)) 978 979 980class ParseIni(object): 981 def __init__(self, config, ini_path, ini_data): # noqa 982 config.toxinipath = ini_path 983 using("tox.ini: {} (pid {})".format(config.toxinipath, os.getpid())) 984 config.toxinidir = config.toxinipath.dirpath() 985 986 self._cfg = py.iniconfig.IniConfig(config.toxinipath, ini_data) 987 previous_line_of = self._cfg.lineof 988 989 def line_of_default_to_zero(section, name=None): 990 at = previous_line_of(section, name=name) 991 if at is None: 992 at = 0 993 return at 994 995 self._cfg.lineof = line_of_default_to_zero 996 config._cfg = self._cfg 997 self.config = config 998 999 prefix = "tox" if ini_path.basename == "setup.cfg" else None 1000 1001 context_name = getcontextname() 1002 if context_name == "jenkins": 1003 reader = SectionReader( 1004 "tox:jenkins", self._cfg, prefix=prefix, fallbacksections=["tox"] 1005 ) 1006 dist_share_default = "{toxworkdir}/distshare" 1007 elif not context_name: 1008 reader = SectionReader("tox", self._cfg, prefix=prefix) 1009 dist_share_default = "{homedir}/.tox/distshare" 1010 else: 1011 raise ValueError("invalid context") 1012 1013 if config.option.hashseed is None: 1014 hash_seed = make_hashseed() 1015 elif config.option.hashseed == "noset": 1016 hash_seed = None 1017 else: 1018 hash_seed = config.option.hashseed 1019 config.hashseed = hash_seed 1020 1021 reader.addsubstitutions(toxinidir=config.toxinidir, homedir=config.homedir) 1022 1023 if config.option.workdir is None: 1024 config.toxworkdir = reader.getpath("toxworkdir", "{toxinidir}/.tox") 1025 else: 1026 config.toxworkdir = config.toxinidir.join(config.option.workdir, abs=True) 1027 1028 if os.path.exists(str(config.toxworkdir)): 1029 config.toxworkdir = config.toxworkdir.realpath() 1030 1031 reader.addsubstitutions(toxworkdir=config.toxworkdir) 1032 config.ignore_basepython_conflict = reader.getbool("ignore_basepython_conflict", False) 1033 1034 config.distdir = reader.getpath("distdir", "{toxworkdir}/dist") 1035 1036 reader.addsubstitutions(distdir=config.distdir) 1037 config.distshare = reader.getpath("distshare", dist_share_default) 1038 config.temp_dir = reader.getpath("temp_dir", "{toxworkdir}/.tmp") 1039 reader.addsubstitutions(distshare=config.distshare) 1040 config.sdistsrc = reader.getpath("sdistsrc", None) 1041 config.setupdir = reader.getpath("setupdir", "{toxinidir}") 1042 config.logdir = config.toxworkdir.join("log") 1043 within_parallel = PARALLEL_ENV_VAR_KEY in os.environ 1044 if not within_parallel: 1045 ensure_empty_dir(config.logdir) 1046 1047 # determine indexserver dictionary 1048 config.indexserver = {"default": IndexServerConfig("default")} 1049 prefix = "indexserver" 1050 for line in reader.getlist(prefix): 1051 name, url = map(lambda x: x.strip(), line.split("=", 1)) 1052 config.indexserver[name] = IndexServerConfig(name, url) 1053 1054 if config.option.skip_missing_interpreters == "config": 1055 val = reader.getbool("skip_missing_interpreters", False) 1056 config.option.skip_missing_interpreters = "true" if val else "false" 1057 1058 override = False 1059 if config.option.indexurl: 1060 for url_def in config.option.indexurl: 1061 m = re.match(r"\W*(\w+)=(\S+)", url_def) 1062 if m is None: 1063 url = url_def 1064 name = "default" 1065 else: 1066 name, url = m.groups() 1067 if not url: 1068 url = None 1069 if name != "ALL": 1070 config.indexserver[name].url = url 1071 else: 1072 override = url 1073 # let ALL override all existing entries 1074 if override: 1075 for name in config.indexserver: 1076 config.indexserver[name] = IndexServerConfig(name, override) 1077 1078 self.handle_provision(config, reader) 1079 1080 self.parse_build_isolation(config, reader) 1081 res = self._getenvdata(reader, config) 1082 config.envlist, all_envs, config.envlist_default, config.envlist_explicit = res 1083 1084 # factors used in config or predefined 1085 known_factors = self._list_section_factors("testenv") 1086 known_factors.update({"py", "python"}) 1087 1088 # factors stated in config envlist 1089 stated_envlist = reader.getstring("envlist", replace=False) 1090 if stated_envlist: 1091 for env in _split_env(stated_envlist): 1092 known_factors.update(env.split("-")) 1093 1094 # configure testenvs 1095 to_do = [] 1096 failures = OrderedDict() 1097 results = {} 1098 cur_self = self 1099 1100 def run(name, section, subs, config): 1101 try: 1102 results[name] = cur_self.make_envconfig(name, section, subs, config) 1103 except Exception as exception: 1104 failures[name] = (exception, traceback.format_exc()) 1105 1106 order = [] 1107 for name in all_envs: 1108 section = "{}{}".format(testenvprefix, name) 1109 factors = set(name.split("-")) 1110 if ( 1111 section in self._cfg 1112 or factors <= known_factors 1113 or all( 1114 tox.PYTHON.PY_FACTORS_RE.match(factor) for factor in factors - known_factors 1115 ) 1116 ): 1117 order.append(name) 1118 thread = Thread(target=run, args=(name, section, reader._subs, config)) 1119 thread.daemon = True 1120 thread.start() 1121 to_do.append(thread) 1122 for thread in to_do: 1123 while thread.is_alive(): 1124 thread.join(timeout=20) 1125 if failures: 1126 raise tox.exception.ConfigError( 1127 "\n".join( 1128 "{} failed with {} at {}".format(key, exc, trace) 1129 for key, (exc, trace) in failures.items() 1130 ) 1131 ) 1132 for name in order: 1133 config.envconfigs[name] = results[name] 1134 all_develop = all( 1135 name in config.envconfigs and config.envconfigs[name].usedevelop 1136 for name in config.envlist 1137 ) 1138 1139 config.skipsdist = reader.getbool("skipsdist", all_develop) 1140 1141 def handle_provision(self, config, reader): 1142 requires_list = reader.getlist("requires") 1143 config.minversion = reader.getstring("minversion", None) 1144 config.provision_tox_env = name = reader.getstring("provision_tox_env", ".tox") 1145 min_version = "tox >= {}".format(config.minversion or tox.__version__) 1146 deps = self.ensure_requires_satisfied(config, requires_list, min_version) 1147 if config.run_provision: 1148 section_name = "testenv:{}".format(name) 1149 if section_name not in self._cfg.sections: 1150 self._cfg.sections[section_name] = {} 1151 self._cfg.sections[section_name]["description"] = "meta tox" 1152 env_config = self.make_envconfig( 1153 name, "{}{}".format(testenvprefix, name), reader._subs, config 1154 ) 1155 env_config.deps = deps 1156 config.envconfigs[config.provision_tox_env] = env_config 1157 raise tox.exception.MissingRequirement(config) 1158 # if provisioning is not on, now we need do a strict argument evaluation 1159 # raise on unknown args 1160 self.config._parser.parse_cli(args=self.config.args, strict=True) 1161 1162 @staticmethod 1163 def ensure_requires_satisfied(config, requires, min_version): 1164 missing_requirements = [] 1165 failed_to_parse = False 1166 deps = [] 1167 exists = set() 1168 for require in requires + [min_version]: 1169 # noinspection PyBroadException 1170 try: 1171 package = pkg_resources.Requirement.parse(require) 1172 if package.project_name not in exists: 1173 deps.append(DepConfig(require, None)) 1174 exists.add(package.project_name) 1175 pkg_resources.get_distribution(package) 1176 except pkg_resources.RequirementParseError as exception: 1177 failed_to_parse = True 1178 error("failed to parse {!r}".format(exception)) 1179 except Exception as exception: 1180 verbosity1("could not satisfy requires {!r}".format(exception)) 1181 missing_requirements.append(str(pkg_resources.Requirement(require))) 1182 if failed_to_parse: 1183 raise tox.exception.BadRequirement() 1184 config.run_provision = bool(len(missing_requirements)) 1185 return deps 1186 1187 def parse_build_isolation(self, config, reader): 1188 config.isolated_build = reader.getbool("isolated_build", False) 1189 config.isolated_build_env = reader.getstring("isolated_build_env", ".package") 1190 if config.isolated_build is True: 1191 name = config.isolated_build_env 1192 section_name = "testenv:{}".format(name) 1193 if section_name not in self._cfg.sections: 1194 self._cfg.sections[section_name] = {} 1195 self._cfg.sections[section_name]["deps"] = "" 1196 self._cfg.sections[section_name]["sitepackages"] = "False" 1197 self._cfg.sections[section_name]["description"] = "isolated packaging environment" 1198 config.envconfigs[name] = self.make_envconfig( 1199 name, "{}{}".format(testenvprefix, name), reader._subs, config 1200 ) 1201 1202 def _list_section_factors(self, section): 1203 factors = set() 1204 if section in self._cfg: 1205 for _, value in self._cfg[section].items(): 1206 exprs = re.findall(r"^([\w{}\.!,-]+)\:\s+", value, re.M) 1207 factors.update(*mapcat(_split_factor_expr_all, exprs)) 1208 return factors 1209 1210 def make_envconfig(self, name, section, subs, config, replace=True): 1211 factors = set(name.split("-")) 1212 reader = SectionReader(section, self._cfg, fallbacksections=["testenv"], factors=factors) 1213 tc = TestenvConfig(name, config, factors, reader) 1214 reader.addsubstitutions( 1215 envname=name, 1216 envbindir=tc.get_envbindir, 1217 envsitepackagesdir=tc.get_envsitepackagesdir, 1218 envpython=tc.get_envpython, 1219 **subs 1220 ) 1221 for env_attr in config._testenv_attr: 1222 atype = env_attr.type 1223 try: 1224 if atype in ("bool", "path", "string", "dict", "dict_setenv", "argv", "argvlist"): 1225 meth = getattr(reader, "get{}".format(atype)) 1226 res = meth(env_attr.name, env_attr.default, replace=replace) 1227 elif atype == "basepython": 1228 no_fallback = name in (config.provision_tox_env,) 1229 res = reader.getstring( 1230 env_attr.name, env_attr.default, replace=replace, no_fallback=no_fallback 1231 ) 1232 elif atype == "space-separated-list": 1233 res = reader.getlist(env_attr.name, sep=" ") 1234 elif atype == "line-list": 1235 res = reader.getlist(env_attr.name, sep="\n") 1236 elif atype == "env-list": 1237 res = reader.getstring(env_attr.name, replace=False) 1238 res = tuple(_split_env(res)) 1239 else: 1240 raise ValueError("unknown type {!r}".format(atype)) 1241 if env_attr.postprocess: 1242 res = env_attr.postprocess(testenv_config=tc, value=res) 1243 except tox.exception.MissingSubstitution as e: 1244 tc.missing_subs.append(e.name) 1245 res = e.FLAG 1246 setattr(tc, env_attr.name, res) 1247 if atype in ("path", "string", "basepython"): 1248 reader.addsubstitutions(**{env_attr.name: res}) 1249 return tc 1250 1251 def _getallenvs(self, reader, extra_env_list=None): 1252 extra_env_list = extra_env_list or [] 1253 env_str = reader.getstring("envlist", replace=False) 1254 env_list = _split_env(env_str) 1255 for env in extra_env_list: 1256 if env not in env_list: 1257 env_list.append(env) 1258 1259 all_envs = OrderedDict((i, None) for i in env_list) 1260 for section in self._cfg: 1261 if section.name.startswith(testenvprefix): 1262 all_envs[section.name[len(testenvprefix) :]] = None 1263 if not all_envs: 1264 all_envs["python"] = None 1265 return list(all_envs.keys()) 1266 1267 def _getenvdata(self, reader, config): 1268 from_option = self.config.option.env 1269 from_environ = os.environ.get("TOXENV") 1270 from_config = reader.getstring("envlist", replace=False) 1271 1272 env_list = [] 1273 envlist_explicit = False 1274 if (from_option and "ALL" in from_option) or ( 1275 not from_option and from_environ and "ALL" in from_environ.split(",") 1276 ): 1277 all_envs = self._getallenvs(reader) 1278 else: 1279 candidates = ( 1280 (os.environ.get(PARALLEL_ENV_VAR_KEY), True), 1281 (from_option, True), 1282 (from_environ, True), 1283 (from_config, False), 1284 ) 1285 env_str, envlist_explicit = next(((i, e) for i, e in candidates if i), ([], False)) 1286 env_list = _split_env(env_str) 1287 all_envs = self._getallenvs(reader, env_list) 1288 1289 if not env_list: 1290 env_list = all_envs 1291 1292 package_env = config.isolated_build_env 1293 if config.isolated_build is True and package_env in all_envs: 1294 all_envs.remove(package_env) 1295 1296 if config.isolated_build is True and package_env in env_list: 1297 msg = "isolated_build_env {} cannot be part of envlist".format(package_env) 1298 raise tox.exception.ConfigError(msg) 1299 return env_list, all_envs, _split_env(from_config), envlist_explicit 1300 1301 1302def _split_env(env): 1303 """if handed a list, action="append" was used for -e """ 1304 if env is None: 1305 return [] 1306 if not isinstance(env, list): 1307 env = [e.split("#", 1)[0].strip() for e in env.split("\n")] 1308 env = ",".join([e for e in env if e]) 1309 env = [env] 1310 return mapcat(_expand_envstr, env) 1311 1312 1313def _is_negated_factor(factor): 1314 return factor.startswith("!") 1315 1316 1317def _base_factor_name(factor): 1318 return factor[1:] if _is_negated_factor(factor) else factor 1319 1320 1321def _split_factor_expr(expr): 1322 def split_single(e): 1323 raw = e.split("-") 1324 included = {_base_factor_name(factor) for factor in raw if not _is_negated_factor(factor)} 1325 excluded = {_base_factor_name(factor) for factor in raw if _is_negated_factor(factor)} 1326 return included, excluded 1327 1328 partial_envs = _expand_envstr(expr) 1329 return [split_single(e) for e in partial_envs] 1330 1331 1332def _split_factor_expr_all(expr): 1333 partial_envs = _expand_envstr(expr) 1334 return [{_base_factor_name(factor) for factor in e.split("-")} for e in partial_envs] 1335 1336 1337def _expand_envstr(envstr): 1338 # split by commas not in groups 1339 tokens = re.split(r"((?:\{[^}]+\})+)|,", envstr) 1340 envlist = ["".join(g).strip() for k, g in itertools.groupby(tokens, key=bool) if k] 1341 1342 def expand(env): 1343 tokens = re.split(r"\{([^}]+)\}", env) 1344 parts = [re.sub(r"\s+", "", token).split(",") for token in tokens] 1345 return ["".join(variant) for variant in itertools.product(*parts)] 1346 1347 return mapcat(expand, envlist) 1348 1349 1350def mapcat(f, seq): 1351 return list(itertools.chain.from_iterable(map(f, seq))) 1352 1353 1354class DepConfig: 1355 def __init__(self, name, indexserver=None): 1356 self.name = name 1357 self.indexserver = indexserver 1358 1359 def __repr__(self): 1360 if self.indexserver: 1361 if self.indexserver.name == "default": 1362 return self.name 1363 return ":{}:{}".format(self.indexserver.name, self.name) 1364 return str(self.name) 1365 1366 1367class IndexServerConfig: 1368 def __init__(self, name, url=None): 1369 self.name = name 1370 self.url = url 1371 1372 def __repr__(self): 1373 return "IndexServerConfig(name={}, url={})".format(self.name, self.url) 1374 1375 1376is_section_substitution = re.compile(r"{\[[^{}\s]+\]\S+?}").match 1377"""Check value matches substitution form of referencing value from other section. 1378 1379E.g. {[base]commands} 1380""" 1381 1382 1383class SectionReader: 1384 def __init__(self, section_name, cfgparser, fallbacksections=None, factors=(), prefix=None): 1385 if prefix is None: 1386 self.section_name = section_name 1387 else: 1388 self.section_name = "{}:{}".format(prefix, section_name) 1389 self._cfg = cfgparser 1390 self.fallbacksections = fallbacksections or [] 1391 self.factors = factors 1392 self._subs = {} 1393 self._subststack = [] 1394 self._setenv = None 1395 1396 def get_environ_value(self, name): 1397 if self._setenv is None: 1398 return os.environ.get(name) 1399 return self._setenv.get(name) 1400 1401 def addsubstitutions(self, _posargs=None, **kw): 1402 self._subs.update(kw) 1403 if _posargs: 1404 self.posargs = _posargs 1405 1406 def getpath(self, name, defaultpath, replace=True): 1407 path = self.getstring(name, defaultpath, replace=replace) 1408 if path is not None: 1409 toxinidir = self._subs["toxinidir"] 1410 return toxinidir.join(path, abs=True) 1411 1412 def getlist(self, name, sep="\n"): 1413 s = self.getstring(name, None) 1414 if s is None: 1415 return [] 1416 return [x.strip() for x in s.split(sep) if x.strip()] 1417 1418 def getdict(self, name, default=None, sep="\n", replace=True): 1419 value = self.getstring(name, None, replace=replace) 1420 return self._getdict(value, default=default, sep=sep, replace=replace) 1421 1422 def getdict_setenv(self, name, default=None, sep="\n", replace=True): 1423 value = self.getstring(name, None, replace=replace, crossonly=True) 1424 definitions = self._getdict(value, default=default, sep=sep, replace=replace) 1425 self._setenv = SetenvDict(definitions, reader=self) 1426 return self._setenv 1427 1428 def _getdict(self, value, default, sep, replace=True): 1429 if value is None or not replace: 1430 return default or {} 1431 1432 d = {} 1433 for line in value.split(sep): 1434 if line.strip(): 1435 name, rest = line.split("=", 1) 1436 d[name.strip()] = rest.strip() 1437 1438 return d 1439 1440 def getbool(self, name, default=None, replace=True): 1441 s = self.getstring(name, default, replace=replace) 1442 if not s or not replace: 1443 s = default 1444 if s is None: 1445 raise KeyError("no config value [{}] {} found".format(self.section_name, name)) 1446 1447 if not isinstance(s, bool): 1448 if s.lower() == "true": 1449 s = True 1450 elif s.lower() == "false": 1451 s = False 1452 else: 1453 raise tox.exception.ConfigError( 1454 "{}: boolean value {!r} needs to be 'True' or 'False'".format(name, s) 1455 ) 1456 return s 1457 1458 def getargvlist(self, name, default="", replace=True): 1459 s = self.getstring(name, default, replace=False) 1460 return _ArgvlistReader.getargvlist(self, s, replace=replace) 1461 1462 def getargv(self, name, default="", replace=True): 1463 return self.getargvlist(name, default, replace=replace)[0] 1464 1465 def getstring(self, name, default=None, replace=True, crossonly=False, no_fallback=False): 1466 x = None 1467 sections = [self.section_name] + ([] if no_fallback else self.fallbacksections) 1468 for s in sections: 1469 try: 1470 x = self._cfg[s][name] 1471 break 1472 except KeyError: 1473 continue 1474 1475 if x is None: 1476 x = default 1477 else: 1478 # It is needed to apply factors before unwrapping 1479 # dependencies, otherwise it can break the substitution 1480 # process. Once they are unwrapped, we call apply factors 1481 # again for those new dependencies. 1482 x = self._apply_factors(x) 1483 x = self._replace_if_needed(x, name, replace, crossonly) 1484 x = self._apply_factors(x) 1485 1486 x = self._replace_if_needed(x, name, replace, crossonly) 1487 return x 1488 1489 def _replace_if_needed(self, x, name, replace, crossonly): 1490 if replace and x and hasattr(x, "replace"): 1491 x = self._replace(x, name=name, crossonly=crossonly) 1492 return x 1493 1494 def _apply_factors(self, s): 1495 def factor_line(line): 1496 m = re.search(r"^([\w{}\.!,-]+)\:\s+(.+)", line) 1497 if not m: 1498 return line 1499 1500 expr, line = m.groups() 1501 if any( 1502 included <= self.factors and not any(x in self.factors for x in excluded) 1503 for included, excluded in _split_factor_expr(expr) 1504 ): 1505 return line 1506 1507 lines = s.strip().splitlines() 1508 return "\n".join(filter(None, map(factor_line, lines))) 1509 1510 def _replace(self, value, name=None, section_name=None, crossonly=False): 1511 if "{" not in value: 1512 return value 1513 1514 section_name = section_name if section_name else self.section_name 1515 self._subststack.append((section_name, name)) 1516 try: 1517 replaced = Replacer(self, crossonly=crossonly).do_replace(value) 1518 assert self._subststack.pop() == (section_name, name) 1519 except tox.exception.MissingSubstitution: 1520 if not section_name.startswith(testenvprefix): 1521 raise tox.exception.ConfigError( 1522 "substitution env:{!r}: unknown or recursive definition in" 1523 " section {!r}.".format(value, section_name) 1524 ) 1525 raise 1526 return replaced 1527 1528 1529class Replacer: 1530 RE_ITEM_REF = re.compile( 1531 r""" 1532 (?<!\\)[{] 1533 (?:(?P<sub_type>[^[:{}]+):)? # optional sub_type for special rules 1534 (?P<substitution_value>(?:\[[^,{}]*\])?[^:,{}]*) # substitution key 1535 (?::(?P<default_value>[^{}]*))? # default value 1536 [}] 1537 """, 1538 re.VERBOSE, 1539 ) 1540 1541 def __init__(self, reader, crossonly=False): 1542 self.reader = reader 1543 self.crossonly = crossonly 1544 1545 def do_replace(self, value): 1546 """ 1547 Recursively expand substitutions starting from the innermost expression 1548 """ 1549 1550 def substitute_once(x): 1551 return self.RE_ITEM_REF.sub(self._replace_match, x) 1552 1553 expanded = substitute_once(value) 1554 1555 while expanded != value: # substitution found 1556 value = expanded 1557 expanded = substitute_once(value) 1558 1559 return expanded 1560 1561 def _replace_match(self, match): 1562 g = match.groupdict() 1563 sub_value = g["substitution_value"] 1564 if self.crossonly: 1565 if sub_value.startswith("["): 1566 return self._substitute_from_other_section(sub_value) 1567 # in crossonly we return all other hits verbatim 1568 start, end = match.span() 1569 return match.string[start:end] 1570 1571 # special case: all empty values means ":" which is os.pathsep 1572 if not any(g.values()): 1573 return os.pathsep 1574 1575 # special case: opts and packages. Leave {opts} and 1576 # {packages} intact, they are replaced manually in 1577 # _venv.VirtualEnv.run_install_command. 1578 if sub_value in ("opts", "packages"): 1579 return "{{{}}}".format(sub_value) 1580 1581 try: 1582 sub_type = g["sub_type"] 1583 except KeyError: 1584 raise tox.exception.ConfigError( 1585 "Malformed substitution; no substitution type provided" 1586 ) 1587 1588 if sub_type == "env": 1589 return self._replace_env(match) 1590 if sub_type == "tty": 1591 if is_interactive(): 1592 return match.group("substitution_value") 1593 return match.group("default_value") 1594 if sub_type is not None: 1595 raise tox.exception.ConfigError( 1596 "No support for the {} substitution type".format(sub_type) 1597 ) 1598 return self._replace_substitution(match) 1599 1600 def _replace_env(self, match): 1601 key = match.group("substitution_value") 1602 if not key: 1603 raise tox.exception.ConfigError("env: requires an environment variable name") 1604 default = match.group("default_value") 1605 value = self.reader.get_environ_value(key) 1606 if value is not None: 1607 return value 1608 if default is not None: 1609 return default 1610 raise tox.exception.MissingSubstitution(key) 1611 1612 def _substitute_from_other_section(self, key): 1613 if key.startswith("[") and "]" in key: 1614 i = key.find("]") 1615 section, item = key[1:i], key[i + 1 :] 1616 cfg = self.reader._cfg 1617 if section in cfg and item in cfg[section]: 1618 if (section, item) in self.reader._subststack: 1619 raise ValueError( 1620 "{} already in {}".format((section, item), self.reader._subststack) 1621 ) 1622 x = str(cfg[section][item]) 1623 return self.reader._replace( 1624 x, name=item, section_name=section, crossonly=self.crossonly 1625 ) 1626 1627 raise tox.exception.ConfigError("substitution key {!r} not found".format(key)) 1628 1629 def _replace_substitution(self, match): 1630 sub_key = match.group("substitution_value") 1631 val = self.reader._subs.get(sub_key, None) 1632 if val is None: 1633 val = self._substitute_from_other_section(sub_key) 1634 if callable(val): 1635 val = val() 1636 return str(val) 1637 1638 1639def is_interactive(): 1640 return sys.stdin.isatty() 1641 1642 1643class _ArgvlistReader: 1644 @classmethod 1645 def getargvlist(cls, reader, value, replace=True): 1646 """Parse ``commands`` argvlist multiline string. 1647 1648 :param SectionReader reader: reader to be used. 1649 :param str value: Content stored by key. 1650 1651 :rtype: list[list[str]] 1652 :raise :class:`tox.exception.ConfigError`: 1653 line-continuation ends nowhere while resolving for specified section 1654 """ 1655 commands = [] 1656 current_command = "" 1657 for line in value.splitlines(): 1658 line = line.rstrip() 1659 if not line: 1660 continue 1661 if line.endswith("\\"): 1662 current_command += " {}".format(line[:-1]) 1663 continue 1664 current_command += line 1665 1666 if is_section_substitution(current_command): 1667 replaced = reader._replace(current_command, crossonly=True) 1668 commands.extend(cls.getargvlist(reader, replaced)) 1669 else: 1670 commands.append(cls.processcommand(reader, current_command, replace)) 1671 current_command = "" 1672 else: 1673 if current_command: 1674 raise tox.exception.ConfigError( 1675 "line-continuation ends nowhere while resolving for [{}] {}".format( 1676 reader.section_name, "commands" 1677 ) 1678 ) 1679 return commands 1680 1681 @classmethod 1682 def processcommand(cls, reader, command, replace=True): 1683 posargs = getattr(reader, "posargs", "") 1684 posargs_string = list2cmdline([x for x in posargs if x]) 1685 1686 # Iterate through each word of the command substituting as 1687 # appropriate to construct the new command string. This 1688 # string is then broken up into exec argv components using 1689 # shlex. 1690 if replace: 1691 newcommand = "" 1692 for word in CommandParser(command).words(): 1693 if word == "{posargs}" or word == "[]": 1694 newcommand += posargs_string 1695 continue 1696 elif word.startswith("{posargs:") and word.endswith("}"): 1697 if posargs: 1698 newcommand += posargs_string 1699 continue 1700 else: 1701 word = word[9:-1] 1702 new_arg = "" 1703 new_word = reader._replace(word) 1704 new_word = reader._replace(new_word) 1705 new_word = new_word.replace("\\{", "{").replace("\\}", "}") 1706 new_arg += new_word 1707 newcommand += new_arg 1708 else: 1709 newcommand = command 1710 1711 # Construct shlex object that will not escape any values, 1712 # use all values as is in argv. 1713 shlexer = shlex.shlex(newcommand, posix=True) 1714 shlexer.whitespace_split = True 1715 shlexer.escape = "" 1716 return list(shlexer) 1717 1718 1719class CommandParser(object): 1720 class State(object): 1721 def __init__(self): 1722 self.word = "" 1723 self.depth = 0 1724 self.yield_words = [] 1725 1726 def __init__(self, command): 1727 self.command = command 1728 1729 def words(self): 1730 ps = CommandParser.State() 1731 1732 def word_has_ended(): 1733 return ( 1734 ( 1735 cur_char in string.whitespace 1736 and ps.word 1737 and ps.word[-1] not in string.whitespace 1738 ) 1739 or (cur_char == "{" and ps.depth == 0 and not ps.word.endswith("\\")) 1740 or (ps.depth == 0 and ps.word and ps.word[-1] == "}") 1741 or (cur_char not in string.whitespace and ps.word and ps.word.strip() == "") 1742 ) 1743 1744 def yield_this_word(): 1745 yieldword = ps.word 1746 ps.word = "" 1747 if yieldword: 1748 ps.yield_words.append(yieldword) 1749 1750 def yield_if_word_ended(): 1751 if word_has_ended(): 1752 yield_this_word() 1753 1754 def accumulate(): 1755 ps.word += cur_char 1756 1757 def push_substitution(): 1758 ps.depth += 1 1759 1760 def pop_substitution(): 1761 ps.depth -= 1 1762 1763 for cur_char in self.command: 1764 if cur_char in string.whitespace: 1765 if ps.depth == 0: 1766 yield_if_word_ended() 1767 accumulate() 1768 elif cur_char == "{": 1769 yield_if_word_ended() 1770 accumulate() 1771 push_substitution() 1772 elif cur_char == "}": 1773 accumulate() 1774 pop_substitution() 1775 else: 1776 yield_if_word_ended() 1777 accumulate() 1778 1779 if ps.word.strip(): 1780 yield_this_word() 1781 return ps.yield_words 1782 1783 1784def getcontextname(): 1785 if any(env in os.environ for env in ["JENKINS_URL", "HUDSON_URL"]): 1786 return "jenkins" 1787 return None 1788