1"""Implements the Astropy TestRunner which is a thin wrapper around pytest.""" 2 3import inspect 4import os 5import glob 6import copy 7import shlex 8import sys 9import tempfile 10import warnings 11import importlib 12from collections import OrderedDict 13from importlib.util import find_spec 14from functools import wraps 15 16from astropy.config.paths import set_temp_config, set_temp_cache 17from astropy.utils import find_current_module 18from astropy.utils.exceptions import AstropyWarning, AstropyDeprecationWarning 19 20__all__ = ['TestRunner', 'TestRunnerBase', 'keyword'] 21 22 23class keyword: 24 """ 25 A decorator to mark a method as keyword argument for the ``TestRunner``. 26 27 Parameters 28 ---------- 29 default_value : `object` 30 The default value for the keyword argument. (Default: `None`) 31 32 priority : `int` 33 keyword argument methods are executed in order of descending priority. 34 """ 35 36 def __init__(self, default_value=None, priority=0): 37 self.default_value = default_value 38 self.priority = priority 39 40 def __call__(self, f): 41 def keyword(*args, **kwargs): 42 return f(*args, **kwargs) 43 44 keyword._default_value = self.default_value 45 keyword._priority = self.priority 46 # Set __doc__ explicitly here rather than using wraps because we want 47 # to keep the function name as keyword so we can inspect it later. 48 keyword.__doc__ = f.__doc__ 49 50 return keyword 51 52 53class TestRunnerBase: 54 """ 55 The base class for the TestRunner. 56 57 A test runner can be constructed by creating a subclass of this class and 58 defining 'keyword' methods. These are methods that have the 59 :class:`~astropy.tests.runner.keyword` decorator, these methods are used to 60 construct allowed keyword arguments to the 61 ``run_tests`` method as a way to allow 62 customization of individual keyword arguments (and associated logic) 63 without having to re-implement the whole 64 ``run_tests`` method. 65 66 Examples 67 -------- 68 69 A simple keyword method:: 70 71 class MyRunner(TestRunnerBase): 72 73 @keyword('default_value'): 74 def spam(self, spam, kwargs): 75 \"\"\" 76 spam : `str` 77 The parameter description for the run_tests docstring. 78 \"\"\" 79 # Return value must be a list with a CLI parameter for pytest. 80 return ['--spam={}'.format(spam)] 81 """ 82 83 def __init__(self, base_path): 84 self.base_path = os.path.abspath(base_path) 85 86 def __new__(cls, *args, **kwargs): 87 # Before constructing the class parse all the methods that have been 88 # decorated with ``keyword``. 89 90 # The objective of this method is to construct a default set of keyword 91 # arguments to the ``run_tests`` method. It does this by inspecting the 92 # methods of the class for functions with the name ``keyword`` which is 93 # the name of the decorator wrapping function. Once it has created this 94 # dictionary, it also formats the docstring of ``run_tests`` to be 95 # comprised of the docstrings for the ``keyword`` methods. 96 97 # To add a keyword argument to the ``run_tests`` method, define a new 98 # method decorated with ``@keyword`` and with the ``self, name, kwargs`` 99 # signature. 100 # Get all 'function' members as the wrapped methods are functions 101 functions = inspect.getmembers(cls, predicate=inspect.isfunction) 102 103 # Filter out anything that's not got the name 'keyword' 104 keywords = filter(lambda func: func[1].__name__ == 'keyword', functions) 105 # Sort all keywords based on the priority flag. 106 sorted_keywords = sorted(keywords, key=lambda x: x[1]._priority, reverse=True) 107 108 cls.keywords = OrderedDict() 109 doc_keywords = "" 110 for name, func in sorted_keywords: 111 # Here we test if the function has been overloaded to return 112 # NotImplemented which is the way to disable arguments on 113 # subclasses. If it has been disabled we need to remove it from the 114 # default keywords dict. We do it in the try except block because 115 # we do not have access to an instance of the class, so this is 116 # going to error unless the method is just doing `return 117 # NotImplemented`. 118 try: 119 # Second argument is False, as it is normally a bool. 120 # The other two are placeholders for objects. 121 if func(None, False, None) is NotImplemented: 122 continue 123 except Exception: 124 pass 125 126 # Construct the default kwargs dict and docstring 127 cls.keywords[name] = func._default_value 128 if func.__doc__: 129 doc_keywords += ' '*8 130 doc_keywords += func.__doc__.strip() 131 doc_keywords += '\n\n' 132 133 cls.run_tests.__doc__ = cls.RUN_TESTS_DOCSTRING.format(keywords=doc_keywords) 134 135 return super().__new__(cls) 136 137 def _generate_args(self, **kwargs): 138 # Update default values with passed kwargs 139 # but don't modify the defaults 140 keywords = copy.deepcopy(self.keywords) 141 keywords.update(kwargs) 142 # Iterate through the keywords (in order of priority) 143 args = [] 144 for keyword in keywords.keys(): 145 func = getattr(self, keyword) 146 result = func(keywords[keyword], keywords) 147 148 # Allow disabling of options in a subclass 149 if result is NotImplemented: 150 raise TypeError(f"run_tests() got an unexpected keyword argument {keyword}") 151 152 # keyword methods must return a list 153 if not isinstance(result, list): 154 raise TypeError(f"{keyword} keyword method must return a list") 155 156 args += result 157 158 return args 159 160 RUN_TESTS_DOCSTRING = \ 161 """ 162 Run the tests for the package. 163 164 This method builds arguments for and then calls ``pytest.main``. 165 166 Parameters 167 ---------- 168{keywords} 169 170 """ 171 172 _required_dependencies = ['pytest', 'pytest_remotedata', 'pytest_doctestplus', 'pytest_astropy_header'] 173 _missing_dependancy_error = ( 174 "Test dependencies are missing: {module}. You should install the " 175 "'pytest-astropy' package (you may need to update the package if you " 176 "have a previous version installed, e.g., " 177 "'pip install pytest-astropy --upgrade' or the equivalent with conda).") 178 179 @classmethod 180 def _has_test_dependencies(cls): # pragma: no cover 181 # Using the test runner will not work without these dependencies, but 182 # pytest-openfiles is optional, so it's not listed here. 183 for module in cls._required_dependencies: 184 spec = find_spec(module) 185 # Checking loader accounts for packages that were uninstalled 186 if spec is None or spec.loader is None: 187 raise RuntimeError( 188 cls._missing_dependancy_error.format(module=module)) 189 190 def run_tests(self, **kwargs): 191 # The following option will include eggs inside a .eggs folder in 192 # sys.path when running the tests. This is possible so that when 193 # running pytest, test dependencies installed via e.g. 194 # tests_requires are available here. This is not an advertised option 195 # since it is only for internal use 196 if kwargs.pop('add_local_eggs_to_path', False): 197 198 # Add each egg to sys.path individually 199 for egg in glob.glob(os.path.join('.eggs', '*.egg')): 200 sys.path.insert(0, egg) 201 202 self._has_test_dependencies() # pragma: no cover 203 204 # The docstring for this method is defined as a class variable. 205 # This allows it to be built for each subclass in __new__. 206 207 # Don't import pytest until it's actually needed to run the tests 208 import pytest 209 210 # Raise error for undefined kwargs 211 allowed_kwargs = set(self.keywords.keys()) 212 passed_kwargs = set(kwargs.keys()) 213 if not passed_kwargs.issubset(allowed_kwargs): 214 wrong_kwargs = list(passed_kwargs.difference(allowed_kwargs)) 215 raise TypeError(f"run_tests() got an unexpected keyword argument {wrong_kwargs[0]}") 216 217 args = self._generate_args(**kwargs) 218 219 if kwargs.get('plugins', None) is not None: 220 plugins = kwargs.pop('plugins') 221 elif self.keywords.get('plugins', None) is not None: 222 plugins = self.keywords['plugins'] 223 else: 224 plugins = [] 225 226 # Override the config locations to not make a new directory nor use 227 # existing cache or config. Note that we need to do this here in 228 # addition to in conftest.py - for users running tests interactively 229 # in e.g. IPython, conftest.py would get read in too late, so we need 230 # to do it here - but at the same time the code here doesn't work when 231 # running tests in parallel mode because this uses subprocesses which 232 # don't know about the temporary config/cache. 233 astropy_config = tempfile.mkdtemp('astropy_config') 234 astropy_cache = tempfile.mkdtemp('astropy_cache') 235 236 # Have to use nested with statements for cross-Python support 237 # Note, using these context managers here is superfluous if the 238 # config_dir or cache_dir options to pytest are in use, but it's 239 # also harmless to nest the contexts 240 with set_temp_config(astropy_config, delete=True): 241 with set_temp_cache(astropy_cache, delete=True): 242 return pytest.main(args=args, plugins=plugins) 243 244 @classmethod 245 def make_test_runner_in(cls, path): 246 """ 247 Constructs a `TestRunner` to run in the given path, and returns a 248 ``test()`` function which takes the same arguments as 249 ``TestRunner.run_tests``. 250 251 The returned ``test()`` function will be defined in the module this 252 was called from. This is used to implement the ``astropy.test()`` 253 function (or the equivalent for affiliated packages). 254 """ 255 256 runner = cls(path) 257 258 @wraps(runner.run_tests, ('__doc__',)) 259 def test(**kwargs): 260 return runner.run_tests(**kwargs) 261 262 module = find_current_module(2) 263 if module is not None: 264 test.__module__ = module.__name__ 265 266 # A somewhat unusual hack, but delete the attached __wrapped__ 267 # attribute--although this is normally used to tell if the function 268 # was wrapped with wraps, on some version of Python this is also 269 # used to determine the signature to display in help() which is 270 # not useful in this case. We don't really care in this case if the 271 # function was wrapped either 272 if hasattr(test, '__wrapped__'): 273 del test.__wrapped__ 274 275 test.__test__ = False 276 return test 277 278 279class TestRunner(TestRunnerBase): 280 """ 281 A test runner for astropy tests 282 """ 283 284 def packages_path(self, packages, base_path, error=None, warning=None): 285 """ 286 Generates the path for multiple packages. 287 288 Parameters 289 ---------- 290 packages : str 291 Comma separated string of packages. 292 base_path : str 293 Base path to the source code or documentation. 294 error : str 295 Error message to be raised as ``ValueError``. Individual package 296 name and path can be accessed by ``{name}`` and ``{path}`` 297 respectively. No error is raised if `None`. (Default: `None`) 298 warning : str 299 Warning message to be issued. Individual package 300 name and path can be accessed by ``{name}`` and ``{path}`` 301 respectively. No warning is issues if `None`. (Default: `None`) 302 303 Returns 304 ------- 305 paths : list of str 306 List of strings of existing package paths. 307 """ 308 packages = packages.split(",") 309 310 paths = [] 311 for package in packages: 312 path = os.path.join( 313 base_path, package.replace('.', os.path.sep)) 314 if not os.path.isdir(path): 315 info = {'name': package, 'path': path} 316 if error is not None: 317 raise ValueError(error.format(**info)) 318 if warning is not None: 319 warnings.warn(warning.format(**info)) 320 else: 321 paths.append(path) 322 323 return paths 324 325 # Increase priority so this warning is displayed first. 326 @keyword(priority=1000) 327 def coverage(self, coverage, kwargs): 328 if coverage: 329 warnings.warn( 330 "The coverage option is ignored on run_tests, since it " 331 "can not be made to work in that context. Use " 332 "'python setup.py test --coverage' instead.", 333 AstropyWarning) 334 335 return [] 336 337 # test_path depends on self.package_path so make sure this runs before 338 # test_path. 339 @keyword(priority=1) 340 def package(self, package, kwargs): 341 """ 342 package : str, optional 343 The name of a specific package to test, e.g. 'io.fits' or 344 'utils'. Accepts comma separated string to specify multiple 345 packages. If nothing is specified all default tests are run. 346 """ 347 if package is None: 348 self.package_path = [self.base_path] 349 else: 350 error_message = ('package to test is not found: {name} ' 351 '(at path {path}).') 352 self.package_path = self.packages_path(package, self.base_path, 353 error=error_message) 354 355 if not kwargs['test_path']: 356 return self.package_path 357 358 return [] 359 360 @keyword() 361 def test_path(self, test_path, kwargs): 362 """ 363 test_path : str, optional 364 Specify location to test by path. May be a single file or 365 directory. Must be specified absolutely or relative to the 366 calling directory. 367 """ 368 all_args = [] 369 # Ensure that the package kwarg has been run. 370 self.package(kwargs['package'], kwargs) 371 if test_path: 372 base, ext = os.path.splitext(test_path) 373 374 if ext in ('.rst', ''): 375 if kwargs['docs_path'] is None: 376 # This shouldn't happen from "python setup.py test" 377 raise ValueError( 378 "Can not test .rst files without a docs_path " 379 "specified.") 380 381 abs_docs_path = os.path.abspath(kwargs['docs_path']) 382 abs_test_path = os.path.abspath( 383 os.path.join(abs_docs_path, os.pardir, test_path)) 384 385 common = os.path.commonprefix((abs_docs_path, abs_test_path)) 386 387 if os.path.exists(abs_test_path) and common == abs_docs_path: 388 # Turn on the doctest_rst plugin 389 all_args.append('--doctest-rst') 390 test_path = abs_test_path 391 392 # Check that the extensions are in the path and not at the end to 393 # support specifying the name of the test, i.e. 394 # test_quantity.py::test_unit 395 if not (os.path.isdir(test_path) or ('.py' in test_path or '.rst' in test_path)): 396 raise ValueError("Test path must be a directory or a path to " 397 "a .py or .rst file") 398 399 return all_args + [test_path] 400 401 return [] 402 403 @keyword() 404 def args(self, args, kwargs): 405 """ 406 args : str, optional 407 Additional arguments to be passed to ``pytest.main`` in the ``args`` 408 keyword argument. 409 """ 410 if args: 411 return shlex.split(args, posix=not sys.platform.startswith('win')) 412 413 return [] 414 415 @keyword(default_value=[]) 416 def plugins(self, plugins, kwargs): 417 """ 418 plugins : list, optional 419 Plugins to be passed to ``pytest.main`` in the ``plugins`` keyword 420 argument. 421 """ 422 # Plugins are handled independently by `run_tests` so we define this 423 # keyword just for the docstring 424 return [] 425 426 @keyword() 427 def verbose(self, verbose, kwargs): 428 """ 429 verbose : bool, optional 430 Convenience option to turn on verbose output from pytest. Passing 431 True is the same as specifying ``-v`` in ``args``. 432 """ 433 if verbose: 434 return ['-v'] 435 436 return [] 437 438 @keyword() 439 def pastebin(self, pastebin, kwargs): 440 """ 441 pastebin : ('failed', 'all', None), optional 442 Convenience option for turning on pytest pastebin output. Set to 443 'failed' to upload info for failed tests, or 'all' to upload info 444 for all tests. 445 """ 446 if pastebin is not None: 447 if pastebin in ['failed', 'all']: 448 return [f'--pastebin={pastebin}'] 449 else: 450 raise ValueError("pastebin should be 'failed' or 'all'") 451 452 return [] 453 454 @keyword(default_value='none') 455 def remote_data(self, remote_data, kwargs): 456 """ 457 remote_data : {'none', 'astropy', 'any'}, optional 458 Controls whether to run tests marked with @pytest.mark.remote_data. This can be 459 set to run no tests with remote data (``none``), only ones that use 460 data from http://data.astropy.org (``astropy``), or all tests that 461 use remote data (``any``). The default is ``none``. 462 """ 463 464 if remote_data is True: 465 remote_data = 'any' 466 elif remote_data is False: 467 remote_data = 'none' 468 elif remote_data not in ('none', 'astropy', 'any'): 469 warnings.warn("The remote_data option should be one of " 470 "none/astropy/any (found {}). For backward-compatibility, " 471 "assuming 'any', but you should change the option to be " 472 "one of the supported ones to avoid issues in " 473 "future.".format(remote_data), 474 AstropyDeprecationWarning) 475 remote_data = 'any' 476 477 return [f'--remote-data={remote_data}'] 478 479 @keyword() 480 def pep8(self, pep8, kwargs): 481 """ 482 pep8 : bool, optional 483 Turn on PEP8 checking via the pytest-pep8 plugin and disable normal 484 tests. Same as specifying ``--pep8 -k pep8`` in ``args``. 485 """ 486 if pep8: 487 try: 488 import pytest_pep8 # pylint: disable=W0611 489 except ImportError: 490 raise ImportError('PEP8 checking requires pytest-pep8 plugin: ' 491 'https://pypi.org/project/pytest-pep8') 492 else: 493 return ['--pep8', '-k', 'pep8'] 494 495 return [] 496 497 @keyword() 498 def pdb(self, pdb, kwargs): 499 """ 500 pdb : bool, optional 501 Turn on PDB post-mortem analysis for failing tests. Same as 502 specifying ``--pdb`` in ``args``. 503 """ 504 if pdb: 505 return ['--pdb'] 506 return [] 507 508 @keyword() 509 def open_files(self, open_files, kwargs): 510 """ 511 open_files : bool, optional 512 Fail when any tests leave files open. Off by default, because 513 this adds extra run time to the test suite. Requires the 514 ``psutil`` package. 515 """ 516 if open_files: 517 if kwargs['parallel'] != 0: 518 raise SystemError( 519 "open file detection may not be used in conjunction with " 520 "parallel testing.") 521 522 try: 523 import psutil # pylint: disable=W0611 524 except ImportError: 525 raise SystemError( 526 "open file detection requested, but psutil package " 527 "is not installed.") 528 529 return ['--open-files'] 530 531 print("Checking for unclosed files") 532 533 return [] 534 535 @keyword(0) 536 def parallel(self, parallel, kwargs): 537 """ 538 parallel : int or 'auto', optional 539 When provided, run the tests in parallel on the specified 540 number of CPUs. If parallel is ``'auto'``, it will use the all 541 the cores on the machine. Requires the ``pytest-xdist`` plugin. 542 """ 543 if parallel != 0: 544 try: 545 from xdist import plugin # noqa 546 except ImportError: 547 raise SystemError( 548 "running tests in parallel requires the pytest-xdist package") 549 550 return ['-n', str(parallel)] 551 552 return [] 553 554 @keyword() 555 def docs_path(self, docs_path, kwargs): 556 """ 557 docs_path : str, optional 558 The path to the documentation .rst files. 559 """ 560 561 paths = [] 562 if docs_path is not None and not kwargs['skip_docs']: 563 if kwargs['package'] is not None: 564 warning_message = ("Can not test .rst docs for {name}, since " 565 "docs path ({path}) does not exist.") 566 paths = self.packages_path(kwargs['package'], docs_path, 567 warning=warning_message) 568 elif not kwargs['test_path']: 569 paths = [docs_path, ] 570 571 if len(paths) and not kwargs['test_path']: 572 paths.append('--doctest-rst') 573 574 return paths 575 576 @keyword() 577 def skip_docs(self, skip_docs, kwargs): 578 """ 579 skip_docs : `bool`, optional 580 When `True`, skips running the doctests in the .rst files. 581 """ 582 # Skip docs is a bool used by docs_path only. 583 return [] 584 585 @keyword() 586 def repeat(self, repeat, kwargs): 587 """ 588 repeat : `int`, optional 589 If set, specifies how many times each test should be run. This is 590 useful for diagnosing sporadic failures. 591 """ 592 if repeat: 593 return [f'--repeat={repeat}'] 594 595 return [] 596 597 # Override run_tests for astropy-specific fixes 598 def run_tests(self, **kwargs): 599 600 # This prevents cyclical import problems that make it 601 # impossible to test packages that define Table types on their 602 # own. 603 from astropy.table import Table # pylint: disable=W0611 604 605 return super().run_tests(**kwargs) 606