1""" 2 sphinx.ext.doctest 3 ~~~~~~~~~~~~~~~~~~ 4 5 Mimic doctest by automatically executing code snippets and checking 6 their results. 7 8 :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS. 9 :license: BSD, see LICENSE for details. 10""" 11 12import doctest 13import re 14import sys 15import time 16import warnings 17from io import StringIO 18from os import path 19from typing import Any, Callable, Dict, Iterable, List, Sequence, Set, Tuple 20 21from docutils import nodes 22from docutils.nodes import Element, Node, TextElement 23from docutils.parsers.rst import directives 24from packaging.specifiers import InvalidSpecifier, SpecifierSet 25from packaging.version import Version 26 27import sphinx 28from sphinx.builders import Builder 29from sphinx.deprecation import RemovedInSphinx40Warning 30from sphinx.locale import __ 31from sphinx.util import logging 32from sphinx.util.console import bold # type: ignore 33from sphinx.util.docutils import SphinxDirective 34from sphinx.util.osutil import relpath 35 36if False: 37 # For type annotation 38 from typing import Type # for python3.5.1 39 40 from sphinx.application import Sphinx 41 42 43logger = logging.getLogger(__name__) 44 45blankline_re = re.compile(r'^\s*<BLANKLINE>', re.MULTILINE) 46doctestopt_re = re.compile(r'#\s*doctest:.+$', re.MULTILINE) 47 48 49def doctest_encode(text: str, encoding: str) -> str: 50 warnings.warn('doctest_encode() is deprecated.', 51 RemovedInSphinx40Warning, stacklevel=2) 52 return text 53 54 55def is_allowed_version(spec: str, version: str) -> bool: 56 """Check `spec` satisfies `version` or not. 57 58 This obeys PEP-440 specifiers: 59 https://www.python.org/dev/peps/pep-0440/#version-specifiers 60 61 Some examples: 62 63 >>> is_allowed_version('3.3', '<=3.5') 64 True 65 >>> is_allowed_version('3.3', '<=3.2') 66 False 67 >>> is_allowed_version('3.3', '>3.2, <4.0') 68 True 69 """ 70 return Version(version) in SpecifierSet(spec) 71 72 73# set up the necessary directives 74 75class TestDirective(SphinxDirective): 76 """ 77 Base class for doctest-related directives. 78 """ 79 80 has_content = True 81 required_arguments = 0 82 optional_arguments = 1 83 final_argument_whitespace = True 84 85 def run(self) -> List[Node]: 86 # use ordinary docutils nodes for test code: they get special attributes 87 # so that our builder recognizes them, and the other builders are happy. 88 code = '\n'.join(self.content) 89 test = None 90 if self.name == 'doctest': 91 if '<BLANKLINE>' in code: 92 # convert <BLANKLINE>s to ordinary blank lines for presentation 93 test = code 94 code = blankline_re.sub('', code) 95 if doctestopt_re.search(code) and 'no-trim-doctest-flags' not in self.options: 96 if not test: 97 test = code 98 code = doctestopt_re.sub('', code) 99 nodetype = nodes.literal_block # type: Type[TextElement] 100 if self.name in ('testsetup', 'testcleanup') or 'hide' in self.options: 101 nodetype = nodes.comment 102 if self.arguments: 103 groups = [x.strip() for x in self.arguments[0].split(',')] 104 else: 105 groups = ['default'] 106 node = nodetype(code, code, testnodetype=self.name, groups=groups) 107 self.set_source_info(node) 108 if test is not None: 109 # only save if it differs from code 110 node['test'] = test 111 if self.name == 'doctest': 112 if self.config.highlight_language in ('py', 'python'): 113 node['language'] = 'pycon' 114 else: 115 node['language'] = 'pycon3' # default 116 elif self.name == 'testcode': 117 if self.config.highlight_language in ('py', 'python'): 118 node['language'] = 'python' 119 else: 120 node['language'] = 'python3' # default 121 elif self.name == 'testoutput': 122 # don't try to highlight output 123 node['language'] = 'none' 124 node['options'] = {} 125 if self.name in ('doctest', 'testoutput') and 'options' in self.options: 126 # parse doctest-like output comparison flags 127 option_strings = self.options['options'].replace(',', ' ').split() 128 for option in option_strings: 129 prefix, option_name = option[0], option[1:] 130 if prefix not in '+-': 131 self.state.document.reporter.warning( 132 __("missing '+' or '-' in '%s' option.") % option, 133 line=self.lineno) 134 continue 135 if option_name not in doctest.OPTIONFLAGS_BY_NAME: 136 self.state.document.reporter.warning( 137 __("'%s' is not a valid option.") % option_name, 138 line=self.lineno) 139 continue 140 flag = doctest.OPTIONFLAGS_BY_NAME[option[1:]] 141 node['options'][flag] = (option[0] == '+') 142 if self.name == 'doctest' and 'pyversion' in self.options: 143 try: 144 spec = self.options['pyversion'] 145 python_version = '.'.join([str(v) for v in sys.version_info[:3]]) 146 if not is_allowed_version(spec, python_version): 147 flag = doctest.OPTIONFLAGS_BY_NAME['SKIP'] 148 node['options'][flag] = True # Skip the test 149 except InvalidSpecifier: 150 self.state.document.reporter.warning( 151 __("'%s' is not a valid pyversion option") % spec, 152 line=self.lineno) 153 if 'skipif' in self.options: 154 node['skipif'] = self.options['skipif'] 155 if 'trim-doctest-flags' in self.options: 156 node['trim_flags'] = True 157 elif 'no-trim-doctest-flags' in self.options: 158 node['trim_flags'] = False 159 return [node] 160 161 162class TestsetupDirective(TestDirective): 163 option_spec = {'skipif': directives.unchanged_required} # type: Dict 164 165 166class TestcleanupDirective(TestDirective): 167 option_spec = {'skipif': directives.unchanged_required} # type: Dict 168 169 170class DoctestDirective(TestDirective): 171 option_spec = { 172 'hide': directives.flag, 173 'no-trim-doctest-flags': directives.flag, 174 'options': directives.unchanged, 175 'pyversion': directives.unchanged_required, 176 'skipif': directives.unchanged_required, 177 'trim-doctest-flags': directives.flag, 178 } 179 180 181class TestcodeDirective(TestDirective): 182 option_spec = { 183 'hide': directives.flag, 184 'no-trim-doctest-flags': directives.flag, 185 'pyversion': directives.unchanged_required, 186 'skipif': directives.unchanged_required, 187 'trim-doctest-flags': directives.flag, 188 } 189 190 191class TestoutputDirective(TestDirective): 192 option_spec = { 193 'hide': directives.flag, 194 'no-trim-doctest-flags': directives.flag, 195 'options': directives.unchanged, 196 'pyversion': directives.unchanged_required, 197 'skipif': directives.unchanged_required, 198 'trim-doctest-flags': directives.flag, 199 } 200 201 202parser = doctest.DocTestParser() 203 204 205# helper classes 206 207class TestGroup: 208 def __init__(self, name: str) -> None: 209 self.name = name 210 self.setup = [] # type: List[TestCode] 211 self.tests = [] # type: List[List[TestCode]] 212 self.cleanup = [] # type: List[TestCode] 213 214 def add_code(self, code: "TestCode", prepend: bool = False) -> None: 215 if code.type == 'testsetup': 216 if prepend: 217 self.setup.insert(0, code) 218 else: 219 self.setup.append(code) 220 elif code.type == 'testcleanup': 221 self.cleanup.append(code) 222 elif code.type == 'doctest': 223 self.tests.append([code]) 224 elif code.type == 'testcode': 225 self.tests.append([code, None]) 226 elif code.type == 'testoutput': 227 if self.tests and len(self.tests[-1]) == 2: 228 self.tests[-1][1] = code 229 else: 230 raise RuntimeError(__('invalid TestCode type')) 231 232 def __repr__(self) -> str: 233 return 'TestGroup(name=%r, setup=%r, cleanup=%r, tests=%r)' % ( 234 self.name, self.setup, self.cleanup, self.tests) 235 236 237class TestCode: 238 def __init__(self, code: str, type: str, filename: str, 239 lineno: int, options: Dict = None) -> None: 240 self.code = code 241 self.type = type 242 self.filename = filename 243 self.lineno = lineno 244 self.options = options or {} 245 246 def __repr__(self) -> str: 247 return 'TestCode(%r, %r, filename=%r, lineno=%r, options=%r)' % ( 248 self.code, self.type, self.filename, self.lineno, self.options) 249 250 251class SphinxDocTestRunner(doctest.DocTestRunner): 252 def summarize(self, out: Callable, verbose: bool = None # type: ignore 253 ) -> Tuple[int, int]: 254 string_io = StringIO() 255 old_stdout = sys.stdout 256 sys.stdout = string_io 257 try: 258 res = super().summarize(verbose) 259 finally: 260 sys.stdout = old_stdout 261 out(string_io.getvalue()) 262 return res 263 264 def _DocTestRunner__patched_linecache_getlines(self, filename: str, 265 module_globals: Any = None) -> Any: 266 # this is overridden from DocTestRunner adding the try-except below 267 m = self._DocTestRunner__LINECACHE_FILENAME_RE.match(filename) # type: ignore 268 if m and m.group('name') == self.test.name: 269 try: 270 example = self.test.examples[int(m.group('examplenum'))] 271 # because we compile multiple doctest blocks with the same name 272 # (viz. the group name) this might, for outer stack frames in a 273 # traceback, get the wrong test which might not have enough examples 274 except IndexError: 275 pass 276 else: 277 return example.source.splitlines(True) 278 return self.save_linecache_getlines(filename, module_globals) # type: ignore 279 280 281# the new builder -- use sphinx-build.py -b doctest to run 282 283class DocTestBuilder(Builder): 284 """ 285 Runs test snippets in the documentation. 286 """ 287 name = 'doctest' 288 epilog = __('Testing of doctests in the sources finished, look at the ' 289 'results in %(outdir)s/output.txt.') 290 291 def init(self) -> None: 292 # default options 293 self.opt = self.config.doctest_default_flags 294 295 # HACK HACK HACK 296 # doctest compiles its snippets with type 'single'. That is nice 297 # for doctest examples but unusable for multi-statement code such 298 # as setup code -- to be able to use doctest error reporting with 299 # that code nevertheless, we monkey-patch the "compile" it uses. 300 doctest.compile = self.compile # type: ignore 301 302 sys.path[0:0] = self.config.doctest_path 303 304 self.type = 'single' 305 306 self.total_failures = 0 307 self.total_tries = 0 308 self.setup_failures = 0 309 self.setup_tries = 0 310 self.cleanup_failures = 0 311 self.cleanup_tries = 0 312 313 date = time.strftime('%Y-%m-%d %H:%M:%S') 314 315 self.outfile = open(path.join(self.outdir, 'output.txt'), 'w', encoding='utf-8') 316 self.outfile.write(('Results of doctest builder run on %s\n' 317 '==================================%s\n') % 318 (date, '=' * len(date))) 319 320 def _out(self, text: str) -> None: 321 logger.info(text, nonl=True) 322 self.outfile.write(text) 323 324 def _warn_out(self, text: str) -> None: 325 if self.app.quiet or self.app.warningiserror: 326 logger.warning(text) 327 else: 328 logger.info(text, nonl=True) 329 self.outfile.write(text) 330 331 def get_target_uri(self, docname: str, typ: str = None) -> str: 332 return '' 333 334 def get_outdated_docs(self) -> Set[str]: 335 return self.env.found_docs 336 337 def finish(self) -> None: 338 # write executive summary 339 def s(v: int) -> str: 340 return 's' if v != 1 else '' 341 repl = (self.total_tries, s(self.total_tries), 342 self.total_failures, s(self.total_failures), 343 self.setup_failures, s(self.setup_failures), 344 self.cleanup_failures, s(self.cleanup_failures)) 345 self._out(''' 346Doctest summary 347=============== 348%5d test%s 349%5d failure%s in tests 350%5d failure%s in setup code 351%5d failure%s in cleanup code 352''' % repl) 353 self.outfile.close() 354 355 if self.total_failures or self.setup_failures or self.cleanup_failures: 356 self.app.statuscode = 1 357 358 def write(self, build_docnames: Iterable[str], updated_docnames: Sequence[str], 359 method: str = 'update') -> None: 360 if build_docnames is None: 361 build_docnames = sorted(self.env.all_docs) 362 363 logger.info(bold('running tests...')) 364 for docname in build_docnames: 365 # no need to resolve the doctree 366 doctree = self.env.get_doctree(docname) 367 self.test_doc(docname, doctree) 368 369 def get_filename_for_node(self, node: Node, docname: str) -> str: 370 """Try to get the file which actually contains the doctest, not the 371 filename of the document it's included in.""" 372 try: 373 filename = relpath(node.source, self.env.srcdir)\ 374 .rsplit(':docstring of ', maxsplit=1)[0] 375 except Exception: 376 filename = self.env.doc2path(docname, base=None) 377 return filename 378 379 @staticmethod 380 def get_line_number(node: Node) -> int: 381 """Get the real line number or admit we don't know.""" 382 # TODO: Work out how to store or calculate real (file-relative) 383 # line numbers for doctest blocks in docstrings. 384 if ':docstring of ' in path.basename(node.source or ''): 385 # The line number is given relative to the stripped docstring, 386 # not the file. This is correct where it is set, in 387 # `docutils.nodes.Node.setup_child`, but Sphinx should report 388 # relative to the file, not the docstring. 389 return None 390 if node.line is not None: 391 # TODO: find the root cause of this off by one error. 392 return node.line - 1 393 return None 394 395 def skipped(self, node: Element) -> bool: 396 if 'skipif' not in node: 397 return False 398 else: 399 condition = node['skipif'] 400 context = {} # type: Dict[str, Any] 401 if self.config.doctest_global_setup: 402 exec(self.config.doctest_global_setup, context) 403 should_skip = eval(condition, context) 404 if self.config.doctest_global_cleanup: 405 exec(self.config.doctest_global_cleanup, context) 406 return should_skip 407 408 def test_doc(self, docname: str, doctree: Node) -> None: 409 groups = {} # type: Dict[str, TestGroup] 410 add_to_all_groups = [] 411 self.setup_runner = SphinxDocTestRunner(verbose=False, 412 optionflags=self.opt) 413 self.test_runner = SphinxDocTestRunner(verbose=False, 414 optionflags=self.opt) 415 self.cleanup_runner = SphinxDocTestRunner(verbose=False, 416 optionflags=self.opt) 417 418 self.test_runner._fakeout = self.setup_runner._fakeout # type: ignore 419 self.cleanup_runner._fakeout = self.setup_runner._fakeout # type: ignore 420 421 if self.config.doctest_test_doctest_blocks: 422 def condition(node: Node) -> bool: 423 return (isinstance(node, (nodes.literal_block, nodes.comment)) and 424 'testnodetype' in node) or \ 425 isinstance(node, nodes.doctest_block) 426 else: 427 def condition(node: Node) -> bool: 428 return isinstance(node, (nodes.literal_block, nodes.comment)) \ 429 and 'testnodetype' in node 430 for node in doctree.traverse(condition): # type: Element 431 if self.skipped(node): 432 continue 433 434 source = node['test'] if 'test' in node else node.astext() 435 filename = self.get_filename_for_node(node, docname) 436 line_number = self.get_line_number(node) 437 if not source: 438 logger.warning(__('no code/output in %s block at %s:%s'), 439 node.get('testnodetype', 'doctest'), 440 filename, line_number) 441 code = TestCode(source, type=node.get('testnodetype', 'doctest'), 442 filename=filename, lineno=line_number, 443 options=node.get('options')) 444 node_groups = node.get('groups', ['default']) 445 if '*' in node_groups: 446 add_to_all_groups.append(code) 447 continue 448 for groupname in node_groups: 449 if groupname not in groups: 450 groups[groupname] = TestGroup(groupname) 451 groups[groupname].add_code(code) 452 for code in add_to_all_groups: 453 for group in groups.values(): 454 group.add_code(code) 455 if self.config.doctest_global_setup: 456 code = TestCode(self.config.doctest_global_setup, 457 'testsetup', filename=None, lineno=0) 458 for group in groups.values(): 459 group.add_code(code, prepend=True) 460 if self.config.doctest_global_cleanup: 461 code = TestCode(self.config.doctest_global_cleanup, 462 'testcleanup', filename=None, lineno=0) 463 for group in groups.values(): 464 group.add_code(code) 465 if not groups: 466 return 467 468 self._out('\nDocument: %s\n----------%s\n' % 469 (docname, '-' * len(docname))) 470 for group in groups.values(): 471 self.test_group(group) 472 # Separately count results from setup code 473 res_f, res_t = self.setup_runner.summarize(self._out, verbose=False) 474 self.setup_failures += res_f 475 self.setup_tries += res_t 476 if self.test_runner.tries: 477 res_f, res_t = self.test_runner.summarize(self._out, verbose=True) 478 self.total_failures += res_f 479 self.total_tries += res_t 480 if self.cleanup_runner.tries: 481 res_f, res_t = self.cleanup_runner.summarize(self._out, 482 verbose=True) 483 self.cleanup_failures += res_f 484 self.cleanup_tries += res_t 485 486 def compile(self, code: str, name: str, type: str, flags: Any, dont_inherit: bool) -> Any: 487 return compile(code, name, self.type, flags, dont_inherit) 488 489 def test_group(self, group: TestGroup) -> None: 490 ns = {} # type: Dict 491 492 def run_setup_cleanup(runner: Any, testcodes: List[TestCode], what: Any) -> bool: 493 examples = [] 494 for testcode in testcodes: 495 example = doctest.Example(testcode.code, '', lineno=testcode.lineno) 496 examples.append(example) 497 if not examples: 498 return True 499 # simulate a doctest with the code 500 sim_doctest = doctest.DocTest(examples, {}, 501 '%s (%s code)' % (group.name, what), 502 testcodes[0].filename, 0, None) 503 sim_doctest.globs = ns 504 old_f = runner.failures 505 self.type = 'exec' # the snippet may contain multiple statements 506 runner.run(sim_doctest, out=self._warn_out, clear_globs=False) 507 if runner.failures > old_f: 508 return False 509 return True 510 511 # run the setup code 512 if not run_setup_cleanup(self.setup_runner, group.setup, 'setup'): 513 # if setup failed, don't run the group 514 return 515 516 # run the tests 517 for code in group.tests: 518 if len(code) == 1: 519 # ordinary doctests (code/output interleaved) 520 try: 521 test = parser.get_doctest(code[0].code, {}, group.name, 522 code[0].filename, code[0].lineno) 523 except Exception: 524 logger.warning(__('ignoring invalid doctest code: %r'), code[0].code, 525 location=(code[0].filename, code[0].lineno)) 526 continue 527 if not test.examples: 528 continue 529 for example in test.examples: 530 # apply directive's comparison options 531 new_opt = code[0].options.copy() 532 new_opt.update(example.options) 533 example.options = new_opt 534 self.type = 'single' # as for ordinary doctests 535 else: 536 # testcode and output separate 537 output = code[1].code if code[1] else '' 538 options = code[1].options if code[1] else {} 539 # disable <BLANKLINE> processing as it is not needed 540 options[doctest.DONT_ACCEPT_BLANKLINE] = True 541 # find out if we're testing an exception 542 m = parser._EXCEPTION_RE.match(output) # type: ignore 543 if m: 544 exc_msg = m.group('msg') 545 else: 546 exc_msg = None 547 example = doctest.Example(code[0].code, output, exc_msg=exc_msg, 548 lineno=code[0].lineno, options=options) 549 test = doctest.DocTest([example], {}, group.name, 550 code[0].filename, code[0].lineno, None) 551 self.type = 'exec' # multiple statements again 552 # DocTest.__init__ copies the globs namespace, which we don't want 553 test.globs = ns 554 # also don't clear the globs namespace after running the doctest 555 self.test_runner.run(test, out=self._warn_out, clear_globs=False) 556 557 # run the cleanup 558 run_setup_cleanup(self.cleanup_runner, group.cleanup, 'cleanup') 559 560 561def setup(app: "Sphinx") -> Dict[str, Any]: 562 app.add_directive('testsetup', TestsetupDirective) 563 app.add_directive('testcleanup', TestcleanupDirective) 564 app.add_directive('doctest', DoctestDirective) 565 app.add_directive('testcode', TestcodeDirective) 566 app.add_directive('testoutput', TestoutputDirective) 567 app.add_builder(DocTestBuilder) 568 # this config value adds to sys.path 569 app.add_config_value('doctest_path', [], False) 570 app.add_config_value('doctest_test_doctest_blocks', 'default', False) 571 app.add_config_value('doctest_global_setup', '', False) 572 app.add_config_value('doctest_global_cleanup', '', False) 573 app.add_config_value( 574 'doctest_default_flags', 575 doctest.DONT_ACCEPT_TRUE_FOR_1 | doctest.ELLIPSIS | doctest.IGNORE_EXCEPTION_DETAIL, 576 False) 577 return {'version': sphinx.__display_version__, 'parallel_read_safe': True} 578