1"""
2    sphinx.ext.doctest
3    ~~~~~~~~~~~~~~~~~~
4
5    Mimic doctest by automatically executing code snippets and checking
6    their results.
7
8    :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
9    :license: BSD, see LICENSE for details.
10"""
11
12import doctest
13import re
14import sys
15import time
16import warnings
17from io import StringIO
18from os import path
19from typing import Any, Callable, Dict, Iterable, List, Sequence, Set, Tuple
20
21from docutils import nodes
22from docutils.nodes import Element, Node, TextElement
23from docutils.parsers.rst import directives
24from packaging.specifiers import InvalidSpecifier, SpecifierSet
25from packaging.version import Version
26
27import sphinx
28from sphinx.builders import Builder
29from sphinx.deprecation import RemovedInSphinx40Warning
30from sphinx.locale import __
31from sphinx.util import logging
32from sphinx.util.console import bold  # type: ignore
33from sphinx.util.docutils import SphinxDirective
34from sphinx.util.osutil import relpath
35
36if False:
37    # For type annotation
38    from typing import Type  # for python3.5.1
39
40    from sphinx.application import Sphinx
41
42
43logger = logging.getLogger(__name__)
44
45blankline_re = re.compile(r'^\s*<BLANKLINE>', re.MULTILINE)
46doctestopt_re = re.compile(r'#\s*doctest:.+$', re.MULTILINE)
47
48
49def doctest_encode(text: str, encoding: str) -> str:
50    warnings.warn('doctest_encode() is deprecated.',
51                  RemovedInSphinx40Warning, stacklevel=2)
52    return text
53
54
55def is_allowed_version(spec: str, version: str) -> bool:
56    """Check `spec` satisfies `version` or not.
57
58    This obeys PEP-440 specifiers:
59    https://www.python.org/dev/peps/pep-0440/#version-specifiers
60
61    Some examples:
62
63        >>> is_allowed_version('3.3', '<=3.5')
64        True
65        >>> is_allowed_version('3.3', '<=3.2')
66        False
67        >>> is_allowed_version('3.3', '>3.2, <4.0')
68        True
69    """
70    return Version(version) in SpecifierSet(spec)
71
72
73# set up the necessary directives
74
75class TestDirective(SphinxDirective):
76    """
77    Base class for doctest-related directives.
78    """
79
80    has_content = True
81    required_arguments = 0
82    optional_arguments = 1
83    final_argument_whitespace = True
84
85    def run(self) -> List[Node]:
86        # use ordinary docutils nodes for test code: they get special attributes
87        # so that our builder recognizes them, and the other builders are happy.
88        code = '\n'.join(self.content)
89        test = None
90        if self.name == 'doctest':
91            if '<BLANKLINE>' in code:
92                # convert <BLANKLINE>s to ordinary blank lines for presentation
93                test = code
94                code = blankline_re.sub('', code)
95            if doctestopt_re.search(code) and 'no-trim-doctest-flags' not in self.options:
96                if not test:
97                    test = code
98                code = doctestopt_re.sub('', code)
99        nodetype = nodes.literal_block  # type: Type[TextElement]
100        if self.name in ('testsetup', 'testcleanup') or 'hide' in self.options:
101            nodetype = nodes.comment
102        if self.arguments:
103            groups = [x.strip() for x in self.arguments[0].split(',')]
104        else:
105            groups = ['default']
106        node = nodetype(code, code, testnodetype=self.name, groups=groups)
107        self.set_source_info(node)
108        if test is not None:
109            # only save if it differs from code
110            node['test'] = test
111        if self.name == 'doctest':
112            if self.config.highlight_language in ('py', 'python'):
113                node['language'] = 'pycon'
114            else:
115                node['language'] = 'pycon3'  # default
116        elif self.name == 'testcode':
117            if self.config.highlight_language in ('py', 'python'):
118                node['language'] = 'python'
119            else:
120                node['language'] = 'python3'  # default
121        elif self.name == 'testoutput':
122            # don't try to highlight output
123            node['language'] = 'none'
124        node['options'] = {}
125        if self.name in ('doctest', 'testoutput') and 'options' in self.options:
126            # parse doctest-like output comparison flags
127            option_strings = self.options['options'].replace(',', ' ').split()
128            for option in option_strings:
129                prefix, option_name = option[0], option[1:]
130                if prefix not in '+-':
131                    self.state.document.reporter.warning(
132                        __("missing '+' or '-' in '%s' option.") % option,
133                        line=self.lineno)
134                    continue
135                if option_name not in doctest.OPTIONFLAGS_BY_NAME:
136                    self.state.document.reporter.warning(
137                        __("'%s' is not a valid option.") % option_name,
138                        line=self.lineno)
139                    continue
140                flag = doctest.OPTIONFLAGS_BY_NAME[option[1:]]
141                node['options'][flag] = (option[0] == '+')
142        if self.name == 'doctest' and 'pyversion' in self.options:
143            try:
144                spec = self.options['pyversion']
145                python_version = '.'.join([str(v) for v in sys.version_info[:3]])
146                if not is_allowed_version(spec, python_version):
147                    flag = doctest.OPTIONFLAGS_BY_NAME['SKIP']
148                    node['options'][flag] = True  # Skip the test
149            except InvalidSpecifier:
150                self.state.document.reporter.warning(
151                    __("'%s' is not a valid pyversion option") % spec,
152                    line=self.lineno)
153        if 'skipif' in self.options:
154            node['skipif'] = self.options['skipif']
155        if 'trim-doctest-flags' in self.options:
156            node['trim_flags'] = True
157        elif 'no-trim-doctest-flags' in self.options:
158            node['trim_flags'] = False
159        return [node]
160
161
162class TestsetupDirective(TestDirective):
163    option_spec = {'skipif': directives.unchanged_required}  # type: Dict
164
165
166class TestcleanupDirective(TestDirective):
167    option_spec = {'skipif': directives.unchanged_required}  # type: Dict
168
169
170class DoctestDirective(TestDirective):
171    option_spec = {
172        'hide': directives.flag,
173        'no-trim-doctest-flags': directives.flag,
174        'options': directives.unchanged,
175        'pyversion': directives.unchanged_required,
176        'skipif': directives.unchanged_required,
177        'trim-doctest-flags': directives.flag,
178    }
179
180
181class TestcodeDirective(TestDirective):
182    option_spec = {
183        'hide': directives.flag,
184        'no-trim-doctest-flags': directives.flag,
185        'pyversion': directives.unchanged_required,
186        'skipif': directives.unchanged_required,
187        'trim-doctest-flags': directives.flag,
188    }
189
190
191class TestoutputDirective(TestDirective):
192    option_spec = {
193        'hide': directives.flag,
194        'no-trim-doctest-flags': directives.flag,
195        'options': directives.unchanged,
196        'pyversion': directives.unchanged_required,
197        'skipif': directives.unchanged_required,
198        'trim-doctest-flags': directives.flag,
199    }
200
201
202parser = doctest.DocTestParser()
203
204
205# helper classes
206
207class TestGroup:
208    def __init__(self, name: str) -> None:
209        self.name = name
210        self.setup = []     # type: List[TestCode]
211        self.tests = []     # type: List[List[TestCode]]
212        self.cleanup = []   # type: List[TestCode]
213
214    def add_code(self, code: "TestCode", prepend: bool = False) -> None:
215        if code.type == 'testsetup':
216            if prepend:
217                self.setup.insert(0, code)
218            else:
219                self.setup.append(code)
220        elif code.type == 'testcleanup':
221            self.cleanup.append(code)
222        elif code.type == 'doctest':
223            self.tests.append([code])
224        elif code.type == 'testcode':
225            self.tests.append([code, None])
226        elif code.type == 'testoutput':
227            if self.tests and len(self.tests[-1]) == 2:
228                self.tests[-1][1] = code
229        else:
230            raise RuntimeError(__('invalid TestCode type'))
231
232    def __repr__(self) -> str:
233        return 'TestGroup(name=%r, setup=%r, cleanup=%r, tests=%r)' % (
234            self.name, self.setup, self.cleanup, self.tests)
235
236
237class TestCode:
238    def __init__(self, code: str, type: str, filename: str,
239                 lineno: int, options: Dict = None) -> None:
240        self.code = code
241        self.type = type
242        self.filename = filename
243        self.lineno = lineno
244        self.options = options or {}
245
246    def __repr__(self) -> str:
247        return 'TestCode(%r, %r, filename=%r, lineno=%r, options=%r)' % (
248            self.code, self.type, self.filename, self.lineno, self.options)
249
250
251class SphinxDocTestRunner(doctest.DocTestRunner):
252    def summarize(self, out: Callable, verbose: bool = None  # type: ignore
253                  ) -> Tuple[int, int]:
254        string_io = StringIO()
255        old_stdout = sys.stdout
256        sys.stdout = string_io
257        try:
258            res = super().summarize(verbose)
259        finally:
260            sys.stdout = old_stdout
261        out(string_io.getvalue())
262        return res
263
264    def _DocTestRunner__patched_linecache_getlines(self, filename: str,
265                                                   module_globals: Any = None) -> Any:
266        # this is overridden from DocTestRunner adding the try-except below
267        m = self._DocTestRunner__LINECACHE_FILENAME_RE.match(filename)  # type: ignore
268        if m and m.group('name') == self.test.name:
269            try:
270                example = self.test.examples[int(m.group('examplenum'))]
271            # because we compile multiple doctest blocks with the same name
272            # (viz. the group name) this might, for outer stack frames in a
273            # traceback, get the wrong test which might not have enough examples
274            except IndexError:
275                pass
276            else:
277                return example.source.splitlines(True)
278        return self.save_linecache_getlines(filename, module_globals)  # type: ignore
279
280
281# the new builder -- use sphinx-build.py -b doctest to run
282
283class DocTestBuilder(Builder):
284    """
285    Runs test snippets in the documentation.
286    """
287    name = 'doctest'
288    epilog = __('Testing of doctests in the sources finished, look at the '
289                'results in %(outdir)s/output.txt.')
290
291    def init(self) -> None:
292        # default options
293        self.opt = self.config.doctest_default_flags
294
295        # HACK HACK HACK
296        # doctest compiles its snippets with type 'single'. That is nice
297        # for doctest examples but unusable for multi-statement code such
298        # as setup code -- to be able to use doctest error reporting with
299        # that code nevertheless, we monkey-patch the "compile" it uses.
300        doctest.compile = self.compile  # type: ignore
301
302        sys.path[0:0] = self.config.doctest_path
303
304        self.type = 'single'
305
306        self.total_failures = 0
307        self.total_tries = 0
308        self.setup_failures = 0
309        self.setup_tries = 0
310        self.cleanup_failures = 0
311        self.cleanup_tries = 0
312
313        date = time.strftime('%Y-%m-%d %H:%M:%S')
314
315        self.outfile = open(path.join(self.outdir, 'output.txt'), 'w', encoding='utf-8')
316        self.outfile.write(('Results of doctest builder run on %s\n'
317                            '==================================%s\n') %
318                           (date, '=' * len(date)))
319
320    def _out(self, text: str) -> None:
321        logger.info(text, nonl=True)
322        self.outfile.write(text)
323
324    def _warn_out(self, text: str) -> None:
325        if self.app.quiet or self.app.warningiserror:
326            logger.warning(text)
327        else:
328            logger.info(text, nonl=True)
329        self.outfile.write(text)
330
331    def get_target_uri(self, docname: str, typ: str = None) -> str:
332        return ''
333
334    def get_outdated_docs(self) -> Set[str]:
335        return self.env.found_docs
336
337    def finish(self) -> None:
338        # write executive summary
339        def s(v: int) -> str:
340            return 's' if v != 1 else ''
341        repl = (self.total_tries, s(self.total_tries),
342                self.total_failures, s(self.total_failures),
343                self.setup_failures, s(self.setup_failures),
344                self.cleanup_failures, s(self.cleanup_failures))
345        self._out('''
346Doctest summary
347===============
348%5d test%s
349%5d failure%s in tests
350%5d failure%s in setup code
351%5d failure%s in cleanup code
352''' % repl)
353        self.outfile.close()
354
355        if self.total_failures or self.setup_failures or self.cleanup_failures:
356            self.app.statuscode = 1
357
358    def write(self, build_docnames: Iterable[str], updated_docnames: Sequence[str],
359              method: str = 'update') -> None:
360        if build_docnames is None:
361            build_docnames = sorted(self.env.all_docs)
362
363        logger.info(bold('running tests...'))
364        for docname in build_docnames:
365            # no need to resolve the doctree
366            doctree = self.env.get_doctree(docname)
367            self.test_doc(docname, doctree)
368
369    def get_filename_for_node(self, node: Node, docname: str) -> str:
370        """Try to get the file which actually contains the doctest, not the
371        filename of the document it's included in."""
372        try:
373            filename = relpath(node.source, self.env.srcdir)\
374                .rsplit(':docstring of ', maxsplit=1)[0]
375        except Exception:
376            filename = self.env.doc2path(docname, base=None)
377        return filename
378
379    @staticmethod
380    def get_line_number(node: Node) -> int:
381        """Get the real line number or admit we don't know."""
382        # TODO:  Work out how to store or calculate real (file-relative)
383        #       line numbers for doctest blocks in docstrings.
384        if ':docstring of ' in path.basename(node.source or ''):
385            # The line number is given relative to the stripped docstring,
386            # not the file.  This is correct where it is set, in
387            # `docutils.nodes.Node.setup_child`, but Sphinx should report
388            # relative to the file, not the docstring.
389            return None
390        if node.line is not None:
391            # TODO: find the root cause of this off by one error.
392            return node.line - 1
393        return None
394
395    def skipped(self, node: Element) -> bool:
396        if 'skipif' not in node:
397            return False
398        else:
399            condition = node['skipif']
400            context = {}  # type: Dict[str, Any]
401            if self.config.doctest_global_setup:
402                exec(self.config.doctest_global_setup, context)
403            should_skip = eval(condition, context)
404            if self.config.doctest_global_cleanup:
405                exec(self.config.doctest_global_cleanup, context)
406            return should_skip
407
408    def test_doc(self, docname: str, doctree: Node) -> None:
409        groups = {}  # type: Dict[str, TestGroup]
410        add_to_all_groups = []
411        self.setup_runner = SphinxDocTestRunner(verbose=False,
412                                                optionflags=self.opt)
413        self.test_runner = SphinxDocTestRunner(verbose=False,
414                                               optionflags=self.opt)
415        self.cleanup_runner = SphinxDocTestRunner(verbose=False,
416                                                  optionflags=self.opt)
417
418        self.test_runner._fakeout = self.setup_runner._fakeout  # type: ignore
419        self.cleanup_runner._fakeout = self.setup_runner._fakeout  # type: ignore
420
421        if self.config.doctest_test_doctest_blocks:
422            def condition(node: Node) -> bool:
423                return (isinstance(node, (nodes.literal_block, nodes.comment)) and
424                        'testnodetype' in node) or \
425                    isinstance(node, nodes.doctest_block)
426        else:
427            def condition(node: Node) -> bool:
428                return isinstance(node, (nodes.literal_block, nodes.comment)) \
429                    and 'testnodetype' in node
430        for node in doctree.traverse(condition):  # type: Element
431            if self.skipped(node):
432                continue
433
434            source = node['test'] if 'test' in node else node.astext()
435            filename = self.get_filename_for_node(node, docname)
436            line_number = self.get_line_number(node)
437            if not source:
438                logger.warning(__('no code/output in %s block at %s:%s'),
439                               node.get('testnodetype', 'doctest'),
440                               filename, line_number)
441            code = TestCode(source, type=node.get('testnodetype', 'doctest'),
442                            filename=filename, lineno=line_number,
443                            options=node.get('options'))
444            node_groups = node.get('groups', ['default'])
445            if '*' in node_groups:
446                add_to_all_groups.append(code)
447                continue
448            for groupname in node_groups:
449                if groupname not in groups:
450                    groups[groupname] = TestGroup(groupname)
451                groups[groupname].add_code(code)
452        for code in add_to_all_groups:
453            for group in groups.values():
454                group.add_code(code)
455        if self.config.doctest_global_setup:
456            code = TestCode(self.config.doctest_global_setup,
457                            'testsetup', filename=None, lineno=0)
458            for group in groups.values():
459                group.add_code(code, prepend=True)
460        if self.config.doctest_global_cleanup:
461            code = TestCode(self.config.doctest_global_cleanup,
462                            'testcleanup', filename=None, lineno=0)
463            for group in groups.values():
464                group.add_code(code)
465        if not groups:
466            return
467
468        self._out('\nDocument: %s\n----------%s\n' %
469                  (docname, '-' * len(docname)))
470        for group in groups.values():
471            self.test_group(group)
472        # Separately count results from setup code
473        res_f, res_t = self.setup_runner.summarize(self._out, verbose=False)
474        self.setup_failures += res_f
475        self.setup_tries += res_t
476        if self.test_runner.tries:
477            res_f, res_t = self.test_runner.summarize(self._out, verbose=True)
478            self.total_failures += res_f
479            self.total_tries += res_t
480        if self.cleanup_runner.tries:
481            res_f, res_t = self.cleanup_runner.summarize(self._out,
482                                                         verbose=True)
483            self.cleanup_failures += res_f
484            self.cleanup_tries += res_t
485
486    def compile(self, code: str, name: str, type: str, flags: Any, dont_inherit: bool) -> Any:
487        return compile(code, name, self.type, flags, dont_inherit)
488
489    def test_group(self, group: TestGroup) -> None:
490        ns = {}  # type: Dict
491
492        def run_setup_cleanup(runner: Any, testcodes: List[TestCode], what: Any) -> bool:
493            examples = []
494            for testcode in testcodes:
495                example = doctest.Example(testcode.code, '', lineno=testcode.lineno)
496                examples.append(example)
497            if not examples:
498                return True
499            # simulate a doctest with the code
500            sim_doctest = doctest.DocTest(examples, {},
501                                          '%s (%s code)' % (group.name, what),
502                                          testcodes[0].filename, 0, None)
503            sim_doctest.globs = ns
504            old_f = runner.failures
505            self.type = 'exec'  # the snippet may contain multiple statements
506            runner.run(sim_doctest, out=self._warn_out, clear_globs=False)
507            if runner.failures > old_f:
508                return False
509            return True
510
511        # run the setup code
512        if not run_setup_cleanup(self.setup_runner, group.setup, 'setup'):
513            # if setup failed, don't run the group
514            return
515
516        # run the tests
517        for code in group.tests:
518            if len(code) == 1:
519                # ordinary doctests (code/output interleaved)
520                try:
521                    test = parser.get_doctest(code[0].code, {}, group.name,
522                                              code[0].filename, code[0].lineno)
523                except Exception:
524                    logger.warning(__('ignoring invalid doctest code: %r'), code[0].code,
525                                   location=(code[0].filename, code[0].lineno))
526                    continue
527                if not test.examples:
528                    continue
529                for example in test.examples:
530                    # apply directive's comparison options
531                    new_opt = code[0].options.copy()
532                    new_opt.update(example.options)
533                    example.options = new_opt
534                self.type = 'single'  # as for ordinary doctests
535            else:
536                # testcode and output separate
537                output = code[1].code if code[1] else ''
538                options = code[1].options if code[1] else {}
539                # disable <BLANKLINE> processing as it is not needed
540                options[doctest.DONT_ACCEPT_BLANKLINE] = True
541                # find out if we're testing an exception
542                m = parser._EXCEPTION_RE.match(output)  # type: ignore
543                if m:
544                    exc_msg = m.group('msg')
545                else:
546                    exc_msg = None
547                example = doctest.Example(code[0].code, output, exc_msg=exc_msg,
548                                          lineno=code[0].lineno, options=options)
549                test = doctest.DocTest([example], {}, group.name,
550                                       code[0].filename, code[0].lineno, None)
551                self.type = 'exec'  # multiple statements again
552            # DocTest.__init__ copies the globs namespace, which we don't want
553            test.globs = ns
554            # also don't clear the globs namespace after running the doctest
555            self.test_runner.run(test, out=self._warn_out, clear_globs=False)
556
557        # run the cleanup
558        run_setup_cleanup(self.cleanup_runner, group.cleanup, 'cleanup')
559
560
561def setup(app: "Sphinx") -> Dict[str, Any]:
562    app.add_directive('testsetup', TestsetupDirective)
563    app.add_directive('testcleanup', TestcleanupDirective)
564    app.add_directive('doctest', DoctestDirective)
565    app.add_directive('testcode', TestcodeDirective)
566    app.add_directive('testoutput', TestoutputDirective)
567    app.add_builder(DocTestBuilder)
568    # this config value adds to sys.path
569    app.add_config_value('doctest_path', [], False)
570    app.add_config_value('doctest_test_doctest_blocks', 'default', False)
571    app.add_config_value('doctest_global_setup', '', False)
572    app.add_config_value('doctest_global_cleanup', '', False)
573    app.add_config_value(
574        'doctest_default_flags',
575        doctest.DONT_ACCEPT_TRUE_FOR_1 | doctest.ELLIPSIS | doctest.IGNORE_EXCEPTION_DETAIL,
576        False)
577    return {'version': sphinx.__display_version__, 'parallel_read_safe': True}
578