1# Module doctest. 2# Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org). 3# Major enhancements and refactoring by: 4# Jim Fulton 5# Edward Loper 6 7# Provided as-is; use at your own risk; no warranty; no promises; enjoy! 8 9r"""Module doctest -- a framework for running examples in docstrings. 10 11In simplest use, end each module M to be tested with: 12 13def _test(): 14 import doctest 15 doctest.testmod() 16 17if __name__ == "__main__": 18 _test() 19 20Then running the module as a script will cause the examples in the 21docstrings to get executed and verified: 22 23python M.py 24 25This won't display anything unless an example fails, in which case the 26failing example(s) and the cause(s) of the failure(s) are printed to stdout 27(why not stderr? because stderr is a lame hack <0.2 wink>), and the final 28line of output is "Test failed.". 29 30Run it with the -v switch instead: 31 32python M.py -v 33 34and a detailed report of all examples tried is printed to stdout, along 35with assorted summaries at the end. 36 37You can force verbose mode by passing "verbose=True" to testmod, or prohibit 38it by passing "verbose=False". In either of those cases, sys.argv is not 39examined by testmod. 40 41There are a variety of other ways to run doctests, including integration 42with the unittest framework, and support for running non-Python text 43files containing doctests. There are also many ways to override parts 44of doctest's default behaviors. See the Library Reference Manual for 45details. 46""" 47 48__docformat__ = 'reStructuredText en' 49 50__all__ = [ 51 # 0, Option Flags 52 'register_optionflag', 53 'DONT_ACCEPT_TRUE_FOR_1', 54 'DONT_ACCEPT_BLANKLINE', 55 'NORMALIZE_WHITESPACE', 56 'ELLIPSIS', 57 'SKIP', 58 'IGNORE_EXCEPTION_DETAIL', 59 'COMPARISON_FLAGS', 60 'REPORT_UDIFF', 61 'REPORT_CDIFF', 62 'REPORT_NDIFF', 63 'REPORT_ONLY_FIRST_FAILURE', 64 'REPORTING_FLAGS', 65 'FAIL_FAST', 66 # 1. Utility Functions 67 # 2. Example & DocTest 68 'Example', 69 'DocTest', 70 # 3. Doctest Parser 71 'DocTestParser', 72 # 4. Doctest Finder 73 'DocTestFinder', 74 # 5. Doctest Runner 75 'DocTestRunner', 76 'OutputChecker', 77 'DocTestFailure', 78 'UnexpectedException', 79 'DebugRunner', 80 # 6. Test Functions 81 'testmod', 82 'testfile', 83 'run_docstring_examples', 84 # 7. Unittest Support 85 'DocTestSuite', 86 'DocFileSuite', 87 'set_unittest_reportflags', 88 # 8. Debugging Support 89 'script_from_examples', 90 'testsource', 91 'debug_src', 92 'debug', 93] 94 95import __future__ 96import difflib 97import inspect 98import linecache 99import os 100import pdb 101import re 102import sys 103import traceback 104import unittest 105from io import StringIO, IncrementalNewlineDecoder 106from collections import namedtuple 107 108TestResults = namedtuple('TestResults', 'failed attempted') 109 110# There are 4 basic classes: 111# - Example: a <source, want> pair, plus an intra-docstring line number. 112# - DocTest: a collection of examples, parsed from a docstring, plus 113# info about where the docstring came from (name, filename, lineno). 114# - DocTestFinder: extracts DocTests from a given object's docstring and 115# its contained objects' docstrings. 116# - DocTestRunner: runs DocTest cases, and accumulates statistics. 117# 118# So the basic picture is: 119# 120# list of: 121# +------+ +---------+ +-------+ 122# |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results| 123# +------+ +---------+ +-------+ 124# | Example | 125# | ... | 126# | Example | 127# +---------+ 128 129# Option constants. 130 131OPTIONFLAGS_BY_NAME = {} 132def register_optionflag(name): 133 # Create a new flag unless `name` is already known. 134 return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME)) 135 136DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1') 137DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE') 138NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE') 139ELLIPSIS = register_optionflag('ELLIPSIS') 140SKIP = register_optionflag('SKIP') 141IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL') 142 143COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 | 144 DONT_ACCEPT_BLANKLINE | 145 NORMALIZE_WHITESPACE | 146 ELLIPSIS | 147 SKIP | 148 IGNORE_EXCEPTION_DETAIL) 149 150REPORT_UDIFF = register_optionflag('REPORT_UDIFF') 151REPORT_CDIFF = register_optionflag('REPORT_CDIFF') 152REPORT_NDIFF = register_optionflag('REPORT_NDIFF') 153REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE') 154FAIL_FAST = register_optionflag('FAIL_FAST') 155 156REPORTING_FLAGS = (REPORT_UDIFF | 157 REPORT_CDIFF | 158 REPORT_NDIFF | 159 REPORT_ONLY_FIRST_FAILURE | 160 FAIL_FAST) 161 162# Special string markers for use in `want` strings: 163BLANKLINE_MARKER = '<BLANKLINE>' 164ELLIPSIS_MARKER = '...' 165 166###################################################################### 167## Table of Contents 168###################################################################### 169# 1. Utility Functions 170# 2. Example & DocTest -- store test cases 171# 3. DocTest Parser -- extracts examples from strings 172# 4. DocTest Finder -- extracts test cases from objects 173# 5. DocTest Runner -- runs test cases 174# 6. Test Functions -- convenient wrappers for testing 175# 7. Unittest Support 176# 8. Debugging Support 177# 9. Example Usage 178 179###################################################################### 180## 1. Utility Functions 181###################################################################### 182 183def _extract_future_flags(globs): 184 """ 185 Return the compiler-flags associated with the future features that 186 have been imported into the given namespace (globs). 187 """ 188 flags = 0 189 for fname in __future__.all_feature_names: 190 feature = globs.get(fname, None) 191 if feature is getattr(__future__, fname): 192 flags |= feature.compiler_flag 193 return flags 194 195def _normalize_module(module, depth=2): 196 """ 197 Return the module specified by `module`. In particular: 198 - If `module` is a module, then return module. 199 - If `module` is a string, then import and return the 200 module with that name. 201 - If `module` is None, then return the calling module. 202 The calling module is assumed to be the module of 203 the stack frame at the given depth in the call stack. 204 """ 205 if inspect.ismodule(module): 206 return module 207 elif isinstance(module, str): 208 return __import__(module, globals(), locals(), ["*"]) 209 elif module is None: 210 return sys.modules[sys._getframe(depth).f_globals['__name__']] 211 else: 212 raise TypeError("Expected a module, string, or None") 213 214def _newline_convert(data): 215 # The IO module provides a handy decoder for universal newline conversion 216 return IncrementalNewlineDecoder(None, True).decode(data, True) 217 218def _load_testfile(filename, package, module_relative, encoding): 219 if module_relative: 220 package = _normalize_module(package, 3) 221 filename = _module_relative_path(package, filename) 222 if (loader := getattr(package, '__loader__', None)) is None: 223 try: 224 loader = package.__spec__.loader 225 except AttributeError: 226 pass 227 if hasattr(loader, 'get_data'): 228 file_contents = loader.get_data(filename) 229 file_contents = file_contents.decode(encoding) 230 # get_data() opens files as 'rb', so one must do the equivalent 231 # conversion as universal newlines would do. 232 return _newline_convert(file_contents), filename 233 with open(filename, encoding=encoding) as f: 234 return f.read(), filename 235 236def _indent(s, indent=4): 237 """ 238 Add the given number of space characters to the beginning of 239 every non-blank line in `s`, and return the result. 240 """ 241 # This regexp matches the start of non-blank lines: 242 return re.sub('(?m)^(?!$)', indent*' ', s) 243 244def _exception_traceback(exc_info): 245 """ 246 Return a string containing a traceback message for the given 247 exc_info tuple (as returned by sys.exc_info()). 248 """ 249 # Get a traceback message. 250 excout = StringIO() 251 exc_type, exc_val, exc_tb = exc_info 252 traceback.print_exception(exc_type, exc_val, exc_tb, file=excout) 253 return excout.getvalue() 254 255# Override some StringIO methods. 256class _SpoofOut(StringIO): 257 def getvalue(self): 258 result = StringIO.getvalue(self) 259 # If anything at all was written, make sure there's a trailing 260 # newline. There's no way for the expected output to indicate 261 # that a trailing newline is missing. 262 if result and not result.endswith("\n"): 263 result += "\n" 264 return result 265 266 def truncate(self, size=None): 267 self.seek(size) 268 StringIO.truncate(self) 269 270# Worst-case linear-time ellipsis matching. 271def _ellipsis_match(want, got): 272 """ 273 Essentially the only subtle case: 274 >>> _ellipsis_match('aa...aa', 'aaa') 275 False 276 """ 277 if ELLIPSIS_MARKER not in want: 278 return want == got 279 280 # Find "the real" strings. 281 ws = want.split(ELLIPSIS_MARKER) 282 assert len(ws) >= 2 283 284 # Deal with exact matches possibly needed at one or both ends. 285 startpos, endpos = 0, len(got) 286 w = ws[0] 287 if w: # starts with exact match 288 if got.startswith(w): 289 startpos = len(w) 290 del ws[0] 291 else: 292 return False 293 w = ws[-1] 294 if w: # ends with exact match 295 if got.endswith(w): 296 endpos -= len(w) 297 del ws[-1] 298 else: 299 return False 300 301 if startpos > endpos: 302 # Exact end matches required more characters than we have, as in 303 # _ellipsis_match('aa...aa', 'aaa') 304 return False 305 306 # For the rest, we only need to find the leftmost non-overlapping 307 # match for each piece. If there's no overall match that way alone, 308 # there's no overall match period. 309 for w in ws: 310 # w may be '' at times, if there are consecutive ellipses, or 311 # due to an ellipsis at the start or end of `want`. That's OK. 312 # Search for an empty string succeeds, and doesn't change startpos. 313 startpos = got.find(w, startpos, endpos) 314 if startpos < 0: 315 return False 316 startpos += len(w) 317 318 return True 319 320def _comment_line(line): 321 "Return a commented form of the given line" 322 line = line.rstrip() 323 if line: 324 return '# '+line 325 else: 326 return '#' 327 328def _strip_exception_details(msg): 329 # Support for IGNORE_EXCEPTION_DETAIL. 330 # Get rid of everything except the exception name; in particular, drop 331 # the possibly dotted module path (if any) and the exception message (if 332 # any). We assume that a colon is never part of a dotted name, or of an 333 # exception name. 334 # E.g., given 335 # "foo.bar.MyError: la di da" 336 # return "MyError" 337 # Or for "abc.def" or "abc.def:\n" return "def". 338 339 start, end = 0, len(msg) 340 # The exception name must appear on the first line. 341 i = msg.find("\n") 342 if i >= 0: 343 end = i 344 # retain up to the first colon (if any) 345 i = msg.find(':', 0, end) 346 if i >= 0: 347 end = i 348 # retain just the exception name 349 i = msg.rfind('.', 0, end) 350 if i >= 0: 351 start = i+1 352 return msg[start: end] 353 354class _OutputRedirectingPdb(pdb.Pdb): 355 """ 356 A specialized version of the python debugger that redirects stdout 357 to a given stream when interacting with the user. Stdout is *not* 358 redirected when traced code is executed. 359 """ 360 def __init__(self, out): 361 self.__out = out 362 self.__debugger_used = False 363 # do not play signal games in the pdb 364 pdb.Pdb.__init__(self, stdout=out, nosigint=True) 365 # still use input() to get user input 366 self.use_rawinput = 1 367 368 def set_trace(self, frame=None): 369 self.__debugger_used = True 370 if frame is None: 371 frame = sys._getframe().f_back 372 pdb.Pdb.set_trace(self, frame) 373 374 def set_continue(self): 375 # Calling set_continue unconditionally would break unit test 376 # coverage reporting, as Bdb.set_continue calls sys.settrace(None). 377 if self.__debugger_used: 378 pdb.Pdb.set_continue(self) 379 380 def trace_dispatch(self, *args): 381 # Redirect stdout to the given stream. 382 save_stdout = sys.stdout 383 sys.stdout = self.__out 384 # Call Pdb's trace dispatch method. 385 try: 386 return pdb.Pdb.trace_dispatch(self, *args) 387 finally: 388 sys.stdout = save_stdout 389 390# [XX] Normalize with respect to os.path.pardir? 391def _module_relative_path(module, test_path): 392 if not inspect.ismodule(module): 393 raise TypeError('Expected a module: %r' % module) 394 if test_path.startswith('/'): 395 raise ValueError('Module-relative files may not have absolute paths') 396 397 # Normalize the path. On Windows, replace "/" with "\". 398 test_path = os.path.join(*(test_path.split('/'))) 399 400 # Find the base directory for the path. 401 if hasattr(module, '__file__'): 402 # A normal module/package 403 basedir = os.path.split(module.__file__)[0] 404 elif module.__name__ == '__main__': 405 # An interactive session. 406 if len(sys.argv)>0 and sys.argv[0] != '': 407 basedir = os.path.split(sys.argv[0])[0] 408 else: 409 basedir = os.curdir 410 else: 411 if hasattr(module, '__path__'): 412 for directory in module.__path__: 413 fullpath = os.path.join(directory, test_path) 414 if os.path.exists(fullpath): 415 return fullpath 416 417 # A module w/o __file__ (this includes builtins) 418 raise ValueError("Can't resolve paths relative to the module " 419 "%r (it has no __file__)" 420 % module.__name__) 421 422 # Combine the base directory and the test path. 423 return os.path.join(basedir, test_path) 424 425###################################################################### 426## 2. Example & DocTest 427###################################################################### 428## - An "example" is a <source, want> pair, where "source" is a 429## fragment of source code, and "want" is the expected output for 430## "source." The Example class also includes information about 431## where the example was extracted from. 432## 433## - A "doctest" is a collection of examples, typically extracted from 434## a string (such as an object's docstring). The DocTest class also 435## includes information about where the string was extracted from. 436 437class Example: 438 """ 439 A single doctest example, consisting of source code and expected 440 output. `Example` defines the following attributes: 441 442 - source: A single Python statement, always ending with a newline. 443 The constructor adds a newline if needed. 444 445 - want: The expected output from running the source code (either 446 from stdout, or a traceback in case of exception). `want` ends 447 with a newline unless it's empty, in which case it's an empty 448 string. The constructor adds a newline if needed. 449 450 - exc_msg: The exception message generated by the example, if 451 the example is expected to generate an exception; or `None` if 452 it is not expected to generate an exception. This exception 453 message is compared against the return value of 454 `traceback.format_exception_only()`. `exc_msg` ends with a 455 newline unless it's `None`. The constructor adds a newline 456 if needed. 457 458 - lineno: The line number within the DocTest string containing 459 this Example where the Example begins. This line number is 460 zero-based, with respect to the beginning of the DocTest. 461 462 - indent: The example's indentation in the DocTest string. 463 I.e., the number of space characters that precede the 464 example's first prompt. 465 466 - options: A dictionary mapping from option flags to True or 467 False, which is used to override default options for this 468 example. Any option flags not contained in this dictionary 469 are left at their default value (as specified by the 470 DocTestRunner's optionflags). By default, no options are set. 471 """ 472 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0, 473 options=None): 474 # Normalize inputs. 475 if not source.endswith('\n'): 476 source += '\n' 477 if want and not want.endswith('\n'): 478 want += '\n' 479 if exc_msg is not None and not exc_msg.endswith('\n'): 480 exc_msg += '\n' 481 # Store properties. 482 self.source = source 483 self.want = want 484 self.lineno = lineno 485 self.indent = indent 486 if options is None: options = {} 487 self.options = options 488 self.exc_msg = exc_msg 489 490 def __eq__(self, other): 491 if type(self) is not type(other): 492 return NotImplemented 493 494 return self.source == other.source and \ 495 self.want == other.want and \ 496 self.lineno == other.lineno and \ 497 self.indent == other.indent and \ 498 self.options == other.options and \ 499 self.exc_msg == other.exc_msg 500 501 def __hash__(self): 502 return hash((self.source, self.want, self.lineno, self.indent, 503 self.exc_msg)) 504 505class DocTest: 506 """ 507 A collection of doctest examples that should be run in a single 508 namespace. Each `DocTest` defines the following attributes: 509 510 - examples: the list of examples. 511 512 - globs: The namespace (aka globals) that the examples should 513 be run in. 514 515 - name: A name identifying the DocTest (typically, the name of 516 the object whose docstring this DocTest was extracted from). 517 518 - filename: The name of the file that this DocTest was extracted 519 from, or `None` if the filename is unknown. 520 521 - lineno: The line number within filename where this DocTest 522 begins, or `None` if the line number is unavailable. This 523 line number is zero-based, with respect to the beginning of 524 the file. 525 526 - docstring: The string that the examples were extracted from, 527 or `None` if the string is unavailable. 528 """ 529 def __init__(self, examples, globs, name, filename, lineno, docstring): 530 """ 531 Create a new DocTest containing the given examples. The 532 DocTest's globals are initialized with a copy of `globs`. 533 """ 534 assert not isinstance(examples, str), \ 535 "DocTest no longer accepts str; use DocTestParser instead" 536 self.examples = examples 537 self.docstring = docstring 538 self.globs = globs.copy() 539 self.name = name 540 self.filename = filename 541 self.lineno = lineno 542 543 def __repr__(self): 544 if len(self.examples) == 0: 545 examples = 'no examples' 546 elif len(self.examples) == 1: 547 examples = '1 example' 548 else: 549 examples = '%d examples' % len(self.examples) 550 return ('<%s %s from %s:%s (%s)>' % 551 (self.__class__.__name__, 552 self.name, self.filename, self.lineno, examples)) 553 554 def __eq__(self, other): 555 if type(self) is not type(other): 556 return NotImplemented 557 558 return self.examples == other.examples and \ 559 self.docstring == other.docstring and \ 560 self.globs == other.globs and \ 561 self.name == other.name and \ 562 self.filename == other.filename and \ 563 self.lineno == other.lineno 564 565 def __hash__(self): 566 return hash((self.docstring, self.name, self.filename, self.lineno)) 567 568 # This lets us sort tests by name: 569 def __lt__(self, other): 570 if not isinstance(other, DocTest): 571 return NotImplemented 572 return ((self.name, self.filename, self.lineno, id(self)) 573 < 574 (other.name, other.filename, other.lineno, id(other))) 575 576###################################################################### 577## 3. DocTestParser 578###################################################################### 579 580class DocTestParser: 581 """ 582 A class used to parse strings containing doctest examples. 583 """ 584 # This regular expression is used to find doctest examples in a 585 # string. It defines three groups: `source` is the source code 586 # (including leading indentation and prompts); `indent` is the 587 # indentation of the first (PS1) line of the source code; and 588 # `want` is the expected output (including leading indentation). 589 _EXAMPLE_RE = re.compile(r''' 590 # Source consists of a PS1 line followed by zero or more PS2 lines. 591 (?P<source> 592 (?:^(?P<indent> [ ]*) >>> .*) # PS1 line 593 (?:\n [ ]* \.\.\. .*)*) # PS2 lines 594 \n? 595 # Want consists of any non-blank lines that do not start with PS1. 596 (?P<want> (?:(?![ ]*$) # Not a blank line 597 (?![ ]*>>>) # Not a line starting with PS1 598 .+$\n? # But any other line 599 )*) 600 ''', re.MULTILINE | re.VERBOSE) 601 602 # A regular expression for handling `want` strings that contain 603 # expected exceptions. It divides `want` into three pieces: 604 # - the traceback header line (`hdr`) 605 # - the traceback stack (`stack`) 606 # - the exception message (`msg`), as generated by 607 # traceback.format_exception_only() 608 # `msg` may have multiple lines. We assume/require that the 609 # exception message is the first non-indented line starting with a word 610 # character following the traceback header line. 611 _EXCEPTION_RE = re.compile(r""" 612 # Grab the traceback header. Different versions of Python have 613 # said different things on the first traceback line. 614 ^(?P<hdr> Traceback\ \( 615 (?: most\ recent\ call\ last 616 | innermost\ last 617 ) \) : 618 ) 619 \s* $ # toss trailing whitespace on the header. 620 (?P<stack> .*?) # don't blink: absorb stuff until... 621 ^ (?P<msg> \w+ .*) # a line *starts* with alphanum. 622 """, re.VERBOSE | re.MULTILINE | re.DOTALL) 623 624 # A callable returning a true value iff its argument is a blank line 625 # or contains a single comment. 626 _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match 627 628 def parse(self, string, name='<string>'): 629 """ 630 Divide the given string into examples and intervening text, 631 and return them as a list of alternating Examples and strings. 632 Line numbers for the Examples are 0-based. The optional 633 argument `name` is a name identifying this string, and is only 634 used for error messages. 635 """ 636 string = string.expandtabs() 637 # If all lines begin with the same indentation, then strip it. 638 min_indent = self._min_indent(string) 639 if min_indent > 0: 640 string = '\n'.join([l[min_indent:] for l in string.split('\n')]) 641 642 output = [] 643 charno, lineno = 0, 0 644 # Find all doctest examples in the string: 645 for m in self._EXAMPLE_RE.finditer(string): 646 # Add the pre-example text to `output`. 647 output.append(string[charno:m.start()]) 648 # Update lineno (lines before this example) 649 lineno += string.count('\n', charno, m.start()) 650 # Extract info from the regexp match. 651 (source, options, want, exc_msg) = \ 652 self._parse_example(m, name, lineno) 653 # Create an Example, and add it to the list. 654 if not self._IS_BLANK_OR_COMMENT(source): 655 output.append( Example(source, want, exc_msg, 656 lineno=lineno, 657 indent=min_indent+len(m.group('indent')), 658 options=options) ) 659 # Update lineno (lines inside this example) 660 lineno += string.count('\n', m.start(), m.end()) 661 # Update charno. 662 charno = m.end() 663 # Add any remaining post-example text to `output`. 664 output.append(string[charno:]) 665 return output 666 667 def get_doctest(self, string, globs, name, filename, lineno): 668 """ 669 Extract all doctest examples from the given string, and 670 collect them into a `DocTest` object. 671 672 `globs`, `name`, `filename`, and `lineno` are attributes for 673 the new `DocTest` object. See the documentation for `DocTest` 674 for more information. 675 """ 676 return DocTest(self.get_examples(string, name), globs, 677 name, filename, lineno, string) 678 679 def get_examples(self, string, name='<string>'): 680 """ 681 Extract all doctest examples from the given string, and return 682 them as a list of `Example` objects. Line numbers are 683 0-based, because it's most common in doctests that nothing 684 interesting appears on the same line as opening triple-quote, 685 and so the first interesting line is called \"line 1\" then. 686 687 The optional argument `name` is a name identifying this 688 string, and is only used for error messages. 689 """ 690 return [x for x in self.parse(string, name) 691 if isinstance(x, Example)] 692 693 def _parse_example(self, m, name, lineno): 694 """ 695 Given a regular expression match from `_EXAMPLE_RE` (`m`), 696 return a pair `(source, want)`, where `source` is the matched 697 example's source code (with prompts and indentation stripped); 698 and `want` is the example's expected output (with indentation 699 stripped). 700 701 `name` is the string's name, and `lineno` is the line number 702 where the example starts; both are used for error messages. 703 """ 704 # Get the example's indentation level. 705 indent = len(m.group('indent')) 706 707 # Divide source into lines; check that they're properly 708 # indented; and then strip their indentation & prompts. 709 source_lines = m.group('source').split('\n') 710 self._check_prompt_blank(source_lines, indent, name, lineno) 711 self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno) 712 source = '\n'.join([sl[indent+4:] for sl in source_lines]) 713 714 # Divide want into lines; check that it's properly indented; and 715 # then strip the indentation. Spaces before the last newline should 716 # be preserved, so plain rstrip() isn't good enough. 717 want = m.group('want') 718 want_lines = want.split('\n') 719 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): 720 del want_lines[-1] # forget final newline & spaces after it 721 self._check_prefix(want_lines, ' '*indent, name, 722 lineno + len(source_lines)) 723 want = '\n'.join([wl[indent:] for wl in want_lines]) 724 725 # If `want` contains a traceback message, then extract it. 726 m = self._EXCEPTION_RE.match(want) 727 if m: 728 exc_msg = m.group('msg') 729 else: 730 exc_msg = None 731 732 # Extract options from the source. 733 options = self._find_options(source, name, lineno) 734 735 return source, options, want, exc_msg 736 737 # This regular expression looks for option directives in the 738 # source code of an example. Option directives are comments 739 # starting with "doctest:". Warning: this may give false 740 # positives for string-literals that contain the string 741 # "#doctest:". Eliminating these false positives would require 742 # actually parsing the string; but we limit them by ignoring any 743 # line containing "#doctest:" that is *followed* by a quote mark. 744 _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$', 745 re.MULTILINE) 746 747 def _find_options(self, source, name, lineno): 748 """ 749 Return a dictionary containing option overrides extracted from 750 option directives in the given source string. 751 752 `name` is the string's name, and `lineno` is the line number 753 where the example starts; both are used for error messages. 754 """ 755 options = {} 756 # (note: with the current regexp, this will match at most once:) 757 for m in self._OPTION_DIRECTIVE_RE.finditer(source): 758 option_strings = m.group(1).replace(',', ' ').split() 759 for option in option_strings: 760 if (option[0] not in '+-' or 761 option[1:] not in OPTIONFLAGS_BY_NAME): 762 raise ValueError('line %r of the doctest for %s ' 763 'has an invalid option: %r' % 764 (lineno+1, name, option)) 765 flag = OPTIONFLAGS_BY_NAME[option[1:]] 766 options[flag] = (option[0] == '+') 767 if options and self._IS_BLANK_OR_COMMENT(source): 768 raise ValueError('line %r of the doctest for %s has an option ' 769 'directive on a line with no example: %r' % 770 (lineno, name, source)) 771 return options 772 773 # This regular expression finds the indentation of every non-blank 774 # line in a string. 775 _INDENT_RE = re.compile(r'^([ ]*)(?=\S)', re.MULTILINE) 776 777 def _min_indent(self, s): 778 "Return the minimum indentation of any non-blank line in `s`" 779 indents = [len(indent) for indent in self._INDENT_RE.findall(s)] 780 if len(indents) > 0: 781 return min(indents) 782 else: 783 return 0 784 785 def _check_prompt_blank(self, lines, indent, name, lineno): 786 """ 787 Given the lines of a source string (including prompts and 788 leading indentation), check to make sure that every prompt is 789 followed by a space character. If any line is not followed by 790 a space character, then raise ValueError. 791 """ 792 for i, line in enumerate(lines): 793 if len(line) >= indent+4 and line[indent+3] != ' ': 794 raise ValueError('line %r of the docstring for %s ' 795 'lacks blank after %s: %r' % 796 (lineno+i+1, name, 797 line[indent:indent+3], line)) 798 799 def _check_prefix(self, lines, prefix, name, lineno): 800 """ 801 Check that every line in the given list starts with the given 802 prefix; if any line does not, then raise a ValueError. 803 """ 804 for i, line in enumerate(lines): 805 if line and not line.startswith(prefix): 806 raise ValueError('line %r of the docstring for %s has ' 807 'inconsistent leading whitespace: %r' % 808 (lineno+i+1, name, line)) 809 810 811###################################################################### 812## 4. DocTest Finder 813###################################################################### 814 815class DocTestFinder: 816 """ 817 A class used to extract the DocTests that are relevant to a given 818 object, from its docstring and the docstrings of its contained 819 objects. Doctests can currently be extracted from the following 820 object types: modules, functions, classes, methods, staticmethods, 821 classmethods, and properties. 822 """ 823 824 def __init__(self, verbose=False, parser=DocTestParser(), 825 recurse=True, exclude_empty=True): 826 """ 827 Create a new doctest finder. 828 829 The optional argument `parser` specifies a class or 830 function that should be used to create new DocTest objects (or 831 objects that implement the same interface as DocTest). The 832 signature for this factory function should match the signature 833 of the DocTest constructor. 834 835 If the optional argument `recurse` is false, then `find` will 836 only examine the given object, and not any contained objects. 837 838 If the optional argument `exclude_empty` is false, then `find` 839 will include tests for objects with empty docstrings. 840 """ 841 self._parser = parser 842 self._verbose = verbose 843 self._recurse = recurse 844 self._exclude_empty = exclude_empty 845 846 def find(self, obj, name=None, module=None, globs=None, extraglobs=None): 847 """ 848 Return a list of the DocTests that are defined by the given 849 object's docstring, or by any of its contained objects' 850 docstrings. 851 852 The optional parameter `module` is the module that contains 853 the given object. If the module is not specified or is None, then 854 the test finder will attempt to automatically determine the 855 correct module. The object's module is used: 856 857 - As a default namespace, if `globs` is not specified. 858 - To prevent the DocTestFinder from extracting DocTests 859 from objects that are imported from other modules. 860 - To find the name of the file containing the object. 861 - To help find the line number of the object within its 862 file. 863 864 Contained objects whose module does not match `module` are ignored. 865 866 If `module` is False, no attempt to find the module will be made. 867 This is obscure, of use mostly in tests: if `module` is False, or 868 is None but cannot be found automatically, then all objects are 869 considered to belong to the (non-existent) module, so all contained 870 objects will (recursively) be searched for doctests. 871 872 The globals for each DocTest is formed by combining `globs` 873 and `extraglobs` (bindings in `extraglobs` override bindings 874 in `globs`). A new copy of the globals dictionary is created 875 for each DocTest. If `globs` is not specified, then it 876 defaults to the module's `__dict__`, if specified, or {} 877 otherwise. If `extraglobs` is not specified, then it defaults 878 to {}. 879 880 """ 881 # If name was not specified, then extract it from the object. 882 if name is None: 883 name = getattr(obj, '__name__', None) 884 if name is None: 885 raise ValueError("DocTestFinder.find: name must be given " 886 "when obj.__name__ doesn't exist: %r" % 887 (type(obj),)) 888 889 # Find the module that contains the given object (if obj is 890 # a module, then module=obj.). Note: this may fail, in which 891 # case module will be None. 892 if module is False: 893 module = None 894 elif module is None: 895 module = inspect.getmodule(obj) 896 897 # Read the module's source code. This is used by 898 # DocTestFinder._find_lineno to find the line number for a 899 # given object's docstring. 900 try: 901 file = inspect.getsourcefile(obj) 902 except TypeError: 903 source_lines = None 904 else: 905 if not file: 906 # Check to see if it's one of our special internal "files" 907 # (see __patched_linecache_getlines). 908 file = inspect.getfile(obj) 909 if not file[0]+file[-2:] == '<]>': file = None 910 if file is None: 911 source_lines = None 912 else: 913 if module is not None: 914 # Supply the module globals in case the module was 915 # originally loaded via a PEP 302 loader and 916 # file is not a valid filesystem path 917 source_lines = linecache.getlines(file, module.__dict__) 918 else: 919 # No access to a loader, so assume it's a normal 920 # filesystem path 921 source_lines = linecache.getlines(file) 922 if not source_lines: 923 source_lines = None 924 925 # Initialize globals, and merge in extraglobs. 926 if globs is None: 927 if module is None: 928 globs = {} 929 else: 930 globs = module.__dict__.copy() 931 else: 932 globs = globs.copy() 933 if extraglobs is not None: 934 globs.update(extraglobs) 935 if '__name__' not in globs: 936 globs['__name__'] = '__main__' # provide a default module name 937 938 # Recursively explore `obj`, extracting DocTests. 939 tests = [] 940 self._find(tests, obj, name, module, source_lines, globs, {}) 941 # Sort the tests by alpha order of names, for consistency in 942 # verbose-mode output. This was a feature of doctest in Pythons 943 # <= 2.3 that got lost by accident in 2.4. It was repaired in 944 # 2.4.4 and 2.5. 945 tests.sort() 946 return tests 947 948 def _from_module(self, module, object): 949 """ 950 Return true if the given object is defined in the given 951 module. 952 """ 953 if module is None: 954 return True 955 elif inspect.getmodule(object) is not None: 956 return module is inspect.getmodule(object) 957 elif inspect.isfunction(object): 958 return module.__dict__ is object.__globals__ 959 elif inspect.ismethoddescriptor(object): 960 if hasattr(object, '__objclass__'): 961 obj_mod = object.__objclass__.__module__ 962 elif hasattr(object, '__module__'): 963 obj_mod = object.__module__ 964 else: 965 return True # [XX] no easy way to tell otherwise 966 return module.__name__ == obj_mod 967 elif inspect.isclass(object): 968 return module.__name__ == object.__module__ 969 elif hasattr(object, '__module__'): 970 return module.__name__ == object.__module__ 971 elif isinstance(object, property): 972 return True # [XX] no way not be sure. 973 else: 974 raise ValueError("object must be a class or function") 975 976 def _is_routine(self, obj): 977 """ 978 Safely unwrap objects and determine if they are functions. 979 """ 980 maybe_routine = obj 981 try: 982 maybe_routine = inspect.unwrap(maybe_routine) 983 except ValueError: 984 pass 985 return inspect.isroutine(maybe_routine) 986 987 def _find(self, tests, obj, name, module, source_lines, globs, seen): 988 """ 989 Find tests for the given object and any contained objects, and 990 add them to `tests`. 991 """ 992 if self._verbose: 993 print('Finding tests in %s' % name) 994 995 # If we've already processed this object, then ignore it. 996 if id(obj) in seen: 997 return 998 seen[id(obj)] = 1 999 1000 # Find a test for this object, and add it to the list of tests. 1001 test = self._get_test(obj, name, module, globs, source_lines) 1002 if test is not None: 1003 tests.append(test) 1004 1005 # Look for tests in a module's contained objects. 1006 if inspect.ismodule(obj) and self._recurse: 1007 for valname, val in obj.__dict__.items(): 1008 valname = '%s.%s' % (name, valname) 1009 1010 # Recurse to functions & classes. 1011 if ((self._is_routine(val) or inspect.isclass(val)) and 1012 self._from_module(module, val)): 1013 self._find(tests, val, valname, module, source_lines, 1014 globs, seen) 1015 1016 # Look for tests in a module's __test__ dictionary. 1017 if inspect.ismodule(obj) and self._recurse: 1018 for valname, val in getattr(obj, '__test__', {}).items(): 1019 if not isinstance(valname, str): 1020 raise ValueError("DocTestFinder.find: __test__ keys " 1021 "must be strings: %r" % 1022 (type(valname),)) 1023 if not (inspect.isroutine(val) or inspect.isclass(val) or 1024 inspect.ismodule(val) or isinstance(val, str)): 1025 raise ValueError("DocTestFinder.find: __test__ values " 1026 "must be strings, functions, methods, " 1027 "classes, or modules: %r" % 1028 (type(val),)) 1029 valname = '%s.__test__.%s' % (name, valname) 1030 self._find(tests, val, valname, module, source_lines, 1031 globs, seen) 1032 1033 # Look for tests in a class's contained objects. 1034 if inspect.isclass(obj) and self._recurse: 1035 for valname, val in obj.__dict__.items(): 1036 # Special handling for staticmethod/classmethod. 1037 if isinstance(val, (staticmethod, classmethod)): 1038 val = val.__func__ 1039 1040 # Recurse to methods, properties, and nested classes. 1041 if ((inspect.isroutine(val) or inspect.isclass(val) or 1042 isinstance(val, property)) and 1043 self._from_module(module, val)): 1044 valname = '%s.%s' % (name, valname) 1045 self._find(tests, val, valname, module, source_lines, 1046 globs, seen) 1047 1048 def _get_test(self, obj, name, module, globs, source_lines): 1049 """ 1050 Return a DocTest for the given object, if it defines a docstring; 1051 otherwise, return None. 1052 """ 1053 # Extract the object's docstring. If it doesn't have one, 1054 # then return None (no test for this object). 1055 if isinstance(obj, str): 1056 docstring = obj 1057 else: 1058 try: 1059 if obj.__doc__ is None: 1060 docstring = '' 1061 else: 1062 docstring = obj.__doc__ 1063 if not isinstance(docstring, str): 1064 docstring = str(docstring) 1065 except (TypeError, AttributeError): 1066 docstring = '' 1067 1068 # Find the docstring's location in the file. 1069 lineno = self._find_lineno(obj, source_lines) 1070 1071 # Don't bother if the docstring is empty. 1072 if self._exclude_empty and not docstring: 1073 return None 1074 1075 # Return a DocTest for this object. 1076 if module is None: 1077 filename = None 1078 else: 1079 # __file__ can be None for namespace packages. 1080 filename = getattr(module, '__file__', None) or module.__name__ 1081 if filename[-4:] == ".pyc": 1082 filename = filename[:-1] 1083 return self._parser.get_doctest(docstring, globs, name, 1084 filename, lineno) 1085 1086 def _find_lineno(self, obj, source_lines): 1087 """ 1088 Return a line number of the given object's docstring. Note: 1089 this method assumes that the object has a docstring. 1090 """ 1091 lineno = None 1092 1093 # Find the line number for modules. 1094 if inspect.ismodule(obj): 1095 lineno = 0 1096 1097 # Find the line number for classes. 1098 # Note: this could be fooled if a class is defined multiple 1099 # times in a single file. 1100 if inspect.isclass(obj): 1101 if source_lines is None: 1102 return None 1103 pat = re.compile(r'^\s*class\s*%s\b' % 1104 getattr(obj, '__name__', '-')) 1105 for i, line in enumerate(source_lines): 1106 if pat.match(line): 1107 lineno = i 1108 break 1109 1110 # Find the line number for functions & methods. 1111 if inspect.ismethod(obj): obj = obj.__func__ 1112 if inspect.isfunction(obj): obj = obj.__code__ 1113 if inspect.istraceback(obj): obj = obj.tb_frame 1114 if inspect.isframe(obj): obj = obj.f_code 1115 if inspect.iscode(obj): 1116 lineno = getattr(obj, 'co_firstlineno', None)-1 1117 1118 # Find the line number where the docstring starts. Assume 1119 # that it's the first line that begins with a quote mark. 1120 # Note: this could be fooled by a multiline function 1121 # signature, where a continuation line begins with a quote 1122 # mark. 1123 if lineno is not None: 1124 if source_lines is None: 1125 return lineno+1 1126 pat = re.compile(r'(^|.*:)\s*\w*("|\')') 1127 for lineno in range(lineno, len(source_lines)): 1128 if pat.match(source_lines[lineno]): 1129 return lineno 1130 1131 # We couldn't find the line number. 1132 return None 1133 1134###################################################################### 1135## 5. DocTest Runner 1136###################################################################### 1137 1138class DocTestRunner: 1139 """ 1140 A class used to run DocTest test cases, and accumulate statistics. 1141 The `run` method is used to process a single DocTest case. It 1142 returns a tuple `(f, t)`, where `t` is the number of test cases 1143 tried, and `f` is the number of test cases that failed. 1144 1145 >>> tests = DocTestFinder().find(_TestClass) 1146 >>> runner = DocTestRunner(verbose=False) 1147 >>> tests.sort(key = lambda test: test.name) 1148 >>> for test in tests: 1149 ... print(test.name, '->', runner.run(test)) 1150 _TestClass -> TestResults(failed=0, attempted=2) 1151 _TestClass.__init__ -> TestResults(failed=0, attempted=2) 1152 _TestClass.get -> TestResults(failed=0, attempted=2) 1153 _TestClass.square -> TestResults(failed=0, attempted=1) 1154 1155 The `summarize` method prints a summary of all the test cases that 1156 have been run by the runner, and returns an aggregated `(f, t)` 1157 tuple: 1158 1159 >>> runner.summarize(verbose=1) 1160 4 items passed all tests: 1161 2 tests in _TestClass 1162 2 tests in _TestClass.__init__ 1163 2 tests in _TestClass.get 1164 1 tests in _TestClass.square 1165 7 tests in 4 items. 1166 7 passed and 0 failed. 1167 Test passed. 1168 TestResults(failed=0, attempted=7) 1169 1170 The aggregated number of tried examples and failed examples is 1171 also available via the `tries` and `failures` attributes: 1172 1173 >>> runner.tries 1174 7 1175 >>> runner.failures 1176 0 1177 1178 The comparison between expected outputs and actual outputs is done 1179 by an `OutputChecker`. This comparison may be customized with a 1180 number of option flags; see the documentation for `testmod` for 1181 more information. If the option flags are insufficient, then the 1182 comparison may also be customized by passing a subclass of 1183 `OutputChecker` to the constructor. 1184 1185 The test runner's display output can be controlled in two ways. 1186 First, an output function (`out) can be passed to 1187 `TestRunner.run`; this function will be called with strings that 1188 should be displayed. It defaults to `sys.stdout.write`. If 1189 capturing the output is not sufficient, then the display output 1190 can be also customized by subclassing DocTestRunner, and 1191 overriding the methods `report_start`, `report_success`, 1192 `report_unexpected_exception`, and `report_failure`. 1193 """ 1194 # This divider string is used to separate failure messages, and to 1195 # separate sections of the summary. 1196 DIVIDER = "*" * 70 1197 1198 def __init__(self, checker=None, verbose=None, optionflags=0): 1199 """ 1200 Create a new test runner. 1201 1202 Optional keyword arg `checker` is the `OutputChecker` that 1203 should be used to compare the expected outputs and actual 1204 outputs of doctest examples. 1205 1206 Optional keyword arg 'verbose' prints lots of stuff if true, 1207 only failures if false; by default, it's true iff '-v' is in 1208 sys.argv. 1209 1210 Optional argument `optionflags` can be used to control how the 1211 test runner compares expected output to actual output, and how 1212 it displays failures. See the documentation for `testmod` for 1213 more information. 1214 """ 1215 self._checker = checker or OutputChecker() 1216 if verbose is None: 1217 verbose = '-v' in sys.argv 1218 self._verbose = verbose 1219 self.optionflags = optionflags 1220 self.original_optionflags = optionflags 1221 1222 # Keep track of the examples we've run. 1223 self.tries = 0 1224 self.failures = 0 1225 self._name2ft = {} 1226 1227 # Create a fake output target for capturing doctest output. 1228 self._fakeout = _SpoofOut() 1229 1230 #///////////////////////////////////////////////////////////////// 1231 # Reporting methods 1232 #///////////////////////////////////////////////////////////////// 1233 1234 def report_start(self, out, test, example): 1235 """ 1236 Report that the test runner is about to process the given 1237 example. (Only displays a message if verbose=True) 1238 """ 1239 if self._verbose: 1240 if example.want: 1241 out('Trying:\n' + _indent(example.source) + 1242 'Expecting:\n' + _indent(example.want)) 1243 else: 1244 out('Trying:\n' + _indent(example.source) + 1245 'Expecting nothing\n') 1246 1247 def report_success(self, out, test, example, got): 1248 """ 1249 Report that the given example ran successfully. (Only 1250 displays a message if verbose=True) 1251 """ 1252 if self._verbose: 1253 out("ok\n") 1254 1255 def report_failure(self, out, test, example, got): 1256 """ 1257 Report that the given example failed. 1258 """ 1259 out(self._failure_header(test, example) + 1260 self._checker.output_difference(example, got, self.optionflags)) 1261 1262 def report_unexpected_exception(self, out, test, example, exc_info): 1263 """ 1264 Report that the given example raised an unexpected exception. 1265 """ 1266 out(self._failure_header(test, example) + 1267 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) 1268 1269 def _failure_header(self, test, example): 1270 out = [self.DIVIDER] 1271 if test.filename: 1272 if test.lineno is not None and example.lineno is not None: 1273 lineno = test.lineno + example.lineno + 1 1274 else: 1275 lineno = '?' 1276 out.append('File "%s", line %s, in %s' % 1277 (test.filename, lineno, test.name)) 1278 else: 1279 out.append('Line %s, in %s' % (example.lineno+1, test.name)) 1280 out.append('Failed example:') 1281 source = example.source 1282 out.append(_indent(source)) 1283 return '\n'.join(out) 1284 1285 #///////////////////////////////////////////////////////////////// 1286 # DocTest Running 1287 #///////////////////////////////////////////////////////////////// 1288 1289 def __run(self, test, compileflags, out): 1290 """ 1291 Run the examples in `test`. Write the outcome of each example 1292 with one of the `DocTestRunner.report_*` methods, using the 1293 writer function `out`. `compileflags` is the set of compiler 1294 flags that should be used to execute examples. Return a tuple 1295 `(f, t)`, where `t` is the number of examples tried, and `f` 1296 is the number of examples that failed. The examples are run 1297 in the namespace `test.globs`. 1298 """ 1299 # Keep track of the number of failures and tries. 1300 failures = tries = 0 1301 1302 # Save the option flags (since option directives can be used 1303 # to modify them). 1304 original_optionflags = self.optionflags 1305 1306 SUCCESS, FAILURE, BOOM = range(3) # `outcome` state 1307 1308 check = self._checker.check_output 1309 1310 # Process each example. 1311 for examplenum, example in enumerate(test.examples): 1312 1313 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress 1314 # reporting after the first failure. 1315 quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and 1316 failures > 0) 1317 1318 # Merge in the example's options. 1319 self.optionflags = original_optionflags 1320 if example.options: 1321 for (optionflag, val) in example.options.items(): 1322 if val: 1323 self.optionflags |= optionflag 1324 else: 1325 self.optionflags &= ~optionflag 1326 1327 # If 'SKIP' is set, then skip this example. 1328 if self.optionflags & SKIP: 1329 continue 1330 1331 # Record that we started this example. 1332 tries += 1 1333 if not quiet: 1334 self.report_start(out, test, example) 1335 1336 # Use a special filename for compile(), so we can retrieve 1337 # the source code during interactive debugging (see 1338 # __patched_linecache_getlines). 1339 filename = '<doctest %s[%d]>' % (test.name, examplenum) 1340 1341 # Run the example in the given context (globs), and record 1342 # any exception that gets raised. (But don't intercept 1343 # keyboard interrupts.) 1344 try: 1345 # Don't blink! This is where the user's code gets run. 1346 exec(compile(example.source, filename, "single", 1347 compileflags, True), test.globs) 1348 self.debugger.set_continue() # ==== Example Finished ==== 1349 exception = None 1350 except KeyboardInterrupt: 1351 raise 1352 except: 1353 exception = sys.exc_info() 1354 self.debugger.set_continue() # ==== Example Finished ==== 1355 1356 got = self._fakeout.getvalue() # the actual output 1357 self._fakeout.truncate(0) 1358 outcome = FAILURE # guilty until proved innocent or insane 1359 1360 # If the example executed without raising any exceptions, 1361 # verify its output. 1362 if exception is None: 1363 if check(example.want, got, self.optionflags): 1364 outcome = SUCCESS 1365 1366 # The example raised an exception: check if it was expected. 1367 else: 1368 exc_msg = traceback.format_exception_only(*exception[:2])[-1] 1369 if not quiet: 1370 got += _exception_traceback(exception) 1371 1372 # If `example.exc_msg` is None, then we weren't expecting 1373 # an exception. 1374 if example.exc_msg is None: 1375 outcome = BOOM 1376 1377 # We expected an exception: see whether it matches. 1378 elif check(example.exc_msg, exc_msg, self.optionflags): 1379 outcome = SUCCESS 1380 1381 # Another chance if they didn't care about the detail. 1382 elif self.optionflags & IGNORE_EXCEPTION_DETAIL: 1383 if check(_strip_exception_details(example.exc_msg), 1384 _strip_exception_details(exc_msg), 1385 self.optionflags): 1386 outcome = SUCCESS 1387 1388 # Report the outcome. 1389 if outcome is SUCCESS: 1390 if not quiet: 1391 self.report_success(out, test, example, got) 1392 elif outcome is FAILURE: 1393 if not quiet: 1394 self.report_failure(out, test, example, got) 1395 failures += 1 1396 elif outcome is BOOM: 1397 if not quiet: 1398 self.report_unexpected_exception(out, test, example, 1399 exception) 1400 failures += 1 1401 else: 1402 assert False, ("unknown outcome", outcome) 1403 1404 if failures and self.optionflags & FAIL_FAST: 1405 break 1406 1407 # Restore the option flags (in case they were modified) 1408 self.optionflags = original_optionflags 1409 1410 # Record and return the number of failures and tries. 1411 self.__record_outcome(test, failures, tries) 1412 return TestResults(failures, tries) 1413 1414 def __record_outcome(self, test, f, t): 1415 """ 1416 Record the fact that the given DocTest (`test`) generated `f` 1417 failures out of `t` tried examples. 1418 """ 1419 f2, t2 = self._name2ft.get(test.name, (0,0)) 1420 self._name2ft[test.name] = (f+f2, t+t2) 1421 self.failures += f 1422 self.tries += t 1423 1424 __LINECACHE_FILENAME_RE = re.compile(r'<doctest ' 1425 r'(?P<name>.+)' 1426 r'\[(?P<examplenum>\d+)\]>$') 1427 def __patched_linecache_getlines(self, filename, module_globals=None): 1428 m = self.__LINECACHE_FILENAME_RE.match(filename) 1429 if m and m.group('name') == self.test.name: 1430 example = self.test.examples[int(m.group('examplenum'))] 1431 return example.source.splitlines(keepends=True) 1432 else: 1433 return self.save_linecache_getlines(filename, module_globals) 1434 1435 def run(self, test, compileflags=None, out=None, clear_globs=True): 1436 """ 1437 Run the examples in `test`, and display the results using the 1438 writer function `out`. 1439 1440 The examples are run in the namespace `test.globs`. If 1441 `clear_globs` is true (the default), then this namespace will 1442 be cleared after the test runs, to help with garbage 1443 collection. If you would like to examine the namespace after 1444 the test completes, then use `clear_globs=False`. 1445 1446 `compileflags` gives the set of flags that should be used by 1447 the Python compiler when running the examples. If not 1448 specified, then it will default to the set of future-import 1449 flags that apply to `globs`. 1450 1451 The output of each example is checked using 1452 `DocTestRunner.check_output`, and the results are formatted by 1453 the `DocTestRunner.report_*` methods. 1454 """ 1455 self.test = test 1456 1457 if compileflags is None: 1458 compileflags = _extract_future_flags(test.globs) 1459 1460 save_stdout = sys.stdout 1461 if out is None: 1462 encoding = save_stdout.encoding 1463 if encoding is None or encoding.lower() == 'utf-8': 1464 out = save_stdout.write 1465 else: 1466 # Use backslashreplace error handling on write 1467 def out(s): 1468 s = str(s.encode(encoding, 'backslashreplace'), encoding) 1469 save_stdout.write(s) 1470 sys.stdout = self._fakeout 1471 1472 # Patch pdb.set_trace to restore sys.stdout during interactive 1473 # debugging (so it's not still redirected to self._fakeout). 1474 # Note that the interactive output will go to *our* 1475 # save_stdout, even if that's not the real sys.stdout; this 1476 # allows us to write test cases for the set_trace behavior. 1477 save_trace = sys.gettrace() 1478 save_set_trace = pdb.set_trace 1479 self.debugger = _OutputRedirectingPdb(save_stdout) 1480 self.debugger.reset() 1481 pdb.set_trace = self.debugger.set_trace 1482 1483 # Patch linecache.getlines, so we can see the example's source 1484 # when we're inside the debugger. 1485 self.save_linecache_getlines = linecache.getlines 1486 linecache.getlines = self.__patched_linecache_getlines 1487 1488 # Make sure sys.displayhook just prints the value to stdout 1489 save_displayhook = sys.displayhook 1490 sys.displayhook = sys.__displayhook__ 1491 1492 try: 1493 return self.__run(test, compileflags, out) 1494 finally: 1495 sys.stdout = save_stdout 1496 pdb.set_trace = save_set_trace 1497 sys.settrace(save_trace) 1498 linecache.getlines = self.save_linecache_getlines 1499 sys.displayhook = save_displayhook 1500 if clear_globs: 1501 test.globs.clear() 1502 import builtins 1503 builtins._ = None 1504 1505 #///////////////////////////////////////////////////////////////// 1506 # Summarization 1507 #///////////////////////////////////////////////////////////////// 1508 def summarize(self, verbose=None): 1509 """ 1510 Print a summary of all the test cases that have been run by 1511 this DocTestRunner, and return a tuple `(f, t)`, where `f` is 1512 the total number of failed examples, and `t` is the total 1513 number of tried examples. 1514 1515 The optional `verbose` argument controls how detailed the 1516 summary is. If the verbosity is not specified, then the 1517 DocTestRunner's verbosity is used. 1518 """ 1519 if verbose is None: 1520 verbose = self._verbose 1521 notests = [] 1522 passed = [] 1523 failed = [] 1524 totalt = totalf = 0 1525 for x in self._name2ft.items(): 1526 name, (f, t) = x 1527 assert f <= t 1528 totalt += t 1529 totalf += f 1530 if t == 0: 1531 notests.append(name) 1532 elif f == 0: 1533 passed.append( (name, t) ) 1534 else: 1535 failed.append(x) 1536 if verbose: 1537 if notests: 1538 print(len(notests), "items had no tests:") 1539 notests.sort() 1540 for thing in notests: 1541 print(" ", thing) 1542 if passed: 1543 print(len(passed), "items passed all tests:") 1544 passed.sort() 1545 for thing, count in passed: 1546 print(" %3d tests in %s" % (count, thing)) 1547 if failed: 1548 print(self.DIVIDER) 1549 print(len(failed), "items had failures:") 1550 failed.sort() 1551 for thing, (f, t) in failed: 1552 print(" %3d of %3d in %s" % (f, t, thing)) 1553 if verbose: 1554 print(totalt, "tests in", len(self._name2ft), "items.") 1555 print(totalt - totalf, "passed and", totalf, "failed.") 1556 if totalf: 1557 print("***Test Failed***", totalf, "failures.") 1558 elif verbose: 1559 print("Test passed.") 1560 return TestResults(totalf, totalt) 1561 1562 #///////////////////////////////////////////////////////////////// 1563 # Backward compatibility cruft to maintain doctest.master. 1564 #///////////////////////////////////////////////////////////////// 1565 def merge(self, other): 1566 d = self._name2ft 1567 for name, (f, t) in other._name2ft.items(): 1568 if name in d: 1569 # Don't print here by default, since doing 1570 # so breaks some of the buildbots 1571 #print("*** DocTestRunner.merge: '" + name + "' in both" \ 1572 # " testers; summing outcomes.") 1573 f2, t2 = d[name] 1574 f = f + f2 1575 t = t + t2 1576 d[name] = f, t 1577 1578class OutputChecker: 1579 """ 1580 A class used to check the whether the actual output from a doctest 1581 example matches the expected output. `OutputChecker` defines two 1582 methods: `check_output`, which compares a given pair of outputs, 1583 and returns true if they match; and `output_difference`, which 1584 returns a string describing the differences between two outputs. 1585 """ 1586 def _toAscii(self, s): 1587 """ 1588 Convert string to hex-escaped ASCII string. 1589 """ 1590 return str(s.encode('ASCII', 'backslashreplace'), "ASCII") 1591 1592 def check_output(self, want, got, optionflags): 1593 """ 1594 Return True iff the actual output from an example (`got`) 1595 matches the expected output (`want`). These strings are 1596 always considered to match if they are identical; but 1597 depending on what option flags the test runner is using, 1598 several non-exact match types are also possible. See the 1599 documentation for `TestRunner` for more information about 1600 option flags. 1601 """ 1602 1603 # If `want` contains hex-escaped character such as "\u1234", 1604 # then `want` is a string of six characters(e.g. [\,u,1,2,3,4]). 1605 # On the other hand, `got` could be another sequence of 1606 # characters such as [\u1234], so `want` and `got` should 1607 # be folded to hex-escaped ASCII string to compare. 1608 got = self._toAscii(got) 1609 want = self._toAscii(want) 1610 1611 # Handle the common case first, for efficiency: 1612 # if they're string-identical, always return true. 1613 if got == want: 1614 return True 1615 1616 # The values True and False replaced 1 and 0 as the return 1617 # value for boolean comparisons in Python 2.3. 1618 if not (optionflags & DONT_ACCEPT_TRUE_FOR_1): 1619 if (got,want) == ("True\n", "1\n"): 1620 return True 1621 if (got,want) == ("False\n", "0\n"): 1622 return True 1623 1624 # <BLANKLINE> can be used as a special sequence to signify a 1625 # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used. 1626 if not (optionflags & DONT_ACCEPT_BLANKLINE): 1627 # Replace <BLANKLINE> in want with a blank line. 1628 want = re.sub(r'(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER), 1629 '', want) 1630 # If a line in got contains only spaces, then remove the 1631 # spaces. 1632 got = re.sub(r'(?m)^[^\S\n]+$', '', got) 1633 if got == want: 1634 return True 1635 1636 # This flag causes doctest to ignore any differences in the 1637 # contents of whitespace strings. Note that this can be used 1638 # in conjunction with the ELLIPSIS flag. 1639 if optionflags & NORMALIZE_WHITESPACE: 1640 got = ' '.join(got.split()) 1641 want = ' '.join(want.split()) 1642 if got == want: 1643 return True 1644 1645 # The ELLIPSIS flag says to let the sequence "..." in `want` 1646 # match any substring in `got`. 1647 if optionflags & ELLIPSIS: 1648 if _ellipsis_match(want, got): 1649 return True 1650 1651 # We didn't find any match; return false. 1652 return False 1653 1654 # Should we do a fancy diff? 1655 def _do_a_fancy_diff(self, want, got, optionflags): 1656 # Not unless they asked for a fancy diff. 1657 if not optionflags & (REPORT_UDIFF | 1658 REPORT_CDIFF | 1659 REPORT_NDIFF): 1660 return False 1661 1662 # If expected output uses ellipsis, a meaningful fancy diff is 1663 # too hard ... or maybe not. In two real-life failures Tim saw, 1664 # a diff was a major help anyway, so this is commented out. 1665 # [todo] _ellipsis_match() knows which pieces do and don't match, 1666 # and could be the basis for a kick-ass diff in this case. 1667 ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want: 1668 ## return False 1669 1670 # ndiff does intraline difference marking, so can be useful even 1671 # for 1-line differences. 1672 if optionflags & REPORT_NDIFF: 1673 return True 1674 1675 # The other diff types need at least a few lines to be helpful. 1676 return want.count('\n') > 2 and got.count('\n') > 2 1677 1678 def output_difference(self, example, got, optionflags): 1679 """ 1680 Return a string describing the differences between the 1681 expected output for a given example (`example`) and the actual 1682 output (`got`). `optionflags` is the set of option flags used 1683 to compare `want` and `got`. 1684 """ 1685 want = example.want 1686 # If <BLANKLINE>s are being used, then replace blank lines 1687 # with <BLANKLINE> in the actual output string. 1688 if not (optionflags & DONT_ACCEPT_BLANKLINE): 1689 got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got) 1690 1691 # Check if we should use diff. 1692 if self._do_a_fancy_diff(want, got, optionflags): 1693 # Split want & got into lines. 1694 want_lines = want.splitlines(keepends=True) 1695 got_lines = got.splitlines(keepends=True) 1696 # Use difflib to find their differences. 1697 if optionflags & REPORT_UDIFF: 1698 diff = difflib.unified_diff(want_lines, got_lines, n=2) 1699 diff = list(diff)[2:] # strip the diff header 1700 kind = 'unified diff with -expected +actual' 1701 elif optionflags & REPORT_CDIFF: 1702 diff = difflib.context_diff(want_lines, got_lines, n=2) 1703 diff = list(diff)[2:] # strip the diff header 1704 kind = 'context diff with expected followed by actual' 1705 elif optionflags & REPORT_NDIFF: 1706 engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK) 1707 diff = list(engine.compare(want_lines, got_lines)) 1708 kind = 'ndiff with -expected +actual' 1709 else: 1710 assert 0, 'Bad diff option' 1711 return 'Differences (%s):\n' % kind + _indent(''.join(diff)) 1712 1713 # If we're not using diff, then simply list the expected 1714 # output followed by the actual output. 1715 if want and got: 1716 return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got)) 1717 elif want: 1718 return 'Expected:\n%sGot nothing\n' % _indent(want) 1719 elif got: 1720 return 'Expected nothing\nGot:\n%s' % _indent(got) 1721 else: 1722 return 'Expected nothing\nGot nothing\n' 1723 1724class DocTestFailure(Exception): 1725 """A DocTest example has failed in debugging mode. 1726 1727 The exception instance has variables: 1728 1729 - test: the DocTest object being run 1730 1731 - example: the Example object that failed 1732 1733 - got: the actual output 1734 """ 1735 def __init__(self, test, example, got): 1736 self.test = test 1737 self.example = example 1738 self.got = got 1739 1740 def __str__(self): 1741 return str(self.test) 1742 1743class UnexpectedException(Exception): 1744 """A DocTest example has encountered an unexpected exception 1745 1746 The exception instance has variables: 1747 1748 - test: the DocTest object being run 1749 1750 - example: the Example object that failed 1751 1752 - exc_info: the exception info 1753 """ 1754 def __init__(self, test, example, exc_info): 1755 self.test = test 1756 self.example = example 1757 self.exc_info = exc_info 1758 1759 def __str__(self): 1760 return str(self.test) 1761 1762class DebugRunner(DocTestRunner): 1763 r"""Run doc tests but raise an exception as soon as there is a failure. 1764 1765 If an unexpected exception occurs, an UnexpectedException is raised. 1766 It contains the test, the example, and the original exception: 1767 1768 >>> runner = DebugRunner(verbose=False) 1769 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', 1770 ... {}, 'foo', 'foo.py', 0) 1771 >>> try: 1772 ... runner.run(test) 1773 ... except UnexpectedException as f: 1774 ... failure = f 1775 1776 >>> failure.test is test 1777 True 1778 1779 >>> failure.example.want 1780 '42\n' 1781 1782 >>> exc_info = failure.exc_info 1783 >>> raise exc_info[1] # Already has the traceback 1784 Traceback (most recent call last): 1785 ... 1786 KeyError 1787 1788 We wrap the original exception to give the calling application 1789 access to the test and example information. 1790 1791 If the output doesn't match, then a DocTestFailure is raised: 1792 1793 >>> test = DocTestParser().get_doctest(''' 1794 ... >>> x = 1 1795 ... >>> x 1796 ... 2 1797 ... ''', {}, 'foo', 'foo.py', 0) 1798 1799 >>> try: 1800 ... runner.run(test) 1801 ... except DocTestFailure as f: 1802 ... failure = f 1803 1804 DocTestFailure objects provide access to the test: 1805 1806 >>> failure.test is test 1807 True 1808 1809 As well as to the example: 1810 1811 >>> failure.example.want 1812 '2\n' 1813 1814 and the actual output: 1815 1816 >>> failure.got 1817 '1\n' 1818 1819 If a failure or error occurs, the globals are left intact: 1820 1821 >>> del test.globs['__builtins__'] 1822 >>> test.globs 1823 {'x': 1} 1824 1825 >>> test = DocTestParser().get_doctest(''' 1826 ... >>> x = 2 1827 ... >>> raise KeyError 1828 ... ''', {}, 'foo', 'foo.py', 0) 1829 1830 >>> runner.run(test) 1831 Traceback (most recent call last): 1832 ... 1833 doctest.UnexpectedException: <DocTest foo from foo.py:0 (2 examples)> 1834 1835 >>> del test.globs['__builtins__'] 1836 >>> test.globs 1837 {'x': 2} 1838 1839 But the globals are cleared if there is no error: 1840 1841 >>> test = DocTestParser().get_doctest(''' 1842 ... >>> x = 2 1843 ... ''', {}, 'foo', 'foo.py', 0) 1844 1845 >>> runner.run(test) 1846 TestResults(failed=0, attempted=1) 1847 1848 >>> test.globs 1849 {} 1850 1851 """ 1852 1853 def run(self, test, compileflags=None, out=None, clear_globs=True): 1854 r = DocTestRunner.run(self, test, compileflags, out, False) 1855 if clear_globs: 1856 test.globs.clear() 1857 return r 1858 1859 def report_unexpected_exception(self, out, test, example, exc_info): 1860 raise UnexpectedException(test, example, exc_info) 1861 1862 def report_failure(self, out, test, example, got): 1863 raise DocTestFailure(test, example, got) 1864 1865###################################################################### 1866## 6. Test Functions 1867###################################################################### 1868# These should be backwards compatible. 1869 1870# For backward compatibility, a global instance of a DocTestRunner 1871# class, updated by testmod. 1872master = None 1873 1874def testmod(m=None, name=None, globs=None, verbose=None, 1875 report=True, optionflags=0, extraglobs=None, 1876 raise_on_error=False, exclude_empty=False): 1877 """m=None, name=None, globs=None, verbose=None, report=True, 1878 optionflags=0, extraglobs=None, raise_on_error=False, 1879 exclude_empty=False 1880 1881 Test examples in docstrings in functions and classes reachable 1882 from module m (or the current module if m is not supplied), starting 1883 with m.__doc__. 1884 1885 Also test examples reachable from dict m.__test__ if it exists and is 1886 not None. m.__test__ maps names to functions, classes and strings; 1887 function and class docstrings are tested even if the name is private; 1888 strings are tested directly, as if they were docstrings. 1889 1890 Return (#failures, #tests). 1891 1892 See help(doctest) for an overview. 1893 1894 Optional keyword arg "name" gives the name of the module; by default 1895 use m.__name__. 1896 1897 Optional keyword arg "globs" gives a dict to be used as the globals 1898 when executing examples; by default, use m.__dict__. A copy of this 1899 dict is actually used for each docstring, so that each docstring's 1900 examples start with a clean slate. 1901 1902 Optional keyword arg "extraglobs" gives a dictionary that should be 1903 merged into the globals that are used to execute examples. By 1904 default, no extra globals are used. This is new in 2.4. 1905 1906 Optional keyword arg "verbose" prints lots of stuff if true, prints 1907 only failures if false; by default, it's true iff "-v" is in sys.argv. 1908 1909 Optional keyword arg "report" prints a summary at the end when true, 1910 else prints nothing at the end. In verbose mode, the summary is 1911 detailed, else very brief (in fact, empty if all tests passed). 1912 1913 Optional keyword arg "optionflags" or's together module constants, 1914 and defaults to 0. This is new in 2.3. Possible values (see the 1915 docs for details): 1916 1917 DONT_ACCEPT_TRUE_FOR_1 1918 DONT_ACCEPT_BLANKLINE 1919 NORMALIZE_WHITESPACE 1920 ELLIPSIS 1921 SKIP 1922 IGNORE_EXCEPTION_DETAIL 1923 REPORT_UDIFF 1924 REPORT_CDIFF 1925 REPORT_NDIFF 1926 REPORT_ONLY_FIRST_FAILURE 1927 1928 Optional keyword arg "raise_on_error" raises an exception on the 1929 first unexpected exception or failure. This allows failures to be 1930 post-mortem debugged. 1931 1932 Advanced tomfoolery: testmod runs methods of a local instance of 1933 class doctest.Tester, then merges the results into (or creates) 1934 global Tester instance doctest.master. Methods of doctest.master 1935 can be called directly too, if you want to do something unusual. 1936 Passing report=0 to testmod is especially useful then, to delay 1937 displaying a summary. Invoke doctest.master.summarize(verbose) 1938 when you're done fiddling. 1939 """ 1940 global master 1941 1942 # If no module was given, then use __main__. 1943 if m is None: 1944 # DWA - m will still be None if this wasn't invoked from the command 1945 # line, in which case the following TypeError is about as good an error 1946 # as we should expect 1947 m = sys.modules.get('__main__') 1948 1949 # Check that we were actually given a module. 1950 if not inspect.ismodule(m): 1951 raise TypeError("testmod: module required; %r" % (m,)) 1952 1953 # If no name was given, then use the module's name. 1954 if name is None: 1955 name = m.__name__ 1956 1957 # Find, parse, and run all tests in the given module. 1958 finder = DocTestFinder(exclude_empty=exclude_empty) 1959 1960 if raise_on_error: 1961 runner = DebugRunner(verbose=verbose, optionflags=optionflags) 1962 else: 1963 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 1964 1965 for test in finder.find(m, name, globs=globs, extraglobs=extraglobs): 1966 runner.run(test) 1967 1968 if report: 1969 runner.summarize() 1970 1971 if master is None: 1972 master = runner 1973 else: 1974 master.merge(runner) 1975 1976 return TestResults(runner.failures, runner.tries) 1977 1978def testfile(filename, module_relative=True, name=None, package=None, 1979 globs=None, verbose=None, report=True, optionflags=0, 1980 extraglobs=None, raise_on_error=False, parser=DocTestParser(), 1981 encoding=None): 1982 """ 1983 Test examples in the given file. Return (#failures, #tests). 1984 1985 Optional keyword arg "module_relative" specifies how filenames 1986 should be interpreted: 1987 1988 - If "module_relative" is True (the default), then "filename" 1989 specifies a module-relative path. By default, this path is 1990 relative to the calling module's directory; but if the 1991 "package" argument is specified, then it is relative to that 1992 package. To ensure os-independence, "filename" should use 1993 "/" characters to separate path segments, and should not 1994 be an absolute path (i.e., it may not begin with "/"). 1995 1996 - If "module_relative" is False, then "filename" specifies an 1997 os-specific path. The path may be absolute or relative (to 1998 the current working directory). 1999 2000 Optional keyword arg "name" gives the name of the test; by default 2001 use the file's basename. 2002 2003 Optional keyword argument "package" is a Python package or the 2004 name of a Python package whose directory should be used as the 2005 base directory for a module relative filename. If no package is 2006 specified, then the calling module's directory is used as the base 2007 directory for module relative filenames. It is an error to 2008 specify "package" if "module_relative" is False. 2009 2010 Optional keyword arg "globs" gives a dict to be used as the globals 2011 when executing examples; by default, use {}. A copy of this dict 2012 is actually used for each docstring, so that each docstring's 2013 examples start with a clean slate. 2014 2015 Optional keyword arg "extraglobs" gives a dictionary that should be 2016 merged into the globals that are used to execute examples. By 2017 default, no extra globals are used. 2018 2019 Optional keyword arg "verbose" prints lots of stuff if true, prints 2020 only failures if false; by default, it's true iff "-v" is in sys.argv. 2021 2022 Optional keyword arg "report" prints a summary at the end when true, 2023 else prints nothing at the end. In verbose mode, the summary is 2024 detailed, else very brief (in fact, empty if all tests passed). 2025 2026 Optional keyword arg "optionflags" or's together module constants, 2027 and defaults to 0. Possible values (see the docs for details): 2028 2029 DONT_ACCEPT_TRUE_FOR_1 2030 DONT_ACCEPT_BLANKLINE 2031 NORMALIZE_WHITESPACE 2032 ELLIPSIS 2033 SKIP 2034 IGNORE_EXCEPTION_DETAIL 2035 REPORT_UDIFF 2036 REPORT_CDIFF 2037 REPORT_NDIFF 2038 REPORT_ONLY_FIRST_FAILURE 2039 2040 Optional keyword arg "raise_on_error" raises an exception on the 2041 first unexpected exception or failure. This allows failures to be 2042 post-mortem debugged. 2043 2044 Optional keyword arg "parser" specifies a DocTestParser (or 2045 subclass) that should be used to extract tests from the files. 2046 2047 Optional keyword arg "encoding" specifies an encoding that should 2048 be used to convert the file to unicode. 2049 2050 Advanced tomfoolery: testmod runs methods of a local instance of 2051 class doctest.Tester, then merges the results into (or creates) 2052 global Tester instance doctest.master. Methods of doctest.master 2053 can be called directly too, if you want to do something unusual. 2054 Passing report=0 to testmod is especially useful then, to delay 2055 displaying a summary. Invoke doctest.master.summarize(verbose) 2056 when you're done fiddling. 2057 """ 2058 global master 2059 2060 if package and not module_relative: 2061 raise ValueError("Package may only be specified for module-" 2062 "relative paths.") 2063 2064 # Relativize the path 2065 text, filename = _load_testfile(filename, package, module_relative, 2066 encoding or "utf-8") 2067 2068 # If no name was given, then use the file's name. 2069 if name is None: 2070 name = os.path.basename(filename) 2071 2072 # Assemble the globals. 2073 if globs is None: 2074 globs = {} 2075 else: 2076 globs = globs.copy() 2077 if extraglobs is not None: 2078 globs.update(extraglobs) 2079 if '__name__' not in globs: 2080 globs['__name__'] = '__main__' 2081 2082 if raise_on_error: 2083 runner = DebugRunner(verbose=verbose, optionflags=optionflags) 2084 else: 2085 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 2086 2087 # Read the file, convert it to a test, and run it. 2088 test = parser.get_doctest(text, globs, name, filename, 0) 2089 runner.run(test) 2090 2091 if report: 2092 runner.summarize() 2093 2094 if master is None: 2095 master = runner 2096 else: 2097 master.merge(runner) 2098 2099 return TestResults(runner.failures, runner.tries) 2100 2101def run_docstring_examples(f, globs, verbose=False, name="NoName", 2102 compileflags=None, optionflags=0): 2103 """ 2104 Test examples in the given object's docstring (`f`), using `globs` 2105 as globals. Optional argument `name` is used in failure messages. 2106 If the optional argument `verbose` is true, then generate output 2107 even if there are no failures. 2108 2109 `compileflags` gives the set of flags that should be used by the 2110 Python compiler when running the examples. If not specified, then 2111 it will default to the set of future-import flags that apply to 2112 `globs`. 2113 2114 Optional keyword arg `optionflags` specifies options for the 2115 testing and output. See the documentation for `testmod` for more 2116 information. 2117 """ 2118 # Find, parse, and run all tests in the given module. 2119 finder = DocTestFinder(verbose=verbose, recurse=False) 2120 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 2121 for test in finder.find(f, name, globs=globs): 2122 runner.run(test, compileflags=compileflags) 2123 2124###################################################################### 2125## 7. Unittest Support 2126###################################################################### 2127 2128_unittest_reportflags = 0 2129 2130def set_unittest_reportflags(flags): 2131 """Sets the unittest option flags. 2132 2133 The old flag is returned so that a runner could restore the old 2134 value if it wished to: 2135 2136 >>> import doctest 2137 >>> old = doctest._unittest_reportflags 2138 >>> doctest.set_unittest_reportflags(REPORT_NDIFF | 2139 ... REPORT_ONLY_FIRST_FAILURE) == old 2140 True 2141 2142 >>> doctest._unittest_reportflags == (REPORT_NDIFF | 2143 ... REPORT_ONLY_FIRST_FAILURE) 2144 True 2145 2146 Only reporting flags can be set: 2147 2148 >>> doctest.set_unittest_reportflags(ELLIPSIS) 2149 Traceback (most recent call last): 2150 ... 2151 ValueError: ('Only reporting flags allowed', 8) 2152 2153 >>> doctest.set_unittest_reportflags(old) == (REPORT_NDIFF | 2154 ... REPORT_ONLY_FIRST_FAILURE) 2155 True 2156 """ 2157 global _unittest_reportflags 2158 2159 if (flags & REPORTING_FLAGS) != flags: 2160 raise ValueError("Only reporting flags allowed", flags) 2161 old = _unittest_reportflags 2162 _unittest_reportflags = flags 2163 return old 2164 2165 2166class DocTestCase(unittest.TestCase): 2167 2168 def __init__(self, test, optionflags=0, setUp=None, tearDown=None, 2169 checker=None): 2170 2171 unittest.TestCase.__init__(self) 2172 self._dt_optionflags = optionflags 2173 self._dt_checker = checker 2174 self._dt_test = test 2175 self._dt_setUp = setUp 2176 self._dt_tearDown = tearDown 2177 2178 def setUp(self): 2179 test = self._dt_test 2180 2181 if self._dt_setUp is not None: 2182 self._dt_setUp(test) 2183 2184 def tearDown(self): 2185 test = self._dt_test 2186 2187 if self._dt_tearDown is not None: 2188 self._dt_tearDown(test) 2189 2190 test.globs.clear() 2191 2192 def runTest(self): 2193 test = self._dt_test 2194 old = sys.stdout 2195 new = StringIO() 2196 optionflags = self._dt_optionflags 2197 2198 if not (optionflags & REPORTING_FLAGS): 2199 # The option flags don't include any reporting flags, 2200 # so add the default reporting flags 2201 optionflags |= _unittest_reportflags 2202 2203 runner = DocTestRunner(optionflags=optionflags, 2204 checker=self._dt_checker, verbose=False) 2205 2206 try: 2207 runner.DIVIDER = "-"*70 2208 failures, tries = runner.run( 2209 test, out=new.write, clear_globs=False) 2210 finally: 2211 sys.stdout = old 2212 2213 if failures: 2214 raise self.failureException(self.format_failure(new.getvalue())) 2215 2216 def format_failure(self, err): 2217 test = self._dt_test 2218 if test.lineno is None: 2219 lineno = 'unknown line number' 2220 else: 2221 lineno = '%s' % test.lineno 2222 lname = '.'.join(test.name.split('.')[-1:]) 2223 return ('Failed doctest test for %s\n' 2224 ' File "%s", line %s, in %s\n\n%s' 2225 % (test.name, test.filename, lineno, lname, err) 2226 ) 2227 2228 def debug(self): 2229 r"""Run the test case without results and without catching exceptions 2230 2231 The unit test framework includes a debug method on test cases 2232 and test suites to support post-mortem debugging. The test code 2233 is run in such a way that errors are not caught. This way a 2234 caller can catch the errors and initiate post-mortem debugging. 2235 2236 The DocTestCase provides a debug method that raises 2237 UnexpectedException errors if there is an unexpected 2238 exception: 2239 2240 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', 2241 ... {}, 'foo', 'foo.py', 0) 2242 >>> case = DocTestCase(test) 2243 >>> try: 2244 ... case.debug() 2245 ... except UnexpectedException as f: 2246 ... failure = f 2247 2248 The UnexpectedException contains the test, the example, and 2249 the original exception: 2250 2251 >>> failure.test is test 2252 True 2253 2254 >>> failure.example.want 2255 '42\n' 2256 2257 >>> exc_info = failure.exc_info 2258 >>> raise exc_info[1] # Already has the traceback 2259 Traceback (most recent call last): 2260 ... 2261 KeyError 2262 2263 If the output doesn't match, then a DocTestFailure is raised: 2264 2265 >>> test = DocTestParser().get_doctest(''' 2266 ... >>> x = 1 2267 ... >>> x 2268 ... 2 2269 ... ''', {}, 'foo', 'foo.py', 0) 2270 >>> case = DocTestCase(test) 2271 2272 >>> try: 2273 ... case.debug() 2274 ... except DocTestFailure as f: 2275 ... failure = f 2276 2277 DocTestFailure objects provide access to the test: 2278 2279 >>> failure.test is test 2280 True 2281 2282 As well as to the example: 2283 2284 >>> failure.example.want 2285 '2\n' 2286 2287 and the actual output: 2288 2289 >>> failure.got 2290 '1\n' 2291 2292 """ 2293 2294 self.setUp() 2295 runner = DebugRunner(optionflags=self._dt_optionflags, 2296 checker=self._dt_checker, verbose=False) 2297 runner.run(self._dt_test, clear_globs=False) 2298 self.tearDown() 2299 2300 def id(self): 2301 return self._dt_test.name 2302 2303 def __eq__(self, other): 2304 if type(self) is not type(other): 2305 return NotImplemented 2306 2307 return self._dt_test == other._dt_test and \ 2308 self._dt_optionflags == other._dt_optionflags and \ 2309 self._dt_setUp == other._dt_setUp and \ 2310 self._dt_tearDown == other._dt_tearDown and \ 2311 self._dt_checker == other._dt_checker 2312 2313 def __hash__(self): 2314 return hash((self._dt_optionflags, self._dt_setUp, self._dt_tearDown, 2315 self._dt_checker)) 2316 2317 def __repr__(self): 2318 name = self._dt_test.name.split('.') 2319 return "%s (%s)" % (name[-1], '.'.join(name[:-1])) 2320 2321 __str__ = object.__str__ 2322 2323 def shortDescription(self): 2324 return "Doctest: " + self._dt_test.name 2325 2326class SkipDocTestCase(DocTestCase): 2327 def __init__(self, module): 2328 self.module = module 2329 DocTestCase.__init__(self, None) 2330 2331 def setUp(self): 2332 self.skipTest("DocTestSuite will not work with -O2 and above") 2333 2334 def test_skip(self): 2335 pass 2336 2337 def shortDescription(self): 2338 return "Skipping tests from %s" % self.module.__name__ 2339 2340 __str__ = shortDescription 2341 2342 2343class _DocTestSuite(unittest.TestSuite): 2344 2345 def _removeTestAtIndex(self, index): 2346 pass 2347 2348 2349def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, 2350 **options): 2351 """ 2352 Convert doctest tests for a module to a unittest test suite. 2353 2354 This converts each documentation string in a module that 2355 contains doctest tests to a unittest test case. If any of the 2356 tests in a doc string fail, then the test case fails. An exception 2357 is raised showing the name of the file containing the test and a 2358 (sometimes approximate) line number. 2359 2360 The `module` argument provides the module to be tested. The argument 2361 can be either a module or a module name. 2362 2363 If no argument is given, the calling module is used. 2364 2365 A number of options may be provided as keyword arguments: 2366 2367 setUp 2368 A set-up function. This is called before running the 2369 tests in each file. The setUp function will be passed a DocTest 2370 object. The setUp function can access the test globals as the 2371 globs attribute of the test passed. 2372 2373 tearDown 2374 A tear-down function. This is called after running the 2375 tests in each file. The tearDown function will be passed a DocTest 2376 object. The tearDown function can access the test globals as the 2377 globs attribute of the test passed. 2378 2379 globs 2380 A dictionary containing initial global variables for the tests. 2381 2382 optionflags 2383 A set of doctest option flags expressed as an integer. 2384 """ 2385 2386 if test_finder is None: 2387 test_finder = DocTestFinder() 2388 2389 module = _normalize_module(module) 2390 tests = test_finder.find(module, globs=globs, extraglobs=extraglobs) 2391 2392 if not tests and sys.flags.optimize >=2: 2393 # Skip doctests when running with -O2 2394 suite = _DocTestSuite() 2395 suite.addTest(SkipDocTestCase(module)) 2396 return suite 2397 2398 tests.sort() 2399 suite = _DocTestSuite() 2400 2401 for test in tests: 2402 if len(test.examples) == 0: 2403 continue 2404 if not test.filename: 2405 filename = module.__file__ 2406 if filename[-4:] == ".pyc": 2407 filename = filename[:-1] 2408 test.filename = filename 2409 suite.addTest(DocTestCase(test, **options)) 2410 2411 return suite 2412 2413class DocFileCase(DocTestCase): 2414 2415 def id(self): 2416 return '_'.join(self._dt_test.name.split('.')) 2417 2418 def __repr__(self): 2419 return self._dt_test.filename 2420 2421 def format_failure(self, err): 2422 return ('Failed doctest test for %s\n File "%s", line 0\n\n%s' 2423 % (self._dt_test.name, self._dt_test.filename, err) 2424 ) 2425 2426def DocFileTest(path, module_relative=True, package=None, 2427 globs=None, parser=DocTestParser(), 2428 encoding=None, **options): 2429 if globs is None: 2430 globs = {} 2431 else: 2432 globs = globs.copy() 2433 2434 if package and not module_relative: 2435 raise ValueError("Package may only be specified for module-" 2436 "relative paths.") 2437 2438 # Relativize the path. 2439 doc, path = _load_testfile(path, package, module_relative, 2440 encoding or "utf-8") 2441 2442 if "__file__" not in globs: 2443 globs["__file__"] = path 2444 2445 # Find the file and read it. 2446 name = os.path.basename(path) 2447 2448 # Convert it to a test, and wrap it in a DocFileCase. 2449 test = parser.get_doctest(doc, globs, name, path, 0) 2450 return DocFileCase(test, **options) 2451 2452def DocFileSuite(*paths, **kw): 2453 """A unittest suite for one or more doctest files. 2454 2455 The path to each doctest file is given as a string; the 2456 interpretation of that string depends on the keyword argument 2457 "module_relative". 2458 2459 A number of options may be provided as keyword arguments: 2460 2461 module_relative 2462 If "module_relative" is True, then the given file paths are 2463 interpreted as os-independent module-relative paths. By 2464 default, these paths are relative to the calling module's 2465 directory; but if the "package" argument is specified, then 2466 they are relative to that package. To ensure os-independence, 2467 "filename" should use "/" characters to separate path 2468 segments, and may not be an absolute path (i.e., it may not 2469 begin with "/"). 2470 2471 If "module_relative" is False, then the given file paths are 2472 interpreted as os-specific paths. These paths may be absolute 2473 or relative (to the current working directory). 2474 2475 package 2476 A Python package or the name of a Python package whose directory 2477 should be used as the base directory for module relative paths. 2478 If "package" is not specified, then the calling module's 2479 directory is used as the base directory for module relative 2480 filenames. It is an error to specify "package" if 2481 "module_relative" is False. 2482 2483 setUp 2484 A set-up function. This is called before running the 2485 tests in each file. The setUp function will be passed a DocTest 2486 object. The setUp function can access the test globals as the 2487 globs attribute of the test passed. 2488 2489 tearDown 2490 A tear-down function. This is called after running the 2491 tests in each file. The tearDown function will be passed a DocTest 2492 object. The tearDown function can access the test globals as the 2493 globs attribute of the test passed. 2494 2495 globs 2496 A dictionary containing initial global variables for the tests. 2497 2498 optionflags 2499 A set of doctest option flags expressed as an integer. 2500 2501 parser 2502 A DocTestParser (or subclass) that should be used to extract 2503 tests from the files. 2504 2505 encoding 2506 An encoding that will be used to convert the files to unicode. 2507 """ 2508 suite = _DocTestSuite() 2509 2510 # We do this here so that _normalize_module is called at the right 2511 # level. If it were called in DocFileTest, then this function 2512 # would be the caller and we might guess the package incorrectly. 2513 if kw.get('module_relative', True): 2514 kw['package'] = _normalize_module(kw.get('package')) 2515 2516 for path in paths: 2517 suite.addTest(DocFileTest(path, **kw)) 2518 2519 return suite 2520 2521###################################################################### 2522## 8. Debugging Support 2523###################################################################### 2524 2525def script_from_examples(s): 2526 r"""Extract script from text with examples. 2527 2528 Converts text with examples to a Python script. Example input is 2529 converted to regular code. Example output and all other words 2530 are converted to comments: 2531 2532 >>> text = ''' 2533 ... Here are examples of simple math. 2534 ... 2535 ... Python has super accurate integer addition 2536 ... 2537 ... >>> 2 + 2 2538 ... 5 2539 ... 2540 ... And very friendly error messages: 2541 ... 2542 ... >>> 1/0 2543 ... To Infinity 2544 ... And 2545 ... Beyond 2546 ... 2547 ... You can use logic if you want: 2548 ... 2549 ... >>> if 0: 2550 ... ... blah 2551 ... ... blah 2552 ... ... 2553 ... 2554 ... Ho hum 2555 ... ''' 2556 2557 >>> print(script_from_examples(text)) 2558 # Here are examples of simple math. 2559 # 2560 # Python has super accurate integer addition 2561 # 2562 2 + 2 2563 # Expected: 2564 ## 5 2565 # 2566 # And very friendly error messages: 2567 # 2568 1/0 2569 # Expected: 2570 ## To Infinity 2571 ## And 2572 ## Beyond 2573 # 2574 # You can use logic if you want: 2575 # 2576 if 0: 2577 blah 2578 blah 2579 # 2580 # Ho hum 2581 <BLANKLINE> 2582 """ 2583 output = [] 2584 for piece in DocTestParser().parse(s): 2585 if isinstance(piece, Example): 2586 # Add the example's source code (strip trailing NL) 2587 output.append(piece.source[:-1]) 2588 # Add the expected output: 2589 want = piece.want 2590 if want: 2591 output.append('# Expected:') 2592 output += ['## '+l for l in want.split('\n')[:-1]] 2593 else: 2594 # Add non-example text. 2595 output += [_comment_line(l) 2596 for l in piece.split('\n')[:-1]] 2597 2598 # Trim junk on both ends. 2599 while output and output[-1] == '#': 2600 output.pop() 2601 while output and output[0] == '#': 2602 output.pop(0) 2603 # Combine the output, and return it. 2604 # Add a courtesy newline to prevent exec from choking (see bug #1172785) 2605 return '\n'.join(output) + '\n' 2606 2607def testsource(module, name): 2608 """Extract the test sources from a doctest docstring as a script. 2609 2610 Provide the module (or dotted name of the module) containing the 2611 test to be debugged and the name (within the module) of the object 2612 with the doc string with tests to be debugged. 2613 """ 2614 module = _normalize_module(module) 2615 tests = DocTestFinder().find(module) 2616 test = [t for t in tests if t.name == name] 2617 if not test: 2618 raise ValueError(name, "not found in tests") 2619 test = test[0] 2620 testsrc = script_from_examples(test.docstring) 2621 return testsrc 2622 2623def debug_src(src, pm=False, globs=None): 2624 """Debug a single doctest docstring, in argument `src`'""" 2625 testsrc = script_from_examples(src) 2626 debug_script(testsrc, pm, globs) 2627 2628def debug_script(src, pm=False, globs=None): 2629 "Debug a test script. `src` is the script, as a string." 2630 import pdb 2631 2632 if globs: 2633 globs = globs.copy() 2634 else: 2635 globs = {} 2636 2637 if pm: 2638 try: 2639 exec(src, globs, globs) 2640 except: 2641 print(sys.exc_info()[1]) 2642 p = pdb.Pdb(nosigint=True) 2643 p.reset() 2644 p.interaction(None, sys.exc_info()[2]) 2645 else: 2646 pdb.Pdb(nosigint=True).run("exec(%r)" % src, globs, globs) 2647 2648def debug(module, name, pm=False): 2649 """Debug a single doctest docstring. 2650 2651 Provide the module (or dotted name of the module) containing the 2652 test to be debugged and the name (within the module) of the object 2653 with the docstring with tests to be debugged. 2654 """ 2655 module = _normalize_module(module) 2656 testsrc = testsource(module, name) 2657 debug_script(testsrc, pm, module.__dict__) 2658 2659###################################################################### 2660## 9. Example Usage 2661###################################################################### 2662class _TestClass: 2663 """ 2664 A pointless class, for sanity-checking of docstring testing. 2665 2666 Methods: 2667 square() 2668 get() 2669 2670 >>> _TestClass(13).get() + _TestClass(-12).get() 2671 1 2672 >>> hex(_TestClass(13).square().get()) 2673 '0xa9' 2674 """ 2675 2676 def __init__(self, val): 2677 """val -> _TestClass object with associated value val. 2678 2679 >>> t = _TestClass(123) 2680 >>> print(t.get()) 2681 123 2682 """ 2683 2684 self.val = val 2685 2686 def square(self): 2687 """square() -> square TestClass's associated value 2688 2689 >>> _TestClass(13).square().get() 2690 169 2691 """ 2692 2693 self.val = self.val ** 2 2694 return self 2695 2696 def get(self): 2697 """get() -> return TestClass's associated value. 2698 2699 >>> x = _TestClass(-42) 2700 >>> print(x.get()) 2701 -42 2702 """ 2703 2704 return self.val 2705 2706__test__ = {"_TestClass": _TestClass, 2707 "string": r""" 2708 Example of a string object, searched as-is. 2709 >>> x = 1; y = 2 2710 >>> x + y, x * y 2711 (3, 2) 2712 """, 2713 2714 "bool-int equivalence": r""" 2715 In 2.2, boolean expressions displayed 2716 0 or 1. By default, we still accept 2717 them. This can be disabled by passing 2718 DONT_ACCEPT_TRUE_FOR_1 to the new 2719 optionflags argument. 2720 >>> 4 == 4 2721 1 2722 >>> 4 == 4 2723 True 2724 >>> 4 > 4 2725 0 2726 >>> 4 > 4 2727 False 2728 """, 2729 2730 "blank lines": r""" 2731 Blank lines can be marked with <BLANKLINE>: 2732 >>> print('foo\n\nbar\n') 2733 foo 2734 <BLANKLINE> 2735 bar 2736 <BLANKLINE> 2737 """, 2738 2739 "ellipsis": r""" 2740 If the ellipsis flag is used, then '...' can be used to 2741 elide substrings in the desired output: 2742 >>> print(list(range(1000))) #doctest: +ELLIPSIS 2743 [0, 1, 2, ..., 999] 2744 """, 2745 2746 "whitespace normalization": r""" 2747 If the whitespace normalization flag is used, then 2748 differences in whitespace are ignored. 2749 >>> print(list(range(30))) #doctest: +NORMALIZE_WHITESPACE 2750 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 2751 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 2752 27, 28, 29] 2753 """, 2754 } 2755 2756 2757def _test(): 2758 import argparse 2759 2760 parser = argparse.ArgumentParser(description="doctest runner") 2761 parser.add_argument('-v', '--verbose', action='store_true', default=False, 2762 help='print very verbose output for all tests') 2763 parser.add_argument('-o', '--option', action='append', 2764 choices=OPTIONFLAGS_BY_NAME.keys(), default=[], 2765 help=('specify a doctest option flag to apply' 2766 ' to the test run; may be specified more' 2767 ' than once to apply multiple options')) 2768 parser.add_argument('-f', '--fail-fast', action='store_true', 2769 help=('stop running tests after first failure (this' 2770 ' is a shorthand for -o FAIL_FAST, and is' 2771 ' in addition to any other -o options)')) 2772 parser.add_argument('file', nargs='+', 2773 help='file containing the tests to run') 2774 args = parser.parse_args() 2775 testfiles = args.file 2776 # Verbose used to be handled by the "inspect argv" magic in DocTestRunner, 2777 # but since we are using argparse we are passing it manually now. 2778 verbose = args.verbose 2779 options = 0 2780 for option in args.option: 2781 options |= OPTIONFLAGS_BY_NAME[option] 2782 if args.fail_fast: 2783 options |= FAIL_FAST 2784 for filename in testfiles: 2785 if filename.endswith(".py"): 2786 # It is a module -- insert its dir into sys.path and try to 2787 # import it. If it is part of a package, that possibly 2788 # won't work because of package imports. 2789 dirname, filename = os.path.split(filename) 2790 sys.path.insert(0, dirname) 2791 m = __import__(filename[:-3]) 2792 del sys.path[0] 2793 failures, _ = testmod(m, verbose=verbose, optionflags=options) 2794 else: 2795 failures, _ = testfile(filename, module_relative=False, 2796 verbose=verbose, optionflags=options) 2797 if failures: 2798 return 1 2799 return 0 2800 2801 2802if __name__ == "__main__": 2803 sys.exit(_test()) 2804