1# Module doctest. 2# Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org). 3# Major enhancements and refactoring by: 4# Jim Fulton 5# Edward Loper 6 7# Provided as-is; use at your own risk; no warranty; no promises; enjoy! 8 9r"""Module doctest -- a framework for running examples in docstrings. 10 11In simplest use, end each module M to be tested with: 12 13def _test(): 14 import doctest 15 doctest.testmod() 16 17if __name__ == "__main__": 18 _test() 19 20Then running the module as a script will cause the examples in the 21docstrings to get executed and verified: 22 23python M.py 24 25This won't display anything unless an example fails, in which case the 26failing example(s) and the cause(s) of the failure(s) are printed to stdout 27(why not stderr? because stderr is a lame hack <0.2 wink>), and the final 28line of output is "Test failed.". 29 30Run it with the -v switch instead: 31 32python M.py -v 33 34and a detailed report of all examples tried is printed to stdout, along 35with assorted summaries at the end. 36 37You can force verbose mode by passing "verbose=True" to testmod, or prohibit 38it by passing "verbose=False". In either of those cases, sys.argv is not 39examined by testmod. 40 41There are a variety of other ways to run doctests, including integration 42with the unittest framework, and support for running non-Python text 43files containing doctests. There are also many ways to override parts 44of doctest's default behaviors. See the Library Reference Manual for 45details. 46""" 47 48__docformat__ = 'reStructuredText en' 49 50__all__ = [ 51 # 0, Option Flags 52 'register_optionflag', 53 'DONT_ACCEPT_TRUE_FOR_1', 54 'DONT_ACCEPT_BLANKLINE', 55 'NORMALIZE_WHITESPACE', 56 'ELLIPSIS', 57 'SKIP', 58 'IGNORE_EXCEPTION_DETAIL', 59 'COMPARISON_FLAGS', 60 'REPORT_UDIFF', 61 'REPORT_CDIFF', 62 'REPORT_NDIFF', 63 'REPORT_ONLY_FIRST_FAILURE', 64 'REPORTING_FLAGS', 65 # 1. Utility Functions 66 # 2. Example & DocTest 67 'Example', 68 'DocTest', 69 # 3. Doctest Parser 70 'DocTestParser', 71 # 4. Doctest Finder 72 'DocTestFinder', 73 # 5. Doctest Runner 74 'DocTestRunner', 75 'OutputChecker', 76 'DocTestFailure', 77 'UnexpectedException', 78 'DebugRunner', 79 # 6. Test Functions 80 'testmod', 81 'testfile', 82 'run_docstring_examples', 83 # 7. Tester 84 'Tester', 85 # 8. Unittest Support 86 'DocTestSuite', 87 'DocFileSuite', 88 'set_unittest_reportflags', 89 # 9. Debugging Support 90 'script_from_examples', 91 'testsource', 92 'debug_src', 93 'debug', 94] 95 96import __future__ 97 98import sys, traceback, inspect, linecache, os, re 99import unittest, difflib, pdb, tempfile 100import warnings 101from StringIO import StringIO 102from collections import namedtuple 103 104TestResults = namedtuple('TestResults', 'failed attempted') 105 106# There are 4 basic classes: 107# - Example: a <source, want> pair, plus an intra-docstring line number. 108# - DocTest: a collection of examples, parsed from a docstring, plus 109# info about where the docstring came from (name, filename, lineno). 110# - DocTestFinder: extracts DocTests from a given object's docstring and 111# its contained objects' docstrings. 112# - DocTestRunner: runs DocTest cases, and accumulates statistics. 113# 114# So the basic picture is: 115# 116# list of: 117# +------+ +---------+ +-------+ 118# |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results| 119# +------+ +---------+ +-------+ 120# | Example | 121# | ... | 122# | Example | 123# +---------+ 124 125# Option constants. 126 127OPTIONFLAGS_BY_NAME = {} 128def register_optionflag(name): 129 # Create a new flag unless `name` is already known. 130 return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME)) 131 132DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1') 133DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE') 134NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE') 135ELLIPSIS = register_optionflag('ELLIPSIS') 136SKIP = register_optionflag('SKIP') 137IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL') 138 139COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 | 140 DONT_ACCEPT_BLANKLINE | 141 NORMALIZE_WHITESPACE | 142 ELLIPSIS | 143 SKIP | 144 IGNORE_EXCEPTION_DETAIL) 145 146REPORT_UDIFF = register_optionflag('REPORT_UDIFF') 147REPORT_CDIFF = register_optionflag('REPORT_CDIFF') 148REPORT_NDIFF = register_optionflag('REPORT_NDIFF') 149REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE') 150 151REPORTING_FLAGS = (REPORT_UDIFF | 152 REPORT_CDIFF | 153 REPORT_NDIFF | 154 REPORT_ONLY_FIRST_FAILURE) 155 156# Special string markers for use in `want` strings: 157BLANKLINE_MARKER = '<BLANKLINE>' 158ELLIPSIS_MARKER = '...' 159 160###################################################################### 161## Table of Contents 162###################################################################### 163# 1. Utility Functions 164# 2. Example & DocTest -- store test cases 165# 3. DocTest Parser -- extracts examples from strings 166# 4. DocTest Finder -- extracts test cases from objects 167# 5. DocTest Runner -- runs test cases 168# 6. Test Functions -- convenient wrappers for testing 169# 7. Tester Class -- for backwards compatibility 170# 8. Unittest Support 171# 9. Debugging Support 172# 10. Example Usage 173 174###################################################################### 175## 1. Utility Functions 176###################################################################### 177 178def _extract_future_flags(globs): 179 """ 180 Return the compiler-flags associated with the future features that 181 have been imported into the given namespace (globs). 182 """ 183 flags = 0 184 for fname in __future__.all_feature_names: 185 feature = globs.get(fname, None) 186 if feature is getattr(__future__, fname): 187 flags |= feature.compiler_flag 188 return flags 189 190def _normalize_module(module, depth=2): 191 """ 192 Return the module specified by `module`. In particular: 193 - If `module` is a module, then return module. 194 - If `module` is a string, then import and return the 195 module with that name. 196 - If `module` is None, then return the calling module. 197 The calling module is assumed to be the module of 198 the stack frame at the given depth in the call stack. 199 """ 200 if inspect.ismodule(module): 201 return module 202 elif isinstance(module, (str, unicode)): 203 return __import__(module, globals(), locals(), ["*"]) 204 elif module is None: 205 return sys.modules[sys._getframe(depth).f_globals['__name__']] 206 else: 207 raise TypeError("Expected a module, string, or None") 208 209def _load_testfile(filename, package, module_relative): 210 if module_relative: 211 package = _normalize_module(package, 3) 212 filename = _module_relative_path(package, filename) 213 if hasattr(package, '__loader__'): 214 if hasattr(package.__loader__, 'get_data'): 215 file_contents = package.__loader__.get_data(filename) 216 # get_data() opens files as 'rb', so one must do the equivalent 217 # conversion as universal newlines would do. 218 return file_contents.replace(os.linesep, '\n'), filename 219 with open(filename) as f: 220 return f.read(), filename 221 222# Use sys.stdout encoding for ouput. 223_encoding = getattr(sys.__stdout__, 'encoding', None) or 'utf-8' 224 225def _indent(s, indent=4): 226 """ 227 Add the given number of space characters to the beginning of 228 every non-blank line in `s`, and return the result. 229 If the string `s` is Unicode, it is encoded using the stdout 230 encoding and the `backslashreplace` error handler. 231 """ 232 if isinstance(s, unicode): 233 s = s.encode(_encoding, 'backslashreplace') 234 # This regexp matches the start of non-blank lines: 235 return re.sub('(?m)^(?!$)', indent*' ', s) 236 237def _exception_traceback(exc_info): 238 """ 239 Return a string containing a traceback message for the given 240 exc_info tuple (as returned by sys.exc_info()). 241 """ 242 # Get a traceback message. 243 excout = StringIO() 244 exc_type, exc_val, exc_tb = exc_info 245 traceback.print_exception(exc_type, exc_val, exc_tb, file=excout) 246 return excout.getvalue() 247 248# Override some StringIO methods. 249class _SpoofOut(StringIO): 250 def getvalue(self): 251 result = StringIO.getvalue(self) 252 # If anything at all was written, make sure there's a trailing 253 # newline. There's no way for the expected output to indicate 254 # that a trailing newline is missing. 255 if result and not result.endswith("\n"): 256 result += "\n" 257 # Prevent softspace from screwing up the next test case, in 258 # case they used print with a trailing comma in an example. 259 if hasattr(self, "softspace"): 260 del self.softspace 261 return result 262 263 def truncate(self, size=None): 264 StringIO.truncate(self, size) 265 if hasattr(self, "softspace"): 266 del self.softspace 267 if not self.buf: 268 # Reset it to an empty string, to make sure it's not unicode. 269 self.buf = '' 270 271# Worst-case linear-time ellipsis matching. 272def _ellipsis_match(want, got): 273 """ 274 Essentially the only subtle case: 275 >>> _ellipsis_match('aa...aa', 'aaa') 276 False 277 """ 278 if ELLIPSIS_MARKER not in want: 279 return want == got 280 281 # Find "the real" strings. 282 ws = want.split(ELLIPSIS_MARKER) 283 assert len(ws) >= 2 284 285 # Deal with exact matches possibly needed at one or both ends. 286 startpos, endpos = 0, len(got) 287 w = ws[0] 288 if w: # starts with exact match 289 if got.startswith(w): 290 startpos = len(w) 291 del ws[0] 292 else: 293 return False 294 w = ws[-1] 295 if w: # ends with exact match 296 if got.endswith(w): 297 endpos -= len(w) 298 del ws[-1] 299 else: 300 return False 301 302 if startpos > endpos: 303 # Exact end matches required more characters than we have, as in 304 # _ellipsis_match('aa...aa', 'aaa') 305 return False 306 307 # For the rest, we only need to find the leftmost non-overlapping 308 # match for each piece. If there's no overall match that way alone, 309 # there's no overall match period. 310 for w in ws: 311 # w may be '' at times, if there are consecutive ellipses, or 312 # due to an ellipsis at the start or end of `want`. That's OK. 313 # Search for an empty string succeeds, and doesn't change startpos. 314 startpos = got.find(w, startpos, endpos) 315 if startpos < 0: 316 return False 317 startpos += len(w) 318 319 return True 320 321def _comment_line(line): 322 "Return a commented form of the given line" 323 line = line.rstrip() 324 if line: 325 return '# '+line 326 else: 327 return '#' 328 329class _OutputRedirectingPdb(pdb.Pdb): 330 """ 331 A specialized version of the python debugger that redirects stdout 332 to a given stream when interacting with the user. Stdout is *not* 333 redirected when traced code is executed. 334 """ 335 def __init__(self, out): 336 self.__out = out 337 self.__debugger_used = False 338 pdb.Pdb.__init__(self, stdout=out) 339 # still use input() to get user input 340 self.use_rawinput = 1 341 342 def set_trace(self, frame=None): 343 self.__debugger_used = True 344 if frame is None: 345 frame = sys._getframe().f_back 346 pdb.Pdb.set_trace(self, frame) 347 348 def set_continue(self): 349 # Calling set_continue unconditionally would break unit test 350 # coverage reporting, as Bdb.set_continue calls sys.settrace(None). 351 if self.__debugger_used: 352 pdb.Pdb.set_continue(self) 353 354 def trace_dispatch(self, *args): 355 # Redirect stdout to the given stream. 356 save_stdout = sys.stdout 357 sys.stdout = self.__out 358 # Call Pdb's trace dispatch method. 359 try: 360 return pdb.Pdb.trace_dispatch(self, *args) 361 finally: 362 sys.stdout = save_stdout 363 364# [XX] Normalize with respect to os.path.pardir? 365def _module_relative_path(module, path): 366 if not inspect.ismodule(module): 367 raise TypeError, 'Expected a module: %r' % module 368 if path.startswith('/'): 369 raise ValueError, 'Module-relative files may not have absolute paths' 370 371 # Find the base directory for the path. 372 if hasattr(module, '__file__'): 373 # A normal module/package 374 basedir = os.path.split(module.__file__)[0] 375 elif module.__name__ == '__main__': 376 # An interactive session. 377 if len(sys.argv)>0 and sys.argv[0] != '': 378 basedir = os.path.split(sys.argv[0])[0] 379 else: 380 basedir = os.curdir 381 else: 382 # A module w/o __file__ (this includes builtins) 383 raise ValueError("Can't resolve paths relative to the module " + 384 module + " (it has no __file__)") 385 386 # Combine the base directory and the path. 387 return os.path.join(basedir, *(path.split('/'))) 388 389###################################################################### 390## 2. Example & DocTest 391###################################################################### 392## - An "example" is a <source, want> pair, where "source" is a 393## fragment of source code, and "want" is the expected output for 394## "source." The Example class also includes information about 395## where the example was extracted from. 396## 397## - A "doctest" is a collection of examples, typically extracted from 398## a string (such as an object's docstring). The DocTest class also 399## includes information about where the string was extracted from. 400 401class Example: 402 """ 403 A single doctest example, consisting of source code and expected 404 output. `Example` defines the following attributes: 405 406 - source: A single Python statement, always ending with a newline. 407 The constructor adds a newline if needed. 408 409 - want: The expected output from running the source code (either 410 from stdout, or a traceback in case of exception). `want` ends 411 with a newline unless it's empty, in which case it's an empty 412 string. The constructor adds a newline if needed. 413 414 - exc_msg: The exception message generated by the example, if 415 the example is expected to generate an exception; or `None` if 416 it is not expected to generate an exception. This exception 417 message is compared against the return value of 418 `traceback.format_exception_only()`. `exc_msg` ends with a 419 newline unless it's `None`. The constructor adds a newline 420 if needed. 421 422 - lineno: The line number within the DocTest string containing 423 this Example where the Example begins. This line number is 424 zero-based, with respect to the beginning of the DocTest. 425 426 - indent: The example's indentation in the DocTest string. 427 I.e., the number of space characters that preceed the 428 example's first prompt. 429 430 - options: A dictionary mapping from option flags to True or 431 False, which is used to override default options for this 432 example. Any option flags not contained in this dictionary 433 are left at their default value (as specified by the 434 DocTestRunner's optionflags). By default, no options are set. 435 """ 436 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0, 437 options=None): 438 # Normalize inputs. 439 if not source.endswith('\n'): 440 source += '\n' 441 if want and not want.endswith('\n'): 442 want += '\n' 443 if exc_msg is not None and not exc_msg.endswith('\n'): 444 exc_msg += '\n' 445 # Store properties. 446 self.source = source 447 self.want = want 448 self.lineno = lineno 449 self.indent = indent 450 if options is None: options = {} 451 self.options = options 452 self.exc_msg = exc_msg 453 454 def __eq__(self, other): 455 if type(self) is not type(other): 456 return NotImplemented 457 458 return self.source == other.source and \ 459 self.want == other.want and \ 460 self.lineno == other.lineno and \ 461 self.indent == other.indent and \ 462 self.options == other.options and \ 463 self.exc_msg == other.exc_msg 464 465 def __ne__(self, other): 466 return not self == other 467 468 def __hash__(self): 469 return hash((self.source, self.want, self.lineno, self.indent, 470 self.exc_msg)) 471 472 473class DocTest: 474 """ 475 A collection of doctest examples that should be run in a single 476 namespace. Each `DocTest` defines the following attributes: 477 478 - examples: the list of examples. 479 480 - globs: The namespace (aka globals) that the examples should 481 be run in. 482 483 - name: A name identifying the DocTest (typically, the name of 484 the object whose docstring this DocTest was extracted from). 485 486 - filename: The name of the file that this DocTest was extracted 487 from, or `None` if the filename is unknown. 488 489 - lineno: The line number within filename where this DocTest 490 begins, or `None` if the line number is unavailable. This 491 line number is zero-based, with respect to the beginning of 492 the file. 493 494 - docstring: The string that the examples were extracted from, 495 or `None` if the string is unavailable. 496 """ 497 def __init__(self, examples, globs, name, filename, lineno, docstring): 498 """ 499 Create a new DocTest containing the given examples. The 500 DocTest's globals are initialized with a copy of `globs`. 501 """ 502 assert not isinstance(examples, basestring), \ 503 "DocTest no longer accepts str; use DocTestParser instead" 504 self.examples = examples 505 self.docstring = docstring 506 self.globs = globs.copy() 507 self.name = name 508 self.filename = filename 509 self.lineno = lineno 510 511 def __repr__(self): 512 if len(self.examples) == 0: 513 examples = 'no examples' 514 elif len(self.examples) == 1: 515 examples = '1 example' 516 else: 517 examples = '%d examples' % len(self.examples) 518 return ('<DocTest %s from %s:%s (%s)>' % 519 (self.name, self.filename, self.lineno, examples)) 520 521 def __eq__(self, other): 522 if type(self) is not type(other): 523 return NotImplemented 524 525 return self.examples == other.examples and \ 526 self.docstring == other.docstring and \ 527 self.globs == other.globs and \ 528 self.name == other.name and \ 529 self.filename == other.filename and \ 530 self.lineno == other.lineno 531 532 def __ne__(self, other): 533 return not self == other 534 535 def __hash__(self): 536 return hash((self.docstring, self.name, self.filename, self.lineno)) 537 538 # This lets us sort tests by name: 539 def __cmp__(self, other): 540 if not isinstance(other, DocTest): 541 return -1 542 return cmp((self.name, self.filename, self.lineno, id(self)), 543 (other.name, other.filename, other.lineno, id(other))) 544 545###################################################################### 546## 3. DocTestParser 547###################################################################### 548 549class DocTestParser: 550 """ 551 A class used to parse strings containing doctest examples. 552 """ 553 # This regular expression is used to find doctest examples in a 554 # string. It defines three groups: `source` is the source code 555 # (including leading indentation and prompts); `indent` is the 556 # indentation of the first (PS1) line of the source code; and 557 # `want` is the expected output (including leading indentation). 558 _EXAMPLE_RE = re.compile(r''' 559 # Source consists of a PS1 line followed by zero or more PS2 lines. 560 (?P<source> 561 (?:^(?P<indent> [ ]*) >>> .*) # PS1 line 562 (?:\n [ ]* \.\.\. .*)*) # PS2 lines 563 \n? 564 # Want consists of any non-blank lines that do not start with PS1. 565 (?P<want> (?:(?![ ]*$) # Not a blank line 566 (?![ ]*>>>) # Not a line starting with PS1 567 .*$\n? # But any other line 568 )*) 569 ''', re.MULTILINE | re.VERBOSE) 570 571 # A regular expression for handling `want` strings that contain 572 # expected exceptions. It divides `want` into three pieces: 573 # - the traceback header line (`hdr`) 574 # - the traceback stack (`stack`) 575 # - the exception message (`msg`), as generated by 576 # traceback.format_exception_only() 577 # `msg` may have multiple lines. We assume/require that the 578 # exception message is the first non-indented line starting with a word 579 # character following the traceback header line. 580 _EXCEPTION_RE = re.compile(r""" 581 # Grab the traceback header. Different versions of Python have 582 # said different things on the first traceback line. 583 ^(?P<hdr> Traceback\ \( 584 (?: most\ recent\ call\ last 585 | innermost\ last 586 ) \) : 587 ) 588 \s* $ # toss trailing whitespace on the header. 589 (?P<stack> .*?) # don't blink: absorb stuff until... 590 ^ (?P<msg> \w+ .*) # a line *starts* with alphanum. 591 """, re.VERBOSE | re.MULTILINE | re.DOTALL) 592 593 # A callable returning a true value iff its argument is a blank line 594 # or contains a single comment. 595 _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match 596 597 def parse(self, string, name='<string>'): 598 """ 599 Divide the given string into examples and intervening text, 600 and return them as a list of alternating Examples and strings. 601 Line numbers for the Examples are 0-based. The optional 602 argument `name` is a name identifying this string, and is only 603 used for error messages. 604 """ 605 string = string.expandtabs() 606 # If all lines begin with the same indentation, then strip it. 607 min_indent = self._min_indent(string) 608 if min_indent > 0: 609 string = '\n'.join([l[min_indent:] for l in string.split('\n')]) 610 611 output = [] 612 charno, lineno = 0, 0 613 # Find all doctest examples in the string: 614 for m in self._EXAMPLE_RE.finditer(string): 615 # Add the pre-example text to `output`. 616 output.append(string[charno:m.start()]) 617 # Update lineno (lines before this example) 618 lineno += string.count('\n', charno, m.start()) 619 # Extract info from the regexp match. 620 (source, options, want, exc_msg) = \ 621 self._parse_example(m, name, lineno) 622 # Create an Example, and add it to the list. 623 if not self._IS_BLANK_OR_COMMENT(source): 624 output.append( Example(source, want, exc_msg, 625 lineno=lineno, 626 indent=min_indent+len(m.group('indent')), 627 options=options) ) 628 # Update lineno (lines inside this example) 629 lineno += string.count('\n', m.start(), m.end()) 630 # Update charno. 631 charno = m.end() 632 # Add any remaining post-example text to `output`. 633 output.append(string[charno:]) 634 return output 635 636 def get_doctest(self, string, globs, name, filename, lineno): 637 """ 638 Extract all doctest examples from the given string, and 639 collect them into a `DocTest` object. 640 641 `globs`, `name`, `filename`, and `lineno` are attributes for 642 the new `DocTest` object. See the documentation for `DocTest` 643 for more information. 644 """ 645 return DocTest(self.get_examples(string, name), globs, 646 name, filename, lineno, string) 647 648 def get_examples(self, string, name='<string>'): 649 """ 650 Extract all doctest examples from the given string, and return 651 them as a list of `Example` objects. Line numbers are 652 0-based, because it's most common in doctests that nothing 653 interesting appears on the same line as opening triple-quote, 654 and so the first interesting line is called \"line 1\" then. 655 656 The optional argument `name` is a name identifying this 657 string, and is only used for error messages. 658 """ 659 return [x for x in self.parse(string, name) 660 if isinstance(x, Example)] 661 662 def _parse_example(self, m, name, lineno): 663 """ 664 Given a regular expression match from `_EXAMPLE_RE` (`m`), 665 return a pair `(source, want)`, where `source` is the matched 666 example's source code (with prompts and indentation stripped); 667 and `want` is the example's expected output (with indentation 668 stripped). 669 670 `name` is the string's name, and `lineno` is the line number 671 where the example starts; both are used for error messages. 672 """ 673 # Get the example's indentation level. 674 indent = len(m.group('indent')) 675 676 # Divide source into lines; check that they're properly 677 # indented; and then strip their indentation & prompts. 678 source_lines = m.group('source').split('\n') 679 self._check_prompt_blank(source_lines, indent, name, lineno) 680 self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno) 681 source = '\n'.join([sl[indent+4:] for sl in source_lines]) 682 683 # Divide want into lines; check that it's properly indented; and 684 # then strip the indentation. Spaces before the last newline should 685 # be preserved, so plain rstrip() isn't good enough. 686 want = m.group('want') 687 want_lines = want.split('\n') 688 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): 689 del want_lines[-1] # forget final newline & spaces after it 690 self._check_prefix(want_lines, ' '*indent, name, 691 lineno + len(source_lines)) 692 want = '\n'.join([wl[indent:] for wl in want_lines]) 693 694 # If `want` contains a traceback message, then extract it. 695 m = self._EXCEPTION_RE.match(want) 696 if m: 697 exc_msg = m.group('msg') 698 else: 699 exc_msg = None 700 701 # Extract options from the source. 702 options = self._find_options(source, name, lineno) 703 704 return source, options, want, exc_msg 705 706 # This regular expression looks for option directives in the 707 # source code of an example. Option directives are comments 708 # starting with "doctest:". Warning: this may give false 709 # positives for string-literals that contain the string 710 # "#doctest:". Eliminating these false positives would require 711 # actually parsing the string; but we limit them by ignoring any 712 # line containing "#doctest:" that is *followed* by a quote mark. 713 _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$', 714 re.MULTILINE) 715 716 def _find_options(self, source, name, lineno): 717 """ 718 Return a dictionary containing option overrides extracted from 719 option directives in the given source string. 720 721 `name` is the string's name, and `lineno` is the line number 722 where the example starts; both are used for error messages. 723 """ 724 options = {} 725 # (note: with the current regexp, this will match at most once:) 726 for m in self._OPTION_DIRECTIVE_RE.finditer(source): 727 option_strings = m.group(1).replace(',', ' ').split() 728 for option in option_strings: 729 if (option[0] not in '+-' or 730 option[1:] not in OPTIONFLAGS_BY_NAME): 731 raise ValueError('line %r of the doctest for %s ' 732 'has an invalid option: %r' % 733 (lineno+1, name, option)) 734 flag = OPTIONFLAGS_BY_NAME[option[1:]] 735 options[flag] = (option[0] == '+') 736 if options and self._IS_BLANK_OR_COMMENT(source): 737 raise ValueError('line %r of the doctest for %s has an option ' 738 'directive on a line with no example: %r' % 739 (lineno, name, source)) 740 return options 741 742 # This regular expression finds the indentation of every non-blank 743 # line in a string. 744 _INDENT_RE = re.compile('^([ ]*)(?=\S)', re.MULTILINE) 745 746 def _min_indent(self, s): 747 "Return the minimum indentation of any non-blank line in `s`" 748 indents = [len(indent) for indent in self._INDENT_RE.findall(s)] 749 if len(indents) > 0: 750 return min(indents) 751 else: 752 return 0 753 754 def _check_prompt_blank(self, lines, indent, name, lineno): 755 """ 756 Given the lines of a source string (including prompts and 757 leading indentation), check to make sure that every prompt is 758 followed by a space character. If any line is not followed by 759 a space character, then raise ValueError. 760 """ 761 for i, line in enumerate(lines): 762 if len(line) >= indent+4 and line[indent+3] != ' ': 763 raise ValueError('line %r of the docstring for %s ' 764 'lacks blank after %s: %r' % 765 (lineno+i+1, name, 766 line[indent:indent+3], line)) 767 768 def _check_prefix(self, lines, prefix, name, lineno): 769 """ 770 Check that every line in the given list starts with the given 771 prefix; if any line does not, then raise a ValueError. 772 """ 773 for i, line in enumerate(lines): 774 if line and not line.startswith(prefix): 775 raise ValueError('line %r of the docstring for %s has ' 776 'inconsistent leading whitespace: %r' % 777 (lineno+i+1, name, line)) 778 779 780###################################################################### 781## 4. DocTest Finder 782###################################################################### 783 784class DocTestFinder: 785 """ 786 A class used to extract the DocTests that are relevant to a given 787 object, from its docstring and the docstrings of its contained 788 objects. Doctests can currently be extracted from the following 789 object types: modules, functions, classes, methods, staticmethods, 790 classmethods, and properties. 791 """ 792 793 def __init__(self, verbose=False, parser=DocTestParser(), 794 recurse=True, exclude_empty=True): 795 """ 796 Create a new doctest finder. 797 798 The optional argument `parser` specifies a class or 799 function that should be used to create new DocTest objects (or 800 objects that implement the same interface as DocTest). The 801 signature for this factory function should match the signature 802 of the DocTest constructor. 803 804 If the optional argument `recurse` is false, then `find` will 805 only examine the given object, and not any contained objects. 806 807 If the optional argument `exclude_empty` is false, then `find` 808 will include tests for objects with empty docstrings. 809 """ 810 self._parser = parser 811 self._verbose = verbose 812 self._recurse = recurse 813 self._exclude_empty = exclude_empty 814 815 def find(self, obj, name=None, module=None, globs=None, extraglobs=None): 816 """ 817 Return a list of the DocTests that are defined by the given 818 object's docstring, or by any of its contained objects' 819 docstrings. 820 821 The optional parameter `module` is the module that contains 822 the given object. If the module is not specified or is None, then 823 the test finder will attempt to automatically determine the 824 correct module. The object's module is used: 825 826 - As a default namespace, if `globs` is not specified. 827 - To prevent the DocTestFinder from extracting DocTests 828 from objects that are imported from other modules. 829 - To find the name of the file containing the object. 830 - To help find the line number of the object within its 831 file. 832 833 Contained objects whose module does not match `module` are ignored. 834 835 If `module` is False, no attempt to find the module will be made. 836 This is obscure, of use mostly in tests: if `module` is False, or 837 is None but cannot be found automatically, then all objects are 838 considered to belong to the (non-existent) module, so all contained 839 objects will (recursively) be searched for doctests. 840 841 The globals for each DocTest is formed by combining `globs` 842 and `extraglobs` (bindings in `extraglobs` override bindings 843 in `globs`). A new copy of the globals dictionary is created 844 for each DocTest. If `globs` is not specified, then it 845 defaults to the module's `__dict__`, if specified, or {} 846 otherwise. If `extraglobs` is not specified, then it defaults 847 to {}. 848 849 """ 850 # If name was not specified, then extract it from the object. 851 if name is None: 852 name = getattr(obj, '__name__', None) 853 if name is None: 854 raise ValueError("DocTestFinder.find: name must be given " 855 "when obj.__name__ doesn't exist: %r" % 856 (type(obj),)) 857 858 # Find the module that contains the given object (if obj is 859 # a module, then module=obj.). Note: this may fail, in which 860 # case module will be None. 861 if module is False: 862 module = None 863 elif module is None: 864 module = inspect.getmodule(obj) 865 866 # Read the module's source code. This is used by 867 # DocTestFinder._find_lineno to find the line number for a 868 # given object's docstring. 869 try: 870 file = inspect.getsourcefile(obj) or inspect.getfile(obj) 871 if module is not None: 872 # Supply the module globals in case the module was 873 # originally loaded via a PEP 302 loader and 874 # file is not a valid filesystem path 875 source_lines = linecache.getlines(file, module.__dict__) 876 else: 877 # No access to a loader, so assume it's a normal 878 # filesystem path 879 source_lines = linecache.getlines(file) 880 if not source_lines: 881 source_lines = None 882 except TypeError: 883 source_lines = None 884 885 # Initialize globals, and merge in extraglobs. 886 if globs is None: 887 if module is None: 888 globs = {} 889 else: 890 globs = module.__dict__.copy() 891 else: 892 globs = globs.copy() 893 if extraglobs is not None: 894 globs.update(extraglobs) 895 if '__name__' not in globs: 896 globs['__name__'] = '__main__' # provide a default module name 897 898 # Recursively expore `obj`, extracting DocTests. 899 tests = [] 900 self._find(tests, obj, name, module, source_lines, globs, {}) 901 # Sort the tests by alpha order of names, for consistency in 902 # verbose-mode output. This was a feature of doctest in Pythons 903 # <= 2.3 that got lost by accident in 2.4. It was repaired in 904 # 2.4.4 and 2.5. 905 tests.sort() 906 return tests 907 908 def _from_module(self, module, object): 909 """ 910 Return true if the given object is defined in the given 911 module. 912 """ 913 if module is None: 914 return True 915 elif inspect.getmodule(object) is not None: 916 return module is inspect.getmodule(object) 917 elif inspect.isfunction(object): 918 return module.__dict__ is object.func_globals 919 elif inspect.isclass(object): 920 return module.__name__ == object.__module__ 921 elif hasattr(object, '__module__'): 922 return module.__name__ == object.__module__ 923 elif isinstance(object, property): 924 return True # [XX] no way not be sure. 925 else: 926 raise ValueError("object must be a class or function") 927 928 def _find(self, tests, obj, name, module, source_lines, globs, seen): 929 """ 930 Find tests for the given object and any contained objects, and 931 add them to `tests`. 932 """ 933 if self._verbose: 934 print 'Finding tests in %s' % name 935 936 # If we've already processed this object, then ignore it. 937 if id(obj) in seen: 938 return 939 seen[id(obj)] = 1 940 941 # Find a test for this object, and add it to the list of tests. 942 test = self._get_test(obj, name, module, globs, source_lines) 943 if test is not None: 944 tests.append(test) 945 946 # Look for tests in a module's contained objects. 947 if inspect.ismodule(obj) and self._recurse: 948 for valname, val in obj.__dict__.items(): 949 valname = '%s.%s' % (name, valname) 950 # Recurse to functions & classes. 951 if ((inspect.isfunction(val) or inspect.isclass(val)) and 952 self._from_module(module, val)): 953 self._find(tests, val, valname, module, source_lines, 954 globs, seen) 955 956 # Look for tests in a module's __test__ dictionary. 957 if inspect.ismodule(obj) and self._recurse: 958 for valname, val in getattr(obj, '__test__', {}).items(): 959 if not isinstance(valname, basestring): 960 raise ValueError("DocTestFinder.find: __test__ keys " 961 "must be strings: %r" % 962 (type(valname),)) 963 if not (inspect.isfunction(val) or inspect.isclass(val) or 964 inspect.ismethod(val) or inspect.ismodule(val) or 965 isinstance(val, basestring)): 966 raise ValueError("DocTestFinder.find: __test__ values " 967 "must be strings, functions, methods, " 968 "classes, or modules: %r" % 969 (type(val),)) 970 valname = '%s.__test__.%s' % (name, valname) 971 self._find(tests, val, valname, module, source_lines, 972 globs, seen) 973 974 # Look for tests in a class's contained objects. 975 if inspect.isclass(obj) and self._recurse: 976 for valname, val in obj.__dict__.items(): 977 # Special handling for staticmethod/classmethod. 978 if isinstance(val, staticmethod): 979 val = getattr(obj, valname) 980 if isinstance(val, classmethod): 981 val = getattr(obj, valname).im_func 982 983 # Recurse to methods, properties, and nested classes. 984 if ((inspect.isfunction(val) or inspect.isclass(val) or 985 isinstance(val, property)) and 986 self._from_module(module, val)): 987 valname = '%s.%s' % (name, valname) 988 self._find(tests, val, valname, module, source_lines, 989 globs, seen) 990 991 def _get_test(self, obj, name, module, globs, source_lines): 992 """ 993 Return a DocTest for the given object, if it defines a docstring; 994 otherwise, return None. 995 """ 996 # Extract the object's docstring. If it doesn't have one, 997 # then return None (no test for this object). 998 if isinstance(obj, basestring): 999 docstring = obj 1000 else: 1001 try: 1002 if obj.__doc__ is None: 1003 docstring = '' 1004 else: 1005 docstring = obj.__doc__ 1006 if not isinstance(docstring, basestring): 1007 docstring = str(docstring) 1008 except (TypeError, AttributeError): 1009 docstring = '' 1010 1011 # Find the docstring's location in the file. 1012 lineno = self._find_lineno(obj, source_lines) 1013 1014 # Don't bother if the docstring is empty. 1015 if self._exclude_empty and not docstring: 1016 return None 1017 1018 # Return a DocTest for this object. 1019 if module is None: 1020 filename = None 1021 else: 1022 filename = getattr(module, '__file__', module.__name__) 1023 if filename[-4:] in (".pyc", ".pyo"): 1024 filename = filename[:-1] 1025 elif filename.endswith('$py.class'): 1026 filename = '%s.py' % filename[:-9] 1027 return self._parser.get_doctest(docstring, globs, name, 1028 filename, lineno) 1029 1030 def _find_lineno(self, obj, source_lines): 1031 """ 1032 Return a line number of the given object's docstring. Note: 1033 this method assumes that the object has a docstring. 1034 """ 1035 lineno = None 1036 1037 # Find the line number for modules. 1038 if inspect.ismodule(obj): 1039 lineno = 0 1040 1041 # Find the line number for classes. 1042 # Note: this could be fooled if a class is defined multiple 1043 # times in a single file. 1044 if inspect.isclass(obj): 1045 if source_lines is None: 1046 return None 1047 pat = re.compile(r'^\s*class\s*%s\b' % 1048 getattr(obj, '__name__', '-')) 1049 for i, line in enumerate(source_lines): 1050 if pat.match(line): 1051 lineno = i 1052 break 1053 1054 # Find the line number for functions & methods. 1055 if inspect.ismethod(obj): obj = obj.im_func 1056 if inspect.isfunction(obj): obj = obj.func_code 1057 if inspect.istraceback(obj): obj = obj.tb_frame 1058 if inspect.isframe(obj): obj = obj.f_code 1059 if inspect.iscode(obj): 1060 lineno = getattr(obj, 'co_firstlineno', None)-1 1061 1062 # Find the line number where the docstring starts. Assume 1063 # that it's the first line that begins with a quote mark. 1064 # Note: this could be fooled by a multiline function 1065 # signature, where a continuation line begins with a quote 1066 # mark. 1067 if lineno is not None: 1068 if source_lines is None: 1069 return lineno+1 1070 pat = re.compile('(^|.*:)\s*\w*("|\')') 1071 for lineno in range(lineno, len(source_lines)): 1072 if pat.match(source_lines[lineno]): 1073 return lineno 1074 1075 # We couldn't find the line number. 1076 return None 1077 1078###################################################################### 1079## 5. DocTest Runner 1080###################################################################### 1081 1082class DocTestRunner: 1083 """ 1084 A class used to run DocTest test cases, and accumulate statistics. 1085 The `run` method is used to process a single DocTest case. It 1086 returns a tuple `(f, t)`, where `t` is the number of test cases 1087 tried, and `f` is the number of test cases that failed. 1088 1089 >>> tests = DocTestFinder().find(_TestClass) 1090 >>> runner = DocTestRunner(verbose=False) 1091 >>> tests.sort(key = lambda test: test.name) 1092 >>> for test in tests: 1093 ... print test.name, '->', runner.run(test) 1094 _TestClass -> TestResults(failed=0, attempted=2) 1095 _TestClass.__init__ -> TestResults(failed=0, attempted=2) 1096 _TestClass.get -> TestResults(failed=0, attempted=2) 1097 _TestClass.square -> TestResults(failed=0, attempted=1) 1098 1099 The `summarize` method prints a summary of all the test cases that 1100 have been run by the runner, and returns an aggregated `(f, t)` 1101 tuple: 1102 1103 >>> runner.summarize(verbose=1) 1104 4 items passed all tests: 1105 2 tests in _TestClass 1106 2 tests in _TestClass.__init__ 1107 2 tests in _TestClass.get 1108 1 tests in _TestClass.square 1109 7 tests in 4 items. 1110 7 passed and 0 failed. 1111 Test passed. 1112 TestResults(failed=0, attempted=7) 1113 1114 The aggregated number of tried examples and failed examples is 1115 also available via the `tries` and `failures` attributes: 1116 1117 >>> runner.tries 1118 7 1119 >>> runner.failures 1120 0 1121 1122 The comparison between expected outputs and actual outputs is done 1123 by an `OutputChecker`. This comparison may be customized with a 1124 number of option flags; see the documentation for `testmod` for 1125 more information. If the option flags are insufficient, then the 1126 comparison may also be customized by passing a subclass of 1127 `OutputChecker` to the constructor. 1128 1129 The test runner's display output can be controlled in two ways. 1130 First, an output function (`out) can be passed to 1131 `TestRunner.run`; this function will be called with strings that 1132 should be displayed. It defaults to `sys.stdout.write`. If 1133 capturing the output is not sufficient, then the display output 1134 can be also customized by subclassing DocTestRunner, and 1135 overriding the methods `report_start`, `report_success`, 1136 `report_unexpected_exception`, and `report_failure`. 1137 """ 1138 # This divider string is used to separate failure messages, and to 1139 # separate sections of the summary. 1140 DIVIDER = "*" * 70 1141 1142 def __init__(self, checker=None, verbose=None, optionflags=0): 1143 """ 1144 Create a new test runner. 1145 1146 Optional keyword arg `checker` is the `OutputChecker` that 1147 should be used to compare the expected outputs and actual 1148 outputs of doctest examples. 1149 1150 Optional keyword arg 'verbose' prints lots of stuff if true, 1151 only failures if false; by default, it's true iff '-v' is in 1152 sys.argv. 1153 1154 Optional argument `optionflags` can be used to control how the 1155 test runner compares expected output to actual output, and how 1156 it displays failures. See the documentation for `testmod` for 1157 more information. 1158 """ 1159 self._checker = checker or OutputChecker() 1160 if verbose is None: 1161 verbose = '-v' in sys.argv 1162 self._verbose = verbose 1163 self.optionflags = optionflags 1164 self.original_optionflags = optionflags 1165 1166 # Keep track of the examples we've run. 1167 self.tries = 0 1168 self.failures = 0 1169 self._name2ft = {} 1170 1171 # Create a fake output target for capturing doctest output. 1172 self._fakeout = _SpoofOut() 1173 1174 #///////////////////////////////////////////////////////////////// 1175 # Reporting methods 1176 #///////////////////////////////////////////////////////////////// 1177 1178 def report_start(self, out, test, example): 1179 """ 1180 Report that the test runner is about to process the given 1181 example. (Only displays a message if verbose=True) 1182 """ 1183 if self._verbose: 1184 if example.want: 1185 out('Trying:\n' + _indent(example.source) + 1186 'Expecting:\n' + _indent(example.want)) 1187 else: 1188 out('Trying:\n' + _indent(example.source) + 1189 'Expecting nothing\n') 1190 1191 def report_success(self, out, test, example, got): 1192 """ 1193 Report that the given example ran successfully. (Only 1194 displays a message if verbose=True) 1195 """ 1196 if self._verbose: 1197 out("ok\n") 1198 1199 def report_failure(self, out, test, example, got): 1200 """ 1201 Report that the given example failed. 1202 """ 1203 out(self._failure_header(test, example) + 1204 self._checker.output_difference(example, got, self.optionflags)) 1205 1206 def report_unexpected_exception(self, out, test, example, exc_info): 1207 """ 1208 Report that the given example raised an unexpected exception. 1209 """ 1210 out(self._failure_header(test, example) + 1211 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) 1212 1213 def _failure_header(self, test, example): 1214 out = [self.DIVIDER] 1215 if test.filename: 1216 if test.lineno is not None and example.lineno is not None: 1217 lineno = test.lineno + example.lineno + 1 1218 else: 1219 lineno = '?' 1220 out.append('File "%s", line %s, in %s' % 1221 (test.filename, lineno, test.name)) 1222 else: 1223 out.append('Line %s, in %s' % (example.lineno+1, test.name)) 1224 out.append('Failed example:') 1225 source = example.source 1226 out.append(_indent(source)) 1227 return '\n'.join(out) 1228 1229 #///////////////////////////////////////////////////////////////// 1230 # DocTest Running 1231 #///////////////////////////////////////////////////////////////// 1232 1233 def __run(self, test, compileflags, out): 1234 """ 1235 Run the examples in `test`. Write the outcome of each example 1236 with one of the `DocTestRunner.report_*` methods, using the 1237 writer function `out`. `compileflags` is the set of compiler 1238 flags that should be used to execute examples. Return a tuple 1239 `(f, t)`, where `t` is the number of examples tried, and `f` 1240 is the number of examples that failed. The examples are run 1241 in the namespace `test.globs`. 1242 """ 1243 # Keep track of the number of failures and tries. 1244 failures = tries = 0 1245 1246 # Save the option flags (since option directives can be used 1247 # to modify them). 1248 original_optionflags = self.optionflags 1249 1250 SUCCESS, FAILURE, BOOM = range(3) # `outcome` state 1251 1252 check = self._checker.check_output 1253 1254 # Process each example. 1255 for examplenum, example in enumerate(test.examples): 1256 1257 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress 1258 # reporting after the first failure. 1259 quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and 1260 failures > 0) 1261 1262 # Merge in the example's options. 1263 self.optionflags = original_optionflags 1264 if example.options: 1265 for (optionflag, val) in example.options.items(): 1266 if val: 1267 self.optionflags |= optionflag 1268 else: 1269 self.optionflags &= ~optionflag 1270 1271 # If 'SKIP' is set, then skip this example. 1272 if self.optionflags & SKIP: 1273 continue 1274 1275 # Record that we started this example. 1276 tries += 1 1277 if not quiet: 1278 self.report_start(out, test, example) 1279 1280 # Use a special filename for compile(), so we can retrieve 1281 # the source code during interactive debugging (see 1282 # __patched_linecache_getlines). 1283 filename = '<doctest %s[%d]>' % (test.name, examplenum) 1284 1285 # Run the example in the given context (globs), and record 1286 # any exception that gets raised. (But don't intercept 1287 # keyboard interrupts.) 1288 try: 1289 # Don't blink! This is where the user's code gets run. 1290 exec compile(example.source, filename, "single", 1291 compileflags, 1) in test.globs 1292 self.debugger.set_continue() # ==== Example Finished ==== 1293 exception = None 1294 except KeyboardInterrupt: 1295 raise 1296 except: 1297 exception = sys.exc_info() 1298 self.debugger.set_continue() # ==== Example Finished ==== 1299 1300 got = self._fakeout.getvalue() # the actual output 1301 self._fakeout.truncate(0) 1302 outcome = FAILURE # guilty until proved innocent or insane 1303 1304 # If the example executed without raising any exceptions, 1305 # verify its output. 1306 if exception is None: 1307 if check(example.want, got, self.optionflags): 1308 outcome = SUCCESS 1309 1310 # The example raised an exception: check if it was expected. 1311 else: 1312 exc_info = sys.exc_info() 1313 exc_msg = traceback.format_exception_only(*exc_info[:2])[-1] 1314 if not quiet: 1315 got += _exception_traceback(exc_info) 1316 1317 # If `example.exc_msg` is None, then we weren't expecting 1318 # an exception. 1319 if example.exc_msg is None: 1320 outcome = BOOM 1321 1322 # We expected an exception: see whether it matches. 1323 elif check(example.exc_msg, exc_msg, self.optionflags): 1324 outcome = SUCCESS 1325 1326 # Another chance if they didn't care about the detail. 1327 elif self.optionflags & IGNORE_EXCEPTION_DETAIL: 1328 m1 = re.match(r'(?:[^:]*\.)?([^:]*:)', example.exc_msg) 1329 m2 = re.match(r'(?:[^:]*\.)?([^:]*:)', exc_msg) 1330 if m1 and m2 and check(m1.group(1), m2.group(1), 1331 self.optionflags): 1332 outcome = SUCCESS 1333 1334 # Report the outcome. 1335 if outcome is SUCCESS: 1336 if not quiet: 1337 self.report_success(out, test, example, got) 1338 elif outcome is FAILURE: 1339 if not quiet: 1340 self.report_failure(out, test, example, got) 1341 failures += 1 1342 elif outcome is BOOM: 1343 if not quiet: 1344 self.report_unexpected_exception(out, test, example, 1345 exc_info) 1346 failures += 1 1347 else: 1348 assert False, ("unknown outcome", outcome) 1349 1350 # Restore the option flags (in case they were modified) 1351 self.optionflags = original_optionflags 1352 1353 # Record and return the number of failures and tries. 1354 self.__record_outcome(test, failures, tries) 1355 return TestResults(failures, tries) 1356 1357 def __record_outcome(self, test, f, t): 1358 """ 1359 Record the fact that the given DocTest (`test`) generated `f` 1360 failures out of `t` tried examples. 1361 """ 1362 f2, t2 = self._name2ft.get(test.name, (0,0)) 1363 self._name2ft[test.name] = (f+f2, t+t2) 1364 self.failures += f 1365 self.tries += t 1366 1367 __LINECACHE_FILENAME_RE = re.compile(r'<doctest ' 1368 r'(?P<name>.+)' 1369 r'\[(?P<examplenum>\d+)\]>$') 1370 def __patched_linecache_getlines(self, filename, module_globals=None): 1371 m = self.__LINECACHE_FILENAME_RE.match(filename) 1372 if m and m.group('name') == self.test.name: 1373 example = self.test.examples[int(m.group('examplenum'))] 1374 source = example.source 1375 if isinstance(source, unicode): 1376 source = source.encode('ascii', 'backslashreplace') 1377 return source.splitlines(True) 1378 else: 1379 return self.save_linecache_getlines(filename, module_globals) 1380 1381 def run(self, test, compileflags=None, out=None, clear_globs=True): 1382 """ 1383 Run the examples in `test`, and display the results using the 1384 writer function `out`. 1385 1386 The examples are run in the namespace `test.globs`. If 1387 `clear_globs` is true (the default), then this namespace will 1388 be cleared after the test runs, to help with garbage 1389 collection. If you would like to examine the namespace after 1390 the test completes, then use `clear_globs=False`. 1391 1392 `compileflags` gives the set of flags that should be used by 1393 the Python compiler when running the examples. If not 1394 specified, then it will default to the set of future-import 1395 flags that apply to `globs`. 1396 1397 The output of each example is checked using 1398 `DocTestRunner.check_output`, and the results are formatted by 1399 the `DocTestRunner.report_*` methods. 1400 """ 1401 self.test = test 1402 1403 if compileflags is None: 1404 compileflags = _extract_future_flags(test.globs) 1405 1406 save_stdout = sys.stdout 1407 if out is None: 1408 out = save_stdout.write 1409 sys.stdout = self._fakeout 1410 1411 # Patch pdb.set_trace to restore sys.stdout during interactive 1412 # debugging (so it's not still redirected to self._fakeout). 1413 # Note that the interactive output will go to *our* 1414 # save_stdout, even if that's not the real sys.stdout; this 1415 # allows us to write test cases for the set_trace behavior. 1416 save_set_trace = pdb.set_trace 1417 self.debugger = _OutputRedirectingPdb(save_stdout) 1418 self.debugger.reset() 1419 pdb.set_trace = self.debugger.set_trace 1420 1421 # Patch linecache.getlines, so we can see the example's source 1422 # when we're inside the debugger. 1423 self.save_linecache_getlines = linecache.getlines 1424 linecache.getlines = self.__patched_linecache_getlines 1425 1426 # Make sure sys.displayhook just prints the value to stdout 1427 save_displayhook = sys.displayhook 1428 sys.displayhook = sys.__displayhook__ 1429 1430 try: 1431 return self.__run(test, compileflags, out) 1432 finally: 1433 sys.stdout = save_stdout 1434 pdb.set_trace = save_set_trace 1435 linecache.getlines = self.save_linecache_getlines 1436 sys.displayhook = save_displayhook 1437 if clear_globs: 1438 test.globs.clear() 1439 1440 #///////////////////////////////////////////////////////////////// 1441 # Summarization 1442 #///////////////////////////////////////////////////////////////// 1443 def summarize(self, verbose=None): 1444 """ 1445 Print a summary of all the test cases that have been run by 1446 this DocTestRunner, and return a tuple `(f, t)`, where `f` is 1447 the total number of failed examples, and `t` is the total 1448 number of tried examples. 1449 1450 The optional `verbose` argument controls how detailed the 1451 summary is. If the verbosity is not specified, then the 1452 DocTestRunner's verbosity is used. 1453 """ 1454 if verbose is None: 1455 verbose = self._verbose 1456 notests = [] 1457 passed = [] 1458 failed = [] 1459 totalt = totalf = 0 1460 for x in self._name2ft.items(): 1461 name, (f, t) = x 1462 assert f <= t 1463 totalt += t 1464 totalf += f 1465 if t == 0: 1466 notests.append(name) 1467 elif f == 0: 1468 passed.append( (name, t) ) 1469 else: 1470 failed.append(x) 1471 if verbose: 1472 if notests: 1473 print len(notests), "items had no tests:" 1474 notests.sort() 1475 for thing in notests: 1476 print " ", thing 1477 if passed: 1478 print len(passed), "items passed all tests:" 1479 passed.sort() 1480 for thing, count in passed: 1481 print " %3d tests in %s" % (count, thing) 1482 if failed: 1483 print self.DIVIDER 1484 print len(failed), "items had failures:" 1485 failed.sort() 1486 for thing, (f, t) in failed: 1487 print " %3d of %3d in %s" % (f, t, thing) 1488 if verbose: 1489 print totalt, "tests in", len(self._name2ft), "items." 1490 print totalt - totalf, "passed and", totalf, "failed." 1491 if totalf: 1492 print "***Test Failed***", totalf, "failures." 1493 elif verbose: 1494 print "Test passed." 1495 return TestResults(totalf, totalt) 1496 1497 #///////////////////////////////////////////////////////////////// 1498 # Backward compatibility cruft to maintain doctest.master. 1499 #///////////////////////////////////////////////////////////////// 1500 def merge(self, other): 1501 d = self._name2ft 1502 for name, (f, t) in other._name2ft.items(): 1503 if name in d: 1504 # Don't print here by default, since doing 1505 # so breaks some of the buildbots 1506 #print "*** DocTestRunner.merge: '" + name + "' in both" \ 1507 # " testers; summing outcomes." 1508 f2, t2 = d[name] 1509 f = f + f2 1510 t = t + t2 1511 d[name] = f, t 1512 1513class OutputChecker: 1514 """ 1515 A class used to check the whether the actual output from a doctest 1516 example matches the expected output. `OutputChecker` defines two 1517 methods: `check_output`, which compares a given pair of outputs, 1518 and returns true if they match; and `output_difference`, which 1519 returns a string describing the differences between two outputs. 1520 """ 1521 def check_output(self, want, got, optionflags): 1522 """ 1523 Return True iff the actual output from an example (`got`) 1524 matches the expected output (`want`). These strings are 1525 always considered to match if they are identical; but 1526 depending on what option flags the test runner is using, 1527 several non-exact match types are also possible. See the 1528 documentation for `TestRunner` for more information about 1529 option flags. 1530 """ 1531 # Handle the common case first, for efficiency: 1532 # if they're string-identical, always return true. 1533 if got == want: 1534 return True 1535 1536 # The values True and False replaced 1 and 0 as the return 1537 # value for boolean comparisons in Python 2.3. 1538 if not (optionflags & DONT_ACCEPT_TRUE_FOR_1): 1539 if (got,want) == ("True\n", "1\n"): 1540 return True 1541 if (got,want) == ("False\n", "0\n"): 1542 return True 1543 1544 # <BLANKLINE> can be used as a special sequence to signify a 1545 # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used. 1546 if not (optionflags & DONT_ACCEPT_BLANKLINE): 1547 # Replace <BLANKLINE> in want with a blank line. 1548 want = re.sub('(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER), 1549 '', want) 1550 # If a line in got contains only spaces, then remove the 1551 # spaces. 1552 got = re.sub('(?m)^\s*?$', '', got) 1553 if got == want: 1554 return True 1555 1556 # This flag causes doctest to ignore any differences in the 1557 # contents of whitespace strings. Note that this can be used 1558 # in conjunction with the ELLIPSIS flag. 1559 if optionflags & NORMALIZE_WHITESPACE: 1560 got = ' '.join(got.split()) 1561 want = ' '.join(want.split()) 1562 if got == want: 1563 return True 1564 1565 # The ELLIPSIS flag says to let the sequence "..." in `want` 1566 # match any substring in `got`. 1567 if optionflags & ELLIPSIS: 1568 if _ellipsis_match(want, got): 1569 return True 1570 1571 # We didn't find any match; return false. 1572 return False 1573 1574 # Should we do a fancy diff? 1575 def _do_a_fancy_diff(self, want, got, optionflags): 1576 # Not unless they asked for a fancy diff. 1577 if not optionflags & (REPORT_UDIFF | 1578 REPORT_CDIFF | 1579 REPORT_NDIFF): 1580 return False 1581 1582 # If expected output uses ellipsis, a meaningful fancy diff is 1583 # too hard ... or maybe not. In two real-life failures Tim saw, 1584 # a diff was a major help anyway, so this is commented out. 1585 # [todo] _ellipsis_match() knows which pieces do and don't match, 1586 # and could be the basis for a kick-ass diff in this case. 1587 ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want: 1588 ## return False 1589 1590 # ndiff does intraline difference marking, so can be useful even 1591 # for 1-line differences. 1592 if optionflags & REPORT_NDIFF: 1593 return True 1594 1595 # The other diff types need at least a few lines to be helpful. 1596 return want.count('\n') > 2 and got.count('\n') > 2 1597 1598 def output_difference(self, example, got, optionflags): 1599 """ 1600 Return a string describing the differences between the 1601 expected output for a given example (`example`) and the actual 1602 output (`got`). `optionflags` is the set of option flags used 1603 to compare `want` and `got`. 1604 """ 1605 want = example.want 1606 # If <BLANKLINE>s are being used, then replace blank lines 1607 # with <BLANKLINE> in the actual output string. 1608 if not (optionflags & DONT_ACCEPT_BLANKLINE): 1609 got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got) 1610 1611 # Check if we should use diff. 1612 if self._do_a_fancy_diff(want, got, optionflags): 1613 # Split want & got into lines. 1614 want_lines = want.splitlines(True) # True == keep line ends 1615 got_lines = got.splitlines(True) 1616 # Use difflib to find their differences. 1617 if optionflags & REPORT_UDIFF: 1618 diff = difflib.unified_diff(want_lines, got_lines, n=2) 1619 diff = list(diff)[2:] # strip the diff header 1620 kind = 'unified diff with -expected +actual' 1621 elif optionflags & REPORT_CDIFF: 1622 diff = difflib.context_diff(want_lines, got_lines, n=2) 1623 diff = list(diff)[2:] # strip the diff header 1624 kind = 'context diff with expected followed by actual' 1625 elif optionflags & REPORT_NDIFF: 1626 engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK) 1627 diff = list(engine.compare(want_lines, got_lines)) 1628 kind = 'ndiff with -expected +actual' 1629 else: 1630 assert 0, 'Bad diff option' 1631 # Remove trailing whitespace on diff output. 1632 diff = [line.rstrip() + '\n' for line in diff] 1633 return 'Differences (%s):\n' % kind + _indent(''.join(diff)) 1634 1635 # If we're not using diff, then simply list the expected 1636 # output followed by the actual output. 1637 if want and got: 1638 return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got)) 1639 elif want: 1640 return 'Expected:\n%sGot nothing\n' % _indent(want) 1641 elif got: 1642 return 'Expected nothing\nGot:\n%s' % _indent(got) 1643 else: 1644 return 'Expected nothing\nGot nothing\n' 1645 1646class DocTestFailure(Exception): 1647 """A DocTest example has failed in debugging mode. 1648 1649 The exception instance has variables: 1650 1651 - test: the DocTest object being run 1652 1653 - example: the Example object that failed 1654 1655 - got: the actual output 1656 """ 1657 def __init__(self, test, example, got): 1658 self.test = test 1659 self.example = example 1660 self.got = got 1661 1662 def __str__(self): 1663 return str(self.test) 1664 1665class UnexpectedException(Exception): 1666 """A DocTest example has encountered an unexpected exception 1667 1668 The exception instance has variables: 1669 1670 - test: the DocTest object being run 1671 1672 - example: the Example object that failed 1673 1674 - exc_info: the exception info 1675 """ 1676 def __init__(self, test, example, exc_info): 1677 self.test = test 1678 self.example = example 1679 self.exc_info = exc_info 1680 1681 def __str__(self): 1682 return str(self.test) 1683 1684class DebugRunner(DocTestRunner): 1685 r"""Run doc tests but raise an exception as soon as there is a failure. 1686 1687 If an unexpected exception occurs, an UnexpectedException is raised. 1688 It contains the test, the example, and the original exception: 1689 1690 >>> runner = DebugRunner(verbose=False) 1691 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', 1692 ... {}, 'foo', 'foo.py', 0) 1693 >>> try: 1694 ... runner.run(test) 1695 ... except UnexpectedException, failure: 1696 ... pass 1697 1698 >>> failure.test is test 1699 True 1700 1701 >>> failure.example.want 1702 '42\n' 1703 1704 >>> exc_info = failure.exc_info 1705 >>> raise exc_info[0], exc_info[1], exc_info[2] 1706 Traceback (most recent call last): 1707 ... 1708 KeyError 1709 1710 We wrap the original exception to give the calling application 1711 access to the test and example information. 1712 1713 If the output doesn't match, then a DocTestFailure is raised: 1714 1715 >>> test = DocTestParser().get_doctest(''' 1716 ... >>> x = 1 1717 ... >>> x 1718 ... 2 1719 ... ''', {}, 'foo', 'foo.py', 0) 1720 1721 >>> try: 1722 ... runner.run(test) 1723 ... except DocTestFailure, failure: 1724 ... pass 1725 1726 DocTestFailure objects provide access to the test: 1727 1728 >>> failure.test is test 1729 True 1730 1731 As well as to the example: 1732 1733 >>> failure.example.want 1734 '2\n' 1735 1736 and the actual output: 1737 1738 >>> failure.got 1739 '1\n' 1740 1741 If a failure or error occurs, the globals are left intact: 1742 1743 >>> if '__builtins__' in test.globs: 1744 ... del test.globs['__builtins__'] 1745 >>> test.globs 1746 {'x': 1} 1747 1748 >>> test = DocTestParser().get_doctest(''' 1749 ... >>> x = 2 1750 ... >>> raise KeyError 1751 ... ''', {}, 'foo', 'foo.py', 0) 1752 1753 >>> runner.run(test) 1754 Traceback (most recent call last): 1755 ... 1756 UnexpectedException: <DocTest foo from foo.py:0 (2 examples)> 1757 1758 >>> if '__builtins__' in test.globs: 1759 ... del test.globs['__builtins__'] 1760 >>> test.globs 1761 {'x': 2} 1762 1763 But the globals are cleared if there is no error: 1764 1765 >>> test = DocTestParser().get_doctest(''' 1766 ... >>> x = 2 1767 ... ''', {}, 'foo', 'foo.py', 0) 1768 1769 >>> runner.run(test) 1770 TestResults(failed=0, attempted=1) 1771 1772 >>> test.globs 1773 {} 1774 1775 """ 1776 1777 def run(self, test, compileflags=None, out=None, clear_globs=True): 1778 r = DocTestRunner.run(self, test, compileflags, out, False) 1779 if clear_globs: 1780 test.globs.clear() 1781 return r 1782 1783 def report_unexpected_exception(self, out, test, example, exc_info): 1784 raise UnexpectedException(test, example, exc_info) 1785 1786 def report_failure(self, out, test, example, got): 1787 raise DocTestFailure(test, example, got) 1788 1789###################################################################### 1790## 6. Test Functions 1791###################################################################### 1792# These should be backwards compatible. 1793 1794# For backward compatibility, a global instance of a DocTestRunner 1795# class, updated by testmod. 1796master = None 1797 1798def testmod(m=None, name=None, globs=None, verbose=None, 1799 report=True, optionflags=0, extraglobs=None, 1800 raise_on_error=False, exclude_empty=False): 1801 """m=None, name=None, globs=None, verbose=None, report=True, 1802 optionflags=0, extraglobs=None, raise_on_error=False, 1803 exclude_empty=False 1804 1805 Test examples in docstrings in functions and classes reachable 1806 from module m (or the current module if m is not supplied), starting 1807 with m.__doc__. 1808 1809 Also test examples reachable from dict m.__test__ if it exists and is 1810 not None. m.__test__ maps names to functions, classes and strings; 1811 function and class docstrings are tested even if the name is private; 1812 strings are tested directly, as if they were docstrings. 1813 1814 Return (#failures, #tests). 1815 1816 See help(doctest) for an overview. 1817 1818 Optional keyword arg "name" gives the name of the module; by default 1819 use m.__name__. 1820 1821 Optional keyword arg "globs" gives a dict to be used as the globals 1822 when executing examples; by default, use m.__dict__. A copy of this 1823 dict is actually used for each docstring, so that each docstring's 1824 examples start with a clean slate. 1825 1826 Optional keyword arg "extraglobs" gives a dictionary that should be 1827 merged into the globals that are used to execute examples. By 1828 default, no extra globals are used. This is new in 2.4. 1829 1830 Optional keyword arg "verbose" prints lots of stuff if true, prints 1831 only failures if false; by default, it's true iff "-v" is in sys.argv. 1832 1833 Optional keyword arg "report" prints a summary at the end when true, 1834 else prints nothing at the end. In verbose mode, the summary is 1835 detailed, else very brief (in fact, empty if all tests passed). 1836 1837 Optional keyword arg "optionflags" or's together module constants, 1838 and defaults to 0. This is new in 2.3. Possible values (see the 1839 docs for details): 1840 1841 DONT_ACCEPT_TRUE_FOR_1 1842 DONT_ACCEPT_BLANKLINE 1843 NORMALIZE_WHITESPACE 1844 ELLIPSIS 1845 SKIP 1846 IGNORE_EXCEPTION_DETAIL 1847 REPORT_UDIFF 1848 REPORT_CDIFF 1849 REPORT_NDIFF 1850 REPORT_ONLY_FIRST_FAILURE 1851 1852 Optional keyword arg "raise_on_error" raises an exception on the 1853 first unexpected exception or failure. This allows failures to be 1854 post-mortem debugged. 1855 1856 Advanced tomfoolery: testmod runs methods of a local instance of 1857 class doctest.Tester, then merges the results into (or creates) 1858 global Tester instance doctest.master. Methods of doctest.master 1859 can be called directly too, if you want to do something unusual. 1860 Passing report=0 to testmod is especially useful then, to delay 1861 displaying a summary. Invoke doctest.master.summarize(verbose) 1862 when you're done fiddling. 1863 """ 1864 global master 1865 1866 # If no module was given, then use __main__. 1867 if m is None: 1868 # DWA - m will still be None if this wasn't invoked from the command 1869 # line, in which case the following TypeError is about as good an error 1870 # as we should expect 1871 m = sys.modules.get('__main__') 1872 1873 # Check that we were actually given a module. 1874 if not inspect.ismodule(m): 1875 raise TypeError("testmod: module required; %r" % (m,)) 1876 1877 # If no name was given, then use the module's name. 1878 if name is None: 1879 name = m.__name__ 1880 1881 # Find, parse, and run all tests in the given module. 1882 finder = DocTestFinder(exclude_empty=exclude_empty) 1883 1884 if raise_on_error: 1885 runner = DebugRunner(verbose=verbose, optionflags=optionflags) 1886 else: 1887 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 1888 1889 for test in finder.find(m, name, globs=globs, extraglobs=extraglobs): 1890 runner.run(test) 1891 1892 if report: 1893 runner.summarize() 1894 1895 if master is None: 1896 master = runner 1897 else: 1898 master.merge(runner) 1899 1900 return TestResults(runner.failures, runner.tries) 1901 1902def testfile(filename, module_relative=True, name=None, package=None, 1903 globs=None, verbose=None, report=True, optionflags=0, 1904 extraglobs=None, raise_on_error=False, parser=DocTestParser(), 1905 encoding=None): 1906 """ 1907 Test examples in the given file. Return (#failures, #tests). 1908 1909 Optional keyword arg "module_relative" specifies how filenames 1910 should be interpreted: 1911 1912 - If "module_relative" is True (the default), then "filename" 1913 specifies a module-relative path. By default, this path is 1914 relative to the calling module's directory; but if the 1915 "package" argument is specified, then it is relative to that 1916 package. To ensure os-independence, "filename" should use 1917 "/" characters to separate path segments, and should not 1918 be an absolute path (i.e., it may not begin with "/"). 1919 1920 - If "module_relative" is False, then "filename" specifies an 1921 os-specific path. The path may be absolute or relative (to 1922 the current working directory). 1923 1924 Optional keyword arg "name" gives the name of the test; by default 1925 use the file's basename. 1926 1927 Optional keyword argument "package" is a Python package or the 1928 name of a Python package whose directory should be used as the 1929 base directory for a module relative filename. If no package is 1930 specified, then the calling module's directory is used as the base 1931 directory for module relative filenames. It is an error to 1932 specify "package" if "module_relative" is False. 1933 1934 Optional keyword arg "globs" gives a dict to be used as the globals 1935 when executing examples; by default, use {}. A copy of this dict 1936 is actually used for each docstring, so that each docstring's 1937 examples start with a clean slate. 1938 1939 Optional keyword arg "extraglobs" gives a dictionary that should be 1940 merged into the globals that are used to execute examples. By 1941 default, no extra globals are used. 1942 1943 Optional keyword arg "verbose" prints lots of stuff if true, prints 1944 only failures if false; by default, it's true iff "-v" is in sys.argv. 1945 1946 Optional keyword arg "report" prints a summary at the end when true, 1947 else prints nothing at the end. In verbose mode, the summary is 1948 detailed, else very brief (in fact, empty if all tests passed). 1949 1950 Optional keyword arg "optionflags" or's together module constants, 1951 and defaults to 0. Possible values (see the docs for details): 1952 1953 DONT_ACCEPT_TRUE_FOR_1 1954 DONT_ACCEPT_BLANKLINE 1955 NORMALIZE_WHITESPACE 1956 ELLIPSIS 1957 SKIP 1958 IGNORE_EXCEPTION_DETAIL 1959 REPORT_UDIFF 1960 REPORT_CDIFF 1961 REPORT_NDIFF 1962 REPORT_ONLY_FIRST_FAILURE 1963 1964 Optional keyword arg "raise_on_error" raises an exception on the 1965 first unexpected exception or failure. This allows failures to be 1966 post-mortem debugged. 1967 1968 Optional keyword arg "parser" specifies a DocTestParser (or 1969 subclass) that should be used to extract tests from the files. 1970 1971 Optional keyword arg "encoding" specifies an encoding that should 1972 be used to convert the file to unicode. 1973 1974 Advanced tomfoolery: testmod runs methods of a local instance of 1975 class doctest.Tester, then merges the results into (or creates) 1976 global Tester instance doctest.master. Methods of doctest.master 1977 can be called directly too, if you want to do something unusual. 1978 Passing report=0 to testmod is especially useful then, to delay 1979 displaying a summary. Invoke doctest.master.summarize(verbose) 1980 when you're done fiddling. 1981 """ 1982 global master 1983 1984 if package and not module_relative: 1985 raise ValueError("Package may only be specified for module-" 1986 "relative paths.") 1987 1988 # Relativize the path 1989 text, filename = _load_testfile(filename, package, module_relative) 1990 1991 # If no name was given, then use the file's name. 1992 if name is None: 1993 name = os.path.basename(filename) 1994 1995 # Assemble the globals. 1996 if globs is None: 1997 globs = {} 1998 else: 1999 globs = globs.copy() 2000 if extraglobs is not None: 2001 globs.update(extraglobs) 2002 if '__name__' not in globs: 2003 globs['__name__'] = '__main__' 2004 2005 if raise_on_error: 2006 runner = DebugRunner(verbose=verbose, optionflags=optionflags) 2007 else: 2008 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 2009 2010 if encoding is not None: 2011 text = text.decode(encoding) 2012 2013 # Read the file, convert it to a test, and run it. 2014 test = parser.get_doctest(text, globs, name, filename, 0) 2015 runner.run(test) 2016 2017 if report: 2018 runner.summarize() 2019 2020 if master is None: 2021 master = runner 2022 else: 2023 master.merge(runner) 2024 2025 return TestResults(runner.failures, runner.tries) 2026 2027def run_docstring_examples(f, globs, verbose=False, name="NoName", 2028 compileflags=None, optionflags=0): 2029 """ 2030 Test examples in the given object's docstring (`f`), using `globs` 2031 as globals. Optional argument `name` is used in failure messages. 2032 If the optional argument `verbose` is true, then generate output 2033 even if there are no failures. 2034 2035 `compileflags` gives the set of flags that should be used by the 2036 Python compiler when running the examples. If not specified, then 2037 it will default to the set of future-import flags that apply to 2038 `globs`. 2039 2040 Optional keyword arg `optionflags` specifies options for the 2041 testing and output. See the documentation for `testmod` for more 2042 information. 2043 """ 2044 # Find, parse, and run all tests in the given module. 2045 finder = DocTestFinder(verbose=verbose, recurse=False) 2046 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 2047 for test in finder.find(f, name, globs=globs): 2048 runner.run(test, compileflags=compileflags) 2049 2050###################################################################### 2051## 7. Tester 2052###################################################################### 2053# This is provided only for backwards compatibility. It's not 2054# actually used in any way. 2055 2056class Tester: 2057 def __init__(self, mod=None, globs=None, verbose=None, optionflags=0): 2058 2059 warnings.warn("class Tester is deprecated; " 2060 "use class doctest.DocTestRunner instead", 2061 DeprecationWarning, stacklevel=2) 2062 if mod is None and globs is None: 2063 raise TypeError("Tester.__init__: must specify mod or globs") 2064 if mod is not None and not inspect.ismodule(mod): 2065 raise TypeError("Tester.__init__: mod must be a module; %r" % 2066 (mod,)) 2067 if globs is None: 2068 globs = mod.__dict__ 2069 self.globs = globs 2070 2071 self.verbose = verbose 2072 self.optionflags = optionflags 2073 self.testfinder = DocTestFinder() 2074 self.testrunner = DocTestRunner(verbose=verbose, 2075 optionflags=optionflags) 2076 2077 def runstring(self, s, name): 2078 test = DocTestParser().get_doctest(s, self.globs, name, None, None) 2079 if self.verbose: 2080 print "Running string", name 2081 (f,t) = self.testrunner.run(test) 2082 if self.verbose: 2083 print f, "of", t, "examples failed in string", name 2084 return TestResults(f,t) 2085 2086 def rundoc(self, object, name=None, module=None): 2087 f = t = 0 2088 tests = self.testfinder.find(object, name, module=module, 2089 globs=self.globs) 2090 for test in tests: 2091 (f2, t2) = self.testrunner.run(test) 2092 (f,t) = (f+f2, t+t2) 2093 return TestResults(f,t) 2094 2095 def rundict(self, d, name, module=None): 2096 import types 2097 m = types.ModuleType(name) 2098 m.__dict__.update(d) 2099 if module is None: 2100 module = False 2101 return self.rundoc(m, name, module) 2102 2103 def run__test__(self, d, name): 2104 import types 2105 m = types.ModuleType(name) 2106 m.__test__ = d 2107 return self.rundoc(m, name) 2108 2109 def summarize(self, verbose=None): 2110 return self.testrunner.summarize(verbose) 2111 2112 def merge(self, other): 2113 self.testrunner.merge(other.testrunner) 2114 2115###################################################################### 2116## 8. Unittest Support 2117###################################################################### 2118 2119_unittest_reportflags = 0 2120 2121def set_unittest_reportflags(flags): 2122 """Sets the unittest option flags. 2123 2124 The old flag is returned so that a runner could restore the old 2125 value if it wished to: 2126 2127 >>> import doctest 2128 >>> old = doctest._unittest_reportflags 2129 >>> doctest.set_unittest_reportflags(REPORT_NDIFF | 2130 ... REPORT_ONLY_FIRST_FAILURE) == old 2131 True 2132 2133 >>> doctest._unittest_reportflags == (REPORT_NDIFF | 2134 ... REPORT_ONLY_FIRST_FAILURE) 2135 True 2136 2137 Only reporting flags can be set: 2138 2139 >>> doctest.set_unittest_reportflags(ELLIPSIS) 2140 Traceback (most recent call last): 2141 ... 2142 ValueError: ('Only reporting flags allowed', 8) 2143 2144 >>> doctest.set_unittest_reportflags(old) == (REPORT_NDIFF | 2145 ... REPORT_ONLY_FIRST_FAILURE) 2146 True 2147 """ 2148 global _unittest_reportflags 2149 2150 if (flags & REPORTING_FLAGS) != flags: 2151 raise ValueError("Only reporting flags allowed", flags) 2152 old = _unittest_reportflags 2153 _unittest_reportflags = flags 2154 return old 2155 2156 2157class DocTestCase(unittest.TestCase): 2158 2159 def __init__(self, test, optionflags=0, setUp=None, tearDown=None, 2160 checker=None): 2161 2162 unittest.TestCase.__init__(self) 2163 self._dt_optionflags = optionflags 2164 self._dt_checker = checker 2165 self._dt_test = test 2166 self._dt_setUp = setUp 2167 self._dt_tearDown = tearDown 2168 2169 def setUp(self): 2170 test = self._dt_test 2171 2172 if self._dt_setUp is not None: 2173 self._dt_setUp(test) 2174 2175 def tearDown(self): 2176 test = self._dt_test 2177 2178 if self._dt_tearDown is not None: 2179 self._dt_tearDown(test) 2180 2181 test.globs.clear() 2182 2183 def runTest(self): 2184 test = self._dt_test 2185 old = sys.stdout 2186 new = StringIO() 2187 optionflags = self._dt_optionflags 2188 2189 if not (optionflags & REPORTING_FLAGS): 2190 # The option flags don't include any reporting flags, 2191 # so add the default reporting flags 2192 optionflags |= _unittest_reportflags 2193 2194 runner = DocTestRunner(optionflags=optionflags, 2195 checker=self._dt_checker, verbose=False) 2196 2197 try: 2198 runner.DIVIDER = "-"*70 2199 failures, tries = runner.run( 2200 test, out=new.write, clear_globs=False) 2201 finally: 2202 sys.stdout = old 2203 2204 if failures: 2205 raise self.failureException(self.format_failure(new.getvalue())) 2206 2207 def format_failure(self, err): 2208 test = self._dt_test 2209 if test.lineno is None: 2210 lineno = 'unknown line number' 2211 else: 2212 lineno = '%s' % test.lineno 2213 lname = '.'.join(test.name.split('.')[-1:]) 2214 return ('Failed doctest test for %s\n' 2215 ' File "%s", line %s, in %s\n\n%s' 2216 % (test.name, test.filename, lineno, lname, err) 2217 ) 2218 2219 def debug(self): 2220 r"""Run the test case without results and without catching exceptions 2221 2222 The unit test framework includes a debug method on test cases 2223 and test suites to support post-mortem debugging. The test code 2224 is run in such a way that errors are not caught. This way a 2225 caller can catch the errors and initiate post-mortem debugging. 2226 2227 The DocTestCase provides a debug method that raises 2228 UnexpectedException errors if there is an unexpected 2229 exception: 2230 2231 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', 2232 ... {}, 'foo', 'foo.py', 0) 2233 >>> case = DocTestCase(test) 2234 >>> try: 2235 ... case.debug() 2236 ... except UnexpectedException, failure: 2237 ... pass 2238 2239 The UnexpectedException contains the test, the example, and 2240 the original exception: 2241 2242 >>> failure.test is test 2243 True 2244 2245 >>> failure.example.want 2246 '42\n' 2247 2248 >>> exc_info = failure.exc_info 2249 >>> raise exc_info[0], exc_info[1], exc_info[2] 2250 Traceback (most recent call last): 2251 ... 2252 KeyError 2253 2254 If the output doesn't match, then a DocTestFailure is raised: 2255 2256 >>> test = DocTestParser().get_doctest(''' 2257 ... >>> x = 1 2258 ... >>> x 2259 ... 2 2260 ... ''', {}, 'foo', 'foo.py', 0) 2261 >>> case = DocTestCase(test) 2262 2263 >>> try: 2264 ... case.debug() 2265 ... except DocTestFailure, failure: 2266 ... pass 2267 2268 DocTestFailure objects provide access to the test: 2269 2270 >>> failure.test is test 2271 True 2272 2273 As well as to the example: 2274 2275 >>> failure.example.want 2276 '2\n' 2277 2278 and the actual output: 2279 2280 >>> failure.got 2281 '1\n' 2282 2283 """ 2284 2285 self.setUp() 2286 runner = DebugRunner(optionflags=self._dt_optionflags, 2287 checker=self._dt_checker, verbose=False) 2288 runner.run(self._dt_test, clear_globs=False) 2289 self.tearDown() 2290 2291 def id(self): 2292 return self._dt_test.name 2293 2294 def __eq__(self, other): 2295 if type(self) is not type(other): 2296 return NotImplemented 2297 2298 return self._dt_test == other._dt_test and \ 2299 self._dt_optionflags == other._dt_optionflags and \ 2300 self._dt_setUp == other._dt_setUp and \ 2301 self._dt_tearDown == other._dt_tearDown and \ 2302 self._dt_checker == other._dt_checker 2303 2304 def __ne__(self, other): 2305 return not self == other 2306 2307 def __hash__(self): 2308 return hash((self._dt_optionflags, self._dt_setUp, self._dt_tearDown, 2309 self._dt_checker)) 2310 2311 def __repr__(self): 2312 name = self._dt_test.name.split('.') 2313 return "%s (%s)" % (name[-1], '.'.join(name[:-1])) 2314 2315 __str__ = __repr__ 2316 2317 def shortDescription(self): 2318 return "Doctest: " + self._dt_test.name 2319 2320class SkipDocTestCase(DocTestCase): 2321 def __init__(self, module): 2322 self.module = module 2323 DocTestCase.__init__(self, None) 2324 2325 def setUp(self): 2326 self.skipTest("DocTestSuite will not work with -O2 and above") 2327 2328 def test_skip(self): 2329 pass 2330 2331 def shortDescription(self): 2332 return "Skipping tests from %s" % self.module.__name__ 2333 2334 __str__ = shortDescription 2335 2336 2337def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, 2338 **options): 2339 """ 2340 Convert doctest tests for a module to a unittest test suite. 2341 2342 This converts each documentation string in a module that 2343 contains doctest tests to a unittest test case. If any of the 2344 tests in a doc string fail, then the test case fails. An exception 2345 is raised showing the name of the file containing the test and a 2346 (sometimes approximate) line number. 2347 2348 The `module` argument provides the module to be tested. The argument 2349 can be either a module or a module name. 2350 2351 If no argument is given, the calling module is used. 2352 2353 A number of options may be provided as keyword arguments: 2354 2355 setUp 2356 A set-up function. This is called before running the 2357 tests in each file. The setUp function will be passed a DocTest 2358 object. The setUp function can access the test globals as the 2359 globs attribute of the test passed. 2360 2361 tearDown 2362 A tear-down function. This is called after running the 2363 tests in each file. The tearDown function will be passed a DocTest 2364 object. The tearDown function can access the test globals as the 2365 globs attribute of the test passed. 2366 2367 globs 2368 A dictionary containing initial global variables for the tests. 2369 2370 optionflags 2371 A set of doctest option flags expressed as an integer. 2372 """ 2373 2374 if test_finder is None: 2375 test_finder = DocTestFinder() 2376 2377 module = _normalize_module(module) 2378 tests = test_finder.find(module, globs=globs, extraglobs=extraglobs) 2379 2380 if not tests and sys.flags.optimize >=2: 2381 # Skip doctests when running with -O2 2382 suite = unittest.TestSuite() 2383 suite.addTest(SkipDocTestCase(module)) 2384 return suite 2385 elif not tests: 2386 # Why do we want to do this? Because it reveals a bug that might 2387 # otherwise be hidden. 2388 # It is probably a bug that this exception is not also raised if the 2389 # number of doctest examples in tests is zero (i.e. if no doctest 2390 # examples were found). However, we should probably not be raising 2391 # an exception at all here, though it is too late to make this change 2392 # for a maintenance release. See also issue #14649. 2393 raise ValueError(module, "has no docstrings") 2394 2395 tests.sort() 2396 suite = unittest.TestSuite() 2397 2398 for test in tests: 2399 if len(test.examples) == 0: 2400 continue 2401 if not test.filename: 2402 filename = module.__file__ 2403 if filename[-4:] in (".pyc", ".pyo"): 2404 filename = filename[:-1] 2405 elif filename.endswith('$py.class'): 2406 filename = '%s.py' % filename[:-9] 2407 test.filename = filename 2408 suite.addTest(DocTestCase(test, **options)) 2409 2410 return suite 2411 2412class DocFileCase(DocTestCase): 2413 2414 def id(self): 2415 return '_'.join(self._dt_test.name.split('.')) 2416 2417 def __repr__(self): 2418 return self._dt_test.filename 2419 __str__ = __repr__ 2420 2421 def format_failure(self, err): 2422 return ('Failed doctest test for %s\n File "%s", line 0\n\n%s' 2423 % (self._dt_test.name, self._dt_test.filename, err) 2424 ) 2425 2426def DocFileTest(path, module_relative=True, package=None, 2427 globs=None, parser=DocTestParser(), 2428 encoding=None, **options): 2429 if globs is None: 2430 globs = {} 2431 else: 2432 globs = globs.copy() 2433 2434 if package and not module_relative: 2435 raise ValueError("Package may only be specified for module-" 2436 "relative paths.") 2437 2438 # Relativize the path. 2439 doc, path = _load_testfile(path, package, module_relative) 2440 2441 if "__file__" not in globs: 2442 globs["__file__"] = path 2443 2444 # Find the file and read it. 2445 name = os.path.basename(path) 2446 2447 # If an encoding is specified, use it to convert the file to unicode 2448 if encoding is not None: 2449 doc = doc.decode(encoding) 2450 2451 # Convert it to a test, and wrap it in a DocFileCase. 2452 test = parser.get_doctest(doc, globs, name, path, 0) 2453 return DocFileCase(test, **options) 2454 2455def DocFileSuite(*paths, **kw): 2456 """A unittest suite for one or more doctest files. 2457 2458 The path to each doctest file is given as a string; the 2459 interpretation of that string depends on the keyword argument 2460 "module_relative". 2461 2462 A number of options may be provided as keyword arguments: 2463 2464 module_relative 2465 If "module_relative" is True, then the given file paths are 2466 interpreted as os-independent module-relative paths. By 2467 default, these paths are relative to the calling module's 2468 directory; but if the "package" argument is specified, then 2469 they are relative to that package. To ensure os-independence, 2470 "filename" should use "/" characters to separate path 2471 segments, and may not be an absolute path (i.e., it may not 2472 begin with "/"). 2473 2474 If "module_relative" is False, then the given file paths are 2475 interpreted as os-specific paths. These paths may be absolute 2476 or relative (to the current working directory). 2477 2478 package 2479 A Python package or the name of a Python package whose directory 2480 should be used as the base directory for module relative paths. 2481 If "package" is not specified, then the calling module's 2482 directory is used as the base directory for module relative 2483 filenames. It is an error to specify "package" if 2484 "module_relative" is False. 2485 2486 setUp 2487 A set-up function. This is called before running the 2488 tests in each file. The setUp function will be passed a DocTest 2489 object. The setUp function can access the test globals as the 2490 globs attribute of the test passed. 2491 2492 tearDown 2493 A tear-down function. This is called after running the 2494 tests in each file. The tearDown function will be passed a DocTest 2495 object. The tearDown function can access the test globals as the 2496 globs attribute of the test passed. 2497 2498 globs 2499 A dictionary containing initial global variables for the tests. 2500 2501 optionflags 2502 A set of doctest option flags expressed as an integer. 2503 2504 parser 2505 A DocTestParser (or subclass) that should be used to extract 2506 tests from the files. 2507 2508 encoding 2509 An encoding that will be used to convert the files to unicode. 2510 """ 2511 suite = unittest.TestSuite() 2512 2513 # We do this here so that _normalize_module is called at the right 2514 # level. If it were called in DocFileTest, then this function 2515 # would be the caller and we might guess the package incorrectly. 2516 if kw.get('module_relative', True): 2517 kw['package'] = _normalize_module(kw.get('package')) 2518 2519 for path in paths: 2520 suite.addTest(DocFileTest(path, **kw)) 2521 2522 return suite 2523 2524###################################################################### 2525## 9. Debugging Support 2526###################################################################### 2527 2528def script_from_examples(s): 2529 r"""Extract script from text with examples. 2530 2531 Converts text with examples to a Python script. Example input is 2532 converted to regular code. Example output and all other words 2533 are converted to comments: 2534 2535 >>> text = ''' 2536 ... Here are examples of simple math. 2537 ... 2538 ... Python has super accurate integer addition 2539 ... 2540 ... >>> 2 + 2 2541 ... 5 2542 ... 2543 ... And very friendly error messages: 2544 ... 2545 ... >>> 1/0 2546 ... To Infinity 2547 ... And 2548 ... Beyond 2549 ... 2550 ... You can use logic if you want: 2551 ... 2552 ... >>> if 0: 2553 ... ... blah 2554 ... ... blah 2555 ... ... 2556 ... 2557 ... Ho hum 2558 ... ''' 2559 2560 >>> print script_from_examples(text) 2561 # Here are examples of simple math. 2562 # 2563 # Python has super accurate integer addition 2564 # 2565 2 + 2 2566 # Expected: 2567 ## 5 2568 # 2569 # And very friendly error messages: 2570 # 2571 1/0 2572 # Expected: 2573 ## To Infinity 2574 ## And 2575 ## Beyond 2576 # 2577 # You can use logic if you want: 2578 # 2579 if 0: 2580 blah 2581 blah 2582 # 2583 # Ho hum 2584 <BLANKLINE> 2585 """ 2586 output = [] 2587 for piece in DocTestParser().parse(s): 2588 if isinstance(piece, Example): 2589 # Add the example's source code (strip trailing NL) 2590 output.append(piece.source[:-1]) 2591 # Add the expected output: 2592 want = piece.want 2593 if want: 2594 output.append('# Expected:') 2595 output += ['## '+l for l in want.split('\n')[:-1]] 2596 else: 2597 # Add non-example text. 2598 output += [_comment_line(l) 2599 for l in piece.split('\n')[:-1]] 2600 2601 # Trim junk on both ends. 2602 while output and output[-1] == '#': 2603 output.pop() 2604 while output and output[0] == '#': 2605 output.pop(0) 2606 # Combine the output, and return it. 2607 # Add a courtesy newline to prevent exec from choking (see bug #1172785) 2608 return '\n'.join(output) + '\n' 2609 2610def testsource(module, name): 2611 """Extract the test sources from a doctest docstring as a script. 2612 2613 Provide the module (or dotted name of the module) containing the 2614 test to be debugged and the name (within the module) of the object 2615 with the doc string with tests to be debugged. 2616 """ 2617 module = _normalize_module(module) 2618 tests = DocTestFinder().find(module) 2619 test = [t for t in tests if t.name == name] 2620 if not test: 2621 raise ValueError(name, "not found in tests") 2622 test = test[0] 2623 testsrc = script_from_examples(test.docstring) 2624 return testsrc 2625 2626def debug_src(src, pm=False, globs=None): 2627 """Debug a single doctest docstring, in argument `src`'""" 2628 testsrc = script_from_examples(src) 2629 debug_script(testsrc, pm, globs) 2630 2631def debug_script(src, pm=False, globs=None): 2632 "Debug a test script. `src` is the script, as a string." 2633 import pdb 2634 2635 # Note that tempfile.NameTemporaryFile() cannot be used. As the 2636 # docs say, a file so created cannot be opened by name a second time 2637 # on modern Windows boxes, and execfile() needs to open it. 2638 srcfilename = tempfile.mktemp(".py", "doctestdebug") 2639 f = open(srcfilename, 'w') 2640 f.write(src) 2641 f.close() 2642 2643 try: 2644 if globs: 2645 globs = globs.copy() 2646 else: 2647 globs = {} 2648 2649 if pm: 2650 try: 2651 execfile(srcfilename, globs, globs) 2652 except: 2653 print sys.exc_info()[1] 2654 pdb.post_mortem(sys.exc_info()[2]) 2655 else: 2656 # Note that %r is vital here. '%s' instead can, e.g., cause 2657 # backslashes to get treated as metacharacters on Windows. 2658 pdb.run("execfile(%r)" % srcfilename, globs, globs) 2659 2660 finally: 2661 os.remove(srcfilename) 2662 2663def debug(module, name, pm=False): 2664 """Debug a single doctest docstring. 2665 2666 Provide the module (or dotted name of the module) containing the 2667 test to be debugged and the name (within the module) of the object 2668 with the docstring with tests to be debugged. 2669 """ 2670 module = _normalize_module(module) 2671 testsrc = testsource(module, name) 2672 debug_script(testsrc, pm, module.__dict__) 2673 2674###################################################################### 2675## 10. Example Usage 2676###################################################################### 2677class _TestClass: 2678 """ 2679 A pointless class, for sanity-checking of docstring testing. 2680 2681 Methods: 2682 square() 2683 get() 2684 2685 >>> _TestClass(13).get() + _TestClass(-12).get() 2686 1 2687 >>> hex(_TestClass(13).square().get()) 2688 '0xa9' 2689 """ 2690 2691 def __init__(self, val): 2692 """val -> _TestClass object with associated value val. 2693 2694 >>> t = _TestClass(123) 2695 >>> print t.get() 2696 123 2697 """ 2698 2699 self.val = val 2700 2701 def square(self): 2702 """square() -> square TestClass's associated value 2703 2704 >>> _TestClass(13).square().get() 2705 169 2706 """ 2707 2708 self.val = self.val ** 2 2709 return self 2710 2711 def get(self): 2712 """get() -> return TestClass's associated value. 2713 2714 >>> x = _TestClass(-42) 2715 >>> print x.get() 2716 -42 2717 """ 2718 2719 return self.val 2720 2721__test__ = {"_TestClass": _TestClass, 2722 "string": r""" 2723 Example of a string object, searched as-is. 2724 >>> x = 1; y = 2 2725 >>> x + y, x * y 2726 (3, 2) 2727 """, 2728 2729 "bool-int equivalence": r""" 2730 In 2.2, boolean expressions displayed 2731 0 or 1. By default, we still accept 2732 them. This can be disabled by passing 2733 DONT_ACCEPT_TRUE_FOR_1 to the new 2734 optionflags argument. 2735 >>> 4 == 4 2736 1 2737 >>> 4 == 4 2738 True 2739 >>> 4 > 4 2740 0 2741 >>> 4 > 4 2742 False 2743 """, 2744 2745 "blank lines": r""" 2746 Blank lines can be marked with <BLANKLINE>: 2747 >>> print 'foo\n\nbar\n' 2748 foo 2749 <BLANKLINE> 2750 bar 2751 <BLANKLINE> 2752 """, 2753 2754 "ellipsis": r""" 2755 If the ellipsis flag is used, then '...' can be used to 2756 elide substrings in the desired output: 2757 >>> print range(1000) #doctest: +ELLIPSIS 2758 [0, 1, 2, ..., 999] 2759 """, 2760 2761 "whitespace normalization": r""" 2762 If the whitespace normalization flag is used, then 2763 differences in whitespace are ignored. 2764 >>> print range(30) #doctest: +NORMALIZE_WHITESPACE 2765 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 2766 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 2767 27, 28, 29] 2768 """, 2769 } 2770 2771 2772def _test(): 2773 testfiles = [arg for arg in sys.argv[1:] if arg and arg[0] != '-'] 2774 if not testfiles: 2775 name = os.path.basename(sys.argv[0]) 2776 if '__loader__' in globals(): # python -m 2777 name, _ = os.path.splitext(name) 2778 print("usage: {0} [-v] file ...".format(name)) 2779 return 2 2780 for filename in testfiles: 2781 if filename.endswith(".py"): 2782 # It is a module -- insert its dir into sys.path and try to 2783 # import it. If it is part of a package, that possibly 2784 # won't work because of package imports. 2785 dirname, filename = os.path.split(filename) 2786 sys.path.insert(0, dirname) 2787 m = __import__(filename[:-3]) 2788 del sys.path[0] 2789 failures, _ = testmod(m) 2790 else: 2791 failures, _ = testfile(filename, module_relative=False) 2792 if failures: 2793 return 1 2794 return 0 2795 2796 2797if __name__ == "__main__": 2798 sys.exit(_test()) 2799