1"""Generic testing tools. 2 3Authors 4------- 5- Fernando Perez <Fernando.Perez@berkeley.edu> 6""" 7 8from __future__ import absolute_import 9 10# Copyright (c) IPython Development Team. 11# Distributed under the terms of the Modified BSD License. 12 13import os 14import re 15import sys 16import tempfile 17 18from contextlib import contextmanager 19from io import StringIO 20from subprocess import Popen, PIPE 21try: 22 from unittest.mock import patch 23except ImportError: 24 # Python 2 compatibility 25 from mock import patch 26 27try: 28 # These tools are used by parts of the runtime, so we make the nose 29 # dependency optional at this point. Nose is a hard dependency to run the 30 # test suite, but NOT to use ipython itself. 31 import nose.tools as nt 32 has_nose = True 33except ImportError: 34 has_nose = False 35 36from traitlets.config.loader import Config 37from IPython.utils.process import get_output_error_code 38from IPython.utils.text import list_strings 39from IPython.utils.io import temp_pyfile, Tee 40from IPython.utils import py3compat 41from IPython.utils.encoding import DEFAULT_ENCODING 42 43from . import decorators as dec 44from . import skipdoctest 45 46 47# The docstring for full_path doctests differently on win32 (different path 48# separator) so just skip the doctest there. The example remains informative. 49doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco 50 51@doctest_deco 52def full_path(startPath,files): 53 """Make full paths for all the listed files, based on startPath. 54 55 Only the base part of startPath is kept, since this routine is typically 56 used with a script's ``__file__`` variable as startPath. The base of startPath 57 is then prepended to all the listed files, forming the output list. 58 59 Parameters 60 ---------- 61 startPath : string 62 Initial path to use as the base for the results. This path is split 63 using os.path.split() and only its first component is kept. 64 65 files : string or list 66 One or more files. 67 68 Examples 69 -------- 70 71 >>> full_path('/foo/bar.py',['a.txt','b.txt']) 72 ['/foo/a.txt', '/foo/b.txt'] 73 74 >>> full_path('/foo',['a.txt','b.txt']) 75 ['/a.txt', '/b.txt'] 76 77 If a single file is given, the output is still a list:: 78 79 >>> full_path('/foo','a.txt') 80 ['/a.txt'] 81 """ 82 83 files = list_strings(files) 84 base = os.path.split(startPath)[0] 85 return [ os.path.join(base,f) for f in files ] 86 87 88def parse_test_output(txt): 89 """Parse the output of a test run and return errors, failures. 90 91 Parameters 92 ---------- 93 txt : str 94 Text output of a test run, assumed to contain a line of one of the 95 following forms:: 96 97 'FAILED (errors=1)' 98 'FAILED (failures=1)' 99 'FAILED (errors=1, failures=1)' 100 101 Returns 102 ------- 103 nerr, nfail 104 number of errors and failures. 105 """ 106 107 err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE) 108 if err_m: 109 nerr = int(err_m.group(1)) 110 nfail = 0 111 return nerr, nfail 112 113 fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE) 114 if fail_m: 115 nerr = 0 116 nfail = int(fail_m.group(1)) 117 return nerr, nfail 118 119 both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt, 120 re.MULTILINE) 121 if both_m: 122 nerr = int(both_m.group(1)) 123 nfail = int(both_m.group(2)) 124 return nerr, nfail 125 126 # If the input didn't match any of these forms, assume no error/failures 127 return 0, 0 128 129 130# So nose doesn't think this is a test 131parse_test_output.__test__ = False 132 133 134def default_argv(): 135 """Return a valid default argv for creating testing instances of ipython""" 136 137 return ['--quick', # so no config file is loaded 138 # Other defaults to minimize side effects on stdout 139 '--colors=NoColor', '--no-term-title','--no-banner', 140 '--autocall=0'] 141 142 143def default_config(): 144 """Return a config object with good defaults for testing.""" 145 config = Config() 146 config.TerminalInteractiveShell.colors = 'NoColor' 147 config.TerminalTerminalInteractiveShell.term_title = False, 148 config.TerminalInteractiveShell.autocall = 0 149 f = tempfile.NamedTemporaryFile(suffix=u'test_hist.sqlite', delete=False) 150 config.HistoryManager.hist_file = f.name 151 f.close() 152 config.HistoryManager.db_cache_size = 10000 153 return config 154 155 156def get_ipython_cmd(as_string=False): 157 """ 158 Return appropriate IPython command line name. By default, this will return 159 a list that can be used with subprocess.Popen, for example, but passing 160 `as_string=True` allows for returning the IPython command as a string. 161 162 Parameters 163 ---------- 164 as_string: bool 165 Flag to allow to return the command as a string. 166 """ 167 ipython_cmd = [sys.executable, "-m", "IPython"] 168 169 if as_string: 170 ipython_cmd = " ".join(ipython_cmd) 171 172 return ipython_cmd 173 174def ipexec(fname, options=None, commands=()): 175 """Utility to call 'ipython filename'. 176 177 Starts IPython with a minimal and safe configuration to make startup as fast 178 as possible. 179 180 Note that this starts IPython in a subprocess! 181 182 Parameters 183 ---------- 184 fname : str 185 Name of file to be executed (should have .py or .ipy extension). 186 187 options : optional, list 188 Extra command-line flags to be passed to IPython. 189 190 commands : optional, list 191 Commands to send in on stdin 192 193 Returns 194 ------- 195 (stdout, stderr) of ipython subprocess. 196 """ 197 if options is None: options = [] 198 199 cmdargs = default_argv() + options 200 201 test_dir = os.path.dirname(__file__) 202 203 ipython_cmd = get_ipython_cmd() 204 # Absolute path for filename 205 full_fname = os.path.join(test_dir, fname) 206 full_cmd = ipython_cmd + cmdargs + [full_fname] 207 env = os.environ.copy() 208 # FIXME: ignore all warnings in ipexec while we have shims 209 # should we keep suppressing warnings here, even after removing shims? 210 env['PYTHONWARNINGS'] = 'ignore' 211 # env.pop('PYTHONWARNINGS', None) # Avoid extraneous warnings appearing on stderr 212 for k, v in env.items(): 213 # Debug a bizarre failure we've seen on Windows: 214 # TypeError: environment can only contain strings 215 if not isinstance(v, str): 216 print(k, v) 217 p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env) 218 out, err = p.communicate(input=py3compat.str_to_bytes('\n'.join(commands)) or None) 219 out, err = py3compat.bytes_to_str(out), py3compat.bytes_to_str(err) 220 # `import readline` causes 'ESC[?1034h' to be output sometimes, 221 # so strip that out before doing comparisons 222 if out: 223 out = re.sub(r'\x1b\[[^h]+h', '', out) 224 return out, err 225 226 227def ipexec_validate(fname, expected_out, expected_err='', 228 options=None, commands=()): 229 """Utility to call 'ipython filename' and validate output/error. 230 231 This function raises an AssertionError if the validation fails. 232 233 Note that this starts IPython in a subprocess! 234 235 Parameters 236 ---------- 237 fname : str 238 Name of the file to be executed (should have .py or .ipy extension). 239 240 expected_out : str 241 Expected stdout of the process. 242 243 expected_err : optional, str 244 Expected stderr of the process. 245 246 options : optional, list 247 Extra command-line flags to be passed to IPython. 248 249 Returns 250 ------- 251 None 252 """ 253 254 import nose.tools as nt 255 256 out, err = ipexec(fname, options, commands) 257 #print 'OUT', out # dbg 258 #print 'ERR', err # dbg 259 # If there are any errors, we must check those befor stdout, as they may be 260 # more informative than simply having an empty stdout. 261 if err: 262 if expected_err: 263 nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines())) 264 else: 265 raise ValueError('Running file %r produced error: %r' % 266 (fname, err)) 267 # If no errors or output on stderr was expected, match stdout 268 nt.assert_equal("\n".join(out.strip().splitlines()), "\n".join(expected_out.strip().splitlines())) 269 270 271class TempFileMixin(object): 272 """Utility class to create temporary Python/IPython files. 273 274 Meant as a mixin class for test cases.""" 275 276 def mktmp(self, src, ext='.py'): 277 """Make a valid python temp file.""" 278 fname, f = temp_pyfile(src, ext) 279 self.tmpfile = f 280 self.fname = fname 281 282 def tearDown(self): 283 if hasattr(self, 'tmpfile'): 284 # If the tmpfile wasn't made because of skipped tests, like in 285 # win32, there's nothing to cleanup. 286 self.tmpfile.close() 287 try: 288 os.unlink(self.fname) 289 except: 290 # On Windows, even though we close the file, we still can't 291 # delete it. I have no clue why 292 pass 293 294 def __enter__(self): 295 return self 296 297 def __exit__(self, exc_type, exc_value, traceback): 298 self.tearDown() 299 300 301pair_fail_msg = ("Testing {0}\n\n" 302 "In:\n" 303 " {1!r}\n" 304 "Expected:\n" 305 " {2!r}\n" 306 "Got:\n" 307 " {3!r}\n") 308def check_pairs(func, pairs): 309 """Utility function for the common case of checking a function with a 310 sequence of input/output pairs. 311 312 Parameters 313 ---------- 314 func : callable 315 The function to be tested. Should accept a single argument. 316 pairs : iterable 317 A list of (input, expected_output) tuples. 318 319 Returns 320 ------- 321 None. Raises an AssertionError if any output does not match the expected 322 value. 323 """ 324 name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>")) 325 for inp, expected in pairs: 326 out = func(inp) 327 assert out == expected, pair_fail_msg.format(name, inp, expected, out) 328 329 330if py3compat.PY3: 331 MyStringIO = StringIO 332else: 333 # In Python 2, stdout/stderr can have either bytes or unicode written to them, 334 # so we need a class that can handle both. 335 class MyStringIO(StringIO): 336 def write(self, s): 337 s = py3compat.cast_unicode(s, encoding=DEFAULT_ENCODING) 338 super(MyStringIO, self).write(s) 339 340_re_type = type(re.compile(r'')) 341 342notprinted_msg = """Did not find {0!r} in printed output (on {1}): 343------- 344{2!s} 345------- 346""" 347 348class AssertPrints(object): 349 """Context manager for testing that code prints certain text. 350 351 Examples 352 -------- 353 >>> with AssertPrints("abc", suppress=False): 354 ... print("abcd") 355 ... print("def") 356 ... 357 abcd 358 def 359 """ 360 def __init__(self, s, channel='stdout', suppress=True): 361 self.s = s 362 if isinstance(self.s, (py3compat.string_types, _re_type)): 363 self.s = [self.s] 364 self.channel = channel 365 self.suppress = suppress 366 367 def __enter__(self): 368 self.orig_stream = getattr(sys, self.channel) 369 self.buffer = MyStringIO() 370 self.tee = Tee(self.buffer, channel=self.channel) 371 setattr(sys, self.channel, self.buffer if self.suppress else self.tee) 372 373 def __exit__(self, etype, value, traceback): 374 try: 375 if value is not None: 376 # If an error was raised, don't check anything else 377 return False 378 self.tee.flush() 379 setattr(sys, self.channel, self.orig_stream) 380 printed = self.buffer.getvalue() 381 for s in self.s: 382 if isinstance(s, _re_type): 383 assert s.search(printed), notprinted_msg.format(s.pattern, self.channel, printed) 384 else: 385 assert s in printed, notprinted_msg.format(s, self.channel, printed) 386 return False 387 finally: 388 self.tee.close() 389 390printed_msg = """Found {0!r} in printed output (on {1}): 391------- 392{2!s} 393------- 394""" 395 396class AssertNotPrints(AssertPrints): 397 """Context manager for checking that certain output *isn't* produced. 398 399 Counterpart of AssertPrints""" 400 def __exit__(self, etype, value, traceback): 401 try: 402 if value is not None: 403 # If an error was raised, don't check anything else 404 self.tee.close() 405 return False 406 self.tee.flush() 407 setattr(sys, self.channel, self.orig_stream) 408 printed = self.buffer.getvalue() 409 for s in self.s: 410 if isinstance(s, _re_type): 411 assert not s.search(printed),printed_msg.format( 412 s.pattern, self.channel, printed) 413 else: 414 assert s not in printed, printed_msg.format( 415 s, self.channel, printed) 416 return False 417 finally: 418 self.tee.close() 419 420@contextmanager 421def mute_warn(): 422 from IPython.utils import warn 423 save_warn = warn.warn 424 warn.warn = lambda *a, **kw: None 425 try: 426 yield 427 finally: 428 warn.warn = save_warn 429 430@contextmanager 431def make_tempfile(name): 432 """ Create an empty, named, temporary file for the duration of the context. 433 """ 434 f = open(name, 'w') 435 f.close() 436 try: 437 yield 438 finally: 439 os.unlink(name) 440 441def fake_input(inputs): 442 """Temporarily replace the input() function to return the given values 443 444 Use as a context manager: 445 446 with fake_input(['result1', 'result2']): 447 ... 448 449 Values are returned in order. If input() is called again after the last value 450 was used, EOFError is raised. 451 """ 452 it = iter(inputs) 453 def mock_input(prompt=''): 454 try: 455 return next(it) 456 except StopIteration: 457 raise EOFError('No more inputs given') 458 459 input_name = '%s.%s' % (py3compat.builtin_mod_name, 460 'input' if py3compat.PY3 else 'raw_input') 461 return patch(input_name, mock_input) 462 463def help_output_test(subcommand=''): 464 """test that `ipython [subcommand] -h` works""" 465 cmd = get_ipython_cmd() + [subcommand, '-h'] 466 out, err, rc = get_output_error_code(cmd) 467 nt.assert_equal(rc, 0, err) 468 nt.assert_not_in("Traceback", err) 469 nt.assert_in("Options", out) 470 nt.assert_in("--help-all", out) 471 return out, err 472 473 474def help_all_output_test(subcommand=''): 475 """test that `ipython [subcommand] --help-all` works""" 476 cmd = get_ipython_cmd() + [subcommand, '--help-all'] 477 out, err, rc = get_output_error_code(cmd) 478 nt.assert_equal(rc, 0, err) 479 nt.assert_not_in("Traceback", err) 480 nt.assert_in("Options", out) 481 nt.assert_in("Class", out) 482 return out, err 483 484