1"""Generic testing tools.
2
3Authors
4-------
5- Fernando Perez <Fernando.Perez@berkeley.edu>
6"""
7
8from __future__ import absolute_import
9
10# Copyright (c) IPython Development Team.
11# Distributed under the terms of the Modified BSD License.
12
13import os
14import re
15import sys
16import tempfile
17
18from contextlib import contextmanager
19from io import StringIO
20from subprocess import Popen, PIPE
21try:
22    from unittest.mock import patch
23except ImportError:
24    # Python 2 compatibility
25    from mock import patch
26
27try:
28    # These tools are used by parts of the runtime, so we make the nose
29    # dependency optional at this point.  Nose is a hard dependency to run the
30    # test suite, but NOT to use ipython itself.
31    import nose.tools as nt
32    has_nose = True
33except ImportError:
34    has_nose = False
35
36from traitlets.config.loader import Config
37from IPython.utils.process import get_output_error_code
38from IPython.utils.text import list_strings
39from IPython.utils.io import temp_pyfile, Tee
40from IPython.utils import py3compat
41from IPython.utils.encoding import DEFAULT_ENCODING
42
43from . import decorators as dec
44from . import skipdoctest
45
46
47# The docstring for full_path doctests differently on win32 (different path
48# separator) so just skip the doctest there.  The example remains informative.
49doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
50
51@doctest_deco
52def full_path(startPath,files):
53    """Make full paths for all the listed files, based on startPath.
54
55    Only the base part of startPath is kept, since this routine is typically
56    used with a script's ``__file__`` variable as startPath. The base of startPath
57    is then prepended to all the listed files, forming the output list.
58
59    Parameters
60    ----------
61    startPath : string
62      Initial path to use as the base for the results.  This path is split
63      using os.path.split() and only its first component is kept.
64
65    files : string or list
66      One or more files.
67
68    Examples
69    --------
70
71    >>> full_path('/foo/bar.py',['a.txt','b.txt'])
72    ['/foo/a.txt', '/foo/b.txt']
73
74    >>> full_path('/foo',['a.txt','b.txt'])
75    ['/a.txt', '/b.txt']
76
77    If a single file is given, the output is still a list::
78
79        >>> full_path('/foo','a.txt')
80        ['/a.txt']
81    """
82
83    files = list_strings(files)
84    base = os.path.split(startPath)[0]
85    return [ os.path.join(base,f) for f in files ]
86
87
88def parse_test_output(txt):
89    """Parse the output of a test run and return errors, failures.
90
91    Parameters
92    ----------
93    txt : str
94      Text output of a test run, assumed to contain a line of one of the
95      following forms::
96
97        'FAILED (errors=1)'
98        'FAILED (failures=1)'
99        'FAILED (errors=1, failures=1)'
100
101    Returns
102    -------
103    nerr, nfail
104      number of errors and failures.
105    """
106
107    err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
108    if err_m:
109        nerr = int(err_m.group(1))
110        nfail = 0
111        return  nerr, nfail
112
113    fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
114    if fail_m:
115        nerr = 0
116        nfail = int(fail_m.group(1))
117        return  nerr, nfail
118
119    both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
120                       re.MULTILINE)
121    if both_m:
122        nerr = int(both_m.group(1))
123        nfail = int(both_m.group(2))
124        return  nerr, nfail
125
126    # If the input didn't match any of these forms, assume no error/failures
127    return 0, 0
128
129
130# So nose doesn't think this is a test
131parse_test_output.__test__ = False
132
133
134def default_argv():
135    """Return a valid default argv for creating testing instances of ipython"""
136
137    return ['--quick', # so no config file is loaded
138            # Other defaults to minimize side effects on stdout
139            '--colors=NoColor', '--no-term-title','--no-banner',
140            '--autocall=0']
141
142
143def default_config():
144    """Return a config object with good defaults for testing."""
145    config = Config()
146    config.TerminalInteractiveShell.colors = 'NoColor'
147    config.TerminalTerminalInteractiveShell.term_title = False,
148    config.TerminalInteractiveShell.autocall = 0
149    f = tempfile.NamedTemporaryFile(suffix=u'test_hist.sqlite', delete=False)
150    config.HistoryManager.hist_file = f.name
151    f.close()
152    config.HistoryManager.db_cache_size = 10000
153    return config
154
155
156def get_ipython_cmd(as_string=False):
157    """
158    Return appropriate IPython command line name. By default, this will return
159    a list that can be used with subprocess.Popen, for example, but passing
160    `as_string=True` allows for returning the IPython command as a string.
161
162    Parameters
163    ----------
164    as_string: bool
165        Flag to allow to return the command as a string.
166    """
167    ipython_cmd = [sys.executable, "-m", "IPython"]
168
169    if as_string:
170        ipython_cmd = " ".join(ipython_cmd)
171
172    return ipython_cmd
173
174def ipexec(fname, options=None, commands=()):
175    """Utility to call 'ipython filename'.
176
177    Starts IPython with a minimal and safe configuration to make startup as fast
178    as possible.
179
180    Note that this starts IPython in a subprocess!
181
182    Parameters
183    ----------
184    fname : str
185      Name of file to be executed (should have .py or .ipy extension).
186
187    options : optional, list
188      Extra command-line flags to be passed to IPython.
189
190    commands : optional, list
191      Commands to send in on stdin
192
193    Returns
194    -------
195    (stdout, stderr) of ipython subprocess.
196    """
197    if options is None: options = []
198
199    cmdargs = default_argv() + options
200
201    test_dir = os.path.dirname(__file__)
202
203    ipython_cmd = get_ipython_cmd()
204    # Absolute path for filename
205    full_fname = os.path.join(test_dir, fname)
206    full_cmd = ipython_cmd + cmdargs + [full_fname]
207    env = os.environ.copy()
208    # FIXME: ignore all warnings in ipexec while we have shims
209    # should we keep suppressing warnings here, even after removing shims?
210    env['PYTHONWARNINGS'] = 'ignore'
211    # env.pop('PYTHONWARNINGS', None)  # Avoid extraneous warnings appearing on stderr
212    for k, v in env.items():
213        # Debug a bizarre failure we've seen on Windows:
214        # TypeError: environment can only contain strings
215        if not isinstance(v, str):
216            print(k, v)
217    p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
218    out, err = p.communicate(input=py3compat.str_to_bytes('\n'.join(commands)) or None)
219    out, err = py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
220    # `import readline` causes 'ESC[?1034h' to be output sometimes,
221    # so strip that out before doing comparisons
222    if out:
223        out = re.sub(r'\x1b\[[^h]+h', '', out)
224    return out, err
225
226
227def ipexec_validate(fname, expected_out, expected_err='',
228                    options=None, commands=()):
229    """Utility to call 'ipython filename' and validate output/error.
230
231    This function raises an AssertionError if the validation fails.
232
233    Note that this starts IPython in a subprocess!
234
235    Parameters
236    ----------
237    fname : str
238      Name of the file to be executed (should have .py or .ipy extension).
239
240    expected_out : str
241      Expected stdout of the process.
242
243    expected_err : optional, str
244      Expected stderr of the process.
245
246    options : optional, list
247      Extra command-line flags to be passed to IPython.
248
249    Returns
250    -------
251    None
252    """
253
254    import nose.tools as nt
255
256    out, err = ipexec(fname, options, commands)
257    #print 'OUT', out  # dbg
258    #print 'ERR', err  # dbg
259    # If there are any errors, we must check those befor stdout, as they may be
260    # more informative than simply having an empty stdout.
261    if err:
262        if expected_err:
263            nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines()))
264        else:
265            raise ValueError('Running file %r produced error: %r' %
266                             (fname, err))
267    # If no errors or output on stderr was expected, match stdout
268    nt.assert_equal("\n".join(out.strip().splitlines()), "\n".join(expected_out.strip().splitlines()))
269
270
271class TempFileMixin(object):
272    """Utility class to create temporary Python/IPython files.
273
274    Meant as a mixin class for test cases."""
275
276    def mktmp(self, src, ext='.py'):
277        """Make a valid python temp file."""
278        fname, f = temp_pyfile(src, ext)
279        self.tmpfile = f
280        self.fname = fname
281
282    def tearDown(self):
283        if hasattr(self, 'tmpfile'):
284            # If the tmpfile wasn't made because of skipped tests, like in
285            # win32, there's nothing to cleanup.
286            self.tmpfile.close()
287            try:
288                os.unlink(self.fname)
289            except:
290                # On Windows, even though we close the file, we still can't
291                # delete it.  I have no clue why
292                pass
293
294    def __enter__(self):
295        return self
296
297    def __exit__(self, exc_type, exc_value, traceback):
298        self.tearDown()
299
300
301pair_fail_msg = ("Testing {0}\n\n"
302                "In:\n"
303                "  {1!r}\n"
304                "Expected:\n"
305                "  {2!r}\n"
306                "Got:\n"
307                "  {3!r}\n")
308def check_pairs(func, pairs):
309    """Utility function for the common case of checking a function with a
310    sequence of input/output pairs.
311
312    Parameters
313    ----------
314    func : callable
315      The function to be tested. Should accept a single argument.
316    pairs : iterable
317      A list of (input, expected_output) tuples.
318
319    Returns
320    -------
321    None. Raises an AssertionError if any output does not match the expected
322    value.
323    """
324    name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
325    for inp, expected in pairs:
326        out = func(inp)
327        assert out == expected, pair_fail_msg.format(name, inp, expected, out)
328
329
330if py3compat.PY3:
331    MyStringIO = StringIO
332else:
333    # In Python 2, stdout/stderr can have either bytes or unicode written to them,
334    # so we need a class that can handle both.
335    class MyStringIO(StringIO):
336        def write(self, s):
337            s = py3compat.cast_unicode(s, encoding=DEFAULT_ENCODING)
338            super(MyStringIO, self).write(s)
339
340_re_type = type(re.compile(r''))
341
342notprinted_msg = """Did not find {0!r} in printed output (on {1}):
343-------
344{2!s}
345-------
346"""
347
348class AssertPrints(object):
349    """Context manager for testing that code prints certain text.
350
351    Examples
352    --------
353    >>> with AssertPrints("abc", suppress=False):
354    ...     print("abcd")
355    ...     print("def")
356    ...
357    abcd
358    def
359    """
360    def __init__(self, s, channel='stdout', suppress=True):
361        self.s = s
362        if isinstance(self.s, (py3compat.string_types, _re_type)):
363            self.s = [self.s]
364        self.channel = channel
365        self.suppress = suppress
366
367    def __enter__(self):
368        self.orig_stream = getattr(sys, self.channel)
369        self.buffer = MyStringIO()
370        self.tee = Tee(self.buffer, channel=self.channel)
371        setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
372
373    def __exit__(self, etype, value, traceback):
374        try:
375            if value is not None:
376                # If an error was raised, don't check anything else
377                return False
378            self.tee.flush()
379            setattr(sys, self.channel, self.orig_stream)
380            printed = self.buffer.getvalue()
381            for s in self.s:
382                if isinstance(s, _re_type):
383                    assert s.search(printed), notprinted_msg.format(s.pattern, self.channel, printed)
384                else:
385                    assert s in printed, notprinted_msg.format(s, self.channel, printed)
386            return False
387        finally:
388            self.tee.close()
389
390printed_msg = """Found {0!r} in printed output (on {1}):
391-------
392{2!s}
393-------
394"""
395
396class AssertNotPrints(AssertPrints):
397    """Context manager for checking that certain output *isn't* produced.
398
399    Counterpart of AssertPrints"""
400    def __exit__(self, etype, value, traceback):
401        try:
402            if value is not None:
403                # If an error was raised, don't check anything else
404                self.tee.close()
405                return False
406            self.tee.flush()
407            setattr(sys, self.channel, self.orig_stream)
408            printed = self.buffer.getvalue()
409            for s in self.s:
410                if isinstance(s, _re_type):
411                    assert not s.search(printed),printed_msg.format(
412                        s.pattern, self.channel, printed)
413                else:
414                    assert s not in printed, printed_msg.format(
415                        s, self.channel, printed)
416            return False
417        finally:
418            self.tee.close()
419
420@contextmanager
421def mute_warn():
422    from IPython.utils import warn
423    save_warn = warn.warn
424    warn.warn = lambda *a, **kw: None
425    try:
426        yield
427    finally:
428        warn.warn = save_warn
429
430@contextmanager
431def make_tempfile(name):
432    """ Create an empty, named, temporary file for the duration of the context.
433    """
434    f = open(name, 'w')
435    f.close()
436    try:
437        yield
438    finally:
439        os.unlink(name)
440
441def fake_input(inputs):
442    """Temporarily replace the input() function to return the given values
443
444    Use as a context manager:
445
446    with fake_input(['result1', 'result2']):
447        ...
448
449    Values are returned in order. If input() is called again after the last value
450    was used, EOFError is raised.
451    """
452    it = iter(inputs)
453    def mock_input(prompt=''):
454        try:
455            return next(it)
456        except StopIteration:
457            raise EOFError('No more inputs given')
458
459    input_name = '%s.%s' % (py3compat.builtin_mod_name,
460                            'input' if py3compat.PY3 else 'raw_input')
461    return patch(input_name, mock_input)
462
463def help_output_test(subcommand=''):
464    """test that `ipython [subcommand] -h` works"""
465    cmd = get_ipython_cmd() + [subcommand, '-h']
466    out, err, rc = get_output_error_code(cmd)
467    nt.assert_equal(rc, 0, err)
468    nt.assert_not_in("Traceback", err)
469    nt.assert_in("Options", out)
470    nt.assert_in("--help-all", out)
471    return out, err
472
473
474def help_all_output_test(subcommand=''):
475    """test that `ipython [subcommand] --help-all` works"""
476    cmd = get_ipython_cmd() + [subcommand, '--help-all']
477    out, err, rc = get_output_error_code(cmd)
478    nt.assert_equal(rc, 0, err)
479    nt.assert_not_in("Traceback", err)
480    nt.assert_in("Options", out)
481    nt.assert_in("Class", out)
482    return out, err
483
484