1"""Miscellaneous utility functions and classes.
2
3This module is used internally by Tornado.  It is not necessarily expected
4that the functions and classes defined here will be useful to other
5applications, but they are documented here in case they are.
6
7The one public-facing part of this module is the `Configurable` class
8and its `~Configurable.configure` method, which becomes a part of the
9interface of its subclasses, including `.AsyncHTTPClient`, `.IOLoop`,
10and `.Resolver`.
11"""
12
13from __future__ import absolute_import, division, print_function
14
15import array
16import atexit
17import os
18import re
19import sys
20import zlib
21
22PY3 = sys.version_info >= (3,)
23
24if PY3:
25    xrange = range
26
27# inspect.getargspec() raises DeprecationWarnings in Python 3.5.
28# The two functions have compatible interfaces for the parts we need.
29if PY3:
30    from inspect import getfullargspec as getargspec
31else:
32    from inspect import getargspec
33
34# Aliases for types that are spelled differently in different Python
35# versions. bytes_type is deprecated and no longer used in Tornado
36# itself but is left in case anyone outside Tornado is using it.
37bytes_type = bytes
38if PY3:
39    unicode_type = str
40    basestring_type = str
41else:
42    # The names unicode and basestring don't exist in py3 so silence flake8.
43    unicode_type = unicode  # noqa
44    basestring_type = basestring  # noqa
45
46
47try:
48    import typing  # noqa
49    from typing import cast
50
51    _ObjectDictBase = typing.Dict[str, typing.Any]
52except ImportError:
53    _ObjectDictBase = dict
54
55    def cast(typ, x):
56        return x
57else:
58    # More imports that are only needed in type comments.
59    import datetime  # noqa
60    import types  # noqa
61    from typing import Any, AnyStr, Union, Optional, Dict, Mapping  # noqa
62    from typing import Tuple, Match, Callable  # noqa
63
64    if PY3:
65        _BaseString = str
66    else:
67        _BaseString = Union[bytes, unicode_type]
68
69
70try:
71    from sys import is_finalizing
72except ImportError:
73    # Emulate it
74    def _get_emulated_is_finalizing():
75        L = []
76        atexit.register(lambda: L.append(None))
77
78        def is_finalizing():
79            # Not referencing any globals here
80            return L != []
81
82        return is_finalizing
83
84    is_finalizing = _get_emulated_is_finalizing()
85
86
87class ObjectDict(_ObjectDictBase):
88    """Makes a dictionary behave like an object, with attribute-style access.
89    """
90    def __getattr__(self, name):
91        # type: (str) -> Any
92        try:
93            return self[name]
94        except KeyError:
95            raise AttributeError(name)
96
97    def __setattr__(self, name, value):
98        # type: (str, Any) -> None
99        self[name] = value
100
101
102class GzipDecompressor(object):
103    """Streaming gzip decompressor.
104
105    The interface is like that of `zlib.decompressobj` (without some of the
106    optional arguments, but it understands gzip headers and checksums.
107    """
108    def __init__(self):
109        # Magic parameter makes zlib module understand gzip header
110        # http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib
111        # This works on cpython and pypy, but not jython.
112        self.decompressobj = zlib.decompressobj(16 + zlib.MAX_WBITS)
113
114    def decompress(self, value, max_length=None):
115        # type: (bytes, Optional[int]) -> bytes
116        """Decompress a chunk, returning newly-available data.
117
118        Some data may be buffered for later processing; `flush` must
119        be called when there is no more input data to ensure that
120        all data was processed.
121
122        If ``max_length`` is given, some input data may be left over
123        in ``unconsumed_tail``; you must retrieve this value and pass
124        it back to a future call to `decompress` if it is not empty.
125        """
126        return self.decompressobj.decompress(value, max_length)
127
128    @property
129    def unconsumed_tail(self):
130        # type: () -> bytes
131        """Returns the unconsumed portion left over
132        """
133        return self.decompressobj.unconsumed_tail
134
135    def flush(self):
136        # type: () -> bytes
137        """Return any remaining buffered data not yet returned by decompress.
138
139        Also checks for errors such as truncated input.
140        No other methods may be called on this object after `flush`.
141        """
142        return self.decompressobj.flush()
143
144
145def import_object(name):
146    # type: (_BaseString) -> Any
147    """Imports an object by name.
148
149    import_object('x') is equivalent to 'import x'.
150    import_object('x.y.z') is equivalent to 'from x.y import z'.
151
152    >>> import tornado.escape
153    >>> import_object('tornado.escape') is tornado.escape
154    True
155    >>> import_object('tornado.escape.utf8') is tornado.escape.utf8
156    True
157    >>> import_object('tornado') is tornado
158    True
159    >>> import_object('tornado.missing_module')
160    Traceback (most recent call last):
161        ...
162    ImportError: No module named missing_module
163    """
164    if not isinstance(name, str):
165        # on python 2 a byte string is required.
166        name = name.encode('utf-8')
167    if name.count('.') == 0:
168        return __import__(name, None, None)
169
170    parts = name.split('.')
171    obj = __import__('.'.join(parts[:-1]), None, None, [parts[-1]], 0)
172    try:
173        return getattr(obj, parts[-1])
174    except AttributeError:
175        raise ImportError("No module named %s" % parts[-1])
176
177
178# Stubs to make mypy happy (and later for actual type-checking).
179def raise_exc_info(exc_info):
180    # type: (Tuple[type, BaseException, types.TracebackType]) -> None
181    pass
182
183
184def exec_in(code, glob, loc=None):
185    # type: (Any, Dict[str, Any], Optional[Mapping[str, Any]]) -> Any
186    if isinstance(code, basestring_type):
187        # exec(string) inherits the caller's future imports; compile
188        # the string first to prevent that.
189        code = compile(code, '<string>', 'exec', dont_inherit=True)
190    exec(code, glob, loc)
191
192
193if PY3:
194    exec("""
195def raise_exc_info(exc_info):
196    try:
197        raise exc_info[1].with_traceback(exc_info[2])
198    finally:
199        exc_info = None
200
201""")
202else:
203    exec("""
204def raise_exc_info(exc_info):
205    raise exc_info[0], exc_info[1], exc_info[2]
206""")
207
208
209def errno_from_exception(e):
210    # type: (BaseException) -> Optional[int]
211    """Provides the errno from an Exception object.
212
213    There are cases that the errno attribute was not set so we pull
214    the errno out of the args but if someone instantiates an Exception
215    without any args you will get a tuple error. So this function
216    abstracts all that behavior to give you a safe way to get the
217    errno.
218    """
219
220    if hasattr(e, 'errno'):
221        return e.errno  # type: ignore
222    elif e.args:
223        return e.args[0]
224    else:
225        return None
226
227
228_alphanum = frozenset(
229    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
230
231
232def _re_unescape_replacement(match):
233    # type: (Match[str]) -> str
234    group = match.group(1)
235    if group[0] in _alphanum:
236        raise ValueError("cannot unescape '\\\\%s'" % group[0])
237    return group
238
239
240_re_unescape_pattern = re.compile(r'\\(.)', re.DOTALL)
241
242
243def re_unescape(s):
244    # type: (str) -> str
245    """Unescape a string escaped by `re.escape`.
246
247    May raise ``ValueError`` for regular expressions which could not
248    have been produced by `re.escape` (for example, strings containing
249    ``\d`` cannot be unescaped).
250
251    .. versionadded:: 4.4
252    """
253    return _re_unescape_pattern.sub(_re_unescape_replacement, s)
254
255
256class Configurable(object):
257    """Base class for configurable interfaces.
258
259    A configurable interface is an (abstract) class whose constructor
260    acts as a factory function for one of its implementation subclasses.
261    The implementation subclass as well as optional keyword arguments to
262    its initializer can be set globally at runtime with `configure`.
263
264    By using the constructor as the factory method, the interface
265    looks like a normal class, `isinstance` works as usual, etc.  This
266    pattern is most useful when the choice of implementation is likely
267    to be a global decision (e.g. when `~select.epoll` is available,
268    always use it instead of `~select.select`), or when a
269    previously-monolithic class has been split into specialized
270    subclasses.
271
272    Configurable subclasses must define the class methods
273    `configurable_base` and `configurable_default`, and use the instance
274    method `initialize` instead of ``__init__``.
275    """
276    __impl_class = None  # type: type
277    __impl_kwargs = None  # type: Dict[str, Any]
278
279    def __new__(cls, *args, **kwargs):
280        base = cls.configurable_base()
281        init_kwargs = {}
282        if cls is base:
283            impl = cls.configured_class()
284            if base.__impl_kwargs:
285                init_kwargs.update(base.__impl_kwargs)
286        else:
287            impl = cls
288        init_kwargs.update(kwargs)
289        instance = super(Configurable, cls).__new__(impl)
290        # initialize vs __init__ chosen for compatibility with AsyncHTTPClient
291        # singleton magic.  If we get rid of that we can switch to __init__
292        # here too.
293        instance.initialize(*args, **init_kwargs)
294        return instance
295
296    @classmethod
297    def configurable_base(cls):
298        # type: () -> Any
299        # TODO: This class needs https://github.com/python/typing/issues/107
300        # to be fully typeable.
301        """Returns the base class of a configurable hierarchy.
302
303        This will normally return the class in which it is defined.
304        (which is *not* necessarily the same as the cls classmethod parameter).
305        """
306        raise NotImplementedError()
307
308    @classmethod
309    def configurable_default(cls):
310        # type: () -> type
311        """Returns the implementation class to be used if none is configured."""
312        raise NotImplementedError()
313
314    def initialize(self):
315        # type: () -> None
316        """Initialize a `Configurable` subclass instance.
317
318        Configurable classes should use `initialize` instead of ``__init__``.
319
320        .. versionchanged:: 4.2
321           Now accepts positional arguments in addition to keyword arguments.
322        """
323
324    @classmethod
325    def configure(cls, impl, **kwargs):
326        # type: (Any, **Any) -> None
327        """Sets the class to use when the base class is instantiated.
328
329        Keyword arguments will be saved and added to the arguments passed
330        to the constructor.  This can be used to set global defaults for
331        some parameters.
332        """
333        base = cls.configurable_base()
334        if isinstance(impl, (str, unicode_type)):
335            impl = import_object(impl)
336        if impl is not None and not issubclass(impl, cls):
337            raise ValueError("Invalid subclass of %s" % cls)
338        base.__impl_class = impl
339        base.__impl_kwargs = kwargs
340
341    @classmethod
342    def configured_class(cls):
343        # type: () -> type
344        """Returns the currently configured class."""
345        base = cls.configurable_base()
346        if cls.__impl_class is None:
347            base.__impl_class = cls.configurable_default()
348        return base.__impl_class
349
350    @classmethod
351    def _save_configuration(cls):
352        # type: () -> Tuple[type, Dict[str, Any]]
353        base = cls.configurable_base()
354        return (base.__impl_class, base.__impl_kwargs)
355
356    @classmethod
357    def _restore_configuration(cls, saved):
358        # type: (Tuple[type, Dict[str, Any]]) -> None
359        base = cls.configurable_base()
360        base.__impl_class = saved[0]
361        base.__impl_kwargs = saved[1]
362
363
364class ArgReplacer(object):
365    """Replaces one value in an ``args, kwargs`` pair.
366
367    Inspects the function signature to find an argument by name
368    whether it is passed by position or keyword.  For use in decorators
369    and similar wrappers.
370    """
371    def __init__(self, func, name):
372        # type: (Callable, str) -> None
373        self.name = name
374        try:
375            self.arg_pos = self._getargnames(func).index(name)
376        except ValueError:
377            # Not a positional parameter
378            self.arg_pos = None
379
380    def _getargnames(self, func):
381        # type: (Callable) -> List[str]
382        try:
383            return getargspec(func).args
384        except TypeError:
385            if hasattr(func, 'func_code'):
386                # Cython-generated code has all the attributes needed
387                # by inspect.getargspec, but the inspect module only
388                # works with ordinary functions. Inline the portion of
389                # getargspec that we need here. Note that for static
390                # functions the @cython.binding(True) decorator must
391                # be used (for methods it works out of the box).
392                code = func.func_code  # type: ignore
393                return code.co_varnames[:code.co_argcount]
394            raise
395
396    def get_old_value(self, args, kwargs, default=None):
397        # type: (List[Any], Dict[str, Any], Any) -> Any
398        """Returns the old value of the named argument without replacing it.
399
400        Returns ``default`` if the argument is not present.
401        """
402        if self.arg_pos is not None and len(args) > self.arg_pos:
403            return args[self.arg_pos]
404        else:
405            return kwargs.get(self.name, default)
406
407    def replace(self, new_value, args, kwargs):
408        # type: (Any, List[Any], Dict[str, Any]) -> Tuple[Any, List[Any], Dict[str, Any]]
409        """Replace the named argument in ``args, kwargs`` with ``new_value``.
410
411        Returns ``(old_value, args, kwargs)``.  The returned ``args`` and
412        ``kwargs`` objects may not be the same as the input objects, or
413        the input objects may be mutated.
414
415        If the named argument was not found, ``new_value`` will be added
416        to ``kwargs`` and None will be returned as ``old_value``.
417        """
418        if self.arg_pos is not None and len(args) > self.arg_pos:
419            # The arg to replace is passed positionally
420            old_value = args[self.arg_pos]
421            args = list(args)  # *args is normally a tuple
422            args[self.arg_pos] = new_value
423        else:
424            # The arg to replace is either omitted or passed by keyword.
425            old_value = kwargs.get(self.name)
426            kwargs[self.name] = new_value
427        return old_value, args, kwargs
428
429
430def timedelta_to_seconds(td):
431    # type: (datetime.timedelta) -> float
432    """Equivalent to td.total_seconds() (introduced in python 2.7)."""
433    return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6)
434
435
436def _websocket_mask_python(mask, data):
437    # type: (bytes, bytes) -> bytes
438    """Websocket masking function.
439
440    `mask` is a `bytes` object of length 4; `data` is a `bytes` object of any length.
441    Returns a `bytes` object of the same length as `data` with the mask applied
442    as specified in section 5.3 of RFC 6455.
443
444    This pure-python implementation may be replaced by an optimized version when available.
445    """
446    mask_arr = array.array("B", mask)
447    unmasked_arr = array.array("B", data)
448    for i in xrange(len(data)):
449        unmasked_arr[i] = unmasked_arr[i] ^ mask_arr[i % 4]
450    if PY3:
451        # tostring was deprecated in py32.  It hasn't been removed,
452        # but since we turn on deprecation warnings in our tests
453        # we need to use the right one.
454        return unmasked_arr.tobytes()
455    else:
456        return unmasked_arr.tostring()
457
458
459if (os.environ.get('TORNADO_NO_EXTENSION') or
460        os.environ.get('TORNADO_EXTENSION') == '0'):
461    # These environment variables exist to make it easier to do performance
462    # comparisons; they are not guaranteed to remain supported in the future.
463    _websocket_mask = _websocket_mask_python
464else:
465    try:
466        from tornado.speedups import websocket_mask as _websocket_mask
467    except ImportError:
468        if os.environ.get('TORNADO_EXTENSION') == '1':
469            raise
470        _websocket_mask = _websocket_mask_python
471
472
473def doctests():
474    import doctest
475    return doctest.DocTestSuite()
476