1"""Miscellaneous utility functions and classes. 2 3This module is used internally by Tornado. It is not necessarily expected 4that the functions and classes defined here will be useful to other 5applications, but they are documented here in case they are. 6 7The one public-facing part of this module is the `Configurable` class 8and its `~Configurable.configure` method, which becomes a part of the 9interface of its subclasses, including `.AsyncHTTPClient`, `.IOLoop`, 10and `.Resolver`. 11""" 12 13from __future__ import absolute_import, division, print_function 14 15import array 16import atexit 17import os 18import re 19import sys 20import zlib 21 22PY3 = sys.version_info >= (3,) 23 24if PY3: 25 xrange = range 26 27# inspect.getargspec() raises DeprecationWarnings in Python 3.5. 28# The two functions have compatible interfaces for the parts we need. 29if PY3: 30 from inspect import getfullargspec as getargspec 31else: 32 from inspect import getargspec 33 34# Aliases for types that are spelled differently in different Python 35# versions. bytes_type is deprecated and no longer used in Tornado 36# itself but is left in case anyone outside Tornado is using it. 37bytes_type = bytes 38if PY3: 39 unicode_type = str 40 basestring_type = str 41else: 42 # The names unicode and basestring don't exist in py3 so silence flake8. 43 unicode_type = unicode # noqa 44 basestring_type = basestring # noqa 45 46 47try: 48 import typing # noqa 49 from typing import cast 50 51 _ObjectDictBase = typing.Dict[str, typing.Any] 52except ImportError: 53 _ObjectDictBase = dict 54 55 def cast(typ, x): 56 return x 57else: 58 # More imports that are only needed in type comments. 59 import datetime # noqa 60 import types # noqa 61 from typing import Any, AnyStr, Union, Optional, Dict, Mapping # noqa 62 from typing import Tuple, Match, Callable # noqa 63 64 if PY3: 65 _BaseString = str 66 else: 67 _BaseString = Union[bytes, unicode_type] 68 69 70try: 71 from sys import is_finalizing 72except ImportError: 73 # Emulate it 74 def _get_emulated_is_finalizing(): 75 L = [] 76 atexit.register(lambda: L.append(None)) 77 78 def is_finalizing(): 79 # Not referencing any globals here 80 return L != [] 81 82 return is_finalizing 83 84 is_finalizing = _get_emulated_is_finalizing() 85 86 87class ObjectDict(_ObjectDictBase): 88 """Makes a dictionary behave like an object, with attribute-style access. 89 """ 90 def __getattr__(self, name): 91 # type: (str) -> Any 92 try: 93 return self[name] 94 except KeyError: 95 raise AttributeError(name) 96 97 def __setattr__(self, name, value): 98 # type: (str, Any) -> None 99 self[name] = value 100 101 102class GzipDecompressor(object): 103 """Streaming gzip decompressor. 104 105 The interface is like that of `zlib.decompressobj` (without some of the 106 optional arguments, but it understands gzip headers and checksums. 107 """ 108 def __init__(self): 109 # Magic parameter makes zlib module understand gzip header 110 # http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib 111 # This works on cpython and pypy, but not jython. 112 self.decompressobj = zlib.decompressobj(16 + zlib.MAX_WBITS) 113 114 def decompress(self, value, max_length=None): 115 # type: (bytes, Optional[int]) -> bytes 116 """Decompress a chunk, returning newly-available data. 117 118 Some data may be buffered for later processing; `flush` must 119 be called when there is no more input data to ensure that 120 all data was processed. 121 122 If ``max_length`` is given, some input data may be left over 123 in ``unconsumed_tail``; you must retrieve this value and pass 124 it back to a future call to `decompress` if it is not empty. 125 """ 126 return self.decompressobj.decompress(value, max_length) 127 128 @property 129 def unconsumed_tail(self): 130 # type: () -> bytes 131 """Returns the unconsumed portion left over 132 """ 133 return self.decompressobj.unconsumed_tail 134 135 def flush(self): 136 # type: () -> bytes 137 """Return any remaining buffered data not yet returned by decompress. 138 139 Also checks for errors such as truncated input. 140 No other methods may be called on this object after `flush`. 141 """ 142 return self.decompressobj.flush() 143 144 145def import_object(name): 146 # type: (_BaseString) -> Any 147 """Imports an object by name. 148 149 import_object('x') is equivalent to 'import x'. 150 import_object('x.y.z') is equivalent to 'from x.y import z'. 151 152 >>> import tornado.escape 153 >>> import_object('tornado.escape') is tornado.escape 154 True 155 >>> import_object('tornado.escape.utf8') is tornado.escape.utf8 156 True 157 >>> import_object('tornado') is tornado 158 True 159 >>> import_object('tornado.missing_module') 160 Traceback (most recent call last): 161 ... 162 ImportError: No module named missing_module 163 """ 164 if not isinstance(name, str): 165 # on python 2 a byte string is required. 166 name = name.encode('utf-8') 167 if name.count('.') == 0: 168 return __import__(name, None, None) 169 170 parts = name.split('.') 171 obj = __import__('.'.join(parts[:-1]), None, None, [parts[-1]], 0) 172 try: 173 return getattr(obj, parts[-1]) 174 except AttributeError: 175 raise ImportError("No module named %s" % parts[-1]) 176 177 178# Stubs to make mypy happy (and later for actual type-checking). 179def raise_exc_info(exc_info): 180 # type: (Tuple[type, BaseException, types.TracebackType]) -> None 181 pass 182 183 184def exec_in(code, glob, loc=None): 185 # type: (Any, Dict[str, Any], Optional[Mapping[str, Any]]) -> Any 186 if isinstance(code, basestring_type): 187 # exec(string) inherits the caller's future imports; compile 188 # the string first to prevent that. 189 code = compile(code, '<string>', 'exec', dont_inherit=True) 190 exec(code, glob, loc) 191 192 193if PY3: 194 exec(""" 195def raise_exc_info(exc_info): 196 try: 197 raise exc_info[1].with_traceback(exc_info[2]) 198 finally: 199 exc_info = None 200 201""") 202else: 203 exec(""" 204def raise_exc_info(exc_info): 205 raise exc_info[0], exc_info[1], exc_info[2] 206""") 207 208 209def errno_from_exception(e): 210 # type: (BaseException) -> Optional[int] 211 """Provides the errno from an Exception object. 212 213 There are cases that the errno attribute was not set so we pull 214 the errno out of the args but if someone instantiates an Exception 215 without any args you will get a tuple error. So this function 216 abstracts all that behavior to give you a safe way to get the 217 errno. 218 """ 219 220 if hasattr(e, 'errno'): 221 return e.errno # type: ignore 222 elif e.args: 223 return e.args[0] 224 else: 225 return None 226 227 228_alphanum = frozenset( 229 "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") 230 231 232def _re_unescape_replacement(match): 233 # type: (Match[str]) -> str 234 group = match.group(1) 235 if group[0] in _alphanum: 236 raise ValueError("cannot unescape '\\\\%s'" % group[0]) 237 return group 238 239 240_re_unescape_pattern = re.compile(r'\\(.)', re.DOTALL) 241 242 243def re_unescape(s): 244 # type: (str) -> str 245 """Unescape a string escaped by `re.escape`. 246 247 May raise ``ValueError`` for regular expressions which could not 248 have been produced by `re.escape` (for example, strings containing 249 ``\d`` cannot be unescaped). 250 251 .. versionadded:: 4.4 252 """ 253 return _re_unescape_pattern.sub(_re_unescape_replacement, s) 254 255 256class Configurable(object): 257 """Base class for configurable interfaces. 258 259 A configurable interface is an (abstract) class whose constructor 260 acts as a factory function for one of its implementation subclasses. 261 The implementation subclass as well as optional keyword arguments to 262 its initializer can be set globally at runtime with `configure`. 263 264 By using the constructor as the factory method, the interface 265 looks like a normal class, `isinstance` works as usual, etc. This 266 pattern is most useful when the choice of implementation is likely 267 to be a global decision (e.g. when `~select.epoll` is available, 268 always use it instead of `~select.select`), or when a 269 previously-monolithic class has been split into specialized 270 subclasses. 271 272 Configurable subclasses must define the class methods 273 `configurable_base` and `configurable_default`, and use the instance 274 method `initialize` instead of ``__init__``. 275 """ 276 __impl_class = None # type: type 277 __impl_kwargs = None # type: Dict[str, Any] 278 279 def __new__(cls, *args, **kwargs): 280 base = cls.configurable_base() 281 init_kwargs = {} 282 if cls is base: 283 impl = cls.configured_class() 284 if base.__impl_kwargs: 285 init_kwargs.update(base.__impl_kwargs) 286 else: 287 impl = cls 288 init_kwargs.update(kwargs) 289 instance = super(Configurable, cls).__new__(impl) 290 # initialize vs __init__ chosen for compatibility with AsyncHTTPClient 291 # singleton magic. If we get rid of that we can switch to __init__ 292 # here too. 293 instance.initialize(*args, **init_kwargs) 294 return instance 295 296 @classmethod 297 def configurable_base(cls): 298 # type: () -> Any 299 # TODO: This class needs https://github.com/python/typing/issues/107 300 # to be fully typeable. 301 """Returns the base class of a configurable hierarchy. 302 303 This will normally return the class in which it is defined. 304 (which is *not* necessarily the same as the cls classmethod parameter). 305 """ 306 raise NotImplementedError() 307 308 @classmethod 309 def configurable_default(cls): 310 # type: () -> type 311 """Returns the implementation class to be used if none is configured.""" 312 raise NotImplementedError() 313 314 def initialize(self): 315 # type: () -> None 316 """Initialize a `Configurable` subclass instance. 317 318 Configurable classes should use `initialize` instead of ``__init__``. 319 320 .. versionchanged:: 4.2 321 Now accepts positional arguments in addition to keyword arguments. 322 """ 323 324 @classmethod 325 def configure(cls, impl, **kwargs): 326 # type: (Any, **Any) -> None 327 """Sets the class to use when the base class is instantiated. 328 329 Keyword arguments will be saved and added to the arguments passed 330 to the constructor. This can be used to set global defaults for 331 some parameters. 332 """ 333 base = cls.configurable_base() 334 if isinstance(impl, (str, unicode_type)): 335 impl = import_object(impl) 336 if impl is not None and not issubclass(impl, cls): 337 raise ValueError("Invalid subclass of %s" % cls) 338 base.__impl_class = impl 339 base.__impl_kwargs = kwargs 340 341 @classmethod 342 def configured_class(cls): 343 # type: () -> type 344 """Returns the currently configured class.""" 345 base = cls.configurable_base() 346 if cls.__impl_class is None: 347 base.__impl_class = cls.configurable_default() 348 return base.__impl_class 349 350 @classmethod 351 def _save_configuration(cls): 352 # type: () -> Tuple[type, Dict[str, Any]] 353 base = cls.configurable_base() 354 return (base.__impl_class, base.__impl_kwargs) 355 356 @classmethod 357 def _restore_configuration(cls, saved): 358 # type: (Tuple[type, Dict[str, Any]]) -> None 359 base = cls.configurable_base() 360 base.__impl_class = saved[0] 361 base.__impl_kwargs = saved[1] 362 363 364class ArgReplacer(object): 365 """Replaces one value in an ``args, kwargs`` pair. 366 367 Inspects the function signature to find an argument by name 368 whether it is passed by position or keyword. For use in decorators 369 and similar wrappers. 370 """ 371 def __init__(self, func, name): 372 # type: (Callable, str) -> None 373 self.name = name 374 try: 375 self.arg_pos = self._getargnames(func).index(name) 376 except ValueError: 377 # Not a positional parameter 378 self.arg_pos = None 379 380 def _getargnames(self, func): 381 # type: (Callable) -> List[str] 382 try: 383 return getargspec(func).args 384 except TypeError: 385 if hasattr(func, 'func_code'): 386 # Cython-generated code has all the attributes needed 387 # by inspect.getargspec, but the inspect module only 388 # works with ordinary functions. Inline the portion of 389 # getargspec that we need here. Note that for static 390 # functions the @cython.binding(True) decorator must 391 # be used (for methods it works out of the box). 392 code = func.func_code # type: ignore 393 return code.co_varnames[:code.co_argcount] 394 raise 395 396 def get_old_value(self, args, kwargs, default=None): 397 # type: (List[Any], Dict[str, Any], Any) -> Any 398 """Returns the old value of the named argument without replacing it. 399 400 Returns ``default`` if the argument is not present. 401 """ 402 if self.arg_pos is not None and len(args) > self.arg_pos: 403 return args[self.arg_pos] 404 else: 405 return kwargs.get(self.name, default) 406 407 def replace(self, new_value, args, kwargs): 408 # type: (Any, List[Any], Dict[str, Any]) -> Tuple[Any, List[Any], Dict[str, Any]] 409 """Replace the named argument in ``args, kwargs`` with ``new_value``. 410 411 Returns ``(old_value, args, kwargs)``. The returned ``args`` and 412 ``kwargs`` objects may not be the same as the input objects, or 413 the input objects may be mutated. 414 415 If the named argument was not found, ``new_value`` will be added 416 to ``kwargs`` and None will be returned as ``old_value``. 417 """ 418 if self.arg_pos is not None and len(args) > self.arg_pos: 419 # The arg to replace is passed positionally 420 old_value = args[self.arg_pos] 421 args = list(args) # *args is normally a tuple 422 args[self.arg_pos] = new_value 423 else: 424 # The arg to replace is either omitted or passed by keyword. 425 old_value = kwargs.get(self.name) 426 kwargs[self.name] = new_value 427 return old_value, args, kwargs 428 429 430def timedelta_to_seconds(td): 431 # type: (datetime.timedelta) -> float 432 """Equivalent to td.total_seconds() (introduced in python 2.7).""" 433 return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6) 434 435 436def _websocket_mask_python(mask, data): 437 # type: (bytes, bytes) -> bytes 438 """Websocket masking function. 439 440 `mask` is a `bytes` object of length 4; `data` is a `bytes` object of any length. 441 Returns a `bytes` object of the same length as `data` with the mask applied 442 as specified in section 5.3 of RFC 6455. 443 444 This pure-python implementation may be replaced by an optimized version when available. 445 """ 446 mask_arr = array.array("B", mask) 447 unmasked_arr = array.array("B", data) 448 for i in xrange(len(data)): 449 unmasked_arr[i] = unmasked_arr[i] ^ mask_arr[i % 4] 450 if PY3: 451 # tostring was deprecated in py32. It hasn't been removed, 452 # but since we turn on deprecation warnings in our tests 453 # we need to use the right one. 454 return unmasked_arr.tobytes() 455 else: 456 return unmasked_arr.tostring() 457 458 459if (os.environ.get('TORNADO_NO_EXTENSION') or 460 os.environ.get('TORNADO_EXTENSION') == '0'): 461 # These environment variables exist to make it easier to do performance 462 # comparisons; they are not guaranteed to remain supported in the future. 463 _websocket_mask = _websocket_mask_python 464else: 465 try: 466 from tornado.speedups import websocket_mask as _websocket_mask 467 except ImportError: 468 if os.environ.get('TORNADO_EXTENSION') == '1': 469 raise 470 _websocket_mask = _websocket_mask_python 471 472 473def doctests(): 474 import doctest 475 return doctest.DocTestSuite() 476