1# -*- coding: utf-8 -*- 2 3""" 4requests.session 5~~~~~~~~~~~~~~~~ 6 7This module provides a Session object to manage and persist settings across 8requests (cookies, auth, proxies). 9 10""" 11import os 12from collections import Mapping 13from datetime import datetime 14 15from .auth import _basic_auth_str 16from .compat import cookielib, OrderedDict, urljoin, urlparse 17from .cookies import ( 18 cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) 19from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT 20from .hooks import default_hooks, dispatch_hook 21from .utils import to_key_val_list, default_headers, to_native_string 22from .exceptions import ( 23 TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) 24from .packages.urllib3._collections import RecentlyUsedContainer 25from .structures import CaseInsensitiveDict 26 27from .adapters import HTTPAdapter 28 29from .utils import ( 30 requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies, 31 get_auth_from_url 32) 33 34from .status_codes import codes 35 36# formerly defined here, reexposed here for backward compatibility 37from .models import REDIRECT_STATI 38 39REDIRECT_CACHE_SIZE = 1000 40 41 42def merge_setting(request_setting, session_setting, dict_class=OrderedDict): 43 """ 44 Determines appropriate setting for a given request, taking into account the 45 explicit setting on that request, and the setting in the session. If a 46 setting is a dictionary, they will be merged together using `dict_class` 47 """ 48 49 if session_setting is None: 50 return request_setting 51 52 if request_setting is None: 53 return session_setting 54 55 # Bypass if not a dictionary (e.g. verify) 56 if not ( 57 isinstance(session_setting, Mapping) and 58 isinstance(request_setting, Mapping) 59 ): 60 return request_setting 61 62 merged_setting = dict_class(to_key_val_list(session_setting)) 63 merged_setting.update(to_key_val_list(request_setting)) 64 65 # Remove keys that are set to None. Extract keys first to avoid altering 66 # the dictionary during iteration. 67 none_keys = [k for (k, v) in merged_setting.items() if v is None] 68 for key in none_keys: 69 del merged_setting[key] 70 71 return merged_setting 72 73 74def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict): 75 """ 76 Properly merges both requests and session hooks. 77 78 This is necessary because when request_hooks == {'response': []}, the 79 merge breaks Session hooks entirely. 80 """ 81 if session_hooks is None or session_hooks.get('response') == []: 82 return request_hooks 83 84 if request_hooks is None or request_hooks.get('response') == []: 85 return session_hooks 86 87 return merge_setting(request_hooks, session_hooks, dict_class) 88 89 90class SessionRedirectMixin(object): 91 def resolve_redirects(self, resp, req, stream=False, timeout=None, 92 verify=True, cert=None, proxies=None, **adapter_kwargs): 93 """Receives a Response. Returns a generator of Responses.""" 94 95 i = 0 96 hist = [] # keep track of history 97 98 while resp.is_redirect: 99 prepared_request = req.copy() 100 101 if i > 0: 102 # Update history and keep track of redirects. 103 hist.append(resp) 104 new_hist = list(hist) 105 resp.history = new_hist 106 107 try: 108 resp.content # Consume socket so it can be released 109 except (ChunkedEncodingError, ContentDecodingError, RuntimeError): 110 resp.raw.read(decode_content=False) 111 112 if i >= self.max_redirects: 113 raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects) 114 115 # Release the connection back into the pool. 116 resp.close() 117 118 url = resp.headers['location'] 119 method = req.method 120 121 # Handle redirection without scheme (see: RFC 1808 Section 4) 122 if url.startswith('//'): 123 parsed_rurl = urlparse(resp.url) 124 url = '%s:%s' % (parsed_rurl.scheme, url) 125 126 # The scheme should be lower case... 127 parsed = urlparse(url) 128 url = parsed.geturl() 129 130 # Facilitate relative 'location' headers, as allowed by RFC 7231. 131 # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') 132 # Compliant with RFC3986, we percent encode the url. 133 if not parsed.netloc: 134 url = urljoin(resp.url, requote_uri(url)) 135 else: 136 url = requote_uri(url) 137 138 prepared_request.url = to_native_string(url) 139 # Cache the url, unless it redirects to itself. 140 if resp.is_permanent_redirect and req.url != prepared_request.url: 141 self.redirect_cache[req.url] = prepared_request.url 142 143 # http://tools.ietf.org/html/rfc7231#section-6.4.4 144 if (resp.status_code == codes.see_other and 145 method != 'HEAD'): 146 method = 'GET' 147 148 # Do what the browsers do, despite standards... 149 # First, turn 302s into GETs. 150 if resp.status_code == codes.found and method != 'HEAD': 151 method = 'GET' 152 153 # Second, if a POST is responded to with a 301, turn it into a GET. 154 # This bizarre behaviour is explained in Issue 1704. 155 if resp.status_code == codes.moved and method == 'POST': 156 method = 'GET' 157 158 prepared_request.method = method 159 160 # https://github.com/kennethreitz/requests/issues/1084 161 if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect): 162 if 'Content-Length' in prepared_request.headers: 163 del prepared_request.headers['Content-Length'] 164 165 prepared_request.body = None 166 167 headers = prepared_request.headers 168 try: 169 del headers['Cookie'] 170 except KeyError: 171 pass 172 173 # Extract any cookies sent on the response to the cookiejar 174 # in the new request. Because we've mutated our copied prepared 175 # request, use the old one that we haven't yet touched. 176 extract_cookies_to_jar(prepared_request._cookies, req, resp.raw) 177 prepared_request._cookies.update(self.cookies) 178 prepared_request.prepare_cookies(prepared_request._cookies) 179 180 # Rebuild auth and proxy information. 181 proxies = self.rebuild_proxies(prepared_request, proxies) 182 self.rebuild_auth(prepared_request, resp) 183 184 # Override the original request. 185 req = prepared_request 186 187 resp = self.send( 188 req, 189 stream=stream, 190 timeout=timeout, 191 verify=verify, 192 cert=cert, 193 proxies=proxies, 194 allow_redirects=False, 195 **adapter_kwargs 196 ) 197 198 extract_cookies_to_jar(self.cookies, prepared_request, resp.raw) 199 200 i += 1 201 yield resp 202 203 def rebuild_auth(self, prepared_request, response): 204 """ 205 When being redirected we may want to strip authentication from the 206 request to avoid leaking credentials. This method intelligently removes 207 and reapplies authentication where possible to avoid credential loss. 208 """ 209 headers = prepared_request.headers 210 url = prepared_request.url 211 212 if 'Authorization' in headers: 213 # If we get redirected to a new host, we should strip out any 214 # authentication headers. 215 original_parsed = urlparse(response.request.url) 216 redirect_parsed = urlparse(url) 217 218 if (original_parsed.hostname != redirect_parsed.hostname): 219 del headers['Authorization'] 220 221 # .netrc might have more auth for us on our new host. 222 new_auth = get_netrc_auth(url) if self.trust_env else None 223 if new_auth is not None: 224 prepared_request.prepare_auth(new_auth) 225 226 return 227 228 def rebuild_proxies(self, prepared_request, proxies): 229 """ 230 This method re-evaluates the proxy configuration by considering the 231 environment variables. If we are redirected to a URL covered by 232 NO_PROXY, we strip the proxy configuration. Otherwise, we set missing 233 proxy keys for this URL (in case they were stripped by a previous 234 redirect). 235 236 This method also replaces the Proxy-Authorization header where 237 necessary. 238 """ 239 headers = prepared_request.headers 240 url = prepared_request.url 241 scheme = urlparse(url).scheme 242 new_proxies = proxies.copy() if proxies is not None else {} 243 244 if self.trust_env and not should_bypass_proxies(url): 245 environ_proxies = get_environ_proxies(url) 246 247 proxy = environ_proxies.get(scheme) 248 249 if proxy: 250 new_proxies.setdefault(scheme, environ_proxies[scheme]) 251 252 if 'Proxy-Authorization' in headers: 253 del headers['Proxy-Authorization'] 254 255 try: 256 username, password = get_auth_from_url(new_proxies[scheme]) 257 except KeyError: 258 username, password = None, None 259 260 if username and password: 261 headers['Proxy-Authorization'] = _basic_auth_str(username, password) 262 263 return new_proxies 264 265 266class Session(SessionRedirectMixin): 267 """A Requests session. 268 269 Provides cookie persistence, connection-pooling, and configuration. 270 271 Basic Usage:: 272 273 >>> import requests 274 >>> s = requests.Session() 275 >>> s.get('http://httpbin.org/get') 276 <Response [200]> 277 278 Or as a context manager:: 279 280 >>> with requests.Session() as s: 281 >>> s.get('http://httpbin.org/get') 282 <Response [200]> 283 """ 284 285 __attrs__ = [ 286 'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify', 287 'cert', 'prefetch', 'adapters', 'stream', 'trust_env', 288 'max_redirects', 289 ] 290 291 def __init__(self): 292 293 #: A case-insensitive dictionary of headers to be sent on each 294 #: :class:`Request <Request>` sent from this 295 #: :class:`Session <Session>`. 296 self.headers = default_headers() 297 298 #: Default Authentication tuple or object to attach to 299 #: :class:`Request <Request>`. 300 self.auth = None 301 302 #: Dictionary mapping protocol or protocol and host to the URL of the proxy 303 #: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to 304 #: be used on each :class:`Request <Request>`. 305 self.proxies = {} 306 307 #: Event-handling hooks. 308 self.hooks = default_hooks() 309 310 #: Dictionary of querystring data to attach to each 311 #: :class:`Request <Request>`. The dictionary values may be lists for 312 #: representing multivalued query parameters. 313 self.params = {} 314 315 #: Stream response content default. 316 self.stream = False 317 318 #: SSL Verification default. 319 self.verify = True 320 321 #: SSL certificate default. 322 self.cert = None 323 324 #: Maximum number of redirects allowed. If the request exceeds this 325 #: limit, a :class:`TooManyRedirects` exception is raised. 326 self.max_redirects = DEFAULT_REDIRECT_LIMIT 327 328 #: Trust environment settings for proxy configuration, default 329 #: authentication and similar. 330 self.trust_env = True 331 332 #: A CookieJar containing all currently outstanding cookies set on this 333 #: session. By default it is a 334 #: :class:`RequestsCookieJar <requests.cookies.RequestsCookieJar>`, but 335 #: may be any other ``cookielib.CookieJar`` compatible object. 336 self.cookies = cookiejar_from_dict({}) 337 338 # Default connection adapters. 339 self.adapters = OrderedDict() 340 self.mount('https://', HTTPAdapter()) 341 self.mount('http://', HTTPAdapter()) 342 343 # Only store 1000 redirects to prevent using infinite memory 344 self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE) 345 346 def __enter__(self): 347 return self 348 349 def __exit__(self, *args): 350 self.close() 351 352 def prepare_request(self, request): 353 """Constructs a :class:`PreparedRequest <PreparedRequest>` for 354 transmission and returns it. The :class:`PreparedRequest` has settings 355 merged from the :class:`Request <Request>` instance and those of the 356 :class:`Session`. 357 358 :param request: :class:`Request` instance to prepare with this 359 session's settings. 360 """ 361 cookies = request.cookies or {} 362 363 # Bootstrap CookieJar. 364 if not isinstance(cookies, cookielib.CookieJar): 365 cookies = cookiejar_from_dict(cookies) 366 367 # Merge with session cookies 368 merged_cookies = merge_cookies( 369 merge_cookies(RequestsCookieJar(), self.cookies), cookies) 370 371 372 # Set environment's basic authentication if not explicitly set. 373 auth = request.auth 374 if self.trust_env and not auth and not self.auth: 375 auth = get_netrc_auth(request.url) 376 377 p = PreparedRequest() 378 p.prepare( 379 method=request.method.upper(), 380 url=request.url, 381 files=request.files, 382 data=request.data, 383 json=request.json, 384 headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), 385 params=merge_setting(request.params, self.params), 386 auth=merge_setting(auth, self.auth), 387 cookies=merged_cookies, 388 hooks=merge_hooks(request.hooks, self.hooks), 389 ) 390 return p 391 392 def request(self, method, url, 393 params=None, 394 data=None, 395 headers=None, 396 cookies=None, 397 files=None, 398 auth=None, 399 timeout=None, 400 allow_redirects=True, 401 proxies=None, 402 hooks=None, 403 stream=None, 404 verify=None, 405 cert=None, 406 json=None): 407 """Constructs a :class:`Request <Request>`, prepares it and sends it. 408 Returns :class:`Response <Response>` object. 409 410 :param method: method for the new :class:`Request` object. 411 :param url: URL for the new :class:`Request` object. 412 :param params: (optional) Dictionary or bytes to be sent in the query 413 string for the :class:`Request`. 414 :param data: (optional) Dictionary, bytes, or file-like object to send 415 in the body of the :class:`Request`. 416 :param json: (optional) json to send in the body of the 417 :class:`Request`. 418 :param headers: (optional) Dictionary of HTTP Headers to send with the 419 :class:`Request`. 420 :param cookies: (optional) Dict or CookieJar object to send with the 421 :class:`Request`. 422 :param files: (optional) Dictionary of ``'filename': file-like-objects`` 423 for multipart encoding upload. 424 :param auth: (optional) Auth tuple or callable to enable 425 Basic/Digest/Custom HTTP Auth. 426 :param timeout: (optional) How long to wait for the server to send 427 data before giving up, as a float, or a :ref:`(connect timeout, 428 read timeout) <timeouts>` tuple. 429 :type timeout: float or tuple 430 :param allow_redirects: (optional) Set to True by default. 431 :type allow_redirects: bool 432 :param proxies: (optional) Dictionary mapping protocol or protocol and 433 hostname to the URL of the proxy. 434 :param stream: (optional) whether to immediately download the response 435 content. Defaults to ``False``. 436 :param verify: (optional) whether the SSL cert will be verified. 437 A CA_BUNDLE path can also be provided. Defaults to ``True``. 438 :param cert: (optional) if String, path to ssl client cert file (.pem). 439 If Tuple, ('cert', 'key') pair. 440 """ 441 # Create the Request. 442 req = Request( 443 method = method.upper(), 444 url = url, 445 headers = headers, 446 files = files, 447 data = data or {}, 448 json = json, 449 params = params or {}, 450 auth = auth, 451 cookies = cookies, 452 hooks = hooks, 453 ) 454 prep = self.prepare_request(req) 455 456 proxies = proxies or {} 457 458 settings = self.merge_environment_settings( 459 prep.url, proxies, stream, verify, cert 460 ) 461 462 # Send the request. 463 send_kwargs = { 464 'timeout': timeout, 465 'allow_redirects': allow_redirects, 466 } 467 send_kwargs.update(settings) 468 resp = self.send(prep, **send_kwargs) 469 470 return resp 471 472 def get(self, url, **kwargs): 473 """Sends a GET request. Returns :class:`Response` object. 474 475 :param url: URL for the new :class:`Request` object. 476 :param \*\*kwargs: Optional arguments that ``request`` takes. 477 """ 478 479 kwargs.setdefault('allow_redirects', True) 480 return self.request('GET', url, **kwargs) 481 482 def options(self, url, **kwargs): 483 """Sends a OPTIONS request. Returns :class:`Response` object. 484 485 :param url: URL for the new :class:`Request` object. 486 :param \*\*kwargs: Optional arguments that ``request`` takes. 487 """ 488 489 kwargs.setdefault('allow_redirects', True) 490 return self.request('OPTIONS', url, **kwargs) 491 492 def head(self, url, **kwargs): 493 """Sends a HEAD request. Returns :class:`Response` object. 494 495 :param url: URL for the new :class:`Request` object. 496 :param \*\*kwargs: Optional arguments that ``request`` takes. 497 """ 498 499 kwargs.setdefault('allow_redirects', False) 500 return self.request('HEAD', url, **kwargs) 501 502 def post(self, url, data=None, json=None, **kwargs): 503 """Sends a POST request. Returns :class:`Response` object. 504 505 :param url: URL for the new :class:`Request` object. 506 :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 507 :param json: (optional) json to send in the body of the :class:`Request`. 508 :param \*\*kwargs: Optional arguments that ``request`` takes. 509 """ 510 511 return self.request('POST', url, data=data, json=json, **kwargs) 512 513 def put(self, url, data=None, **kwargs): 514 """Sends a PUT request. Returns :class:`Response` object. 515 516 :param url: URL for the new :class:`Request` object. 517 :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 518 :param \*\*kwargs: Optional arguments that ``request`` takes. 519 """ 520 521 return self.request('PUT', url, data=data, **kwargs) 522 523 def patch(self, url, data=None, **kwargs): 524 """Sends a PATCH request. Returns :class:`Response` object. 525 526 :param url: URL for the new :class:`Request` object. 527 :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 528 :param \*\*kwargs: Optional arguments that ``request`` takes. 529 """ 530 531 return self.request('PATCH', url, data=data, **kwargs) 532 533 def delete(self, url, **kwargs): 534 """Sends a DELETE request. Returns :class:`Response` object. 535 536 :param url: URL for the new :class:`Request` object. 537 :param \*\*kwargs: Optional arguments that ``request`` takes. 538 """ 539 540 return self.request('DELETE', url, **kwargs) 541 542 def send(self, request, **kwargs): 543 """Send a given PreparedRequest.""" 544 # Set defaults that the hooks can utilize to ensure they always have 545 # the correct parameters to reproduce the previous request. 546 kwargs.setdefault('stream', self.stream) 547 kwargs.setdefault('verify', self.verify) 548 kwargs.setdefault('cert', self.cert) 549 kwargs.setdefault('proxies', self.proxies) 550 551 # It's possible that users might accidentally send a Request object. 552 # Guard against that specific failure case. 553 if not isinstance(request, PreparedRequest): 554 raise ValueError('You can only send PreparedRequests.') 555 556 checked_urls = set() 557 while request.url in self.redirect_cache: 558 checked_urls.add(request.url) 559 new_url = self.redirect_cache.get(request.url) 560 if new_url in checked_urls: 561 break 562 request.url = new_url 563 564 # Set up variables needed for resolve_redirects and dispatching of hooks 565 allow_redirects = kwargs.pop('allow_redirects', True) 566 stream = kwargs.get('stream') 567 hooks = request.hooks 568 569 # Get the appropriate adapter to use 570 adapter = self.get_adapter(url=request.url) 571 572 # Start time (approximately) of the request 573 start = datetime.utcnow() 574 575 # Send the request 576 r = adapter.send(request, **kwargs) 577 578 # Total elapsed time of the request (approximately) 579 r.elapsed = datetime.utcnow() - start 580 581 # Response manipulation hooks 582 r = dispatch_hook('response', hooks, r, **kwargs) 583 584 # Persist cookies 585 if r.history: 586 587 # If the hooks create history then we want those cookies too 588 for resp in r.history: 589 extract_cookies_to_jar(self.cookies, resp.request, resp.raw) 590 591 extract_cookies_to_jar(self.cookies, request, r.raw) 592 593 # Redirect resolving generator. 594 gen = self.resolve_redirects(r, request, **kwargs) 595 596 # Resolve redirects if allowed. 597 history = [resp for resp in gen] if allow_redirects else [] 598 599 # Shuffle things around if there's history. 600 if history: 601 # Insert the first (original) request at the start 602 history.insert(0, r) 603 # Get the last request made 604 r = history.pop() 605 r.history = history 606 607 if not stream: 608 r.content 609 610 return r 611 612 def merge_environment_settings(self, url, proxies, stream, verify, cert): 613 """Check the environment and merge it with some settings.""" 614 # Gather clues from the surrounding environment. 615 if self.trust_env: 616 # Set environment's proxies. 617 env_proxies = get_environ_proxies(url) or {} 618 for (k, v) in env_proxies.items(): 619 proxies.setdefault(k, v) 620 621 # Look for requests environment configuration and be compatible 622 # with cURL. 623 if verify is True or verify is None: 624 verify = (os.environ.get('REQUESTS_CA_BUNDLE') or 625 os.environ.get('CURL_CA_BUNDLE')) 626 627 # Merge all the kwargs. 628 proxies = merge_setting(proxies, self.proxies) 629 stream = merge_setting(stream, self.stream) 630 verify = merge_setting(verify, self.verify) 631 cert = merge_setting(cert, self.cert) 632 633 return {'verify': verify, 'proxies': proxies, 'stream': stream, 634 'cert': cert} 635 636 def get_adapter(self, url): 637 """Returns the appropriate connection adapter for the given URL.""" 638 for (prefix, adapter) in self.adapters.items(): 639 640 if url.lower().startswith(prefix): 641 return adapter 642 643 # Nothing matches :-/ 644 raise InvalidSchema("No connection adapters were found for '%s'" % url) 645 646 def close(self): 647 """Closes all adapters and as such the session""" 648 for v in self.adapters.values(): 649 v.close() 650 651 def mount(self, prefix, adapter): 652 """Registers a connection adapter to a prefix. 653 654 Adapters are sorted in descending order by key length.""" 655 656 self.adapters[prefix] = adapter 657 keys_to_move = [k for k in self.adapters if len(k) < len(prefix)] 658 659 for key in keys_to_move: 660 self.adapters[key] = self.adapters.pop(key) 661 662 def __getstate__(self): 663 state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) 664 state['redirect_cache'] = dict(self.redirect_cache) 665 return state 666 667 def __setstate__(self, state): 668 redirect_cache = state.pop('redirect_cache', {}) 669 for attr, value in state.items(): 670 setattr(self, attr, value) 671 672 self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE) 673 for redirect, to in redirect_cache.items(): 674 self.redirect_cache[redirect] = to 675 676 677def session(): 678 """Returns a :class:`Session` for context-management.""" 679 680 return Session() 681