1# -*- coding: utf-8 -*- 2 3""" 4requests.models 5~~~~~~~~~~~~~~~ 6 7This module contains the primary objects that power Requests. 8""" 9 10import collections 11import datetime 12import sys 13 14# Import encoding now, to avoid implicit import later. 15# Implicit import within threads may cause LookupError when standard library is in a ZIP, 16# such as in Embedded Python. See https://github.com/requests/requests/issues/3578. 17import encodings.idna 18 19from pip9._vendor.urllib3.fields import RequestField 20from pip9._vendor.urllib3.filepost import encode_multipart_formdata 21from pip9._vendor.urllib3.util import parse_url 22from pip9._vendor.urllib3.exceptions import ( 23 DecodeError, ReadTimeoutError, ProtocolError, LocationParseError) 24 25from io import UnsupportedOperation 26from .hooks import default_hooks 27from .structures import CaseInsensitiveDict 28 29from .auth import HTTPBasicAuth 30from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar 31from .exceptions import ( 32 HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, 33 ContentDecodingError, ConnectionError, StreamConsumedError) 34from ._internal_utils import to_native_string, unicode_is_ascii 35from .utils import ( 36 guess_filename, get_auth_from_url, requote_uri, 37 stream_decode_response_unicode, to_key_val_list, parse_header_links, 38 iter_slices, guess_json_utf, super_len, check_header_validity) 39from .compat import ( 40 cookielib, urlunparse, urlsplit, urlencode, str, bytes, 41 is_py2, chardet, builtin_str, basestring) 42from .compat import json as complexjson 43from .status_codes import codes 44 45#: The set of HTTP status codes that indicate an automatically 46#: processable redirect. 47REDIRECT_STATI = ( 48 codes.moved, # 301 49 codes.found, # 302 50 codes.other, # 303 51 codes.temporary_redirect, # 307 52 codes.permanent_redirect, # 308 53) 54 55DEFAULT_REDIRECT_LIMIT = 30 56CONTENT_CHUNK_SIZE = 10 * 1024 57ITER_CHUNK_SIZE = 512 58 59 60class RequestEncodingMixin(object): 61 @property 62 def path_url(self): 63 """Build the path URL to use.""" 64 65 url = [] 66 67 p = urlsplit(self.url) 68 69 path = p.path 70 if not path: 71 path = '/' 72 73 url.append(path) 74 75 query = p.query 76 if query: 77 url.append('?') 78 url.append(query) 79 80 return ''.join(url) 81 82 @staticmethod 83 def _encode_params(data): 84 """Encode parameters in a piece of data. 85 86 Will successfully encode parameters when passed as a dict or a list of 87 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary 88 if parameters are supplied as a dict. 89 """ 90 91 if isinstance(data, (str, bytes)): 92 return data 93 elif hasattr(data, 'read'): 94 return data 95 elif hasattr(data, '__iter__'): 96 result = [] 97 for k, vs in to_key_val_list(data): 98 if isinstance(vs, basestring) or not hasattr(vs, '__iter__'): 99 vs = [vs] 100 for v in vs: 101 if v is not None: 102 result.append( 103 (k.encode('utf-8') if isinstance(k, str) else k, 104 v.encode('utf-8') if isinstance(v, str) else v)) 105 return urlencode(result, doseq=True) 106 else: 107 return data 108 109 @staticmethod 110 def _encode_files(files, data): 111 """Build the body for a multipart/form-data request. 112 113 Will successfully encode files when passed as a dict or a list of 114 tuples. Order is retained if data is a list of tuples but arbitrary 115 if parameters are supplied as a dict. 116 The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype) 117 or 4-tuples (filename, fileobj, contentype, custom_headers). 118 """ 119 if (not files): 120 raise ValueError("Files must be provided.") 121 elif isinstance(data, basestring): 122 raise ValueError("Data must not be a string.") 123 124 new_fields = [] 125 fields = to_key_val_list(data or {}) 126 files = to_key_val_list(files or {}) 127 128 for field, val in fields: 129 if isinstance(val, basestring) or not hasattr(val, '__iter__'): 130 val = [val] 131 for v in val: 132 if v is not None: 133 # Don't call str() on bytestrings: in Py3 it all goes wrong. 134 if not isinstance(v, bytes): 135 v = str(v) 136 137 new_fields.append( 138 (field.decode('utf-8') if isinstance(field, bytes) else field, 139 v.encode('utf-8') if isinstance(v, str) else v)) 140 141 for (k, v) in files: 142 # support for explicit filename 143 ft = None 144 fh = None 145 if isinstance(v, (tuple, list)): 146 if len(v) == 2: 147 fn, fp = v 148 elif len(v) == 3: 149 fn, fp, ft = v 150 else: 151 fn, fp, ft, fh = v 152 else: 153 fn = guess_filename(v) or k 154 fp = v 155 156 if isinstance(fp, (str, bytes, bytearray)): 157 fdata = fp 158 else: 159 fdata = fp.read() 160 161 rf = RequestField(name=k, data=fdata, filename=fn, headers=fh) 162 rf.make_multipart(content_type=ft) 163 new_fields.append(rf) 164 165 body, content_type = encode_multipart_formdata(new_fields) 166 167 return body, content_type 168 169 170class RequestHooksMixin(object): 171 def register_hook(self, event, hook): 172 """Properly register a hook.""" 173 174 if event not in self.hooks: 175 raise ValueError('Unsupported event specified, with event name "%s"' % (event)) 176 177 if isinstance(hook, collections.Callable): 178 self.hooks[event].append(hook) 179 elif hasattr(hook, '__iter__'): 180 self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable)) 181 182 def deregister_hook(self, event, hook): 183 """Deregister a previously registered hook. 184 Returns True if the hook existed, False if not. 185 """ 186 187 try: 188 self.hooks[event].remove(hook) 189 return True 190 except ValueError: 191 return False 192 193 194class Request(RequestHooksMixin): 195 """A user-created :class:`Request <Request>` object. 196 197 Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server. 198 199 :param method: HTTP method to use. 200 :param url: URL to send. 201 :param headers: dictionary of headers to send. 202 :param files: dictionary of {filename: fileobject} files to multipart upload. 203 :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. 204 :param json: json for the body to attach to the request (if files or data is not specified). 205 :param params: dictionary of URL parameters to append to the URL. 206 :param auth: Auth handler or (user, pass) tuple. 207 :param cookies: dictionary or CookieJar of cookies to attach to this request. 208 :param hooks: dictionary of callback hooks, for internal usage. 209 210 Usage:: 211 212 >>> import requests 213 >>> req = requests.Request('GET', 'http://httpbin.org/get') 214 >>> req.prepare() 215 <PreparedRequest [GET]> 216 """ 217 218 def __init__(self, 219 method=None, url=None, headers=None, files=None, data=None, 220 params=None, auth=None, cookies=None, hooks=None, json=None): 221 222 # Default empty dicts for dict params. 223 data = [] if data is None else data 224 files = [] if files is None else files 225 headers = {} if headers is None else headers 226 params = {} if params is None else params 227 hooks = {} if hooks is None else hooks 228 229 self.hooks = default_hooks() 230 for (k, v) in list(hooks.items()): 231 self.register_hook(event=k, hook=v) 232 233 self.method = method 234 self.url = url 235 self.headers = headers 236 self.files = files 237 self.data = data 238 self.json = json 239 self.params = params 240 self.auth = auth 241 self.cookies = cookies 242 243 def __repr__(self): 244 return '<Request [%s]>' % (self.method) 245 246 def prepare(self): 247 """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it.""" 248 p = PreparedRequest() 249 p.prepare( 250 method=self.method, 251 url=self.url, 252 headers=self.headers, 253 files=self.files, 254 data=self.data, 255 json=self.json, 256 params=self.params, 257 auth=self.auth, 258 cookies=self.cookies, 259 hooks=self.hooks, 260 ) 261 return p 262 263 264class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): 265 """The fully mutable :class:`PreparedRequest <PreparedRequest>` object, 266 containing the exact bytes that will be sent to the server. 267 268 Generated from either a :class:`Request <Request>` object or manually. 269 270 Usage:: 271 272 >>> import requests 273 >>> req = requests.Request('GET', 'http://httpbin.org/get') 274 >>> r = req.prepare() 275 <PreparedRequest [GET]> 276 277 >>> s = requests.Session() 278 >>> s.send(r) 279 <Response [200]> 280 """ 281 282 def __init__(self): 283 #: HTTP verb to send to the server. 284 self.method = None 285 #: HTTP URL to send the request to. 286 self.url = None 287 #: dictionary of HTTP headers. 288 self.headers = None 289 # The `CookieJar` used to create the Cookie header will be stored here 290 # after prepare_cookies is called 291 self._cookies = None 292 #: request body to send to the server. 293 self.body = None 294 #: dictionary of callback hooks, for internal usage. 295 self.hooks = default_hooks() 296 #: integer denoting starting position of a readable file-like body. 297 self._body_position = None 298 299 def prepare(self, 300 method=None, url=None, headers=None, files=None, data=None, 301 params=None, auth=None, cookies=None, hooks=None, json=None): 302 """Prepares the entire request with the given parameters.""" 303 304 self.prepare_method(method) 305 self.prepare_url(url, params) 306 self.prepare_headers(headers) 307 self.prepare_cookies(cookies) 308 self.prepare_body(data, files, json) 309 self.prepare_auth(auth, url) 310 311 # Note that prepare_auth must be last to enable authentication schemes 312 # such as OAuth to work on a fully prepared request. 313 314 # This MUST go after prepare_auth. Authenticators could add a hook 315 self.prepare_hooks(hooks) 316 317 def __repr__(self): 318 return '<PreparedRequest [%s]>' % (self.method) 319 320 def copy(self): 321 p = PreparedRequest() 322 p.method = self.method 323 p.url = self.url 324 p.headers = self.headers.copy() if self.headers is not None else None 325 p._cookies = _copy_cookie_jar(self._cookies) 326 p.body = self.body 327 p.hooks = self.hooks 328 p._body_position = self._body_position 329 return p 330 331 def prepare_method(self, method): 332 """Prepares the given HTTP method.""" 333 self.method = method 334 if self.method is not None: 335 self.method = to_native_string(self.method.upper()) 336 337 @staticmethod 338 def _get_idna_encoded_host(host): 339 import idna 340 341 try: 342 host = idna.encode(host, uts46=True).decode('utf-8') 343 except idna.IDNAError: 344 raise UnicodeError 345 return host 346 347 def prepare_url(self, url, params): 348 """Prepares the given HTTP URL.""" 349 #: Accept objects that have string representations. 350 #: We're unable to blindly call unicode/str functions 351 #: as this will include the bytestring indicator (b'') 352 #: on python 3.x. 353 #: https://github.com/requests/requests/pull/2238 354 if isinstance(url, bytes): 355 url = url.decode('utf8') 356 else: 357 url = unicode(url) if is_py2 else str(url) 358 359 # Remove leading whitespaces from url 360 url = url.lstrip() 361 362 # Don't do any URL preparation for non-HTTP schemes like `mailto`, 363 # `data` etc to work around exceptions from `url_parse`, which 364 # handles RFC 3986 only. 365 if ':' in url and not url.lower().startswith('http'): 366 self.url = url 367 return 368 369 # Support for unicode domain names and paths. 370 try: 371 scheme, auth, host, port, path, query, fragment = parse_url(url) 372 except LocationParseError as e: 373 raise InvalidURL(*e.args) 374 375 if not scheme: 376 error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?") 377 error = error.format(to_native_string(url, 'utf8')) 378 379 raise MissingSchema(error) 380 381 if not host: 382 raise InvalidURL("Invalid URL %r: No host supplied" % url) 383 384 # In general, we want to try IDNA encoding the hostname if the string contains 385 # non-ASCII characters. This allows users to automatically get the correct IDNA 386 # behaviour. For strings containing only ASCII characters, we need to also verify 387 # it doesn't start with a wildcard (*), before allowing the unencoded hostname. 388 if not unicode_is_ascii(host): 389 try: 390 host = self._get_idna_encoded_host(host) 391 except UnicodeError: 392 raise InvalidURL('URL has an invalid label.') 393 elif host.startswith(u'*'): 394 raise InvalidURL('URL has an invalid label.') 395 396 # Carefully reconstruct the network location 397 netloc = auth or '' 398 if netloc: 399 netloc += '@' 400 netloc += host 401 if port: 402 netloc += ':' + str(port) 403 404 # Bare domains aren't valid URLs. 405 if not path: 406 path = '/' 407 408 if is_py2: 409 if isinstance(scheme, str): 410 scheme = scheme.encode('utf-8') 411 if isinstance(netloc, str): 412 netloc = netloc.encode('utf-8') 413 if isinstance(path, str): 414 path = path.encode('utf-8') 415 if isinstance(query, str): 416 query = query.encode('utf-8') 417 if isinstance(fragment, str): 418 fragment = fragment.encode('utf-8') 419 420 if isinstance(params, (str, bytes)): 421 params = to_native_string(params) 422 423 enc_params = self._encode_params(params) 424 if enc_params: 425 if query: 426 query = '%s&%s' % (query, enc_params) 427 else: 428 query = enc_params 429 430 url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment])) 431 self.url = url 432 433 def prepare_headers(self, headers): 434 """Prepares the given HTTP headers.""" 435 436 self.headers = CaseInsensitiveDict() 437 if headers: 438 for header in headers.items(): 439 # Raise exception on invalid header value. 440 check_header_validity(header) 441 name, value = header 442 self.headers[to_native_string(name)] = value 443 444 def prepare_body(self, data, files, json=None): 445 """Prepares the given HTTP body data.""" 446 447 # Check if file, fo, generator, iterator. 448 # If not, run through normal process. 449 450 # Nottin' on you. 451 body = None 452 content_type = None 453 454 if not data and json is not None: 455 # urllib3 requires a bytes-like body. Python 2's json.dumps 456 # provides this natively, but Python 3 gives a Unicode string. 457 content_type = 'application/json' 458 body = complexjson.dumps(json) 459 if not isinstance(body, bytes): 460 body = body.encode('utf-8') 461 462 is_stream = all([ 463 hasattr(data, '__iter__'), 464 not isinstance(data, (basestring, list, tuple, collections.Mapping)) 465 ]) 466 467 try: 468 length = super_len(data) 469 except (TypeError, AttributeError, UnsupportedOperation): 470 length = None 471 472 if is_stream: 473 body = data 474 475 if getattr(body, 'tell', None) is not None: 476 # Record the current file position before reading. 477 # This will allow us to rewind a file in the event 478 # of a redirect. 479 try: 480 self._body_position = body.tell() 481 except (IOError, OSError): 482 # This differentiates from None, allowing us to catch 483 # a failed `tell()` later when trying to rewind the body 484 self._body_position = object() 485 486 if files: 487 raise NotImplementedError('Streamed bodies and files are mutually exclusive.') 488 489 if length: 490 self.headers['Content-Length'] = builtin_str(length) 491 else: 492 self.headers['Transfer-Encoding'] = 'chunked' 493 else: 494 # Multi-part file uploads. 495 if files: 496 (body, content_type) = self._encode_files(files, data) 497 else: 498 if data: 499 body = self._encode_params(data) 500 if isinstance(data, basestring) or hasattr(data, 'read'): 501 content_type = None 502 else: 503 content_type = 'application/x-www-form-urlencoded' 504 505 self.prepare_content_length(body) 506 507 # Add content-type if it wasn't explicitly provided. 508 if content_type and ('content-type' not in self.headers): 509 self.headers['Content-Type'] = content_type 510 511 self.body = body 512 513 def prepare_content_length(self, body): 514 """Prepare Content-Length header based on request method and body""" 515 if body is not None: 516 length = super_len(body) 517 if length: 518 # If length exists, set it. Otherwise, we fallback 519 # to Transfer-Encoding: chunked. 520 self.headers['Content-Length'] = builtin_str(length) 521 elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None: 522 # Set Content-Length to 0 for methods that can have a body 523 # but don't provide one. (i.e. not GET or HEAD) 524 self.headers['Content-Length'] = '0' 525 526 def prepare_auth(self, auth, url=''): 527 """Prepares the given HTTP auth data.""" 528 529 # If no Auth is explicitly provided, extract it from the URL first. 530 if auth is None: 531 url_auth = get_auth_from_url(self.url) 532 auth = url_auth if any(url_auth) else None 533 534 if auth: 535 if isinstance(auth, tuple) and len(auth) == 2: 536 # special-case basic HTTP auth 537 auth = HTTPBasicAuth(*auth) 538 539 # Allow auth to make its changes. 540 r = auth(self) 541 542 # Update self to reflect the auth changes. 543 self.__dict__.update(r.__dict__) 544 545 # Recompute Content-Length 546 self.prepare_content_length(self.body) 547 548 def prepare_cookies(self, cookies): 549 """Prepares the given HTTP cookie data. 550 551 This function eventually generates a ``Cookie`` header from the 552 given cookies using cookielib. Due to cookielib's design, the header 553 will not be regenerated if it already exists, meaning this function 554 can only be called once for the life of the 555 :class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls 556 to ``prepare_cookies`` will have no actual effect, unless the "Cookie" 557 header is removed beforehand. 558 """ 559 if isinstance(cookies, cookielib.CookieJar): 560 self._cookies = cookies 561 else: 562 self._cookies = cookiejar_from_dict(cookies) 563 564 cookie_header = get_cookie_header(self._cookies, self) 565 if cookie_header is not None: 566 self.headers['Cookie'] = cookie_header 567 568 def prepare_hooks(self, hooks): 569 """Prepares the given hooks.""" 570 # hooks can be passed as None to the prepare method and to this 571 # method. To prevent iterating over None, simply use an empty list 572 # if hooks is False-y 573 hooks = hooks or [] 574 for event in hooks: 575 self.register_hook(event, hooks[event]) 576 577 578class Response(object): 579 """The :class:`Response <Response>` object, which contains a 580 server's response to an HTTP request. 581 """ 582 583 __attrs__ = [ 584 '_content', 'status_code', 'headers', 'url', 'history', 585 'encoding', 'reason', 'cookies', 'elapsed', 'request' 586 ] 587 588 def __init__(self): 589 self._content = False 590 self._content_consumed = False 591 self._next = None 592 593 #: Integer Code of responded HTTP Status, e.g. 404 or 200. 594 self.status_code = None 595 596 #: Case-insensitive Dictionary of Response Headers. 597 #: For example, ``headers['content-encoding']`` will return the 598 #: value of a ``'Content-Encoding'`` response header. 599 self.headers = CaseInsensitiveDict() 600 601 #: File-like object representation of response (for advanced usage). 602 #: Use of ``raw`` requires that ``stream=True`` be set on the request. 603 # This requirement does not apply for use internally to Requests. 604 self.raw = None 605 606 #: Final URL location of Response. 607 self.url = None 608 609 #: Encoding to decode with when accessing r.text. 610 self.encoding = None 611 612 #: A list of :class:`Response <Response>` objects from 613 #: the history of the Request. Any redirect responses will end 614 #: up here. The list is sorted from the oldest to the most recent request. 615 self.history = [] 616 617 #: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK". 618 self.reason = None 619 620 #: A CookieJar of Cookies the server sent back. 621 self.cookies = cookiejar_from_dict({}) 622 623 #: The amount of time elapsed between sending the request 624 #: and the arrival of the response (as a timedelta). 625 #: This property specifically measures the time taken between sending 626 #: the first byte of the request and finishing parsing the headers. It 627 #: is therefore unaffected by consuming the response content or the 628 #: value of the ``stream`` keyword argument. 629 self.elapsed = datetime.timedelta(0) 630 631 #: The :class:`PreparedRequest <PreparedRequest>` object to which this 632 #: is a response. 633 self.request = None 634 635 def __enter__(self): 636 return self 637 638 def __exit__(self, *args): 639 self.close() 640 641 def __getstate__(self): 642 # Consume everything; accessing the content attribute makes 643 # sure the content has been fully read. 644 if not self._content_consumed: 645 self.content 646 647 return dict( 648 (attr, getattr(self, attr, None)) 649 for attr in self.__attrs__ 650 ) 651 652 def __setstate__(self, state): 653 for name, value in state.items(): 654 setattr(self, name, value) 655 656 # pickled objects do not have .raw 657 setattr(self, '_content_consumed', True) 658 setattr(self, 'raw', None) 659 660 def __repr__(self): 661 return '<Response [%s]>' % (self.status_code) 662 663 def __bool__(self): 664 """Returns True if :attr:`status_code` is less than 400. 665 666 This attribute checks if the status code of the response is between 667 400 and 600 to see if there was a client error or a server error. If 668 the status code, is between 200 and 400, this will return True. This 669 is **not** a check to see if the response code is ``200 OK``. 670 """ 671 return self.ok 672 673 def __nonzero__(self): 674 """Returns True if :attr:`status_code` is less than 400. 675 676 This attribute checks if the status code of the response is between 677 400 and 600 to see if there was a client error or a server error. If 678 the status code, is between 200 and 400, this will return True. This 679 is **not** a check to see if the response code is ``200 OK``. 680 """ 681 return self.ok 682 683 def __iter__(self): 684 """Allows you to use a response as an iterator.""" 685 return self.iter_content(128) 686 687 @property 688 def ok(self): 689 """Returns True if :attr:`status_code` is less than 400. 690 691 This attribute checks if the status code of the response is between 692 400 and 600 to see if there was a client error or a server error. If 693 the status code, is between 200 and 400, this will return True. This 694 is **not** a check to see if the response code is ``200 OK``. 695 """ 696 try: 697 self.raise_for_status() 698 except HTTPError: 699 return False 700 return True 701 702 @property 703 def is_redirect(self): 704 """True if this Response is a well-formed HTTP redirect that could have 705 been processed automatically (by :meth:`Session.resolve_redirects`). 706 """ 707 return ('location' in self.headers and self.status_code in REDIRECT_STATI) 708 709 @property 710 def is_permanent_redirect(self): 711 """True if this Response one of the permanent versions of redirect.""" 712 return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect)) 713 714 @property 715 def next(self): 716 """Returns a PreparedRequest for the next request in a redirect chain, if there is one.""" 717 return self._next 718 719 @property 720 def apparent_encoding(self): 721 """The apparent encoding, provided by the chardet library.""" 722 return chardet.detect(self.content)['encoding'] 723 724 def iter_content(self, chunk_size=1, decode_unicode=False): 725 """Iterates over the response data. When stream=True is set on the 726 request, this avoids reading the content at once into memory for 727 large responses. The chunk size is the number of bytes it should 728 read into memory. This is not necessarily the length of each item 729 returned as decoding can take place. 730 731 chunk_size must be of type int or None. A value of None will 732 function differently depending on the value of `stream`. 733 stream=True will read data as it arrives in whatever size the 734 chunks are received. If stream=False, data is returned as 735 a single chunk. 736 737 If decode_unicode is True, content will be decoded using the best 738 available encoding based on the response. 739 """ 740 741 def generate(): 742 # Special case for urllib3. 743 if hasattr(self.raw, 'stream'): 744 try: 745 for chunk in self.raw.stream(chunk_size, decode_content=True): 746 yield chunk 747 except ProtocolError as e: 748 raise ChunkedEncodingError(e) 749 except DecodeError as e: 750 raise ContentDecodingError(e) 751 except ReadTimeoutError as e: 752 raise ConnectionError(e) 753 else: 754 # Standard file-like object. 755 while True: 756 chunk = self.raw.read(chunk_size) 757 if not chunk: 758 break 759 yield chunk 760 761 self._content_consumed = True 762 763 if self._content_consumed and isinstance(self._content, bool): 764 raise StreamConsumedError() 765 elif chunk_size is not None and not isinstance(chunk_size, int): 766 raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) 767 # simulate reading small chunks of the content 768 reused_chunks = iter_slices(self._content, chunk_size) 769 770 stream_chunks = generate() 771 772 chunks = reused_chunks if self._content_consumed else stream_chunks 773 774 if decode_unicode: 775 chunks = stream_decode_response_unicode(chunks, self) 776 777 return chunks 778 779 def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): 780 """Iterates over the response data, one line at a time. When 781 stream=True is set on the request, this avoids reading the 782 content at once into memory for large responses. 783 784 .. note:: This method is not reentrant safe. 785 """ 786 787 pending = None 788 789 for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode): 790 791 if pending is not None: 792 chunk = pending + chunk 793 794 if delimiter: 795 lines = chunk.split(delimiter) 796 else: 797 lines = chunk.splitlines() 798 799 if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: 800 pending = lines.pop() 801 else: 802 pending = None 803 804 for line in lines: 805 yield line 806 807 if pending is not None: 808 yield pending 809 810 @property 811 def content(self): 812 """Content of the response, in bytes.""" 813 814 if self._content is False: 815 # Read the contents. 816 if self._content_consumed: 817 raise RuntimeError( 818 'The content for this response was already consumed') 819 820 if self.status_code == 0 or self.raw is None: 821 self._content = None 822 else: 823 self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes() 824 825 self._content_consumed = True 826 # don't need to release the connection; that's been handled by urllib3 827 # since we exhausted the data. 828 return self._content 829 830 @property 831 def text(self): 832 """Content of the response, in unicode. 833 834 If Response.encoding is None, encoding will be guessed using 835 ``chardet``. 836 837 The encoding of the response content is determined based solely on HTTP 838 headers, following RFC 2616 to the letter. If you can take advantage of 839 non-HTTP knowledge to make a better guess at the encoding, you should 840 set ``r.encoding`` appropriately before accessing this property. 841 """ 842 843 # Try charset from content-type 844 content = None 845 encoding = self.encoding 846 847 if not self.content: 848 return str('') 849 850 # Fallback to auto-detected encoding. 851 if self.encoding is None: 852 encoding = self.apparent_encoding 853 854 # Decode unicode from given encoding. 855 try: 856 content = str(self.content, encoding, errors='replace') 857 except (LookupError, TypeError): 858 # A LookupError is raised if the encoding was not found which could 859 # indicate a misspelling or similar mistake. 860 # 861 # A TypeError can be raised if encoding is None 862 # 863 # So we try blindly encoding. 864 content = str(self.content, errors='replace') 865 866 return content 867 868 def json(self, **kwargs): 869 r"""Returns the json-encoded content of a response, if any. 870 871 :param \*\*kwargs: Optional arguments that ``json.loads`` takes. 872 :raises ValueError: If the response body does not contain valid json. 873 """ 874 875 if not self.encoding and self.content and len(self.content) > 3: 876 # No encoding set. JSON RFC 4627 section 3 states we should expect 877 # UTF-8, -16 or -32. Detect which one to use; If the detection or 878 # decoding fails, fall back to `self.text` (using chardet to make 879 # a best guess). 880 encoding = guess_json_utf(self.content) 881 if encoding is not None: 882 try: 883 return complexjson.loads( 884 self.content.decode(encoding), **kwargs 885 ) 886 except UnicodeDecodeError: 887 # Wrong UTF codec detected; usually because it's not UTF-8 888 # but some other 8-bit codec. This is an RFC violation, 889 # and the server didn't bother to tell us what codec *was* 890 # used. 891 pass 892 return complexjson.loads(self.text, **kwargs) 893 894 @property 895 def links(self): 896 """Returns the parsed header links of the response, if any.""" 897 898 header = self.headers.get('link') 899 900 # l = MultiDict() 901 l = {} 902 903 if header: 904 links = parse_header_links(header) 905 906 for link in links: 907 key = link.get('rel') or link.get('url') 908 l[key] = link 909 910 return l 911 912 def raise_for_status(self): 913 """Raises stored :class:`HTTPError`, if one occurred.""" 914 915 http_error_msg = '' 916 if isinstance(self.reason, bytes): 917 # We attempt to decode utf-8 first because some servers 918 # choose to localize their reason strings. If the string 919 # isn't utf-8, we fall back to iso-8859-1 for all other 920 # encodings. (See PR #3538) 921 try: 922 reason = self.reason.decode('utf-8') 923 except UnicodeDecodeError: 924 reason = self.reason.decode('iso-8859-1') 925 else: 926 reason = self.reason 927 928 if 400 <= self.status_code < 500: 929 http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url) 930 931 elif 500 <= self.status_code < 600: 932 http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url) 933 934 if http_error_msg: 935 raise HTTPError(http_error_msg, response=self) 936 937 def close(self): 938 """Releases the connection back to the pool. Once this method has been 939 called the underlying ``raw`` object must not be accessed again. 940 941 *Note: Should not normally need to be called explicitly.* 942 """ 943 if not self._content_consumed: 944 self.raw.close() 945 946 release_conn = getattr(self.raw, 'release_conn', None) 947 if release_conn is not None: 948 release_conn() 949