1from __future__ import absolute_import
2
3import errno
4import logging
5import socket
6import sys
7import warnings
8from socket import error as SocketError
9from socket import timeout as SocketTimeout
10
11from .connection import (
12    BaseSSLError,
13    BrokenPipeError,
14    DummyConnection,
15    HTTPConnection,
16    HTTPException,
17    HTTPSConnection,
18    VerifiedHTTPSConnection,
19    port_by_scheme,
20)
21from .exceptions import (
22    ClosedPoolError,
23    EmptyPoolError,
24    HeaderParsingError,
25    HostChangedError,
26    InsecureRequestWarning,
27    LocationValueError,
28    MaxRetryError,
29    NewConnectionError,
30    ProtocolError,
31    ProxyError,
32    ReadTimeoutError,
33    SSLError,
34    TimeoutError,
35)
36from .packages import six
37from .packages.six.moves import queue
38from .packages.ssl_match_hostname import CertificateError
39from .request import RequestMethods
40from .response import HTTPResponse
41from .util.connection import is_connection_dropped
42from .util.proxy import connection_requires_http_tunnel
43from .util.queue import LifoQueue
44from .util.request import set_file_position
45from .util.response import assert_header_parsing
46from .util.retry import Retry
47from .util.timeout import Timeout
48from .util.url import Url, _encode_target
49from .util.url import _normalize_host as normalize_host
50from .util.url import get_host, parse_url
51
52xrange = six.moves.xrange
53
54log = logging.getLogger(__name__)
55
56_Default = object()
57
58
59# Pool objects
60class ConnectionPool(object):
61    """
62    Base class for all connection pools, such as
63    :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
64
65    .. note::
66       ConnectionPool.urlopen() does not normalize or percent-encode target URIs
67       which is useful if your target server doesn't support percent-encoded
68       target URIs.
69    """
70
71    scheme = None
72    QueueCls = LifoQueue
73
74    def __init__(self, host, port=None):
75        if not host:
76            raise LocationValueError("No host specified.")
77
78        self.host = _normalize_host(host, scheme=self.scheme)
79        self._proxy_host = host.lower()
80        self.port = port
81
82    def __str__(self):
83        return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port)
84
85    def __enter__(self):
86        return self
87
88    def __exit__(self, exc_type, exc_val, exc_tb):
89        self.close()
90        # Return False to re-raise any potential exceptions
91        return False
92
93    def close(self):
94        """
95        Close all pooled connections and disable the pool.
96        """
97        pass
98
99
100# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
101_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
102
103
104class HTTPConnectionPool(ConnectionPool, RequestMethods):
105    """
106    Thread-safe connection pool for one host.
107
108    :param host:
109        Host used for this HTTP Connection (e.g. "localhost"), passed into
110        :class:`http.client.HTTPConnection`.
111
112    :param port:
113        Port used for this HTTP Connection (None is equivalent to 80), passed
114        into :class:`http.client.HTTPConnection`.
115
116    :param strict:
117        Causes BadStatusLine to be raised if the status line can't be parsed
118        as a valid HTTP/1.0 or 1.1 status line, passed into
119        :class:`http.client.HTTPConnection`.
120
121        .. note::
122           Only works in Python 2. This parameter is ignored in Python 3.
123
124    :param timeout:
125        Socket timeout in seconds for each individual connection. This can
126        be a float or integer, which sets the timeout for the HTTP request,
127        or an instance of :class:`urllib3.util.Timeout` which gives you more
128        fine-grained control over request timeouts. After the constructor has
129        been parsed, this is always a `urllib3.util.Timeout` object.
130
131    :param maxsize:
132        Number of connections to save that can be reused. More than 1 is useful
133        in multithreaded situations. If ``block`` is set to False, more
134        connections will be created but they will not be saved once they've
135        been used.
136
137    :param block:
138        If set to True, no more than ``maxsize`` connections will be used at
139        a time. When no free connections are available, the call will block
140        until a connection has been released. This is a useful side effect for
141        particular multithreaded situations where one does not want to use more
142        than maxsize connections per host to prevent flooding.
143
144    :param headers:
145        Headers to include with all requests, unless other headers are given
146        explicitly.
147
148    :param retries:
149        Retry configuration to use by default with requests in this pool.
150
151    :param _proxy:
152        Parsed proxy URL, should not be used directly, instead, see
153        :class:`urllib3.ProxyManager`
154
155    :param _proxy_headers:
156        A dictionary with proxy headers, should not be used directly,
157        instead, see :class:`urllib3.ProxyManager`
158
159    :param \\**conn_kw:
160        Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
161        :class:`urllib3.connection.HTTPSConnection` instances.
162    """
163
164    scheme = "http"
165    ConnectionCls = HTTPConnection
166    ResponseCls = HTTPResponse
167
168    def __init__(
169        self,
170        host,
171        port=None,
172        strict=False,
173        timeout=Timeout.DEFAULT_TIMEOUT,
174        maxsize=1,
175        block=False,
176        headers=None,
177        retries=None,
178        _proxy=None,
179        _proxy_headers=None,
180        _proxy_config=None,
181        **conn_kw
182    ):
183        ConnectionPool.__init__(self, host, port)
184        RequestMethods.__init__(self, headers)
185
186        self.strict = strict
187
188        if not isinstance(timeout, Timeout):
189            timeout = Timeout.from_float(timeout)
190
191        if retries is None:
192            retries = Retry.DEFAULT
193
194        self.timeout = timeout
195        self.retries = retries
196
197        self.pool = self.QueueCls(maxsize)
198        self.block = block
199
200        self.proxy = _proxy
201        self.proxy_headers = _proxy_headers or {}
202        self.proxy_config = _proxy_config
203
204        # Fill the queue up so that doing get() on it will block properly
205        for _ in xrange(maxsize):
206            self.pool.put(None)
207
208        # These are mostly for testing and debugging purposes.
209        self.num_connections = 0
210        self.num_requests = 0
211        self.conn_kw = conn_kw
212
213        if self.proxy:
214            # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
215            # We cannot know if the user has added default socket options, so we cannot replace the
216            # list.
217            self.conn_kw.setdefault("socket_options", [])
218
219            self.conn_kw["proxy"] = self.proxy
220            self.conn_kw["proxy_config"] = self.proxy_config
221
222    def _new_conn(self):
223        """
224        Return a fresh :class:`HTTPConnection`.
225        """
226        self.num_connections += 1
227        log.debug(
228            "Starting new HTTP connection (%d): %s:%s",
229            self.num_connections,
230            self.host,
231            self.port or "80",
232        )
233
234        conn = self.ConnectionCls(
235            host=self.host,
236            port=self.port,
237            timeout=self.timeout.connect_timeout,
238            strict=self.strict,
239            **self.conn_kw
240        )
241        return conn
242
243    def _get_conn(self, timeout=None):
244        """
245        Get a connection. Will return a pooled connection if one is available.
246
247        If no connections are available and :prop:`.block` is ``False``, then a
248        fresh connection is returned.
249
250        :param timeout:
251            Seconds to wait before giving up and raising
252            :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
253            :prop:`.block` is ``True``.
254        """
255        conn = None
256        try:
257            conn = self.pool.get(block=self.block, timeout=timeout)
258
259        except AttributeError:  # self.pool is None
260            raise ClosedPoolError(self, "Pool is closed.")
261
262        except queue.Empty:
263            if self.block:
264                raise EmptyPoolError(
265                    self,
266                    "Pool reached maximum size and no more connections are allowed.",
267                )
268            pass  # Oh well, we'll create a new connection then
269
270        # If this is a persistent connection, check if it got disconnected
271        if conn and is_connection_dropped(conn):
272            log.debug("Resetting dropped connection: %s", self.host)
273            conn.close()
274            if getattr(conn, "auto_open", 1) == 0:
275                # This is a proxied connection that has been mutated by
276                # http.client._tunnel() and cannot be reused (since it would
277                # attempt to bypass the proxy)
278                conn = None
279
280        return conn or self._new_conn()
281
282    def _put_conn(self, conn):
283        """
284        Put a connection back into the pool.
285
286        :param conn:
287            Connection object for the current host and port as returned by
288            :meth:`._new_conn` or :meth:`._get_conn`.
289
290        If the pool is already full, the connection is closed and discarded
291        because we exceeded maxsize. If connections are discarded frequently,
292        then maxsize should be increased.
293
294        If the pool is closed, then the connection will be closed and discarded.
295        """
296        try:
297            self.pool.put(conn, block=False)
298            return  # Everything is dandy, done.
299        except AttributeError:
300            # self.pool is None.
301            pass
302        except queue.Full:
303            # This should never happen if self.block == True
304            log.warning("Connection pool is full, discarding connection: %s", self.host)
305
306        # Connection never got put back into the pool, close it.
307        if conn:
308            conn.close()
309
310    def _validate_conn(self, conn):
311        """
312        Called right before a request is made, after the socket is created.
313        """
314        pass
315
316    def _prepare_proxy(self, conn):
317        # Nothing to do for HTTP connections.
318        pass
319
320    def _get_timeout(self, timeout):
321        """Helper that always returns a :class:`urllib3.util.Timeout`"""
322        if timeout is _Default:
323            return self.timeout.clone()
324
325        if isinstance(timeout, Timeout):
326            return timeout.clone()
327        else:
328            # User passed us an int/float. This is for backwards compatibility,
329            # can be removed later
330            return Timeout.from_float(timeout)
331
332    def _raise_timeout(self, err, url, timeout_value):
333        """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
334
335        if isinstance(err, SocketTimeout):
336            raise ReadTimeoutError(
337                self, url, "Read timed out. (read timeout=%s)" % timeout_value
338            )
339
340        # See the above comment about EAGAIN in Python 3. In Python 2 we have
341        # to specifically catch it and throw the timeout error
342        if hasattr(err, "errno") and err.errno in _blocking_errnos:
343            raise ReadTimeoutError(
344                self, url, "Read timed out. (read timeout=%s)" % timeout_value
345            )
346
347        # Catch possible read timeouts thrown as SSL errors. If not the
348        # case, rethrow the original. We need to do this because of:
349        # http://bugs.python.org/issue10272
350        if "timed out" in str(err) or "did not complete (read)" in str(
351            err
352        ):  # Python < 2.7.4
353            raise ReadTimeoutError(
354                self, url, "Read timed out. (read timeout=%s)" % timeout_value
355            )
356
357    def _make_request(
358        self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw
359    ):
360        """
361        Perform a request on a given urllib connection object taken from our
362        pool.
363
364        :param conn:
365            a connection from one of our connection pools
366
367        :param timeout:
368            Socket timeout in seconds for the request. This can be a
369            float or integer, which will set the same timeout value for
370            the socket connect and the socket read, or an instance of
371            :class:`urllib3.util.Timeout`, which gives you more fine-grained
372            control over your timeouts.
373        """
374        self.num_requests += 1
375
376        timeout_obj = self._get_timeout(timeout)
377        timeout_obj.start_connect()
378        conn.timeout = timeout_obj.connect_timeout
379
380        # Trigger any extra validation we need to do.
381        try:
382            self._validate_conn(conn)
383        except (SocketTimeout, BaseSSLError) as e:
384            # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
385            self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
386            raise
387
388        # conn.request() calls http.client.*.request, not the method in
389        # urllib3.request. It also calls makefile (recv) on the socket.
390        try:
391            if chunked:
392                conn.request_chunked(method, url, **httplib_request_kw)
393            else:
394                conn.request(method, url, **httplib_request_kw)
395
396        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is
397        # legitimately able to close the connection after sending a valid response.
398        # With this behaviour, the received response is still readable.
399        except BrokenPipeError:
400            # Python 3
401            pass
402        except IOError as e:
403            # Python 2 and macOS/Linux
404            # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS
405            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
406            if e.errno not in {
407                errno.EPIPE,
408                errno.ESHUTDOWN,
409                errno.EPROTOTYPE,
410            }:
411                raise
412
413        # Reset the timeout for the recv() on the socket
414        read_timeout = timeout_obj.read_timeout
415
416        # App Engine doesn't have a sock attr
417        if getattr(conn, "sock", None):
418            # In Python 3 socket.py will catch EAGAIN and return None when you
419            # try and read into the file pointer created by http.client, which
420            # instead raises a BadStatusLine exception. Instead of catching
421            # the exception and assuming all BadStatusLine exceptions are read
422            # timeouts, check for a zero timeout before making the request.
423            if read_timeout == 0:
424                raise ReadTimeoutError(
425                    self, url, "Read timed out. (read timeout=%s)" % read_timeout
426                )
427            if read_timeout is Timeout.DEFAULT_TIMEOUT:
428                conn.sock.settimeout(socket.getdefaulttimeout())
429            else:  # None or a value
430                conn.sock.settimeout(read_timeout)
431
432        # Receive the response from the server
433        try:
434            try:
435                # Python 2.7, use buffering of HTTP responses
436                httplib_response = conn.getresponse(buffering=True)
437            except TypeError:
438                # Python 3
439                try:
440                    httplib_response = conn.getresponse()
441                except BaseException as e:
442                    # Remove the TypeError from the exception chain in
443                    # Python 3 (including for exceptions like SystemExit).
444                    # Otherwise it looks like a bug in the code.
445                    six.raise_from(e, None)
446        except (SocketTimeout, BaseSSLError, SocketError) as e:
447            self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
448            raise
449
450        # AppEngine doesn't have a version attr.
451        http_version = getattr(conn, "_http_vsn_str", "HTTP/?")
452        log.debug(
453            '%s://%s:%s "%s %s %s" %s %s',
454            self.scheme,
455            self.host,
456            self.port,
457            method,
458            url,
459            http_version,
460            httplib_response.status,
461            httplib_response.length,
462        )
463
464        try:
465            assert_header_parsing(httplib_response.msg)
466        except (HeaderParsingError, TypeError) as hpe:  # Platform-specific: Python 3
467            log.warning(
468                "Failed to parse headers (url=%s): %s",
469                self._absolute_url(url),
470                hpe,
471                exc_info=True,
472            )
473
474        return httplib_response
475
476    def _absolute_url(self, path):
477        return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url
478
479    def close(self):
480        """
481        Close all pooled connections and disable the pool.
482        """
483        if self.pool is None:
484            return
485        # Disable access to the pool
486        old_pool, self.pool = self.pool, None
487
488        try:
489            while True:
490                conn = old_pool.get(block=False)
491                if conn:
492                    conn.close()
493
494        except queue.Empty:
495            pass  # Done.
496
497    def is_same_host(self, url):
498        """
499        Check if the given ``url`` is a member of the same host as this
500        connection pool.
501        """
502        if url.startswith("/"):
503            return True
504
505        # TODO: Add optional support for socket.gethostbyname checking.
506        scheme, host, port = get_host(url)
507        if host is not None:
508            host = _normalize_host(host, scheme=scheme)
509
510        # Use explicit default port for comparison when none is given
511        if self.port and not port:
512            port = port_by_scheme.get(scheme)
513        elif not self.port and port == port_by_scheme.get(scheme):
514            port = None
515
516        return (scheme, host, port) == (self.scheme, self.host, self.port)
517
518    def urlopen(
519        self,
520        method,
521        url,
522        body=None,
523        headers=None,
524        retries=None,
525        redirect=True,
526        assert_same_host=True,
527        timeout=_Default,
528        pool_timeout=None,
529        release_conn=None,
530        chunked=False,
531        body_pos=None,
532        **response_kw
533    ):
534        """
535        Get a connection from the pool and perform an HTTP request. This is the
536        lowest level call for making a request, so you'll need to specify all
537        the raw details.
538
539        .. note::
540
541           More commonly, it's appropriate to use a convenience method provided
542           by :class:`.RequestMethods`, such as :meth:`request`.
543
544        .. note::
545
546           `release_conn` will only behave as expected if
547           `preload_content=False` because we want to make
548           `preload_content=False` the default behaviour someday soon without
549           breaking backwards compatibility.
550
551        :param method:
552            HTTP request method (such as GET, POST, PUT, etc.)
553
554        :param url:
555            The URL to perform the request on.
556
557        :param body:
558            Data to send in the request body, either :class:`str`, :class:`bytes`,
559            an iterable of :class:`str`/:class:`bytes`, or a file-like object.
560
561        :param headers:
562            Dictionary of custom headers to send, such as User-Agent,
563            If-None-Match, etc. If None, pool headers are used. If provided,
564            these headers completely replace any pool-specific headers.
565
566        :param retries:
567            Configure the number of retries to allow before raising a
568            :class:`~urllib3.exceptions.MaxRetryError` exception.
569
570            Pass ``None`` to retry until you receive a response. Pass a
571            :class:`~urllib3.util.retry.Retry` object for fine-grained control
572            over different types of retries.
573            Pass an integer number to retry connection errors that many times,
574            but no other types of errors. Pass zero to never retry.
575
576            If ``False``, then retries are disabled and any exception is raised
577            immediately. Also, instead of raising a MaxRetryError on redirects,
578            the redirect response will be returned.
579
580        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
581
582        :param redirect:
583            If True, automatically handle redirects (status codes 301, 302,
584            303, 307, 308). Each redirect counts as a retry. Disabling retries
585            will disable redirect, too.
586
587        :param assert_same_host:
588            If ``True``, will make sure that the host of the pool requests is
589            consistent else will raise HostChangedError. When ``False``, you can
590            use the pool on an HTTP proxy and request foreign hosts.
591
592        :param timeout:
593            If specified, overrides the default timeout for this one
594            request. It may be a float (in seconds) or an instance of
595            :class:`urllib3.util.Timeout`.
596
597        :param pool_timeout:
598            If set and the pool is set to block=True, then this method will
599            block for ``pool_timeout`` seconds and raise EmptyPoolError if no
600            connection is available within the time period.
601
602        :param release_conn:
603            If False, then the urlopen call will not release the connection
604            back into the pool once a response is received (but will release if
605            you read the entire contents of the response such as when
606            `preload_content=True`). This is useful if you're not preloading
607            the response's content immediately. You will need to call
608            ``r.release_conn()`` on the response ``r`` to return the connection
609            back into the pool. If None, it takes the value of
610            ``response_kw.get('preload_content', True)``.
611
612        :param chunked:
613            If True, urllib3 will send the body using chunked transfer
614            encoding. Otherwise, urllib3 will send the body using the standard
615            content-length form. Defaults to False.
616
617        :param int body_pos:
618            Position to seek to in file-like body in the event of a retry or
619            redirect. Typically this won't need to be set because urllib3 will
620            auto-populate the value when needed.
621
622        :param \\**response_kw:
623            Additional parameters are passed to
624            :meth:`urllib3.response.HTTPResponse.from_httplib`
625        """
626
627        parsed_url = parse_url(url)
628        destination_scheme = parsed_url.scheme
629
630        if headers is None:
631            headers = self.headers
632
633        if not isinstance(retries, Retry):
634            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
635
636        if release_conn is None:
637            release_conn = response_kw.get("preload_content", True)
638
639        # Check host
640        if assert_same_host and not self.is_same_host(url):
641            raise HostChangedError(self, url, retries)
642
643        # Ensure that the URL we're connecting to is properly encoded
644        if url.startswith("/"):
645            url = six.ensure_str(_encode_target(url))
646        else:
647            url = six.ensure_str(parsed_url.url)
648
649        conn = None
650
651        # Track whether `conn` needs to be released before
652        # returning/raising/recursing. Update this variable if necessary, and
653        # leave `release_conn` constant throughout the function. That way, if
654        # the function recurses, the original value of `release_conn` will be
655        # passed down into the recursive call, and its value will be respected.
656        #
657        # See issue #651 [1] for details.
658        #
659        # [1] <https://github.com/urllib3/urllib3/issues/651>
660        release_this_conn = release_conn
661
662        http_tunnel_required = connection_requires_http_tunnel(
663            self.proxy, self.proxy_config, destination_scheme
664        )
665
666        # Merge the proxy headers. Only done when not using HTTP CONNECT. We
667        # have to copy the headers dict so we can safely change it without those
668        # changes being reflected in anyone else's copy.
669        if not http_tunnel_required:
670            headers = headers.copy()
671            headers.update(self.proxy_headers)
672
673        # Must keep the exception bound to a separate variable or else Python 3
674        # complains about UnboundLocalError.
675        err = None
676
677        # Keep track of whether we cleanly exited the except block. This
678        # ensures we do proper cleanup in finally.
679        clean_exit = False
680
681        # Rewind body position, if needed. Record current position
682        # for future rewinds in the event of a redirect/retry.
683        body_pos = set_file_position(body, body_pos)
684
685        try:
686            # Request a connection from the queue.
687            timeout_obj = self._get_timeout(timeout)
688            conn = self._get_conn(timeout=pool_timeout)
689
690            conn.timeout = timeout_obj.connect_timeout
691
692            is_new_proxy_conn = self.proxy is not None and not getattr(
693                conn, "sock", None
694            )
695            if is_new_proxy_conn and http_tunnel_required:
696                self._prepare_proxy(conn)
697
698            # Make the request on the httplib connection object.
699            httplib_response = self._make_request(
700                conn,
701                method,
702                url,
703                timeout=timeout_obj,
704                body=body,
705                headers=headers,
706                chunked=chunked,
707            )
708
709            # If we're going to release the connection in ``finally:``, then
710            # the response doesn't need to know about the connection. Otherwise
711            # it will also try to release it and we'll have a double-release
712            # mess.
713            response_conn = conn if not release_conn else None
714
715            # Pass method to Response for length checking
716            response_kw["request_method"] = method
717
718            # Import httplib's response into our own wrapper object
719            response = self.ResponseCls.from_httplib(
720                httplib_response,
721                pool=self,
722                connection=response_conn,
723                retries=retries,
724                **response_kw
725            )
726
727            # Everything went great!
728            clean_exit = True
729
730        except EmptyPoolError:
731            # Didn't get a connection from the pool, no need to clean up
732            clean_exit = True
733            release_this_conn = False
734            raise
735
736        except (
737            TimeoutError,
738            HTTPException,
739            SocketError,
740            ProtocolError,
741            BaseSSLError,
742            SSLError,
743            CertificateError,
744        ) as e:
745            # Discard the connection for these exceptions. It will be
746            # replaced during the next _get_conn() call.
747            clean_exit = False
748            if isinstance(e, (BaseSSLError, CertificateError)):
749                e = SSLError(e)
750            elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy:
751                e = ProxyError("Cannot connect to proxy.", e)
752            elif isinstance(e, (SocketError, HTTPException)):
753                e = ProtocolError("Connection aborted.", e)
754
755            retries = retries.increment(
756                method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
757            )
758            retries.sleep()
759
760            # Keep track of the error for the retry warning.
761            err = e
762
763        finally:
764            if not clean_exit:
765                # We hit some kind of exception, handled or otherwise. We need
766                # to throw the connection away unless explicitly told not to.
767                # Close the connection, set the variable to None, and make sure
768                # we put the None back in the pool to avoid leaking it.
769                conn = conn and conn.close()
770                release_this_conn = True
771
772            if release_this_conn:
773                # Put the connection back to be reused. If the connection is
774                # expired then it will be None, which will get replaced with a
775                # fresh connection during _get_conn.
776                self._put_conn(conn)
777
778        if not conn:
779            # Try again
780            log.warning(
781                "Retrying (%r) after connection broken by '%r': %s", retries, err, url
782            )
783            return self.urlopen(
784                method,
785                url,
786                body,
787                headers,
788                retries,
789                redirect,
790                assert_same_host,
791                timeout=timeout,
792                pool_timeout=pool_timeout,
793                release_conn=release_conn,
794                chunked=chunked,
795                body_pos=body_pos,
796                **response_kw
797            )
798
799        # Handle redirect?
800        redirect_location = redirect and response.get_redirect_location()
801        if redirect_location:
802            if response.status == 303:
803                method = "GET"
804
805            try:
806                retries = retries.increment(method, url, response=response, _pool=self)
807            except MaxRetryError:
808                if retries.raise_on_redirect:
809                    response.drain_conn()
810                    raise
811                return response
812
813            response.drain_conn()
814            retries.sleep_for_retry(response)
815            log.debug("Redirecting %s -> %s", url, redirect_location)
816            return self.urlopen(
817                method,
818                redirect_location,
819                body,
820                headers,
821                retries=retries,
822                redirect=redirect,
823                assert_same_host=assert_same_host,
824                timeout=timeout,
825                pool_timeout=pool_timeout,
826                release_conn=release_conn,
827                chunked=chunked,
828                body_pos=body_pos,
829                **response_kw
830            )
831
832        # Check if we should retry the HTTP response.
833        has_retry_after = bool(response.getheader("Retry-After"))
834        if retries.is_retry(method, response.status, has_retry_after):
835            try:
836                retries = retries.increment(method, url, response=response, _pool=self)
837            except MaxRetryError:
838                if retries.raise_on_status:
839                    response.drain_conn()
840                    raise
841                return response
842
843            response.drain_conn()
844            retries.sleep(response)
845            log.debug("Retry: %s", url)
846            return self.urlopen(
847                method,
848                url,
849                body,
850                headers,
851                retries=retries,
852                redirect=redirect,
853                assert_same_host=assert_same_host,
854                timeout=timeout,
855                pool_timeout=pool_timeout,
856                release_conn=release_conn,
857                chunked=chunked,
858                body_pos=body_pos,
859                **response_kw
860            )
861
862        return response
863
864
865class HTTPSConnectionPool(HTTPConnectionPool):
866    """
867    Same as :class:`.HTTPConnectionPool`, but HTTPS.
868
869    :class:`.HTTPSConnection` uses one of ``assert_fingerprint``,
870    ``assert_hostname`` and ``host`` in this order to verify connections.
871    If ``assert_hostname`` is False, no verification is done.
872
873    The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
874    ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
875    is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
876    the connection socket into an SSL socket.
877    """
878
879    scheme = "https"
880    ConnectionCls = HTTPSConnection
881
882    def __init__(
883        self,
884        host,
885        port=None,
886        strict=False,
887        timeout=Timeout.DEFAULT_TIMEOUT,
888        maxsize=1,
889        block=False,
890        headers=None,
891        retries=None,
892        _proxy=None,
893        _proxy_headers=None,
894        key_file=None,
895        cert_file=None,
896        cert_reqs=None,
897        key_password=None,
898        ca_certs=None,
899        ssl_version=None,
900        assert_hostname=None,
901        assert_fingerprint=None,
902        ca_cert_dir=None,
903        **conn_kw
904    ):
905
906        HTTPConnectionPool.__init__(
907            self,
908            host,
909            port,
910            strict,
911            timeout,
912            maxsize,
913            block,
914            headers,
915            retries,
916            _proxy,
917            _proxy_headers,
918            **conn_kw
919        )
920
921        self.key_file = key_file
922        self.cert_file = cert_file
923        self.cert_reqs = cert_reqs
924        self.key_password = key_password
925        self.ca_certs = ca_certs
926        self.ca_cert_dir = ca_cert_dir
927        self.ssl_version = ssl_version
928        self.assert_hostname = assert_hostname
929        self.assert_fingerprint = assert_fingerprint
930
931    def _prepare_conn(self, conn):
932        """
933        Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket`
934        and establish the tunnel if proxy is used.
935        """
936
937        if isinstance(conn, VerifiedHTTPSConnection):
938            conn.set_cert(
939                key_file=self.key_file,
940                key_password=self.key_password,
941                cert_file=self.cert_file,
942                cert_reqs=self.cert_reqs,
943                ca_certs=self.ca_certs,
944                ca_cert_dir=self.ca_cert_dir,
945                assert_hostname=self.assert_hostname,
946                assert_fingerprint=self.assert_fingerprint,
947            )
948            conn.ssl_version = self.ssl_version
949        return conn
950
951    def _prepare_proxy(self, conn):
952        """
953        Establishes a tunnel connection through HTTP CONNECT.
954
955        Tunnel connection is established early because otherwise httplib would
956        improperly set Host: header to proxy's IP:port.
957        """
958
959        conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
960
961        if self.proxy.scheme == "https":
962            conn.tls_in_tls_required = True
963
964        conn.connect()
965
966    def _new_conn(self):
967        """
968        Return a fresh :class:`http.client.HTTPSConnection`.
969        """
970        self.num_connections += 1
971        log.debug(
972            "Starting new HTTPS connection (%d): %s:%s",
973            self.num_connections,
974            self.host,
975            self.port or "443",
976        )
977
978        if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
979            raise SSLError(
980                "Can't connect to HTTPS URL because the SSL module is not available."
981            )
982
983        actual_host = self.host
984        actual_port = self.port
985        if self.proxy is not None:
986            actual_host = self.proxy.host
987            actual_port = self.proxy.port
988
989        conn = self.ConnectionCls(
990            host=actual_host,
991            port=actual_port,
992            timeout=self.timeout.connect_timeout,
993            strict=self.strict,
994            cert_file=self.cert_file,
995            key_file=self.key_file,
996            key_password=self.key_password,
997            **self.conn_kw
998        )
999
1000        return self._prepare_conn(conn)
1001
1002    def _validate_conn(self, conn):
1003        """
1004        Called right before a request is made, after the socket is created.
1005        """
1006        super(HTTPSConnectionPool, self)._validate_conn(conn)
1007
1008        # Force connect early to allow us to validate the connection.
1009        if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`
1010            conn.connect()
1011
1012        if not conn.is_verified:
1013            warnings.warn(
1014                (
1015                    "Unverified HTTPS request is being made to host '%s'. "
1016                    "Adding certificate verification is strongly advised. See: "
1017                    "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html"
1018                    "#ssl-warnings" % conn.host
1019                ),
1020                InsecureRequestWarning,
1021            )
1022
1023        if getattr(conn, "proxy_is_verified", None) is False:
1024            warnings.warn(
1025                (
1026                    "Unverified HTTPS connection done to an HTTPS proxy. "
1027                    "Adding certificate verification is strongly advised. See: "
1028                    "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html"
1029                    "#ssl-warnings"
1030                ),
1031                InsecureRequestWarning,
1032            )
1033
1034
1035def connection_from_url(url, **kw):
1036    """
1037    Given a url, return an :class:`.ConnectionPool` instance of its host.
1038
1039    This is a shortcut for not having to parse out the scheme, host, and port
1040    of the url before creating an :class:`.ConnectionPool` instance.
1041
1042    :param url:
1043        Absolute URL string that must include the scheme. Port is optional.
1044
1045    :param \\**kw:
1046        Passes additional parameters to the constructor of the appropriate
1047        :class:`.ConnectionPool`. Useful for specifying things like
1048        timeout, maxsize, headers, etc.
1049
1050    Example::
1051
1052        >>> conn = connection_from_url('http://google.com/')
1053        >>> r = conn.request('GET', '/')
1054    """
1055    scheme, host, port = get_host(url)
1056    port = port or port_by_scheme.get(scheme, 80)
1057    if scheme == "https":
1058        return HTTPSConnectionPool(host, port=port, **kw)
1059    else:
1060        return HTTPConnectionPool(host, port=port, **kw)
1061
1062
1063def _normalize_host(host, scheme):
1064    """
1065    Normalize hosts for comparisons and use with sockets.
1066    """
1067
1068    host = normalize_host(host, scheme)
1069
1070    # httplib doesn't like it when we include brackets in IPv6 addresses
1071    # Specifically, if we include brackets but also pass the port then
1072    # httplib crazily doubles up the square brackets on the Host header.
1073    # Instead, we need to make sure we never pass ``None`` as the port.
1074    # However, for backward compatibility reasons we can't actually
1075    # *assert* that.  See http://bugs.python.org/issue28539
1076    if host.startswith("[") and host.endswith("]"):
1077        host = host[1:-1]
1078    return host
1079