1""" 2This module provides a pool manager that uses Google App Engine's 3`URLFetch Service <https://cloud.google.com/appengine/docs/python/urlfetch>`_. 4 5Example usage:: 6 7 from urllib3 import PoolManager 8 from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox 9 10 if is_appengine_sandbox(): 11 # AppEngineManager uses AppEngine's URLFetch API behind the scenes 12 http = AppEngineManager() 13 else: 14 # PoolManager uses a socket-level API behind the scenes 15 http = PoolManager() 16 17 r = http.request('GET', 'https://google.com/') 18 19There are `limitations <https://cloud.google.com/appengine/docs/python/\ 20urlfetch/#Python_Quotas_and_limits>`_ to the URLFetch service and it may not be 21the best choice for your application. There are three options for using 22urllib3 on Google App Engine: 23 241. You can use :class:`AppEngineManager` with URLFetch. URLFetch is 25 cost-effective in many circumstances as long as your usage is within the 26 limitations. 272. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets. 28 Sockets also have `limitations and restrictions 29 <https://cloud.google.com/appengine/docs/python/sockets/\ 30 #limitations-and-restrictions>`_ and have a lower free quota than URLFetch. 31 To use sockets, be sure to specify the following in your ``app.yaml``:: 32 33 env_variables: 34 GAE_USE_SOCKETS_HTTPLIB : 'true' 35 363. If you are using `App Engine Flexible 37<https://cloud.google.com/appengine/docs/flexible/>`_, you can use the standard 38:class:`PoolManager` without any configuration or special environment variables. 39""" 40 41from __future__ import absolute_import 42 43import io 44import logging 45import warnings 46 47from ..exceptions import ( 48 HTTPError, 49 HTTPWarning, 50 MaxRetryError, 51 ProtocolError, 52 SSLError, 53 TimeoutError, 54) 55from ..packages.six.moves.urllib.parse import urljoin 56from ..request import RequestMethods 57from ..response import HTTPResponse 58from ..util.retry import Retry 59from ..util.timeout import Timeout 60from . import _appengine_environ 61 62try: 63 from google.appengine.api import urlfetch 64except ImportError: 65 urlfetch = None 66 67 68log = logging.getLogger(__name__) 69 70 71class AppEnginePlatformWarning(HTTPWarning): 72 pass 73 74 75class AppEnginePlatformError(HTTPError): 76 pass 77 78 79class AppEngineManager(RequestMethods): 80 """ 81 Connection manager for Google App Engine sandbox applications. 82 83 This manager uses the URLFetch service directly instead of using the 84 emulated httplib, and is subject to URLFetch limitations as described in 85 the App Engine documentation `here 86 <https://cloud.google.com/appengine/docs/python/urlfetch>`_. 87 88 Notably it will raise an :class:`AppEnginePlatformError` if: 89 * URLFetch is not available. 90 * If you attempt to use this on App Engine Flexible, as full socket 91 support is available. 92 * If a request size is more than 10 megabytes. 93 * If a response size is more than 32 megabytes. 94 * If you use an unsupported request method such as OPTIONS. 95 96 Beyond those cases, it will raise normal urllib3 errors. 97 """ 98 99 def __init__( 100 self, 101 headers=None, 102 retries=None, 103 validate_certificate=True, 104 urlfetch_retries=True, 105 ): 106 if not urlfetch: 107 raise AppEnginePlatformError( 108 "URLFetch is not available in this environment." 109 ) 110 111 warnings.warn( 112 "urllib3 is using URLFetch on Google App Engine sandbox instead " 113 "of sockets. To use sockets directly instead of URLFetch see " 114 "https://urllib3.readthedocs.io/en/1.26.x/reference/urllib3.contrib.html.", 115 AppEnginePlatformWarning, 116 ) 117 118 RequestMethods.__init__(self, headers) 119 self.validate_certificate = validate_certificate 120 self.urlfetch_retries = urlfetch_retries 121 122 self.retries = retries or Retry.DEFAULT 123 124 def __enter__(self): 125 return self 126 127 def __exit__(self, exc_type, exc_val, exc_tb): 128 # Return False to re-raise any potential exceptions 129 return False 130 131 def urlopen( 132 self, 133 method, 134 url, 135 body=None, 136 headers=None, 137 retries=None, 138 redirect=True, 139 timeout=Timeout.DEFAULT_TIMEOUT, 140 **response_kw 141 ): 142 143 retries = self._get_retries(retries, redirect) 144 145 try: 146 follow_redirects = redirect and retries.redirect != 0 and retries.total 147 response = urlfetch.fetch( 148 url, 149 payload=body, 150 method=method, 151 headers=headers or {}, 152 allow_truncated=False, 153 follow_redirects=self.urlfetch_retries and follow_redirects, 154 deadline=self._get_absolute_timeout(timeout), 155 validate_certificate=self.validate_certificate, 156 ) 157 except urlfetch.DeadlineExceededError as e: 158 raise TimeoutError(self, e) 159 160 except urlfetch.InvalidURLError as e: 161 if "too large" in str(e): 162 raise AppEnginePlatformError( 163 "URLFetch request too large, URLFetch only " 164 "supports requests up to 10mb in size.", 165 e, 166 ) 167 raise ProtocolError(e) 168 169 except urlfetch.DownloadError as e: 170 if "Too many redirects" in str(e): 171 raise MaxRetryError(self, url, reason=e) 172 raise ProtocolError(e) 173 174 except urlfetch.ResponseTooLargeError as e: 175 raise AppEnginePlatformError( 176 "URLFetch response too large, URLFetch only supports" 177 "responses up to 32mb in size.", 178 e, 179 ) 180 181 except urlfetch.SSLCertificateError as e: 182 raise SSLError(e) 183 184 except urlfetch.InvalidMethodError as e: 185 raise AppEnginePlatformError( 186 "URLFetch does not support method: %s" % method, e 187 ) 188 189 http_response = self._urlfetch_response_to_http_response( 190 response, retries=retries, **response_kw 191 ) 192 193 # Handle redirect? 194 redirect_location = redirect and http_response.get_redirect_location() 195 if redirect_location: 196 # Check for redirect response 197 if self.urlfetch_retries and retries.raise_on_redirect: 198 raise MaxRetryError(self, url, "too many redirects") 199 else: 200 if http_response.status == 303: 201 method = "GET" 202 203 try: 204 retries = retries.increment( 205 method, url, response=http_response, _pool=self 206 ) 207 except MaxRetryError: 208 if retries.raise_on_redirect: 209 raise MaxRetryError(self, url, "too many redirects") 210 return http_response 211 212 retries.sleep_for_retry(http_response) 213 log.debug("Redirecting %s -> %s", url, redirect_location) 214 redirect_url = urljoin(url, redirect_location) 215 return self.urlopen( 216 method, 217 redirect_url, 218 body, 219 headers, 220 retries=retries, 221 redirect=redirect, 222 timeout=timeout, 223 **response_kw 224 ) 225 226 # Check if we should retry the HTTP response. 227 has_retry_after = bool(http_response.getheader("Retry-After")) 228 if retries.is_retry(method, http_response.status, has_retry_after): 229 retries = retries.increment(method, url, response=http_response, _pool=self) 230 log.debug("Retry: %s", url) 231 retries.sleep(http_response) 232 return self.urlopen( 233 method, 234 url, 235 body=body, 236 headers=headers, 237 retries=retries, 238 redirect=redirect, 239 timeout=timeout, 240 **response_kw 241 ) 242 243 return http_response 244 245 def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): 246 247 if is_prod_appengine(): 248 # Production GAE handles deflate encoding automatically, but does 249 # not remove the encoding header. 250 content_encoding = urlfetch_resp.headers.get("content-encoding") 251 252 if content_encoding == "deflate": 253 del urlfetch_resp.headers["content-encoding"] 254 255 transfer_encoding = urlfetch_resp.headers.get("transfer-encoding") 256 # We have a full response's content, 257 # so let's make sure we don't report ourselves as chunked data. 258 if transfer_encoding == "chunked": 259 encodings = transfer_encoding.split(",") 260 encodings.remove("chunked") 261 urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings) 262 263 original_response = HTTPResponse( 264 # In order for decoding to work, we must present the content as 265 # a file-like object. 266 body=io.BytesIO(urlfetch_resp.content), 267 msg=urlfetch_resp.header_msg, 268 headers=urlfetch_resp.headers, 269 status=urlfetch_resp.status_code, 270 **response_kw 271 ) 272 273 return HTTPResponse( 274 body=io.BytesIO(urlfetch_resp.content), 275 headers=urlfetch_resp.headers, 276 status=urlfetch_resp.status_code, 277 original_response=original_response, 278 **response_kw 279 ) 280 281 def _get_absolute_timeout(self, timeout): 282 if timeout is Timeout.DEFAULT_TIMEOUT: 283 return None # Defer to URLFetch's default. 284 if isinstance(timeout, Timeout): 285 if timeout._read is not None or timeout._connect is not None: 286 warnings.warn( 287 "URLFetch does not support granular timeout settings, " 288 "reverting to total or default URLFetch timeout.", 289 AppEnginePlatformWarning, 290 ) 291 return timeout.total 292 return timeout 293 294 def _get_retries(self, retries, redirect): 295 if not isinstance(retries, Retry): 296 retries = Retry.from_int(retries, redirect=redirect, default=self.retries) 297 298 if retries.connect or retries.read or retries.redirect: 299 warnings.warn( 300 "URLFetch only supports total retries and does not " 301 "recognize connect, read, or redirect retry parameters.", 302 AppEnginePlatformWarning, 303 ) 304 305 return retries 306 307 308# Alias methods from _appengine_environ to maintain public API interface. 309 310is_appengine = _appengine_environ.is_appengine 311is_appengine_sandbox = _appengine_environ.is_appengine_sandbox 312is_local_appengine = _appengine_environ.is_local_appengine 313is_prod_appengine = _appengine_environ.is_prod_appengine 314is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms 315