1""" 2This module provides a pool manager that uses Google App Engine's 3`URLFetch Service <https://cloud.google.com/appengine/docs/python/urlfetch>`_. 4 5Example usage:: 6 7 from urllib3 import PoolManager 8 from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox 9 10 if is_appengine_sandbox(): 11 # AppEngineManager uses AppEngine's URLFetch API behind the scenes 12 http = AppEngineManager() 13 else: 14 # PoolManager uses a socket-level API behind the scenes 15 http = PoolManager() 16 17 r = http.request('GET', 'https://google.com/') 18 19There are `limitations <https://cloud.google.com/appengine/docs/python/\ 20urlfetch/#Python_Quotas_and_limits>`_ to the URLFetch service and it may not be 21the best choice for your application. There are three options for using 22urllib3 on Google App Engine: 23 241. You can use :class:`AppEngineManager` with URLFetch. URLFetch is 25 cost-effective in many circumstances as long as your usage is within the 26 limitations. 272. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets. 28 Sockets also have `limitations and restrictions 29 <https://cloud.google.com/appengine/docs/python/sockets/\ 30 #limitations-and-restrictions>`_ and have a lower free quota than URLFetch. 31 To use sockets, be sure to specify the following in your ``app.yaml``:: 32 33 env_variables: 34 GAE_USE_SOCKETS_HTTPLIB : 'true' 35 363. If you are using `App Engine Flexible 37<https://cloud.google.com/appengine/docs/flexible/>`_, you can use the standard 38:class:`PoolManager` without any configuration or special environment variables. 39""" 40 41from __future__ import absolute_import 42import logging 43import os 44import warnings 45from ..packages.six.moves.urllib.parse import urljoin 46 47from ..exceptions import ( 48 HTTPError, 49 HTTPWarning, 50 MaxRetryError, 51 ProtocolError, 52 TimeoutError, 53 SSLError 54) 55 56from ..packages.six import BytesIO 57from ..request import RequestMethods 58from ..response import HTTPResponse 59from ..util.timeout import Timeout 60from ..util.retry import Retry 61 62try: 63 from google.appengine.api import urlfetch 64except ImportError: 65 urlfetch = None 66 67 68log = logging.getLogger(__name__) 69 70 71class AppEnginePlatformWarning(HTTPWarning): 72 pass 73 74 75class AppEnginePlatformError(HTTPError): 76 pass 77 78 79class AppEngineManager(RequestMethods): 80 """ 81 Connection manager for Google App Engine sandbox applications. 82 83 This manager uses the URLFetch service directly instead of using the 84 emulated httplib, and is subject to URLFetch limitations as described in 85 the App Engine documentation `here 86 <https://cloud.google.com/appengine/docs/python/urlfetch>`_. 87 88 Notably it will raise an :class:`AppEnginePlatformError` if: 89 * URLFetch is not available. 90 * If you attempt to use this on App Engine Flexible, as full socket 91 support is available. 92 * If a request size is more than 10 megabytes. 93 * If a response size is more than 32 megabtyes. 94 * If you use an unsupported request method such as OPTIONS. 95 96 Beyond those cases, it will raise normal urllib3 errors. 97 """ 98 99 def __init__(self, headers=None, retries=None, validate_certificate=True, 100 urlfetch_retries=True): 101 if not urlfetch: 102 raise AppEnginePlatformError( 103 "URLFetch is not available in this environment.") 104 105 if is_prod_appengine_mvms(): 106 raise AppEnginePlatformError( 107 "Use normal urllib3.PoolManager instead of AppEngineManager" 108 "on Managed VMs, as using URLFetch is not necessary in " 109 "this environment.") 110 111 warnings.warn( 112 "urllib3 is using URLFetch on Google App Engine sandbox instead " 113 "of sockets. To use sockets directly instead of URLFetch see " 114 "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.", 115 AppEnginePlatformWarning) 116 117 RequestMethods.__init__(self, headers) 118 self.validate_certificate = validate_certificate 119 self.urlfetch_retries = urlfetch_retries 120 121 self.retries = retries or Retry.DEFAULT 122 123 def __enter__(self): 124 return self 125 126 def __exit__(self, exc_type, exc_val, exc_tb): 127 # Return False to re-raise any potential exceptions 128 return False 129 130 def urlopen(self, method, url, body=None, headers=None, 131 retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT, 132 **response_kw): 133 134 retries = self._get_retries(retries, redirect) 135 136 try: 137 follow_redirects = ( 138 redirect and 139 retries.redirect != 0 and 140 retries.total) 141 response = urlfetch.fetch( 142 url, 143 payload=body, 144 method=method, 145 headers=headers or {}, 146 allow_truncated=False, 147 follow_redirects=self.urlfetch_retries and follow_redirects, 148 deadline=self._get_absolute_timeout(timeout), 149 validate_certificate=self.validate_certificate, 150 ) 151 except urlfetch.DeadlineExceededError as e: 152 raise TimeoutError(self, e) 153 154 except urlfetch.InvalidURLError as e: 155 if 'too large' in str(e): 156 raise AppEnginePlatformError( 157 "URLFetch request too large, URLFetch only " 158 "supports requests up to 10mb in size.", e) 159 raise ProtocolError(e) 160 161 except urlfetch.DownloadError as e: 162 if 'Too many redirects' in str(e): 163 raise MaxRetryError(self, url, reason=e) 164 raise ProtocolError(e) 165 166 except urlfetch.ResponseTooLargeError as e: 167 raise AppEnginePlatformError( 168 "URLFetch response too large, URLFetch only supports" 169 "responses up to 32mb in size.", e) 170 171 except urlfetch.SSLCertificateError as e: 172 raise SSLError(e) 173 174 except urlfetch.InvalidMethodError as e: 175 raise AppEnginePlatformError( 176 "URLFetch does not support method: %s" % method, e) 177 178 http_response = self._urlfetch_response_to_http_response( 179 response, retries=retries, **response_kw) 180 181 # Handle redirect? 182 redirect_location = redirect and http_response.get_redirect_location() 183 if redirect_location: 184 # Check for redirect response 185 if (self.urlfetch_retries and retries.raise_on_redirect): 186 raise MaxRetryError(self, url, "too many redirects") 187 else: 188 if http_response.status == 303: 189 method = 'GET' 190 191 try: 192 retries = retries.increment(method, url, response=http_response, _pool=self) 193 except MaxRetryError: 194 if retries.raise_on_redirect: 195 raise MaxRetryError(self, url, "too many redirects") 196 return http_response 197 198 retries.sleep_for_retry(http_response) 199 log.debug("Redirecting %s -> %s", url, redirect_location) 200 redirect_url = urljoin(url, redirect_location) 201 return self.urlopen( 202 method, redirect_url, body, headers, 203 retries=retries, redirect=redirect, 204 timeout=timeout, **response_kw) 205 206 # Check if we should retry the HTTP response. 207 has_retry_after = bool(http_response.getheader('Retry-After')) 208 if retries.is_retry(method, http_response.status, has_retry_after): 209 retries = retries.increment( 210 method, url, response=http_response, _pool=self) 211 log.debug("Retry: %s", url) 212 retries.sleep(http_response) 213 return self.urlopen( 214 method, url, 215 body=body, headers=headers, 216 retries=retries, redirect=redirect, 217 timeout=timeout, **response_kw) 218 219 return http_response 220 221 def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): 222 223 if is_prod_appengine(): 224 # Production GAE handles deflate encoding automatically, but does 225 # not remove the encoding header. 226 content_encoding = urlfetch_resp.headers.get('content-encoding') 227 228 if content_encoding == 'deflate': 229 del urlfetch_resp.headers['content-encoding'] 230 231 transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') 232 # We have a full response's content, 233 # so let's make sure we don't report ourselves as chunked data. 234 if transfer_encoding == 'chunked': 235 encodings = transfer_encoding.split(",") 236 encodings.remove('chunked') 237 urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) 238 239 return HTTPResponse( 240 # In order for decoding to work, we must present the content as 241 # a file-like object. 242 body=BytesIO(urlfetch_resp.content), 243 headers=urlfetch_resp.headers, 244 status=urlfetch_resp.status_code, 245 **response_kw 246 ) 247 248 def _get_absolute_timeout(self, timeout): 249 if timeout is Timeout.DEFAULT_TIMEOUT: 250 return None # Defer to URLFetch's default. 251 if isinstance(timeout, Timeout): 252 if timeout._read is not None or timeout._connect is not None: 253 warnings.warn( 254 "URLFetch does not support granular timeout settings, " 255 "reverting to total or default URLFetch timeout.", 256 AppEnginePlatformWarning) 257 return timeout.total 258 return timeout 259 260 def _get_retries(self, retries, redirect): 261 if not isinstance(retries, Retry): 262 retries = Retry.from_int( 263 retries, redirect=redirect, default=self.retries) 264 265 if retries.connect or retries.read or retries.redirect: 266 warnings.warn( 267 "URLFetch only supports total retries and does not " 268 "recognize connect, read, or redirect retry parameters.", 269 AppEnginePlatformWarning) 270 271 return retries 272 273 274def is_appengine(): 275 return (is_local_appengine() or 276 is_prod_appengine() or 277 is_prod_appengine_mvms()) 278 279 280def is_appengine_sandbox(): 281 return is_appengine() and not is_prod_appengine_mvms() 282 283 284def is_local_appengine(): 285 return ('APPENGINE_RUNTIME' in os.environ and 286 'Development/' in os.environ['SERVER_SOFTWARE']) 287 288 289def is_prod_appengine(): 290 return ('APPENGINE_RUNTIME' in os.environ and 291 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and 292 not is_prod_appengine_mvms()) 293 294 295def is_prod_appengine_mvms(): 296 return os.environ.get('GAE_VM', False) == 'true' 297