1# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"). You 4# may not use this file except in compliance with the License. A copy of 5# the License is located at 6# 7# http://aws.amazon.com/apache2.0/ 8# 9# or in the "license" file accompanying this file. This file is 10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF 11# ANY KIND, either express or implied. See the License for the specific 12# language governing permissions and limitations under the License. 13"""Internal module to help with normalizing botocore client args. 14 15This module (and all function/classes within this module) should be 16considered internal, and *not* a public API. 17 18""" 19import copy 20import logging 21import socket 22 23import botocore.exceptions 24import botocore.serialize 25import botocore.utils 26from botocore.signers import RequestSigner 27from botocore.config import Config 28from botocore.endpoint import EndpointCreator 29 30 31logger = logging.getLogger(__name__) 32 33 34VALID_REGIONAL_ENDPOINTS_CONFIG = [ 35 'legacy', 36 'regional', 37] 38LEGACY_GLOBAL_STS_REGIONS = [ 39 'ap-northeast-1', 40 'ap-south-1', 41 'ap-southeast-1', 42 'ap-southeast-2', 43 'aws-global', 44 'ca-central-1', 45 'eu-central-1', 46 'eu-north-1', 47 'eu-west-1', 48 'eu-west-2', 49 'eu-west-3', 50 'sa-east-1', 51 'us-east-1', 52 'us-east-2', 53 'us-west-1', 54 'us-west-2', 55] 56 57 58class ClientArgsCreator(object): 59 def __init__(self, event_emitter, user_agent, response_parser_factory, 60 loader, exceptions_factory, config_store): 61 self._event_emitter = event_emitter 62 self._user_agent = user_agent 63 self._response_parser_factory = response_parser_factory 64 self._loader = loader 65 self._exceptions_factory = exceptions_factory 66 self._config_store = config_store 67 68 def get_client_args(self, service_model, region_name, is_secure, 69 endpoint_url, verify, credentials, scoped_config, 70 client_config, endpoint_bridge): 71 final_args = self.compute_client_args( 72 service_model, client_config, endpoint_bridge, region_name, 73 endpoint_url, is_secure, scoped_config) 74 75 service_name = final_args['service_name'] 76 parameter_validation = final_args['parameter_validation'] 77 endpoint_config = final_args['endpoint_config'] 78 protocol = final_args['protocol'] 79 config_kwargs = final_args['config_kwargs'] 80 s3_config = final_args['s3_config'] 81 partition = endpoint_config['metadata'].get('partition', None) 82 socket_options = final_args['socket_options'] 83 84 signing_region = endpoint_config['signing_region'] 85 endpoint_region_name = endpoint_config['region_name'] 86 87 event_emitter = copy.copy(self._event_emitter) 88 signer = RequestSigner( 89 service_model.service_id, signing_region, 90 endpoint_config['signing_name'], 91 endpoint_config['signature_version'], 92 credentials, event_emitter 93 ) 94 95 config_kwargs['s3'] = s3_config 96 new_config = Config(**config_kwargs) 97 endpoint_creator = EndpointCreator(event_emitter) 98 99 endpoint = endpoint_creator.create_endpoint( 100 service_model, region_name=endpoint_region_name, 101 endpoint_url=endpoint_config['endpoint_url'], verify=verify, 102 response_parser_factory=self._response_parser_factory, 103 max_pool_connections=new_config.max_pool_connections, 104 proxies=new_config.proxies, 105 timeout=(new_config.connect_timeout, new_config.read_timeout), 106 socket_options=socket_options, 107 client_cert=new_config.client_cert) 108 109 serializer = botocore.serialize.create_serializer( 110 protocol, parameter_validation) 111 response_parser = botocore.parsers.create_parser(protocol) 112 return { 113 'serializer': serializer, 114 'endpoint': endpoint, 115 'response_parser': response_parser, 116 'event_emitter': event_emitter, 117 'request_signer': signer, 118 'service_model': service_model, 119 'loader': self._loader, 120 'client_config': new_config, 121 'partition': partition, 122 'exceptions_factory': self._exceptions_factory 123 } 124 125 def compute_client_args(self, service_model, client_config, 126 endpoint_bridge, region_name, endpoint_url, 127 is_secure, scoped_config): 128 service_name = service_model.endpoint_prefix 129 protocol = service_model.metadata['protocol'] 130 parameter_validation = True 131 if client_config and not client_config.parameter_validation: 132 parameter_validation = False 133 elif scoped_config: 134 raw_value = scoped_config.get('parameter_validation') 135 if raw_value is not None: 136 parameter_validation = botocore.utils.ensure_boolean(raw_value) 137 138 # Override the user agent if specified in the client config. 139 user_agent = self._user_agent 140 if client_config is not None: 141 if client_config.user_agent is not None: 142 user_agent = client_config.user_agent 143 if client_config.user_agent_extra is not None: 144 user_agent += ' %s' % client_config.user_agent_extra 145 146 s3_config = self.compute_s3_config(client_config) 147 endpoint_config = self._compute_endpoint_config( 148 service_name=service_name, 149 region_name=region_name, 150 endpoint_url=endpoint_url, 151 is_secure=is_secure, 152 endpoint_bridge=endpoint_bridge, 153 s3_config=s3_config, 154 ) 155 # Create a new client config to be passed to the client based 156 # on the final values. We do not want the user to be able 157 # to try to modify an existing client with a client config. 158 config_kwargs = dict( 159 region_name=endpoint_config['region_name'], 160 signature_version=endpoint_config['signature_version'], 161 user_agent=user_agent) 162 if client_config is not None: 163 config_kwargs.update( 164 connect_timeout=client_config.connect_timeout, 165 read_timeout=client_config.read_timeout, 166 max_pool_connections=client_config.max_pool_connections, 167 proxies=client_config.proxies, 168 retries=client_config.retries, 169 client_cert=client_config.client_cert, 170 inject_host_prefix=client_config.inject_host_prefix, 171 ) 172 self._compute_retry_config(config_kwargs) 173 s3_config = self.compute_s3_config(client_config) 174 return { 175 'service_name': service_name, 176 'parameter_validation': parameter_validation, 177 'user_agent': user_agent, 178 'endpoint_config': endpoint_config, 179 'protocol': protocol, 180 'config_kwargs': config_kwargs, 181 's3_config': s3_config, 182 'socket_options': self._compute_socket_options(scoped_config) 183 } 184 185 def compute_s3_config(self, client_config): 186 s3_configuration = self._config_store.get_config_variable('s3') 187 188 # Next specific client config values takes precedence over 189 # specific values in the scoped config. 190 if client_config is not None: 191 if client_config.s3 is not None: 192 if s3_configuration is None: 193 s3_configuration = client_config.s3 194 else: 195 # The current s3_configuration dictionary may be 196 # from a source that only should be read from so 197 # we want to be safe and just make a copy of it to modify 198 # before it actually gets updated. 199 s3_configuration = s3_configuration.copy() 200 s3_configuration.update(client_config.s3) 201 202 return s3_configuration 203 204 def _compute_endpoint_config(self, service_name, region_name, endpoint_url, 205 is_secure, endpoint_bridge, s3_config): 206 resolve_endpoint_kwargs = { 207 'service_name': service_name, 208 'region_name': region_name, 209 'endpoint_url': endpoint_url, 210 'is_secure': is_secure, 211 'endpoint_bridge': endpoint_bridge, 212 } 213 if service_name == 's3': 214 return self._compute_s3_endpoint_config( 215 s3_config=s3_config, **resolve_endpoint_kwargs) 216 if service_name == 'sts': 217 return self._compute_sts_endpoint_config(**resolve_endpoint_kwargs) 218 return self._resolve_endpoint(**resolve_endpoint_kwargs) 219 220 def _compute_s3_endpoint_config(self, s3_config, 221 **resolve_endpoint_kwargs): 222 force_s3_global = self._should_force_s3_global( 223 resolve_endpoint_kwargs['region_name'], s3_config) 224 if force_s3_global: 225 resolve_endpoint_kwargs['region_name'] = None 226 endpoint_config = self._resolve_endpoint(**resolve_endpoint_kwargs) 227 self._set_region_if_custom_s3_endpoint( 228 endpoint_config, resolve_endpoint_kwargs['endpoint_bridge']) 229 # For backwards compatibility reasons, we want to make sure the 230 # client.meta.region_name will remain us-east-1 if we forced the 231 # endpoint to be the global region. Specifically, if this value 232 # changes to aws-global, it breaks logic where a user is checking 233 # for us-east-1 as the global endpoint such as in creating buckets. 234 if force_s3_global and endpoint_config['region_name'] == 'aws-global': 235 endpoint_config['region_name'] = 'us-east-1' 236 return endpoint_config 237 238 def _should_force_s3_global(self, region_name, s3_config): 239 s3_regional_config = 'legacy' 240 if s3_config and 'us_east_1_regional_endpoint' in s3_config: 241 s3_regional_config = s3_config['us_east_1_regional_endpoint'] 242 self._validate_s3_regional_config(s3_regional_config) 243 return ( 244 s3_regional_config == 'legacy' and 245 region_name in ['us-east-1', None] 246 ) 247 248 def _validate_s3_regional_config(self, config_val): 249 if config_val not in VALID_REGIONAL_ENDPOINTS_CONFIG: 250 raise botocore.exceptions.\ 251 InvalidS3UsEast1RegionalEndpointConfigError( 252 s3_us_east_1_regional_endpoint_config=config_val) 253 254 def _set_region_if_custom_s3_endpoint(self, endpoint_config, 255 endpoint_bridge): 256 # If a user is providing a custom URL, the endpoint resolver will 257 # refuse to infer a signing region. If we want to default to s3v4, 258 # we have to account for this. 259 if endpoint_config['signing_region'] is None \ 260 and endpoint_config['region_name'] is None: 261 endpoint = endpoint_bridge.resolve('s3') 262 endpoint_config['signing_region'] = endpoint['signing_region'] 263 endpoint_config['region_name'] = endpoint['region_name'] 264 265 def _compute_sts_endpoint_config(self, **resolve_endpoint_kwargs): 266 endpoint_config = self._resolve_endpoint(**resolve_endpoint_kwargs) 267 if self._should_set_global_sts_endpoint( 268 resolve_endpoint_kwargs['region_name'], 269 resolve_endpoint_kwargs['endpoint_url']): 270 self._set_global_sts_endpoint( 271 endpoint_config, resolve_endpoint_kwargs['is_secure']) 272 return endpoint_config 273 274 def _should_set_global_sts_endpoint(self, region_name, endpoint_url): 275 if endpoint_url: 276 return False 277 return ( 278 self._get_sts_regional_endpoints_config() == 'legacy' and 279 region_name in LEGACY_GLOBAL_STS_REGIONS 280 ) 281 282 def _get_sts_regional_endpoints_config(self): 283 sts_regional_endpoints_config = self._config_store.get_config_variable( 284 'sts_regional_endpoints') 285 if not sts_regional_endpoints_config: 286 sts_regional_endpoints_config = 'legacy' 287 if sts_regional_endpoints_config not in \ 288 VALID_REGIONAL_ENDPOINTS_CONFIG: 289 raise botocore.exceptions.InvalidSTSRegionalEndpointsConfigError( 290 sts_regional_endpoints_config=sts_regional_endpoints_config) 291 return sts_regional_endpoints_config 292 293 def _set_global_sts_endpoint(self, endpoint_config, is_secure): 294 scheme = 'https' if is_secure else 'http' 295 endpoint_config['endpoint_url'] = '%s://sts.amazonaws.com' % scheme 296 endpoint_config['signing_region'] = 'us-east-1' 297 298 def _resolve_endpoint(self, service_name, region_name, 299 endpoint_url, is_secure, endpoint_bridge): 300 return endpoint_bridge.resolve( 301 service_name, region_name, endpoint_url, is_secure) 302 303 def _compute_socket_options(self, scoped_config): 304 # This disables Nagle's algorithm and is the default socket options 305 # in urllib3. 306 socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] 307 if scoped_config: 308 # Enables TCP Keepalive if specified in shared config file. 309 if self._ensure_boolean(scoped_config.get('tcp_keepalive', False)): 310 socket_options.append( 311 (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)) 312 return socket_options 313 314 def _compute_retry_config(self, config_kwargs): 315 self._compute_retry_max_attempts(config_kwargs) 316 self._compute_retry_mode(config_kwargs) 317 318 def _compute_retry_max_attempts(self, config_kwargs): 319 # There's a pre-existing max_attempts client config value that actually 320 # means max *retry* attempts. There's also a `max_attempts` we pull 321 # from the config store that means *total attempts*, which includes the 322 # intitial request. We can't change what `max_attempts` means in 323 # client config so we try to normalize everything to a new 324 # "total_max_attempts" variable. We ensure that after this, the only 325 # configuration for "max attempts" is the 'total_max_attempts' key. 326 # An explicitly provided max_attempts in the client config 327 # overrides everything. 328 retries = config_kwargs.get('retries') 329 if retries is not None: 330 if 'total_max_attempts' in retries: 331 retries.pop('max_attempts', None) 332 return 333 if 'max_attempts' in retries: 334 value = retries.pop('max_attempts') 335 # client config max_attempts means total retries so we 336 # have to add one for 'total_max_attempts' to account 337 # for the initial request. 338 retries['total_max_attempts'] = value + 1 339 return 340 # Otherwise we'll check the config store which checks env vars, 341 # config files, etc. There is no default value for max_attempts 342 # so if this returns None and we don't set a default value here. 343 max_attempts = self._config_store.get_config_variable('max_attempts') 344 if max_attempts is not None: 345 if retries is None: 346 retries = {} 347 config_kwargs['retries'] = retries 348 retries['total_max_attempts'] = max_attempts 349 350 def _compute_retry_mode(self, config_kwargs): 351 retries = config_kwargs.get('retries') 352 if retries is None: 353 retries = {} 354 config_kwargs['retries'] = retries 355 elif 'mode' in retries: 356 # If there's a retry mode explicitly set in the client config 357 # that overrides everything. 358 return 359 retry_mode = self._config_store.get_config_variable('retry_mode') 360 if retry_mode is None: 361 retry_mode = 'legacy' 362 retries['mode'] = retry_mode 363 364 def _ensure_boolean(self, val): 365 if isinstance(val, bool): 366 return val 367 else: 368 return val.lower() == 'true' 369