1# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"). You
4# may not use this file except in compliance with the License. A copy of
5# the License is located at
6#
7# http://aws.amazon.com/apache2.0/
8#
9# or in the "license" file accompanying this file. This file is
10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11# ANY KIND, either express or implied. See the License for the specific
12# language governing permissions and limitations under the License.
13"""Internal module to help with normalizing botocore client args.
14
15This module (and all function/classes within this module) should be
16considered internal, and *not* a public API.
17
18"""
19import copy
20import logging
21import socket
22
23import botocore.exceptions
24import botocore.serialize
25import botocore.utils
26from botocore.signers import RequestSigner
27from botocore.config import Config
28from botocore.endpoint import EndpointCreator
29
30
31logger = logging.getLogger(__name__)
32
33
34VALID_REGIONAL_ENDPOINTS_CONFIG = [
35    'legacy',
36    'regional',
37]
38LEGACY_GLOBAL_STS_REGIONS = [
39    'ap-northeast-1',
40    'ap-south-1',
41    'ap-southeast-1',
42    'ap-southeast-2',
43    'aws-global',
44    'ca-central-1',
45    'eu-central-1',
46    'eu-north-1',
47    'eu-west-1',
48    'eu-west-2',
49    'eu-west-3',
50    'sa-east-1',
51    'us-east-1',
52    'us-east-2',
53    'us-west-1',
54    'us-west-2',
55]
56
57
58class ClientArgsCreator(object):
59    def __init__(self, event_emitter, user_agent, response_parser_factory,
60                 loader, exceptions_factory, config_store):
61        self._event_emitter = event_emitter
62        self._user_agent = user_agent
63        self._response_parser_factory = response_parser_factory
64        self._loader = loader
65        self._exceptions_factory = exceptions_factory
66        self._config_store = config_store
67
68    def get_client_args(self, service_model, region_name, is_secure,
69                        endpoint_url, verify, credentials, scoped_config,
70                        client_config, endpoint_bridge):
71        final_args = self.compute_client_args(
72            service_model, client_config, endpoint_bridge, region_name,
73            endpoint_url, is_secure, scoped_config)
74
75        service_name = final_args['service_name']
76        parameter_validation = final_args['parameter_validation']
77        endpoint_config = final_args['endpoint_config']
78        protocol = final_args['protocol']
79        config_kwargs = final_args['config_kwargs']
80        s3_config = final_args['s3_config']
81        partition = endpoint_config['metadata'].get('partition', None)
82        socket_options = final_args['socket_options']
83
84        signing_region = endpoint_config['signing_region']
85        endpoint_region_name = endpoint_config['region_name']
86
87        event_emitter = copy.copy(self._event_emitter)
88        signer = RequestSigner(
89            service_model.service_id, signing_region,
90            endpoint_config['signing_name'],
91            endpoint_config['signature_version'],
92            credentials, event_emitter
93        )
94
95        config_kwargs['s3'] = s3_config
96        new_config = Config(**config_kwargs)
97        endpoint_creator = EndpointCreator(event_emitter)
98
99        endpoint = endpoint_creator.create_endpoint(
100            service_model, region_name=endpoint_region_name,
101            endpoint_url=endpoint_config['endpoint_url'], verify=verify,
102            response_parser_factory=self._response_parser_factory,
103            max_pool_connections=new_config.max_pool_connections,
104            proxies=new_config.proxies,
105            timeout=(new_config.connect_timeout, new_config.read_timeout),
106            socket_options=socket_options,
107            client_cert=new_config.client_cert)
108
109        serializer = botocore.serialize.create_serializer(
110            protocol, parameter_validation)
111        response_parser = botocore.parsers.create_parser(protocol)
112        return {
113            'serializer': serializer,
114            'endpoint': endpoint,
115            'response_parser': response_parser,
116            'event_emitter': event_emitter,
117            'request_signer': signer,
118            'service_model': service_model,
119            'loader': self._loader,
120            'client_config': new_config,
121            'partition': partition,
122            'exceptions_factory': self._exceptions_factory
123        }
124
125    def compute_client_args(self, service_model, client_config,
126                            endpoint_bridge, region_name, endpoint_url,
127                            is_secure, scoped_config):
128        service_name = service_model.endpoint_prefix
129        protocol = service_model.metadata['protocol']
130        parameter_validation = True
131        if client_config and not client_config.parameter_validation:
132            parameter_validation = False
133        elif scoped_config:
134            raw_value = scoped_config.get('parameter_validation')
135            if raw_value is not None:
136                parameter_validation = botocore.utils.ensure_boolean(raw_value)
137
138        # Override the user agent if specified in the client config.
139        user_agent = self._user_agent
140        if client_config is not None:
141            if client_config.user_agent is not None:
142                user_agent = client_config.user_agent
143            if client_config.user_agent_extra is not None:
144                user_agent += ' %s' % client_config.user_agent_extra
145
146        s3_config = self.compute_s3_config(client_config)
147        endpoint_config = self._compute_endpoint_config(
148            service_name=service_name,
149            region_name=region_name,
150            endpoint_url=endpoint_url,
151            is_secure=is_secure,
152            endpoint_bridge=endpoint_bridge,
153            s3_config=s3_config,
154        )
155        # Create a new client config to be passed to the client based
156        # on the final values. We do not want the user to be able
157        # to try to modify an existing client with a client config.
158        config_kwargs = dict(
159            region_name=endpoint_config['region_name'],
160            signature_version=endpoint_config['signature_version'],
161            user_agent=user_agent)
162        if client_config is not None:
163            config_kwargs.update(
164                connect_timeout=client_config.connect_timeout,
165                read_timeout=client_config.read_timeout,
166                max_pool_connections=client_config.max_pool_connections,
167                proxies=client_config.proxies,
168                retries=client_config.retries,
169                client_cert=client_config.client_cert,
170                inject_host_prefix=client_config.inject_host_prefix,
171            )
172        self._compute_retry_config(config_kwargs)
173        s3_config = self.compute_s3_config(client_config)
174        return {
175            'service_name': service_name,
176            'parameter_validation': parameter_validation,
177            'user_agent': user_agent,
178            'endpoint_config': endpoint_config,
179            'protocol': protocol,
180            'config_kwargs': config_kwargs,
181            's3_config': s3_config,
182            'socket_options': self._compute_socket_options(scoped_config)
183        }
184
185    def compute_s3_config(self, client_config):
186        s3_configuration = self._config_store.get_config_variable('s3')
187
188        # Next specific client config values takes precedence over
189        # specific values in the scoped config.
190        if client_config is not None:
191            if client_config.s3 is not None:
192                if s3_configuration is None:
193                    s3_configuration = client_config.s3
194                else:
195                    # The current s3_configuration dictionary may be
196                    # from a source that only should be read from so
197                    # we want to be safe and just make a copy of it to modify
198                    # before it actually gets updated.
199                    s3_configuration = s3_configuration.copy()
200                    s3_configuration.update(client_config.s3)
201
202        return s3_configuration
203
204    def _compute_endpoint_config(self, service_name, region_name, endpoint_url,
205                                 is_secure, endpoint_bridge, s3_config):
206        resolve_endpoint_kwargs = {
207            'service_name': service_name,
208            'region_name': region_name,
209            'endpoint_url': endpoint_url,
210            'is_secure': is_secure,
211            'endpoint_bridge': endpoint_bridge,
212        }
213        if service_name == 's3':
214            return self._compute_s3_endpoint_config(
215                s3_config=s3_config, **resolve_endpoint_kwargs)
216        if service_name == 'sts':
217            return self._compute_sts_endpoint_config(**resolve_endpoint_kwargs)
218        return self._resolve_endpoint(**resolve_endpoint_kwargs)
219
220    def _compute_s3_endpoint_config(self, s3_config,
221                                    **resolve_endpoint_kwargs):
222        force_s3_global = self._should_force_s3_global(
223            resolve_endpoint_kwargs['region_name'], s3_config)
224        if force_s3_global:
225            resolve_endpoint_kwargs['region_name'] = None
226        endpoint_config = self._resolve_endpoint(**resolve_endpoint_kwargs)
227        self._set_region_if_custom_s3_endpoint(
228            endpoint_config, resolve_endpoint_kwargs['endpoint_bridge'])
229        # For backwards compatibility reasons, we want to make sure the
230        # client.meta.region_name will remain us-east-1 if we forced the
231        # endpoint to be the global region. Specifically, if this value
232        # changes to aws-global, it breaks logic where a user is checking
233        # for us-east-1 as the global endpoint such as in creating buckets.
234        if force_s3_global and endpoint_config['region_name'] == 'aws-global':
235            endpoint_config['region_name'] = 'us-east-1'
236        return endpoint_config
237
238    def _should_force_s3_global(self, region_name, s3_config):
239        s3_regional_config = 'legacy'
240        if s3_config and 'us_east_1_regional_endpoint' in s3_config:
241            s3_regional_config = s3_config['us_east_1_regional_endpoint']
242            self._validate_s3_regional_config(s3_regional_config)
243        return (
244            s3_regional_config == 'legacy' and
245            region_name in ['us-east-1', None]
246        )
247
248    def _validate_s3_regional_config(self, config_val):
249        if config_val not in VALID_REGIONAL_ENDPOINTS_CONFIG:
250            raise botocore.exceptions.\
251                InvalidS3UsEast1RegionalEndpointConfigError(
252                    s3_us_east_1_regional_endpoint_config=config_val)
253
254    def _set_region_if_custom_s3_endpoint(self, endpoint_config,
255                                          endpoint_bridge):
256        # If a user is providing a custom URL, the endpoint resolver will
257        # refuse to infer a signing region. If we want to default to s3v4,
258        # we have to account for this.
259        if endpoint_config['signing_region'] is None \
260                and endpoint_config['region_name'] is None:
261            endpoint = endpoint_bridge.resolve('s3')
262            endpoint_config['signing_region'] = endpoint['signing_region']
263            endpoint_config['region_name'] = endpoint['region_name']
264
265    def _compute_sts_endpoint_config(self, **resolve_endpoint_kwargs):
266        endpoint_config = self._resolve_endpoint(**resolve_endpoint_kwargs)
267        if self._should_set_global_sts_endpoint(
268                resolve_endpoint_kwargs['region_name'],
269                resolve_endpoint_kwargs['endpoint_url']):
270            self._set_global_sts_endpoint(
271                endpoint_config, resolve_endpoint_kwargs['is_secure'])
272        return endpoint_config
273
274    def _should_set_global_sts_endpoint(self, region_name, endpoint_url):
275        if endpoint_url:
276            return False
277        return (
278            self._get_sts_regional_endpoints_config() == 'legacy' and
279            region_name in LEGACY_GLOBAL_STS_REGIONS
280        )
281
282    def _get_sts_regional_endpoints_config(self):
283        sts_regional_endpoints_config = self._config_store.get_config_variable(
284            'sts_regional_endpoints')
285        if not sts_regional_endpoints_config:
286            sts_regional_endpoints_config = 'legacy'
287        if sts_regional_endpoints_config not in \
288                VALID_REGIONAL_ENDPOINTS_CONFIG:
289            raise botocore.exceptions.InvalidSTSRegionalEndpointsConfigError(
290                sts_regional_endpoints_config=sts_regional_endpoints_config)
291        return sts_regional_endpoints_config
292
293    def _set_global_sts_endpoint(self, endpoint_config, is_secure):
294        scheme = 'https' if is_secure else 'http'
295        endpoint_config['endpoint_url'] = '%s://sts.amazonaws.com' % scheme
296        endpoint_config['signing_region'] = 'us-east-1'
297
298    def _resolve_endpoint(self, service_name, region_name,
299                          endpoint_url, is_secure, endpoint_bridge):
300        return endpoint_bridge.resolve(
301            service_name, region_name, endpoint_url, is_secure)
302
303    def _compute_socket_options(self, scoped_config):
304        # This disables Nagle's algorithm and is the default socket options
305        # in urllib3.
306        socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
307        if scoped_config:
308            # Enables TCP Keepalive if specified in shared config file.
309            if self._ensure_boolean(scoped_config.get('tcp_keepalive', False)):
310                socket_options.append(
311                    (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1))
312        return socket_options
313
314    def _compute_retry_config(self, config_kwargs):
315        self._compute_retry_max_attempts(config_kwargs)
316        self._compute_retry_mode(config_kwargs)
317
318    def _compute_retry_max_attempts(self, config_kwargs):
319        # There's a pre-existing max_attempts client config value that actually
320        # means max *retry* attempts.  There's also a `max_attempts` we pull
321        # from the config store that means *total attempts*, which includes the
322        # intitial request.  We can't change what `max_attempts` means in
323        # client config so we try to normalize everything to a new
324        # "total_max_attempts" variable.  We ensure that after this, the only
325        # configuration for "max attempts" is the 'total_max_attempts' key.
326        # An explicitly provided max_attempts in the client config
327        # overrides everything.
328        retries = config_kwargs.get('retries')
329        if retries is not None:
330            if 'total_max_attempts' in retries:
331                retries.pop('max_attempts', None)
332                return
333            if 'max_attempts' in retries:
334                value = retries.pop('max_attempts')
335                # client config max_attempts means total retries so we
336                # have to add one for 'total_max_attempts' to account
337                # for the initial request.
338                retries['total_max_attempts'] = value + 1
339                return
340        # Otherwise we'll check the config store which checks env vars,
341        # config files, etc.  There is no default value for max_attempts
342        # so if this returns None and we don't set a default value here.
343        max_attempts = self._config_store.get_config_variable('max_attempts')
344        if max_attempts is not None:
345            if retries is None:
346                retries = {}
347                config_kwargs['retries'] = retries
348            retries['total_max_attempts'] = max_attempts
349
350    def _compute_retry_mode(self, config_kwargs):
351        retries = config_kwargs.get('retries')
352        if retries is None:
353            retries = {}
354            config_kwargs['retries'] = retries
355        elif 'mode' in retries:
356            # If there's a retry mode explicitly set in the client config
357            # that overrides everything.
358            return
359        retry_mode = self._config_store.get_config_variable('retry_mode')
360        if retry_mode is None:
361            retry_mode = 'legacy'
362        retries['mode'] = retry_mode
363
364    def _ensure_boolean(self, val):
365        if isinstance(val, bool):
366            return val
367        else:
368            return val.lower() == 'true'
369