1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
|
# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""Internal module to help with normalizing botocore client args.
This module (and all function/classes within this module) should be
considered internal, and *not* a public API.
"""
import copy
import logging
import socket
import botocore.serialize
import botocore.utils
from botocore.signers import RequestSigner
from botocore.config import Config
from botocore.endpoint import EndpointCreator
logger = logging.getLogger(__name__)
class ClientArgsCreator(object):
def __init__(self, event_emitter, user_agent, response_parser_factory,
loader, exceptions_factory):
self._event_emitter = event_emitter
self._user_agent = user_agent
self._response_parser_factory = response_parser_factory
self._loader = loader
self._exceptions_factory = exceptions_factory
def get_client_args(self, service_model, region_name, is_secure,
endpoint_url, verify, credentials, scoped_config,
client_config, endpoint_bridge):
final_args = self.compute_client_args(
service_model, client_config, endpoint_bridge, region_name,
endpoint_url, is_secure, scoped_config)
service_name = final_args['service_name']
parameter_validation = final_args['parameter_validation']
endpoint_config = final_args['endpoint_config']
protocol = final_args['protocol']
config_kwargs = final_args['config_kwargs']
s3_config = final_args['s3_config']
partition = endpoint_config['metadata'].get('partition', None)
socket_options = final_args['socket_options']
signing_region = endpoint_config['signing_region']
endpoint_region_name = endpoint_config['region_name']
if signing_region is None and endpoint_region_name is None:
signing_region, endpoint_region_name = \
self._get_default_s3_region(service_name, endpoint_bridge)
config_kwargs['region_name'] = endpoint_region_name
event_emitter = copy.copy(self._event_emitter)
signer = RequestSigner(
service_model.service_id, signing_region,
endpoint_config['signing_name'],
endpoint_config['signature_version'],
credentials, event_emitter
)
config_kwargs['s3'] = s3_config
new_config = Config(**config_kwargs)
endpoint_creator = EndpointCreator(event_emitter)
endpoint = endpoint_creator.create_endpoint(
service_model, region_name=endpoint_region_name,
endpoint_url=endpoint_config['endpoint_url'], verify=verify,
response_parser_factory=self._response_parser_factory,
max_pool_connections=new_config.max_pool_connections,
proxies=new_config.proxies,
timeout=(new_config.connect_timeout, new_config.read_timeout),
socket_options=socket_options,
client_cert=new_config.client_cert)
serializer = botocore.serialize.create_serializer(
protocol, parameter_validation)
response_parser = botocore.parsers.create_parser(protocol)
return {
'serializer': serializer,
'endpoint': endpoint,
'response_parser': response_parser,
'event_emitter': event_emitter,
'request_signer': signer,
'service_model': service_model,
'loader': self._loader,
'client_config': new_config,
'partition': partition,
'exceptions_factory': self._exceptions_factory
}
def compute_client_args(self, service_model, client_config,
endpoint_bridge, region_name, endpoint_url,
is_secure, scoped_config):
service_name = service_model.endpoint_prefix
protocol = service_model.metadata['protocol']
parameter_validation = True
if client_config and not client_config.parameter_validation:
parameter_validation = False
elif scoped_config:
raw_value = scoped_config.get('parameter_validation')
if raw_value is not None:
parameter_validation = botocore.utils.ensure_boolean(raw_value)
endpoint_config = endpoint_bridge.resolve(
service_name, region_name, endpoint_url, is_secure)
# Override the user agent if specified in the client config.
user_agent = self._user_agent
if client_config is not None:
if client_config.user_agent is not None:
user_agent = client_config.user_agent
if client_config.user_agent_extra is not None:
user_agent += ' %s' % client_config.user_agent_extra
# Create a new client config to be passed to the client based
# on the final values. We do not want the user to be able
# to try to modify an existing client with a client config.
config_kwargs = dict(
region_name=endpoint_config['region_name'],
signature_version=endpoint_config['signature_version'],
user_agent=user_agent)
if client_config is not None:
config_kwargs.update(
connect_timeout=client_config.connect_timeout,
read_timeout=client_config.read_timeout,
max_pool_connections=client_config.max_pool_connections,
proxies=client_config.proxies,
retries=client_config.retries,
client_cert=client_config.client_cert,
inject_host_prefix=client_config.inject_host_prefix,
)
s3_config = self.compute_s3_config(scoped_config,
client_config)
return {
'service_name': service_name,
'parameter_validation': parameter_validation,
'user_agent': user_agent,
'endpoint_config': endpoint_config,
'protocol': protocol,
'config_kwargs': config_kwargs,
's3_config': s3_config,
'socket_options': self._compute_socket_options(scoped_config)
}
def compute_s3_config(self, scoped_config, client_config):
s3_configuration = None
# Check the scoped config first.
if scoped_config is not None:
s3_configuration = scoped_config.get('s3')
# Until we have proper validation of the config file (including
# nested types), we have to account for the fact that the s3
# key could be parsed as a string, e.g 's3 = foo'.
# In the case we'll ignore the key for now.
if not isinstance(s3_configuration, dict):
logger.debug("The s3 config key is not a dictionary type, "
"ignoring its value of: %s", s3_configuration)
s3_configuration = None
# Convert logic for several s3 keys in the scoped config
# so that the various strings map to the appropriate boolean value.
if s3_configuration:
boolean_keys = ['use_accelerate_endpoint',
'use_dualstack_endpoint',
'payload_signing_enabled']
s3_configuration = self._convert_config_to_bool(
s3_configuration, boolean_keys)
# Next specific client config values takes precedence over
# specific values in the scoped config.
if client_config is not None:
if client_config.s3 is not None:
if s3_configuration is None:
s3_configuration = client_config.s3
else:
# The current s3_configuration dictionary may be
# from a source that only should be read from so
# we want to be safe and just make a copy of it to modify
# before it actually gets updated.
s3_configuration = s3_configuration.copy()
s3_configuration.update(client_config.s3)
return s3_configuration
def _convert_config_to_bool(self, config_dict, keys):
# Make sure any further modifications to this section of the config
# will not affect the scoped config by making a copy of it.
config_copy = config_dict.copy()
present_keys = [k for k in keys if k in config_copy]
for key in present_keys:
config_copy[key] = botocore.utils.ensure_boolean(config_copy[key])
return config_copy
def _get_default_s3_region(self, service_name, endpoint_bridge):
# If a user is providing a custom URL, the endpoint resolver will
# refuse to infer a signing region. If we want to default to s3v4,
# we have to account for this.
if service_name == 's3':
endpoint = endpoint_bridge.resolve('s3')
return endpoint['signing_region'], endpoint['region_name']
return None, None
def _compute_socket_options(self, scoped_config):
# This disables Nagle's algorithm and is the default socket options
# in urllib3.
socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
if scoped_config:
# Enables TCP Keepalive if specified in shared config file.
if self._ensure_boolean(scoped_config.get('tcp_keepalive', False)):
socket_options.append(
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1))
return socket_options
def _ensure_boolean(self, val):
if isinstance(val, bool):
return val
else:
return val.lower() == 'true'
|
