1"""
2Boto3 Common Utils
3=================
4
5Note: This module depends on the dicts packed by the loader and,
6therefore, must be accessed via the loader or from the __utils__ dict.
7
8This module provides common functionality for the boto execution modules.
9The expected usage is to call `apply_funcs` from the `__virtual__` function
10of the module. This will bring properly initilized partials of  `_get_conn`
11and `_cache_id` into the module's namespace.
12
13Example Usage:
14
15    .. code-block:: python
16
17        def __virtual__():
18            __utils__['boto.apply_funcs'](__name__, 'vpc')
19
20        def test():
21            conn = _get_conn()
22            vpc_id = _cache_id('test-vpc')
23
24.. versionadded:: 2015.8.0
25"""
26
27
28import hashlib
29import logging
30import sys
31from functools import partial
32
33import salt.loader.context
34import salt.utils.stringutils
35import salt.utils.versions
36from salt.exceptions import SaltInvocationError
37
38# pylint: disable=import-error
39try:
40    # pylint: disable=import-error
41    import boto
42    import boto3
43    import boto.exception
44    import boto3.session
45    import botocore  # pylint: disable=W0611
46
47    # pylint: enable=import-error
48    logging.getLogger("boto3").setLevel(logging.CRITICAL)
49    HAS_BOTO = True
50except ImportError:
51    HAS_BOTO = False
52# pylint: enable=import-error
53
54
55log = logging.getLogger(__name__)
56
57__virtualname__ = "boto3"
58__salt_loader__ = salt.loader.context.LoaderContext()
59__context__ = __salt_loader__.named_context("__context__", {})
60
61
62def __virtual__():
63    """
64    Only load if boto libraries exist and if boto libraries are greater than
65    a given version.
66    """
67    has_boto = salt.utils.versions.check_boto_reqs()
68    if has_boto is True:
69        return __virtualname__
70    return has_boto
71
72
73def _option(value):
74    """
75    Look up the value for an option.
76    """
77    if value in __opts__:
78        return __opts__[value]
79    master_opts = __pillar__.get("master", {})
80    if value in master_opts:
81        return master_opts[value]
82    if value in __pillar__:
83        return __pillar__[value]
84
85
86def _get_profile(service, region, key, keyid, profile):
87    if profile:
88        if isinstance(profile, str):
89            _profile = _option(profile)
90        elif isinstance(profile, dict):
91            _profile = profile
92        key = _profile.get("key", None)
93        keyid = _profile.get("keyid", None)
94        region = _profile.get("region", None)
95
96    if not region and _option(service + ".region"):
97        region = _option(service + ".region")
98
99    if not region:
100        region = "us-east-1"
101        log.info("Assuming default region %s", region)
102
103    if not key and _option(service + ".key"):
104        key = _option(service + ".key")
105    if not keyid and _option(service + ".keyid"):
106        keyid = _option(service + ".keyid")
107
108    label = "boto_{}:".format(service)
109    if keyid:
110        hash_string = region + keyid + key
111        hash_string = salt.utils.stringutils.to_bytes(hash_string)
112        cxkey = label + hashlib.md5(hash_string).hexdigest()
113    else:
114        cxkey = label + region
115
116    return (cxkey, region, key, keyid)
117
118
119def cache_id(
120    service,
121    name,
122    sub_resource=None,
123    resource_id=None,
124    invalidate=False,
125    region=None,
126    key=None,
127    keyid=None,
128    profile=None,
129):
130    """
131    Cache, invalidate, or retrieve an AWS resource id keyed by name.
132
133    .. code-block:: python
134
135        __utils__['boto.cache_id']('ec2', 'myinstance',
136                                   'i-a1b2c3',
137                                   profile='custom_profile')
138    """
139
140    cxkey, _, _, _ = _get_profile(service, region, key, keyid, profile)
141    if sub_resource:
142        cxkey = "{}:{}:{}:id".format(cxkey, sub_resource, name)
143    else:
144        cxkey = "{}:{}:id".format(cxkey, name)
145
146    if invalidate:
147        if cxkey in __context__:
148            del __context__[cxkey]
149            return True
150        elif resource_id in __context__.values():
151            ctx = {k: v for k, v in __context__.items() if v != resource_id}
152            __context__.clear()
153            __context__.update(ctx)
154            return True
155        else:
156            return False
157    if resource_id:
158        __context__[cxkey] = resource_id
159        return True
160
161    return __context__.get(cxkey)
162
163
164def cache_id_func(service):
165    """
166    Returns a partial `cache_id` function for the provided service.
167
168    .. code-block:: python
169
170        cache_id = __utils__['boto.cache_id_func']('ec2')
171        cache_id('myinstance', 'i-a1b2c3')
172        instance_id = cache_id('myinstance')
173    """
174    return partial(cache_id, service)
175
176
177def get_connection(
178    service, module=None, region=None, key=None, keyid=None, profile=None
179):
180    """
181    Return a boto connection for the service.
182
183    .. code-block:: python
184
185        conn = __utils__['boto.get_connection']('ec2', profile='custom_profile')
186    """
187
188    module = module or service
189
190    cxkey, region, key, keyid = _get_profile(service, region, key, keyid, profile)
191    cxkey = cxkey + ":conn3"
192
193    if cxkey in __context__:
194        return __context__[cxkey]
195
196    try:
197        session = boto3.session.Session(
198            aws_access_key_id=keyid, aws_secret_access_key=key, region_name=region
199        )
200        if session is None:
201            raise SaltInvocationError('Region "{}" is not valid.'.format(region))
202        conn = session.client(module)
203        if conn is None:
204            raise SaltInvocationError('Region "{}" is not valid.'.format(region))
205    except boto.exception.NoAuthHandlerFound:
206        raise SaltInvocationError(
207            "No authentication credentials found when "
208            "attempting to make boto {} connection to "
209            'region "{}".'.format(service, region)
210        )
211    __context__[cxkey] = conn
212    return conn
213
214
215def get_connection_func(service, module=None):
216    """
217    Returns a partial `get_connection` function for the provided service.
218
219    .. code-block:: python
220
221        get_conn = __utils__['boto.get_connection_func']('ec2')
222        conn = get_conn()
223    """
224    return partial(get_connection, service, module=module)
225
226
227def get_region(service, region, profile):
228    """
229    Retrieve the region for a particular AWS service based on configured region and/or profile.
230    """
231    _, region, _, _ = _get_profile(service, region, None, None, profile)
232
233    return region
234
235
236def get_error(e):
237    # The returns from boto modules vary greatly between modules. We need to
238    # assume that none of the data we're looking for exists.
239    aws = {}
240
241    message = ""
242    message = e.args[0]
243
244    r = {"message": message}
245    if aws:
246        r["aws"] = aws
247    return r
248
249
250def exactly_n(l, n=1):
251    """
252    Tests that exactly N items in an iterable are "truthy" (neither None,
253    False, nor 0).
254    """
255    i = iter(l)
256    return all(any(i) for j in range(n)) and not any(i)
257
258
259def exactly_one(l):
260    return exactly_n(l)
261
262
263def assign_funcs(
264    modname,
265    service,
266    module=None,
267    get_conn_funcname="_get_conn",
268    cache_id_funcname="_cache_id",
269    exactly_one_funcname="_exactly_one",
270):
271    """
272    Assign _get_conn and _cache_id functions to the named module.
273
274    .. code-block:: python
275
276        _utils__['boto.assign_partials'](__name__, 'ec2')
277    """
278    mod = sys.modules[modname]
279    setattr(mod, get_conn_funcname, get_connection_func(service, module=module))
280    setattr(mod, cache_id_funcname, cache_id_func(service))
281
282    # TODO: Remove this and import salt.utils.data.exactly_one into boto_* modules instead
283    # Leaving this way for now so boto modules can be back ported
284    if exactly_one_funcname is not None:
285        setattr(mod, exactly_one_funcname, exactly_one)
286
287
288def paged_call(function, *args, **kwargs):
289    """Retrieve full set of values from a boto3 API call that may truncate
290    its results, yielding each page as it is obtained.
291    """
292    marker_flag = kwargs.pop("marker_flag", "NextMarker")
293    marker_arg = kwargs.pop("marker_arg", "Marker")
294    while True:
295        ret = function(*args, **kwargs)
296        marker = ret.get(marker_flag)
297        yield ret
298        if not marker:
299            break
300        kwargs[marker_arg] = marker
301
302
303def ordered(obj):
304    if isinstance(obj, (list, tuple)):
305        return sorted(ordered(x) for x in obj)
306    elif isinstance(obj, dict):
307        return {str(k) if isinstance(k, str) else k: ordered(v) for k, v in obj.items()}
308    elif isinstance(obj, str):
309        return str(obj)
310    return obj
311
312
313def json_objs_equal(left, right):
314    """Compare two parsed JSON objects, given non-ordering in JSON objects"""
315    return ordered(left) == ordered(right)
316