1""" 2Boto3 Common Utils 3================= 4 5Note: This module depends on the dicts packed by the loader and, 6therefore, must be accessed via the loader or from the __utils__ dict. 7 8This module provides common functionality for the boto execution modules. 9The expected usage is to call `apply_funcs` from the `__virtual__` function 10of the module. This will bring properly initilized partials of `_get_conn` 11and `_cache_id` into the module's namespace. 12 13Example Usage: 14 15 .. code-block:: python 16 17 def __virtual__(): 18 __utils__['boto.apply_funcs'](__name__, 'vpc') 19 20 def test(): 21 conn = _get_conn() 22 vpc_id = _cache_id('test-vpc') 23 24.. versionadded:: 2015.8.0 25""" 26 27 28import hashlib 29import logging 30import sys 31from functools import partial 32 33import salt.loader.context 34import salt.utils.stringutils 35import salt.utils.versions 36from salt.exceptions import SaltInvocationError 37 38# pylint: disable=import-error 39try: 40 # pylint: disable=import-error 41 import boto 42 import boto3 43 import boto.exception 44 import boto3.session 45 import botocore # pylint: disable=W0611 46 47 # pylint: enable=import-error 48 logging.getLogger("boto3").setLevel(logging.CRITICAL) 49 HAS_BOTO = True 50except ImportError: 51 HAS_BOTO = False 52# pylint: enable=import-error 53 54 55log = logging.getLogger(__name__) 56 57__virtualname__ = "boto3" 58__salt_loader__ = salt.loader.context.LoaderContext() 59__context__ = __salt_loader__.named_context("__context__", {}) 60 61 62def __virtual__(): 63 """ 64 Only load if boto libraries exist and if boto libraries are greater than 65 a given version. 66 """ 67 has_boto = salt.utils.versions.check_boto_reqs() 68 if has_boto is True: 69 return __virtualname__ 70 return has_boto 71 72 73def _option(value): 74 """ 75 Look up the value for an option. 76 """ 77 if value in __opts__: 78 return __opts__[value] 79 master_opts = __pillar__.get("master", {}) 80 if value in master_opts: 81 return master_opts[value] 82 if value in __pillar__: 83 return __pillar__[value] 84 85 86def _get_profile(service, region, key, keyid, profile): 87 if profile: 88 if isinstance(profile, str): 89 _profile = _option(profile) 90 elif isinstance(profile, dict): 91 _profile = profile 92 key = _profile.get("key", None) 93 keyid = _profile.get("keyid", None) 94 region = _profile.get("region", None) 95 96 if not region and _option(service + ".region"): 97 region = _option(service + ".region") 98 99 if not region: 100 region = "us-east-1" 101 log.info("Assuming default region %s", region) 102 103 if not key and _option(service + ".key"): 104 key = _option(service + ".key") 105 if not keyid and _option(service + ".keyid"): 106 keyid = _option(service + ".keyid") 107 108 label = "boto_{}:".format(service) 109 if keyid: 110 hash_string = region + keyid + key 111 hash_string = salt.utils.stringutils.to_bytes(hash_string) 112 cxkey = label + hashlib.md5(hash_string).hexdigest() 113 else: 114 cxkey = label + region 115 116 return (cxkey, region, key, keyid) 117 118 119def cache_id( 120 service, 121 name, 122 sub_resource=None, 123 resource_id=None, 124 invalidate=False, 125 region=None, 126 key=None, 127 keyid=None, 128 profile=None, 129): 130 """ 131 Cache, invalidate, or retrieve an AWS resource id keyed by name. 132 133 .. code-block:: python 134 135 __utils__['boto.cache_id']('ec2', 'myinstance', 136 'i-a1b2c3', 137 profile='custom_profile') 138 """ 139 140 cxkey, _, _, _ = _get_profile(service, region, key, keyid, profile) 141 if sub_resource: 142 cxkey = "{}:{}:{}:id".format(cxkey, sub_resource, name) 143 else: 144 cxkey = "{}:{}:id".format(cxkey, name) 145 146 if invalidate: 147 if cxkey in __context__: 148 del __context__[cxkey] 149 return True 150 elif resource_id in __context__.values(): 151 ctx = {k: v for k, v in __context__.items() if v != resource_id} 152 __context__.clear() 153 __context__.update(ctx) 154 return True 155 else: 156 return False 157 if resource_id: 158 __context__[cxkey] = resource_id 159 return True 160 161 return __context__.get(cxkey) 162 163 164def cache_id_func(service): 165 """ 166 Returns a partial `cache_id` function for the provided service. 167 168 .. code-block:: python 169 170 cache_id = __utils__['boto.cache_id_func']('ec2') 171 cache_id('myinstance', 'i-a1b2c3') 172 instance_id = cache_id('myinstance') 173 """ 174 return partial(cache_id, service) 175 176 177def get_connection( 178 service, module=None, region=None, key=None, keyid=None, profile=None 179): 180 """ 181 Return a boto connection for the service. 182 183 .. code-block:: python 184 185 conn = __utils__['boto.get_connection']('ec2', profile='custom_profile') 186 """ 187 188 module = module or service 189 190 cxkey, region, key, keyid = _get_profile(service, region, key, keyid, profile) 191 cxkey = cxkey + ":conn3" 192 193 if cxkey in __context__: 194 return __context__[cxkey] 195 196 try: 197 session = boto3.session.Session( 198 aws_access_key_id=keyid, aws_secret_access_key=key, region_name=region 199 ) 200 if session is None: 201 raise SaltInvocationError('Region "{}" is not valid.'.format(region)) 202 conn = session.client(module) 203 if conn is None: 204 raise SaltInvocationError('Region "{}" is not valid.'.format(region)) 205 except boto.exception.NoAuthHandlerFound: 206 raise SaltInvocationError( 207 "No authentication credentials found when " 208 "attempting to make boto {} connection to " 209 'region "{}".'.format(service, region) 210 ) 211 __context__[cxkey] = conn 212 return conn 213 214 215def get_connection_func(service, module=None): 216 """ 217 Returns a partial `get_connection` function for the provided service. 218 219 .. code-block:: python 220 221 get_conn = __utils__['boto.get_connection_func']('ec2') 222 conn = get_conn() 223 """ 224 return partial(get_connection, service, module=module) 225 226 227def get_region(service, region, profile): 228 """ 229 Retrieve the region for a particular AWS service based on configured region and/or profile. 230 """ 231 _, region, _, _ = _get_profile(service, region, None, None, profile) 232 233 return region 234 235 236def get_error(e): 237 # The returns from boto modules vary greatly between modules. We need to 238 # assume that none of the data we're looking for exists. 239 aws = {} 240 241 message = "" 242 message = e.args[0] 243 244 r = {"message": message} 245 if aws: 246 r["aws"] = aws 247 return r 248 249 250def exactly_n(l, n=1): 251 """ 252 Tests that exactly N items in an iterable are "truthy" (neither None, 253 False, nor 0). 254 """ 255 i = iter(l) 256 return all(any(i) for j in range(n)) and not any(i) 257 258 259def exactly_one(l): 260 return exactly_n(l) 261 262 263def assign_funcs( 264 modname, 265 service, 266 module=None, 267 get_conn_funcname="_get_conn", 268 cache_id_funcname="_cache_id", 269 exactly_one_funcname="_exactly_one", 270): 271 """ 272 Assign _get_conn and _cache_id functions to the named module. 273 274 .. code-block:: python 275 276 _utils__['boto.assign_partials'](__name__, 'ec2') 277 """ 278 mod = sys.modules[modname] 279 setattr(mod, get_conn_funcname, get_connection_func(service, module=module)) 280 setattr(mod, cache_id_funcname, cache_id_func(service)) 281 282 # TODO: Remove this and import salt.utils.data.exactly_one into boto_* modules instead 283 # Leaving this way for now so boto modules can be back ported 284 if exactly_one_funcname is not None: 285 setattr(mod, exactly_one_funcname, exactly_one) 286 287 288def paged_call(function, *args, **kwargs): 289 """Retrieve full set of values from a boto3 API call that may truncate 290 its results, yielding each page as it is obtained. 291 """ 292 marker_flag = kwargs.pop("marker_flag", "NextMarker") 293 marker_arg = kwargs.pop("marker_arg", "Marker") 294 while True: 295 ret = function(*args, **kwargs) 296 marker = ret.get(marker_flag) 297 yield ret 298 if not marker: 299 break 300 kwargs[marker_arg] = marker 301 302 303def ordered(obj): 304 if isinstance(obj, (list, tuple)): 305 return sorted(ordered(x) for x in obj) 306 elif isinstance(obj, dict): 307 return {str(k) if isinstance(k, str) else k: ordered(v) for k, v in obj.items()} 308 elif isinstance(obj, str): 309 return str(obj) 310 return obj 311 312 313def json_objs_equal(left, right): 314 """Compare two parsed JSON objects, given non-ordering in JSON objects""" 315 return ordered(left) == ordered(right) 316