1""" 2Boto Common Utils 3================= 4 5Note: This module depends on the dicts packed by the loader and, 6therefore, must be accessed via the loader or from the __utils__ dict. 7 8This module provides common functionality for the boto execution modules. 9The expected usage is to call `assign_funcs` from the `__virtual__` function 10of the module. This will bring properly initialized partials of `_get_conn` 11and `_cache_id` into the module's namespace. 12 13Example Usage: 14 15 .. code-block:: python 16 17 def __virtual__(): 18 __utils__['boto.assign_funcs'](__name__, 'vpc') 19 20 def test(): 21 conn = _get_conn() 22 vpc_id = _cache_id('test-vpc') 23 24.. versionadded:: 2015.8.0 25""" 26 27 28import hashlib 29import logging 30import sys 31from functools import partial 32 33import salt.loader.context 34import salt.utils.stringutils 35import salt.utils.versions 36from salt.exceptions import SaltInvocationError 37from salt.loader import minion_mods 38 39# pylint: disable=import-error 40try: 41 # pylint: disable=import-error 42 import boto 43 import boto.exception 44 45 # pylint: enable=import-error 46 logging.getLogger("boto").setLevel(logging.CRITICAL) 47 HAS_BOTO = True 48except ImportError: 49 HAS_BOTO = False 50# pylint: enable=import-error 51 52 53log = logging.getLogger(__name__) 54 55__salt__ = None 56__virtualname__ = "boto" 57__salt_loader__ = salt.loader.context.LoaderContext() 58__context__ = __salt_loader__.named_context("__context__", {}) 59 60 61def __virtual__(): 62 """ 63 Only load if boto libraries exist and if boto libraries are greater than 64 a given version. 65 """ 66 has_boto_requirements = salt.utils.versions.check_boto_reqs(check_boto3=False) 67 if has_boto_requirements is True: 68 global __salt__ 69 if not __salt__: 70 __salt__ = minion_mods(__opts__) 71 return __virtualname__ 72 return has_boto_requirements 73 74 75def _get_profile(service, region, key, keyid, profile): 76 if profile: 77 if isinstance(profile, str): 78 _profile = __salt__["config.option"](profile) 79 elif isinstance(profile, dict): 80 _profile = profile 81 key = _profile.get("key", None) 82 keyid = _profile.get("keyid", None) 83 region = _profile.get("region", region or None) 84 if not region and __salt__["config.option"](service + ".region"): 85 region = __salt__["config.option"](service + ".region") 86 87 if not region: 88 region = "us-east-1" 89 if not key and __salt__["config.option"](service + ".key"): 90 key = __salt__["config.option"](service + ".key") 91 if not keyid and __salt__["config.option"](service + ".keyid"): 92 keyid = __salt__["config.option"](service + ".keyid") 93 94 label = "boto_{}:".format(service) 95 if keyid: 96 hash_string = region + keyid + key 97 hash_string = salt.utils.stringutils.to_bytes(hash_string) 98 cxkey = label + hashlib.md5(hash_string).hexdigest() 99 else: 100 cxkey = label + region 101 102 return (cxkey, region, key, keyid) 103 104 105def cache_id( 106 service, 107 name, 108 sub_resource=None, 109 resource_id=None, 110 invalidate=False, 111 region=None, 112 key=None, 113 keyid=None, 114 profile=None, 115): 116 """ 117 Cache, invalidate, or retrieve an AWS resource id keyed by name. 118 119 .. code-block:: python 120 121 __utils__['boto.cache_id']('ec2', 'myinstance', 122 'i-a1b2c3', 123 profile='custom_profile') 124 """ 125 126 cxkey, _, _, _ = _get_profile(service, region, key, keyid, profile) 127 if sub_resource: 128 cxkey = "{}:{}:{}:id".format(cxkey, sub_resource, name) 129 else: 130 cxkey = "{}:{}:id".format(cxkey, name) 131 132 if invalidate: 133 if cxkey in __context__: 134 del __context__[cxkey] 135 return True 136 elif resource_id in __context__.values(): 137 ctx = {k: v for k, v in __context__.items() if v != resource_id} 138 __context__.clear() 139 __context__.update(ctx) 140 return True 141 else: 142 return False 143 if resource_id: 144 __context__[cxkey] = resource_id 145 return True 146 147 return __context__.get(cxkey) 148 149 150def cache_id_func(service): 151 """ 152 Returns a partial ``cache_id`` function for the provided service. 153 154 .. code-block:: python 155 156 cache_id = __utils__['boto.cache_id_func']('ec2') 157 cache_id('myinstance', 'i-a1b2c3') 158 instance_id = cache_id('myinstance') 159 """ 160 return partial(cache_id, service) 161 162 163def get_connection( 164 service, module=None, region=None, key=None, keyid=None, profile=None 165): 166 """ 167 Return a boto connection for the service. 168 169 .. code-block:: python 170 171 conn = __utils__['boto.get_connection']('ec2', profile='custom_profile') 172 """ 173 174 module = str(module or service) 175 module, submodule = ("boto." + module).rsplit(".", 1) 176 177 svc_mod = getattr(__import__(module, fromlist=[submodule]), submodule) 178 179 cxkey, region, key, keyid = _get_profile(service, region, key, keyid, profile) 180 cxkey = cxkey + ":conn" 181 182 if cxkey in __context__: 183 return __context__[cxkey] 184 185 try: 186 conn = svc_mod.connect_to_region( 187 region, aws_access_key_id=keyid, aws_secret_access_key=key 188 ) 189 if conn is None: 190 raise SaltInvocationError('Region "{}" is not valid.'.format(region)) 191 except boto.exception.NoAuthHandlerFound: 192 raise SaltInvocationError( 193 "No authentication credentials found when " 194 "attempting to make boto {} connection to " 195 'region "{}".'.format(service, region) 196 ) 197 __context__[cxkey] = conn 198 return conn 199 200 201def get_connection_func(service, module=None): 202 """ 203 Returns a partial ``get_connection`` function for the provided service. 204 205 .. code-block:: python 206 207 get_conn = __utils__['boto.get_connection_func']('ec2') 208 conn = get_conn() 209 """ 210 return partial(get_connection, service, module=module) 211 212 213def get_error(e): 214 # The returns from boto modules vary greatly between modules. We need to 215 # assume that none of the data we're looking for exists. 216 aws = {} 217 if hasattr(e, "status"): 218 aws["status"] = e.status 219 if hasattr(e, "reason"): 220 aws["reason"] = e.reason 221 if hasattr(e, "message") and e.message != "": 222 aws["message"] = e.message 223 if hasattr(e, "error_code") and e.error_code is not None: 224 aws["code"] = e.error_code 225 226 if "message" in aws and "reason" in aws: 227 message = "{}: {}".format(aws["reason"], aws["message"]) 228 elif "message" in aws: 229 message = aws["message"] 230 elif "reason" in aws: 231 message = aws["reason"] 232 else: 233 message = "" 234 r = {"message": message} 235 if aws: 236 r["aws"] = aws 237 return r 238 239 240def exactly_n(l, n=1): 241 """ 242 Tests that exactly N items in an iterable are "truthy" (neither None, 243 False, nor 0). 244 """ 245 i = iter(l) 246 return all(any(i) for j in range(n)) and not any(i) 247 248 249def exactly_one(l): 250 return exactly_n(l) 251 252 253def assign_funcs(modname, service, module=None, pack=None): 254 """ 255 Assign _get_conn and _cache_id functions to the named module. 256 257 .. code-block:: python 258 259 __utils__['boto.assign_partials'](__name__, 'ec2') 260 """ 261 if pack: 262 global __salt__ # pylint: disable=W0601 263 __salt__ = pack 264 mod = sys.modules[modname] 265 setattr(mod, "_get_conn", get_connection_func(service, module=module)) 266 setattr(mod, "_cache_id", cache_id_func(service)) 267 268 # TODO: Remove this and import salt.utils.data.exactly_one into boto_* modules instead 269 # Leaving this way for now so boto modules can be back ported 270 setattr(mod, "_exactly_one", exactly_one) 271 272 273def paged_call(function, *args, **kwargs): 274 """ 275 Retrieve full set of values from a boto API call that may truncate 276 its results, yielding each page as it is obtained. 277 """ 278 marker_flag = kwargs.pop("marker_flag", "marker") 279 marker_arg = kwargs.pop("marker_flag", "marker") 280 while True: 281 ret = function(*args, **kwargs) 282 marker = ret.get(marker_flag) 283 yield ret 284 if not marker: 285 break 286 kwargs[marker_arg] = marker 287