1""" 2Utilities for working with etcd 3 4.. versionadded:: 2014.7.0 5 6:depends: - python-etcd 7 8This library sets up a client object for etcd, using the configuration passed 9into the client() function. Normally, this is __opts__. Optionally, a profile 10may be passed in. The following configurations are both valid: 11 12.. code-block:: yaml 13 14 # No profile name 15 etcd.host: 127.0.0.1 16 etcd.port: 2379 17 etcd.username: larry # Optional; requires etcd.password to be set 18 etcd.password: 123pass # Optional; requires etcd.username to be set 19 etcd.ca: /path/to/your/ca_cert/ca.pem # Optional 20 etcd.client_key: /path/to/your/client_key/client-key.pem # Optional; requires etcd.ca and etcd.client_cert to be set 21 etcd.client_cert: /path/to/your/client_cert/client.pem # Optional; requires etcd.ca and etcd.client_key to be set 22 23 # One or more profiles defined 24 my_etcd_config: 25 etcd.host: 127.0.0.1 26 etcd.port: 2379 27 etcd.username: larry # Optional; requires etcd.password to be set 28 etcd.password: 123pass # Optional; requires etcd.username to be set 29 etcd.ca: /path/to/your/ca_cert/ca.pem # Optional 30 etcd.client_key: /path/to/your/client_key/client-key.pem # Optional; requires etcd.ca and etcd.client_cert to be set 31 etcd.client_cert: /path/to/your/client_cert/client.pem # Optional; requires etcd.ca and etcd.client_key to be set 32 33Once configured, the client() function is passed a set of opts, and optionally, 34the name of a profile to be used. 35 36.. code-block:: python 37 38 import salt.utils.etcd_utils 39 client = salt.utils.etcd_utils.client(__opts__, profile='my_etcd_config') 40 41You may also use the newer syntax and bypass the generator function. 42 43.. code-block:: python 44 45 import salt.utils.etcd_utils 46 client = salt.utils.etcd_utils.EtcdClient(__opts__, profile='my_etcd_config') 47 48It should be noted that some usages of etcd require a profile to be specified, 49rather than top-level configurations. This being the case, it is better to 50always use a named configuration profile, as shown above. 51""" 52 53import logging 54 55from salt.exceptions import CommandExecutionError 56 57try: 58 import etcd 59 from urllib3.exceptions import ReadTimeoutError, MaxRetryError 60 61 HAS_LIBS = True 62except ImportError: 63 HAS_LIBS = False 64 65# Set up logging 66log = logging.getLogger(__name__) 67 68 69class EtcdUtilWatchTimeout(Exception): 70 """ 71 A watch timed out without returning a result 72 """ 73 74 75class EtcdClient: 76 def __init__( 77 self, 78 opts, 79 profile=None, 80 host=None, 81 port=None, 82 username=None, 83 password=None, 84 ca=None, 85 client_key=None, 86 client_cert=None, 87 **kwargs 88 ): 89 opts_pillar = opts.get("pillar", {}) 90 opts_master = opts_pillar.get("master", {}) 91 92 opts_merged = {} 93 opts_merged.update(opts_master) 94 opts_merged.update(opts_pillar) 95 opts_merged.update(opts) 96 97 if profile: 98 self.conf = opts_merged.get(profile, {}) 99 else: 100 self.conf = opts_merged 101 102 host = host or self.conf.get("etcd.host", "127.0.0.1") 103 port = port or self.conf.get("etcd.port", 2379) 104 username = username or self.conf.get("etcd.username") 105 password = password or self.conf.get("etcd.password") 106 ca_cert = ca or self.conf.get("etcd.ca") 107 cli_key = client_key or self.conf.get("etcd.client_key") 108 cli_cert = client_cert or self.conf.get("etcd.client_cert") 109 110 auth = {} 111 if username and password: 112 auth = { 113 "username": str(username), 114 "password": str(password), 115 } 116 117 certs = {} 118 if ca_cert and not (cli_cert or cli_key): 119 certs = {"ca_cert": str(ca_cert), "protocol": "https"} 120 121 if ca_cert and cli_cert and cli_key: 122 cert = (cli_cert, cli_key) 123 certs = { 124 "ca_cert": str(ca_cert), 125 "cert": cert, 126 "protocol": "https", 127 } 128 129 xargs = auth.copy() 130 xargs.update(certs) 131 132 if HAS_LIBS: 133 self.client = etcd.Client(host, port, **xargs) 134 else: 135 raise CommandExecutionError( 136 "(unable to import etcd, module most likely not installed)" 137 ) 138 139 def watch(self, key, recurse=False, timeout=0, index=None): 140 ret = {"key": key, "value": None, "changed": False, "mIndex": 0, "dir": False} 141 try: 142 result = self.read( 143 key, recursive=recurse, wait=True, timeout=timeout, waitIndex=index 144 ) 145 except EtcdUtilWatchTimeout: 146 try: 147 result = self.read(key) 148 except etcd.EtcdKeyNotFound: 149 log.debug("etcd: key was not created while watching") 150 return ret 151 except ValueError: 152 return {} 153 if result and getattr(result, "dir"): 154 ret["dir"] = True 155 ret["value"] = getattr(result, "value") 156 ret["mIndex"] = getattr(result, "modifiedIndex") 157 return ret 158 except (etcd.EtcdConnectionFailed, MaxRetryError): 159 # This gets raised when we can't contact etcd at all 160 log.error( 161 "etcd: failed to perform 'watch' operation on key %s due to connection" 162 " error", 163 key, 164 ) 165 return {} 166 except ValueError: 167 return {} 168 169 if result is None: 170 return {} 171 172 if recurse: 173 ret["key"] = getattr(result, "key", None) 174 ret["value"] = getattr(result, "value", None) 175 ret["dir"] = getattr(result, "dir", None) 176 ret["changed"] = True 177 ret["mIndex"] = getattr(result, "modifiedIndex") 178 return ret 179 180 def get(self, key, recurse=False): 181 try: 182 result = self.read(key, recursive=recurse) 183 except etcd.EtcdKeyNotFound: 184 # etcd already logged that the key wasn't found, no need to do 185 # anything here but return 186 return None 187 except etcd.EtcdConnectionFailed: 188 log.error( 189 "etcd: failed to perform 'get' operation on key %s due to connection" 190 " error", 191 key, 192 ) 193 return None 194 except ValueError: 195 return None 196 197 return getattr(result, "value", None) 198 199 def read(self, key, recursive=False, wait=False, timeout=None, waitIndex=None): 200 try: 201 if waitIndex: 202 result = self.client.read( 203 key, 204 recursive=recursive, 205 wait=wait, 206 timeout=timeout, 207 waitIndex=waitIndex, 208 ) 209 else: 210 result = self.client.read( 211 key, recursive=recursive, wait=wait, timeout=timeout 212 ) 213 except (etcd.EtcdConnectionFailed, etcd.EtcdKeyNotFound) as err: 214 log.error("etcd: %s", err) 215 raise 216 except ReadTimeoutError: 217 # For some reason, we have to catch this directly. It falls through 218 # from python-etcd because it's trying to catch 219 # urllib3.exceptions.ReadTimeoutError and strangely, doesn't catch. 220 # This can occur from a watch timeout that expires, so it may be 'expected' 221 # behavior. See issue #28553 222 if wait: 223 # Wait timeouts will throw ReadTimeoutError, which isn't bad 224 log.debug("etcd: Timed out while executing a wait") 225 raise EtcdUtilWatchTimeout("Watch on {} timed out".format(key)) 226 log.error("etcd: Timed out") 227 raise etcd.EtcdConnectionFailed("Connection failed") 228 except MaxRetryError as err: 229 # Same issue as ReadTimeoutError. When it 'works', python-etcd 230 # throws EtcdConnectionFailed, so we'll do that for it. 231 log.error("etcd: Could not connect") 232 raise etcd.EtcdConnectionFailed("Could not connect to etcd server") 233 except etcd.EtcdException as err: 234 # EtcdValueError inherits from ValueError, so we don't want to accidentally 235 # catch this below on ValueError and give a bogus error message 236 log.error("etcd: %s", err) 237 raise 238 except ValueError: 239 # python-etcd doesn't fully support python 2.6 and ends up throwing this for *any* exception because 240 # it uses the newer {} format syntax 241 log.error( 242 "etcd: error. python-etcd does not fully support python 2.6, no error" 243 " information available" 244 ) 245 raise 246 except Exception as err: # pylint: disable=broad-except 247 log.error("etcd: uncaught exception %s", err) 248 raise 249 return result 250 251 def _flatten(self, data, path=""): 252 if not data: 253 return {path: {}} 254 path = path.strip("/") 255 flat = {} 256 for k, v in data.items(): 257 k = k.strip("/") 258 if path: 259 p = "/{}/{}".format(path, k) 260 else: 261 p = "/{}".format(k) 262 if isinstance(v, dict): 263 ret = self._flatten(v, p) 264 flat.update(ret) 265 else: 266 flat[p] = v 267 return flat 268 269 def update(self, fields, path=""): 270 if not isinstance(fields, dict): 271 log.error("etcd.update: fields is not type dict") 272 return None 273 fields = self._flatten(fields, path) 274 keys = {} 275 for k, v in fields.items(): 276 is_dir = False 277 if isinstance(v, dict): 278 is_dir = True 279 keys[k] = self.write(k, v, directory=is_dir) 280 return keys 281 282 def set(self, key, value, ttl=None, directory=False): 283 return self.write(key, value, ttl=ttl, directory=directory) 284 285 def write(self, key, value, ttl=None, directory=False): 286 if directory: 287 return self.write_directory(key, value, ttl) 288 return self.write_file(key, value, ttl) 289 290 def write_file(self, key, value, ttl=None): 291 try: 292 result = self.client.write(key, value, ttl=ttl, dir=False) 293 except (etcd.EtcdNotFile, etcd.EtcdRootReadOnly, ValueError) as err: 294 # If EtcdNotFile is raised, then this key is a directory and 295 # really this is a name collision. 296 log.error("etcd: %s", err) 297 return None 298 except MaxRetryError as err: 299 log.error("etcd: Could not connect to etcd server: %s", err) 300 return None 301 except Exception as err: # pylint: disable=broad-except 302 log.error("etcd: uncaught exception %s", err) 303 raise 304 305 return getattr(result, "value") 306 307 def write_directory(self, key, value, ttl=None): 308 if value is not None: 309 log.info("etcd: non-empty value passed for directory: %s", value) 310 try: 311 # directories can't have values, but have to have it passed 312 result = self.client.write(key, None, ttl=ttl, dir=True) 313 except etcd.EtcdNotFile: 314 # When a directory already exists, python-etcd raises an EtcdNotFile 315 # exception. In this case, we just catch and return True for success. 316 log.info("etcd: directory already exists: %s", key) 317 return True 318 except (etcd.EtcdNotDir, etcd.EtcdRootReadOnly, ValueError) as err: 319 # If EtcdNotDir is raised, then the specified path is a file and 320 # thus this is an error. 321 log.error("etcd: %s", err) 322 return None 323 except MaxRetryError as err: 324 log.error("etcd: Could not connect to etcd server: %s", err) 325 return None 326 except Exception as err: # pylint: disable=broad-except 327 log.error("etcd: uncaught exception %s", err) 328 raise 329 330 return getattr(result, "dir") 331 332 def ls(self, path): 333 ret = {} 334 try: 335 items = self.read(path) 336 except (etcd.EtcdKeyNotFound, ValueError): 337 return {} 338 except etcd.EtcdConnectionFailed: 339 log.error( 340 "etcd: failed to perform 'ls' operation on path %s due to connection" 341 " error", 342 path, 343 ) 344 return None 345 346 for item in items.children: 347 if item.dir is True: 348 if item.key == path: 349 continue 350 dir_name = "{}/".format(item.key) 351 ret[dir_name] = {} 352 else: 353 ret[item.key] = item.value 354 return {path: ret} 355 356 def rm(self, key, recurse=False): 357 return self.delete(key, recurse) 358 359 def delete(self, key, recursive=False): 360 try: 361 if self.client.delete(key, recursive=recursive): 362 return True 363 else: 364 return False 365 except ( 366 etcd.EtcdNotFile, 367 etcd.EtcdRootReadOnly, 368 etcd.EtcdDirNotEmpty, 369 etcd.EtcdKeyNotFound, 370 ValueError, 371 ) as err: 372 log.error("etcd: %s", err) 373 return None 374 except MaxRetryError as err: 375 log.error("etcd: Could not connect to etcd server: %s", err) 376 return None 377 except Exception as err: # pylint: disable=broad-except 378 log.error("etcd: uncaught exception %s", err) 379 raise 380 381 def tree(self, path): 382 """ 383 .. versionadded:: 2014.7.0 384 385 Recurse through etcd and return all values 386 """ 387 ret = {} 388 try: 389 items = self.read(path) 390 except (etcd.EtcdKeyNotFound, ValueError): 391 return None 392 except etcd.EtcdConnectionFailed: 393 log.error( 394 "etcd: failed to perform 'tree' operation on path %s due to connection" 395 " error", 396 path, 397 ) 398 return None 399 400 for item in items.children: 401 comps = str(item.key).split("/") 402 if item.dir is True: 403 if item.key == path: 404 continue 405 ret[comps[-1]] = self.tree(item.key) 406 else: 407 ret[comps[-1]] = item.value 408 return ret 409 410 411def get_conn(opts, profile=None, **kwargs): 412 client = EtcdClient(opts, profile, **kwargs) 413 return client 414 415 416def tree(client, path): 417 return client.tree(path) 418