1"""
2Utilities for working with etcd
3
4.. versionadded:: 2014.7.0
5
6:depends:  - python-etcd
7
8This library sets up a client object for etcd, using the configuration passed
9into the client() function. Normally, this is __opts__. Optionally, a profile
10may be passed in. The following configurations are both valid:
11
12.. code-block:: yaml
13
14    # No profile name
15    etcd.host: 127.0.0.1
16    etcd.port: 2379
17    etcd.username: larry  # Optional; requires etcd.password to be set
18    etcd.password: 123pass  # Optional; requires etcd.username to be set
19    etcd.ca: /path/to/your/ca_cert/ca.pem # Optional
20    etcd.client_key: /path/to/your/client_key/client-key.pem # Optional; requires etcd.ca and etcd.client_cert to be set
21    etcd.client_cert: /path/to/your/client_cert/client.pem # Optional; requires etcd.ca and etcd.client_key to be set
22
23    # One or more profiles defined
24    my_etcd_config:
25      etcd.host: 127.0.0.1
26      etcd.port: 2379
27      etcd.username: larry  # Optional; requires etcd.password to be set
28      etcd.password: 123pass  # Optional; requires etcd.username to be set
29      etcd.ca: /path/to/your/ca_cert/ca.pem # Optional
30      etcd.client_key: /path/to/your/client_key/client-key.pem # Optional; requires etcd.ca and etcd.client_cert to be set
31      etcd.client_cert: /path/to/your/client_cert/client.pem # Optional; requires etcd.ca and etcd.client_key to be set
32
33Once configured, the client() function is passed a set of opts, and optionally,
34the name of a profile to be used.
35
36.. code-block:: python
37
38    import salt.utils.etcd_utils
39    client = salt.utils.etcd_utils.client(__opts__, profile='my_etcd_config')
40
41You may also use the newer syntax and bypass the generator function.
42
43.. code-block:: python
44
45    import salt.utils.etcd_utils
46    client = salt.utils.etcd_utils.EtcdClient(__opts__, profile='my_etcd_config')
47
48It should be noted that some usages of etcd require a profile to be specified,
49rather than top-level configurations. This being the case, it is better to
50always use a named configuration profile, as shown above.
51"""
52
53import logging
54
55from salt.exceptions import CommandExecutionError
56
57try:
58    import etcd
59    from urllib3.exceptions import ReadTimeoutError, MaxRetryError
60
61    HAS_LIBS = True
62except ImportError:
63    HAS_LIBS = False
64
65# Set up logging
66log = logging.getLogger(__name__)
67
68
69class EtcdUtilWatchTimeout(Exception):
70    """
71    A watch timed out without returning a result
72    """
73
74
75class EtcdClient:
76    def __init__(
77        self,
78        opts,
79        profile=None,
80        host=None,
81        port=None,
82        username=None,
83        password=None,
84        ca=None,
85        client_key=None,
86        client_cert=None,
87        **kwargs
88    ):
89        opts_pillar = opts.get("pillar", {})
90        opts_master = opts_pillar.get("master", {})
91
92        opts_merged = {}
93        opts_merged.update(opts_master)
94        opts_merged.update(opts_pillar)
95        opts_merged.update(opts)
96
97        if profile:
98            self.conf = opts_merged.get(profile, {})
99        else:
100            self.conf = opts_merged
101
102        host = host or self.conf.get("etcd.host", "127.0.0.1")
103        port = port or self.conf.get("etcd.port", 2379)
104        username = username or self.conf.get("etcd.username")
105        password = password or self.conf.get("etcd.password")
106        ca_cert = ca or self.conf.get("etcd.ca")
107        cli_key = client_key or self.conf.get("etcd.client_key")
108        cli_cert = client_cert or self.conf.get("etcd.client_cert")
109
110        auth = {}
111        if username and password:
112            auth = {
113                "username": str(username),
114                "password": str(password),
115            }
116
117        certs = {}
118        if ca_cert and not (cli_cert or cli_key):
119            certs = {"ca_cert": str(ca_cert), "protocol": "https"}
120
121        if ca_cert and cli_cert and cli_key:
122            cert = (cli_cert, cli_key)
123            certs = {
124                "ca_cert": str(ca_cert),
125                "cert": cert,
126                "protocol": "https",
127            }
128
129        xargs = auth.copy()
130        xargs.update(certs)
131
132        if HAS_LIBS:
133            self.client = etcd.Client(host, port, **xargs)
134        else:
135            raise CommandExecutionError(
136                "(unable to import etcd, module most likely not installed)"
137            )
138
139    def watch(self, key, recurse=False, timeout=0, index=None):
140        ret = {"key": key, "value": None, "changed": False, "mIndex": 0, "dir": False}
141        try:
142            result = self.read(
143                key, recursive=recurse, wait=True, timeout=timeout, waitIndex=index
144            )
145        except EtcdUtilWatchTimeout:
146            try:
147                result = self.read(key)
148            except etcd.EtcdKeyNotFound:
149                log.debug("etcd: key was not created while watching")
150                return ret
151            except ValueError:
152                return {}
153            if result and getattr(result, "dir"):
154                ret["dir"] = True
155            ret["value"] = getattr(result, "value")
156            ret["mIndex"] = getattr(result, "modifiedIndex")
157            return ret
158        except (etcd.EtcdConnectionFailed, MaxRetryError):
159            # This gets raised when we can't contact etcd at all
160            log.error(
161                "etcd: failed to perform 'watch' operation on key %s due to connection"
162                " error",
163                key,
164            )
165            return {}
166        except ValueError:
167            return {}
168
169        if result is None:
170            return {}
171
172        if recurse:
173            ret["key"] = getattr(result, "key", None)
174        ret["value"] = getattr(result, "value", None)
175        ret["dir"] = getattr(result, "dir", None)
176        ret["changed"] = True
177        ret["mIndex"] = getattr(result, "modifiedIndex")
178        return ret
179
180    def get(self, key, recurse=False):
181        try:
182            result = self.read(key, recursive=recurse)
183        except etcd.EtcdKeyNotFound:
184            # etcd already logged that the key wasn't found, no need to do
185            # anything here but return
186            return None
187        except etcd.EtcdConnectionFailed:
188            log.error(
189                "etcd: failed to perform 'get' operation on key %s due to connection"
190                " error",
191                key,
192            )
193            return None
194        except ValueError:
195            return None
196
197        return getattr(result, "value", None)
198
199    def read(self, key, recursive=False, wait=False, timeout=None, waitIndex=None):
200        try:
201            if waitIndex:
202                result = self.client.read(
203                    key,
204                    recursive=recursive,
205                    wait=wait,
206                    timeout=timeout,
207                    waitIndex=waitIndex,
208                )
209            else:
210                result = self.client.read(
211                    key, recursive=recursive, wait=wait, timeout=timeout
212                )
213        except (etcd.EtcdConnectionFailed, etcd.EtcdKeyNotFound) as err:
214            log.error("etcd: %s", err)
215            raise
216        except ReadTimeoutError:
217            # For some reason, we have to catch this directly.  It falls through
218            # from python-etcd because it's trying to catch
219            # urllib3.exceptions.ReadTimeoutError and strangely, doesn't catch.
220            # This can occur from a watch timeout that expires, so it may be 'expected'
221            # behavior. See issue #28553
222            if wait:
223                # Wait timeouts will throw ReadTimeoutError, which isn't bad
224                log.debug("etcd: Timed out while executing a wait")
225                raise EtcdUtilWatchTimeout("Watch on {} timed out".format(key))
226            log.error("etcd: Timed out")
227            raise etcd.EtcdConnectionFailed("Connection failed")
228        except MaxRetryError as err:
229            # Same issue as ReadTimeoutError.  When it 'works', python-etcd
230            # throws EtcdConnectionFailed, so we'll do that for it.
231            log.error("etcd: Could not connect")
232            raise etcd.EtcdConnectionFailed("Could not connect to etcd server")
233        except etcd.EtcdException as err:
234            # EtcdValueError inherits from ValueError, so we don't want to accidentally
235            # catch this below on ValueError and give a bogus error message
236            log.error("etcd: %s", err)
237            raise
238        except ValueError:
239            # python-etcd doesn't fully support python 2.6 and ends up throwing this for *any* exception because
240            # it uses the newer {} format syntax
241            log.error(
242                "etcd: error. python-etcd does not fully support python 2.6, no error"
243                " information available"
244            )
245            raise
246        except Exception as err:  # pylint: disable=broad-except
247            log.error("etcd: uncaught exception %s", err)
248            raise
249        return result
250
251    def _flatten(self, data, path=""):
252        if not data:
253            return {path: {}}
254        path = path.strip("/")
255        flat = {}
256        for k, v in data.items():
257            k = k.strip("/")
258            if path:
259                p = "/{}/{}".format(path, k)
260            else:
261                p = "/{}".format(k)
262            if isinstance(v, dict):
263                ret = self._flatten(v, p)
264                flat.update(ret)
265            else:
266                flat[p] = v
267        return flat
268
269    def update(self, fields, path=""):
270        if not isinstance(fields, dict):
271            log.error("etcd.update: fields is not type dict")
272            return None
273        fields = self._flatten(fields, path)
274        keys = {}
275        for k, v in fields.items():
276            is_dir = False
277            if isinstance(v, dict):
278                is_dir = True
279            keys[k] = self.write(k, v, directory=is_dir)
280        return keys
281
282    def set(self, key, value, ttl=None, directory=False):
283        return self.write(key, value, ttl=ttl, directory=directory)
284
285    def write(self, key, value, ttl=None, directory=False):
286        if directory:
287            return self.write_directory(key, value, ttl)
288        return self.write_file(key, value, ttl)
289
290    def write_file(self, key, value, ttl=None):
291        try:
292            result = self.client.write(key, value, ttl=ttl, dir=False)
293        except (etcd.EtcdNotFile, etcd.EtcdRootReadOnly, ValueError) as err:
294            # If EtcdNotFile is raised, then this key is a directory and
295            # really this is a name collision.
296            log.error("etcd: %s", err)
297            return None
298        except MaxRetryError as err:
299            log.error("etcd: Could not connect to etcd server: %s", err)
300            return None
301        except Exception as err:  # pylint: disable=broad-except
302            log.error("etcd: uncaught exception %s", err)
303            raise
304
305        return getattr(result, "value")
306
307    def write_directory(self, key, value, ttl=None):
308        if value is not None:
309            log.info("etcd: non-empty value passed for directory: %s", value)
310        try:
311            # directories can't have values, but have to have it passed
312            result = self.client.write(key, None, ttl=ttl, dir=True)
313        except etcd.EtcdNotFile:
314            # When a directory already exists, python-etcd raises an EtcdNotFile
315            # exception. In this case, we just catch and return True for success.
316            log.info("etcd: directory already exists: %s", key)
317            return True
318        except (etcd.EtcdNotDir, etcd.EtcdRootReadOnly, ValueError) as err:
319            # If EtcdNotDir is raised, then the specified path is a file and
320            # thus this is an error.
321            log.error("etcd: %s", err)
322            return None
323        except MaxRetryError as err:
324            log.error("etcd: Could not connect to etcd server: %s", err)
325            return None
326        except Exception as err:  # pylint: disable=broad-except
327            log.error("etcd: uncaught exception %s", err)
328            raise
329
330        return getattr(result, "dir")
331
332    def ls(self, path):
333        ret = {}
334        try:
335            items = self.read(path)
336        except (etcd.EtcdKeyNotFound, ValueError):
337            return {}
338        except etcd.EtcdConnectionFailed:
339            log.error(
340                "etcd: failed to perform 'ls' operation on path %s due to connection"
341                " error",
342                path,
343            )
344            return None
345
346        for item in items.children:
347            if item.dir is True:
348                if item.key == path:
349                    continue
350                dir_name = "{}/".format(item.key)
351                ret[dir_name] = {}
352            else:
353                ret[item.key] = item.value
354        return {path: ret}
355
356    def rm(self, key, recurse=False):
357        return self.delete(key, recurse)
358
359    def delete(self, key, recursive=False):
360        try:
361            if self.client.delete(key, recursive=recursive):
362                return True
363            else:
364                return False
365        except (
366            etcd.EtcdNotFile,
367            etcd.EtcdRootReadOnly,
368            etcd.EtcdDirNotEmpty,
369            etcd.EtcdKeyNotFound,
370            ValueError,
371        ) as err:
372            log.error("etcd: %s", err)
373            return None
374        except MaxRetryError as err:
375            log.error("etcd: Could not connect to etcd server: %s", err)
376            return None
377        except Exception as err:  # pylint: disable=broad-except
378            log.error("etcd: uncaught exception %s", err)
379            raise
380
381    def tree(self, path):
382        """
383        .. versionadded:: 2014.7.0
384
385        Recurse through etcd and return all values
386        """
387        ret = {}
388        try:
389            items = self.read(path)
390        except (etcd.EtcdKeyNotFound, ValueError):
391            return None
392        except etcd.EtcdConnectionFailed:
393            log.error(
394                "etcd: failed to perform 'tree' operation on path %s due to connection"
395                " error",
396                path,
397            )
398            return None
399
400        for item in items.children:
401            comps = str(item.key).split("/")
402            if item.dir is True:
403                if item.key == path:
404                    continue
405                ret[comps[-1]] = self.tree(item.key)
406            else:
407                ret[comps[-1]] = item.value
408        return ret
409
410
411def get_conn(opts, profile=None, **kwargs):
412    client = EtcdClient(opts, profile, **kwargs)
413    return client
414
415
416def tree(client, path):
417    return client.tree(path)
418