1# (c) 2014, Michael DeHaan <michael.dehaan@gmail.com>
2# (c) 2018, Ansible Project
3#
4# This file is part of Ansible
5#
6# Ansible is free software: you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation, either version 3 of the License, or
9# (at your option) any later version.
10#
11# Ansible is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License
17# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
18from __future__ import (absolute_import, division, print_function)
19__metaclass__ = type
20
21import copy
22import os
23import time
24import errno
25from abc import ABCMeta, abstractmethod
26
27from ansible import constants as C
28from ansible.errors import AnsibleError
29from ansible.module_utils.six import with_metaclass
30from ansible.module_utils._text import to_bytes, to_text
31from ansible.module_utils.common._collections_compat import MutableMapping
32from ansible.plugins import AnsiblePlugin
33from ansible.plugins.loader import cache_loader
34from ansible.utils.collection_loader import resource_from_fqcr
35from ansible.utils.display import Display
36from ansible.vars.fact_cache import FactCache as RealFactCache
37
38display = Display()
39
40
41class FactCache(RealFactCache):
42    """
43    This is for backwards compatibility.  Will be removed after deprecation.  It was removed as it
44    wasn't actually part of the cache plugin API.  It's actually the code to make use of cache
45    plugins, not the cache plugin itself.  Subclassing it wouldn't yield a usable Cache Plugin and
46    there was no facility to use it as anything else.
47    """
48    def __init__(self, *args, **kwargs):
49        display.deprecated('ansible.plugins.cache.FactCache has been moved to'
50                           ' ansible.vars.fact_cache.FactCache.  If you are looking for the class'
51                           ' to subclass for a cache plugin, you want'
52                           ' ansible.plugins.cache.BaseCacheModule or one of its subclasses.',
53                           version='2.12', collection_name='ansible.builtin')
54        super(FactCache, self).__init__(*args, **kwargs)
55
56
57class BaseCacheModule(AnsiblePlugin):
58
59    # Backwards compat only.  Just import the global display instead
60    _display = display
61
62    def __init__(self, *args, **kwargs):
63        # Third party code is not using cache_loader to load plugin - fall back to previous behavior
64        if not hasattr(self, '_load_name'):
65            display.deprecated('Rather than importing custom CacheModules directly, use ansible.plugins.loader.cache_loader',
66                               version='2.14', collection_name='ansible.builtin')
67            self._load_name = self.__module__.split('.')[-1]
68            self._load_name = resource_from_fqcr(self.__module__)
69        super(BaseCacheModule, self).__init__()
70        self.set_options(var_options=args, direct=kwargs)
71
72    @abstractmethod
73    def get(self, key):
74        pass
75
76    @abstractmethod
77    def set(self, key, value):
78        pass
79
80    @abstractmethod
81    def keys(self):
82        pass
83
84    @abstractmethod
85    def contains(self, key):
86        pass
87
88    @abstractmethod
89    def delete(self, key):
90        pass
91
92    @abstractmethod
93    def flush(self):
94        pass
95
96    @abstractmethod
97    def copy(self):
98        pass
99
100
101class BaseFileCacheModule(BaseCacheModule):
102    """
103    A caching module backed by file based storage.
104    """
105    def __init__(self, *args, **kwargs):
106
107        try:
108            super(BaseFileCacheModule, self).__init__(*args, **kwargs)
109            self._cache_dir = self._get_cache_connection(self.get_option('_uri'))
110            self._timeout = float(self.get_option('_timeout'))
111        except KeyError:
112            self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION)
113            self._timeout = float(C.CACHE_PLUGIN_TIMEOUT)
114        self.plugin_name = resource_from_fqcr(self.__module__)
115        self._cache = {}
116        self.validate_cache_connection()
117
118    def _get_cache_connection(self, source):
119        if source:
120            try:
121                return os.path.expanduser(os.path.expandvars(source))
122            except TypeError:
123                pass
124
125    def validate_cache_connection(self):
126        if not self._cache_dir:
127            raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option "
128                               "to be set (to a writeable directory path)" % self.plugin_name)
129
130        if not os.path.exists(self._cache_dir):
131            try:
132                os.makedirs(self._cache_dir)
133            except (OSError, IOError) as e:
134                raise AnsibleError("error in '%s' cache plugin while trying to create cache dir %s : %s" % (self.plugin_name, self._cache_dir, to_bytes(e)))
135        else:
136            for x in (os.R_OK, os.W_OK, os.X_OK):
137                if not os.access(self._cache_dir, x):
138                    raise AnsibleError("error in '%s' cache, configured path (%s) does not have necessary permissions (rwx), disabling plugin" % (
139                        self.plugin_name, self._cache_dir))
140
141    def _get_cache_file_name(self, key):
142        prefix = self.get_option('_prefix')
143        if prefix:
144            cachefile = "%s/%s%s" % (self._cache_dir, prefix, key)
145        else:
146            cachefile = "%s/%s" % (self._cache_dir, key)
147        return cachefile
148
149    def get(self, key):
150        """ This checks the in memory cache first as the fact was not expired at 'gather time'
151        and it would be problematic if the key did expire after some long running tasks and
152        user gets 'undefined' error in the same play """
153
154        if key not in self._cache:
155
156            if self.has_expired(key) or key == "":
157                raise KeyError
158
159            cachefile = self._get_cache_file_name(key)
160            try:
161                value = self._load(cachefile)
162                self._cache[key] = value
163            except ValueError as e:
164                display.warning("error in '%s' cache plugin while trying to read %s : %s. "
165                                "Most likely a corrupt file, so erasing and failing." % (self.plugin_name, cachefile, to_bytes(e)))
166                self.delete(key)
167                raise AnsibleError("The cache file %s was corrupt, or did not otherwise contain valid data. "
168                                   "It has been removed, so you can re-run your command now." % cachefile)
169            except (OSError, IOError) as e:
170                display.warning("error in '%s' cache plugin while trying to read %s : %s" % (self.plugin_name, cachefile, to_bytes(e)))
171                raise KeyError
172            except Exception as e:
173                raise AnsibleError("Error while decoding the cache file %s: %s" % (cachefile, to_bytes(e)))
174
175        return self._cache.get(key)
176
177    def set(self, key, value):
178
179        self._cache[key] = value
180
181        cachefile = self._get_cache_file_name(key)
182        try:
183            self._dump(value, cachefile)
184        except (OSError, IOError) as e:
185            display.warning("error in '%s' cache plugin while trying to write to %s : %s" % (self.plugin_name, cachefile, to_bytes(e)))
186
187    def has_expired(self, key):
188
189        if self._timeout == 0:
190            return False
191
192        cachefile = self._get_cache_file_name(key)
193        try:
194            st = os.stat(cachefile)
195        except (OSError, IOError) as e:
196            if e.errno == errno.ENOENT:
197                return False
198            else:
199                display.warning("error in '%s' cache plugin while trying to stat %s : %s" % (self.plugin_name, cachefile, to_bytes(e)))
200                return False
201
202        if time.time() - st.st_mtime <= self._timeout:
203            return False
204
205        if key in self._cache:
206            del self._cache[key]
207        return True
208
209    def keys(self):
210        keys = []
211        for k in os.listdir(self._cache_dir):
212            if not (k.startswith('.') or self.has_expired(k)):
213                keys.append(k)
214        return keys
215
216    def contains(self, key):
217        cachefile = self._get_cache_file_name(key)
218
219        if key in self._cache:
220            return True
221
222        if self.has_expired(key):
223            return False
224        try:
225            os.stat(cachefile)
226            return True
227        except (OSError, IOError) as e:
228            if e.errno == errno.ENOENT:
229                return False
230            else:
231                display.warning("error in '%s' cache plugin while trying to stat %s : %s" % (self.plugin_name, cachefile, to_bytes(e)))
232
233    def delete(self, key):
234        try:
235            del self._cache[key]
236        except KeyError:
237            pass
238        try:
239            os.remove(self._get_cache_file_name(key))
240        except (OSError, IOError):
241            pass  # TODO: only pass on non existing?
242
243    def flush(self):
244        self._cache = {}
245        for key in self.keys():
246            self.delete(key)
247
248    def copy(self):
249        ret = dict()
250        for key in self.keys():
251            ret[key] = self.get(key)
252        return ret
253
254    @abstractmethod
255    def _load(self, filepath):
256        """
257        Read data from a filepath and return it as a value
258
259        :arg filepath: The filepath to read from.
260        :returns: The value stored in the filepath
261
262        This method reads from the file on disk and takes care of any parsing
263        and transformation of the data before returning it.  The value
264        returned should be what Ansible would expect if it were uncached data.
265
266        .. note:: Filehandles have advantages but calling code doesn't know
267            whether this file is text or binary, should be decoded, or accessed via
268            a library function.  Therefore the API uses a filepath and opens
269            the file inside of the method.
270        """
271        pass
272
273    @abstractmethod
274    def _dump(self, value, filepath):
275        """
276        Write data to a filepath
277
278        :arg value: The value to store
279        :arg filepath: The filepath to store it at
280        """
281        pass
282
283
284class CachePluginAdjudicator(MutableMapping):
285    """
286    Intermediary between a cache dictionary and a CacheModule
287    """
288    def __init__(self, plugin_name='memory', **kwargs):
289        self._cache = {}
290        self._retrieved = {}
291
292        self._plugin = cache_loader.get(plugin_name, **kwargs)
293        if not self._plugin:
294            raise AnsibleError('Unable to load the cache plugin (%s).' % plugin_name)
295
296        self._plugin_name = plugin_name
297
298    def update_cache_if_changed(self):
299        if self._retrieved != self._cache:
300            self.set_cache()
301
302    def set_cache(self):
303        for top_level_cache_key in self._cache.keys():
304            self._plugin.set(top_level_cache_key, self._cache[top_level_cache_key])
305        self._retrieved = copy.deepcopy(self._cache)
306
307    def load_whole_cache(self):
308        for key in self._plugin.keys():
309            self._cache[key] = self._plugin.get(key)
310
311    def __repr__(self):
312        return to_text(self._cache)
313
314    def __iter__(self):
315        return iter(self.keys())
316
317    def __len__(self):
318        return len(self.keys())
319
320    def _do_load_key(self, key):
321        load = False
322        if all([
323            key not in self._cache,
324            key not in self._retrieved,
325            self._plugin_name != 'memory',
326            self._plugin.contains(key),
327        ]):
328            load = True
329        return load
330
331    def __getitem__(self, key):
332        if self._do_load_key(key):
333            try:
334                self._cache[key] = self._plugin.get(key)
335            except KeyError:
336                pass
337            else:
338                self._retrieved[key] = self._cache[key]
339        return self._cache[key]
340
341    def get(self, key, default=None):
342        if self._do_load_key(key):
343            try:
344                self._cache[key] = self._plugin.get(key)
345            except KeyError as e:
346                pass
347            else:
348                self._retrieved[key] = self._cache[key]
349        return self._cache.get(key, default)
350
351    def items(self):
352        return self._cache.items()
353
354    def values(self):
355        return self._cache.values()
356
357    def keys(self):
358        return self._cache.keys()
359
360    def pop(self, key, *args):
361        if args:
362            return self._cache.pop(key, args[0])
363        return self._cache.pop(key)
364
365    def __delitem__(self, key):
366        del self._cache[key]
367
368    def __setitem__(self, key, value):
369        self._cache[key] = value
370
371    def flush(self):
372        self._plugin.flush()
373        self._cache = {}
374
375    def update(self, value):
376        self._cache.update(value)
377