1# (c) 2014, Michael DeHaan <michael.dehaan@gmail.com> 2# (c) 2018, Ansible Project 3# 4# This file is part of Ansible 5# 6# Ansible is free software: you can redistribute it and/or modify 7# it under the terms of the GNU General Public License as published by 8# the Free Software Foundation, either version 3 of the License, or 9# (at your option) any later version. 10# 11# Ansible is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with Ansible. If not, see <http://www.gnu.org/licenses/>. 18from __future__ import (absolute_import, division, print_function) 19__metaclass__ = type 20 21import copy 22import os 23import time 24import errno 25from abc import ABCMeta, abstractmethod 26 27from ansible import constants as C 28from ansible.errors import AnsibleError 29from ansible.module_utils.six import with_metaclass 30from ansible.module_utils._text import to_bytes, to_text 31from ansible.module_utils.common._collections_compat import MutableMapping 32from ansible.plugins import AnsiblePlugin 33from ansible.plugins.loader import cache_loader 34from ansible.utils.collection_loader import resource_from_fqcr 35from ansible.utils.display import Display 36from ansible.vars.fact_cache import FactCache as RealFactCache 37 38display = Display() 39 40 41class FactCache(RealFactCache): 42 """ 43 This is for backwards compatibility. Will be removed after deprecation. It was removed as it 44 wasn't actually part of the cache plugin API. It's actually the code to make use of cache 45 plugins, not the cache plugin itself. Subclassing it wouldn't yield a usable Cache Plugin and 46 there was no facility to use it as anything else. 47 """ 48 def __init__(self, *args, **kwargs): 49 display.deprecated('ansible.plugins.cache.FactCache has been moved to' 50 ' ansible.vars.fact_cache.FactCache. If you are looking for the class' 51 ' to subclass for a cache plugin, you want' 52 ' ansible.plugins.cache.BaseCacheModule or one of its subclasses.', 53 version='2.12', collection_name='ansible.builtin') 54 super(FactCache, self).__init__(*args, **kwargs) 55 56 57class BaseCacheModule(AnsiblePlugin): 58 59 # Backwards compat only. Just import the global display instead 60 _display = display 61 62 def __init__(self, *args, **kwargs): 63 # Third party code is not using cache_loader to load plugin - fall back to previous behavior 64 if not hasattr(self, '_load_name'): 65 display.deprecated('Rather than importing custom CacheModules directly, use ansible.plugins.loader.cache_loader', 66 version='2.14', collection_name='ansible.builtin') 67 self._load_name = self.__module__.split('.')[-1] 68 self._load_name = resource_from_fqcr(self.__module__) 69 super(BaseCacheModule, self).__init__() 70 self.set_options(var_options=args, direct=kwargs) 71 72 @abstractmethod 73 def get(self, key): 74 pass 75 76 @abstractmethod 77 def set(self, key, value): 78 pass 79 80 @abstractmethod 81 def keys(self): 82 pass 83 84 @abstractmethod 85 def contains(self, key): 86 pass 87 88 @abstractmethod 89 def delete(self, key): 90 pass 91 92 @abstractmethod 93 def flush(self): 94 pass 95 96 @abstractmethod 97 def copy(self): 98 pass 99 100 101class BaseFileCacheModule(BaseCacheModule): 102 """ 103 A caching module backed by file based storage. 104 """ 105 def __init__(self, *args, **kwargs): 106 107 try: 108 super(BaseFileCacheModule, self).__init__(*args, **kwargs) 109 self._cache_dir = self._get_cache_connection(self.get_option('_uri')) 110 self._timeout = float(self.get_option('_timeout')) 111 except KeyError: 112 self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION) 113 self._timeout = float(C.CACHE_PLUGIN_TIMEOUT) 114 self.plugin_name = resource_from_fqcr(self.__module__) 115 self._cache = {} 116 self.validate_cache_connection() 117 118 def _get_cache_connection(self, source): 119 if source: 120 try: 121 return os.path.expanduser(os.path.expandvars(source)) 122 except TypeError: 123 pass 124 125 def validate_cache_connection(self): 126 if not self._cache_dir: 127 raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option " 128 "to be set (to a writeable directory path)" % self.plugin_name) 129 130 if not os.path.exists(self._cache_dir): 131 try: 132 os.makedirs(self._cache_dir) 133 except (OSError, IOError) as e: 134 raise AnsibleError("error in '%s' cache plugin while trying to create cache dir %s : %s" % (self.plugin_name, self._cache_dir, to_bytes(e))) 135 else: 136 for x in (os.R_OK, os.W_OK, os.X_OK): 137 if not os.access(self._cache_dir, x): 138 raise AnsibleError("error in '%s' cache, configured path (%s) does not have necessary permissions (rwx), disabling plugin" % ( 139 self.plugin_name, self._cache_dir)) 140 141 def _get_cache_file_name(self, key): 142 prefix = self.get_option('_prefix') 143 if prefix: 144 cachefile = "%s/%s%s" % (self._cache_dir, prefix, key) 145 else: 146 cachefile = "%s/%s" % (self._cache_dir, key) 147 return cachefile 148 149 def get(self, key): 150 """ This checks the in memory cache first as the fact was not expired at 'gather time' 151 and it would be problematic if the key did expire after some long running tasks and 152 user gets 'undefined' error in the same play """ 153 154 if key not in self._cache: 155 156 if self.has_expired(key) or key == "": 157 raise KeyError 158 159 cachefile = self._get_cache_file_name(key) 160 try: 161 value = self._load(cachefile) 162 self._cache[key] = value 163 except ValueError as e: 164 display.warning("error in '%s' cache plugin while trying to read %s : %s. " 165 "Most likely a corrupt file, so erasing and failing." % (self.plugin_name, cachefile, to_bytes(e))) 166 self.delete(key) 167 raise AnsibleError("The cache file %s was corrupt, or did not otherwise contain valid data. " 168 "It has been removed, so you can re-run your command now." % cachefile) 169 except (OSError, IOError) as e: 170 display.warning("error in '%s' cache plugin while trying to read %s : %s" % (self.plugin_name, cachefile, to_bytes(e))) 171 raise KeyError 172 except Exception as e: 173 raise AnsibleError("Error while decoding the cache file %s: %s" % (cachefile, to_bytes(e))) 174 175 return self._cache.get(key) 176 177 def set(self, key, value): 178 179 self._cache[key] = value 180 181 cachefile = self._get_cache_file_name(key) 182 try: 183 self._dump(value, cachefile) 184 except (OSError, IOError) as e: 185 display.warning("error in '%s' cache plugin while trying to write to %s : %s" % (self.plugin_name, cachefile, to_bytes(e))) 186 187 def has_expired(self, key): 188 189 if self._timeout == 0: 190 return False 191 192 cachefile = self._get_cache_file_name(key) 193 try: 194 st = os.stat(cachefile) 195 except (OSError, IOError) as e: 196 if e.errno == errno.ENOENT: 197 return False 198 else: 199 display.warning("error in '%s' cache plugin while trying to stat %s : %s" % (self.plugin_name, cachefile, to_bytes(e))) 200 return False 201 202 if time.time() - st.st_mtime <= self._timeout: 203 return False 204 205 if key in self._cache: 206 del self._cache[key] 207 return True 208 209 def keys(self): 210 keys = [] 211 for k in os.listdir(self._cache_dir): 212 if not (k.startswith('.') or self.has_expired(k)): 213 keys.append(k) 214 return keys 215 216 def contains(self, key): 217 cachefile = self._get_cache_file_name(key) 218 219 if key in self._cache: 220 return True 221 222 if self.has_expired(key): 223 return False 224 try: 225 os.stat(cachefile) 226 return True 227 except (OSError, IOError) as e: 228 if e.errno == errno.ENOENT: 229 return False 230 else: 231 display.warning("error in '%s' cache plugin while trying to stat %s : %s" % (self.plugin_name, cachefile, to_bytes(e))) 232 233 def delete(self, key): 234 try: 235 del self._cache[key] 236 except KeyError: 237 pass 238 try: 239 os.remove(self._get_cache_file_name(key)) 240 except (OSError, IOError): 241 pass # TODO: only pass on non existing? 242 243 def flush(self): 244 self._cache = {} 245 for key in self.keys(): 246 self.delete(key) 247 248 def copy(self): 249 ret = dict() 250 for key in self.keys(): 251 ret[key] = self.get(key) 252 return ret 253 254 @abstractmethod 255 def _load(self, filepath): 256 """ 257 Read data from a filepath and return it as a value 258 259 :arg filepath: The filepath to read from. 260 :returns: The value stored in the filepath 261 262 This method reads from the file on disk and takes care of any parsing 263 and transformation of the data before returning it. The value 264 returned should be what Ansible would expect if it were uncached data. 265 266 .. note:: Filehandles have advantages but calling code doesn't know 267 whether this file is text or binary, should be decoded, or accessed via 268 a library function. Therefore the API uses a filepath and opens 269 the file inside of the method. 270 """ 271 pass 272 273 @abstractmethod 274 def _dump(self, value, filepath): 275 """ 276 Write data to a filepath 277 278 :arg value: The value to store 279 :arg filepath: The filepath to store it at 280 """ 281 pass 282 283 284class CachePluginAdjudicator(MutableMapping): 285 """ 286 Intermediary between a cache dictionary and a CacheModule 287 """ 288 def __init__(self, plugin_name='memory', **kwargs): 289 self._cache = {} 290 self._retrieved = {} 291 292 self._plugin = cache_loader.get(plugin_name, **kwargs) 293 if not self._plugin: 294 raise AnsibleError('Unable to load the cache plugin (%s).' % plugin_name) 295 296 self._plugin_name = plugin_name 297 298 def update_cache_if_changed(self): 299 if self._retrieved != self._cache: 300 self.set_cache() 301 302 def set_cache(self): 303 for top_level_cache_key in self._cache.keys(): 304 self._plugin.set(top_level_cache_key, self._cache[top_level_cache_key]) 305 self._retrieved = copy.deepcopy(self._cache) 306 307 def load_whole_cache(self): 308 for key in self._plugin.keys(): 309 self._cache[key] = self._plugin.get(key) 310 311 def __repr__(self): 312 return to_text(self._cache) 313 314 def __iter__(self): 315 return iter(self.keys()) 316 317 def __len__(self): 318 return len(self.keys()) 319 320 def _do_load_key(self, key): 321 load = False 322 if all([ 323 key not in self._cache, 324 key not in self._retrieved, 325 self._plugin_name != 'memory', 326 self._plugin.contains(key), 327 ]): 328 load = True 329 return load 330 331 def __getitem__(self, key): 332 if self._do_load_key(key): 333 try: 334 self._cache[key] = self._plugin.get(key) 335 except KeyError: 336 pass 337 else: 338 self._retrieved[key] = self._cache[key] 339 return self._cache[key] 340 341 def get(self, key, default=None): 342 if self._do_load_key(key): 343 try: 344 self._cache[key] = self._plugin.get(key) 345 except KeyError as e: 346 pass 347 else: 348 self._retrieved[key] = self._cache[key] 349 return self._cache.get(key, default) 350 351 def items(self): 352 return self._cache.items() 353 354 def values(self): 355 return self._cache.values() 356 357 def keys(self): 358 return self._cache.keys() 359 360 def pop(self, key, *args): 361 if args: 362 return self._cache.pop(key, args[0]) 363 return self._cache.pop(key) 364 365 def __delitem__(self, key): 366 del self._cache[key] 367 368 def __setitem__(self, key, value): 369 self._cache[key] = value 370 371 def flush(self): 372 self._plugin.flush() 373 self._cache = {} 374 375 def update(self, value): 376 self._cache.update(value) 377