1"""Container and Namespace classes"""
2import errno
3
4from ._compat import pickle, anydbm, add_metaclass, PYVER, unicode_text
5
6import beaker.util as util
7import logging
8import os
9import time
10
11from beaker.exceptions import CreationAbortedError, MissingCacheParameter
12from beaker.synchronization import _threading, file_synchronizer, \
13     mutex_synchronizer, NameLock, null_synchronizer
14
15__all__ = ['Value', 'Container', 'ContainerContext',
16           'MemoryContainer', 'DBMContainer', 'NamespaceManager',
17           'MemoryNamespaceManager', 'DBMNamespaceManager', 'FileContainer',
18           'OpenResourceNamespaceManager',
19           'FileNamespaceManager', 'CreationAbortedError']
20
21
22logger = logging.getLogger('beaker.container')
23if logger.isEnabledFor(logging.DEBUG):
24    debug = logger.debug
25else:
26    def debug(message, *args):
27        pass
28
29
30class NamespaceManager(object):
31    """Handles dictionary operations and locking for a namespace of
32    values.
33
34    :class:`.NamespaceManager` provides a dictionary-like interface,
35    implementing ``__getitem__()``, ``__setitem__()``, and
36    ``__contains__()``, as well as functions related to lock
37    acquisition.
38
39    The implementation for setting and retrieving the namespace data is
40    handled by subclasses.
41
42    NamespaceManager may be used alone, or may be accessed by
43    one or more :class:`.Value` objects.  :class:`.Value` objects provide per-key
44    services like expiration times and automatic recreation of values.
45
46    Multiple NamespaceManagers created with a particular name will all
47    share access to the same underlying datasource and will attempt to
48    synchronize against a common mutex object.  The scope of this
49    sharing may be within a single process or across multiple
50    processes, depending on the type of NamespaceManager used.
51
52    The NamespaceManager itself is generally threadsafe, except in the
53    case of the DBMNamespaceManager in conjunction with the gdbm dbm
54    implementation.
55
56    """
57
58    @classmethod
59    def _init_dependencies(cls):
60        """Initialize module-level dependent libraries required
61        by this :class:`.NamespaceManager`."""
62
63    def __init__(self, namespace):
64        self._init_dependencies()
65        self.namespace = namespace
66
67    def get_creation_lock(self, key):
68        """Return a locking object that is used to synchronize
69        multiple threads or processes which wish to generate a new
70        cache value.
71
72        This function is typically an instance of
73        :class:`.FileSynchronizer`, :class:`.ConditionSynchronizer`,
74        or :class:`.null_synchronizer`.
75
76        The creation lock is only used when a requested value
77        does not exist, or has been expired, and is only used
78        by the :class:`.Value` key-management object in conjunction
79        with a "createfunc" value-creation function.
80
81        """
82        raise NotImplementedError()
83
84    def do_remove(self):
85        """Implement removal of the entire contents of this
86        :class:`.NamespaceManager`.
87
88        e.g. for a file-based namespace, this would remove
89        all the files.
90
91        The front-end to this method is the
92        :meth:`.NamespaceManager.remove` method.
93
94        """
95        raise NotImplementedError()
96
97    def acquire_read_lock(self):
98        """Establish a read lock.
99
100        This operation is called before a key is read.    By
101        default the function does nothing.
102
103        """
104
105    def release_read_lock(self):
106        """Release a read lock.
107
108        This operation is called after a key is read.    By
109        default the function does nothing.
110
111        """
112
113    def acquire_write_lock(self, wait=True, replace=False):
114        """Establish a write lock.
115
116        This operation is called before a key is written.
117        A return value of ``True`` indicates the lock has
118        been acquired.
119
120        By default the function returns ``True`` unconditionally.
121
122        'replace' is a hint indicating the full contents
123        of the namespace may be safely discarded. Some backends
124        may implement this (i.e. file backend won't unpickle the
125        current contents).
126
127        """
128        return True
129
130    def release_write_lock(self):
131        """Release a write lock.
132
133        This operation is called after a new value is written.
134        By default this function does nothing.
135
136        """
137
138    def has_key(self, key):
139        """Return ``True`` if the given key is present in this
140        :class:`.Namespace`.
141        """
142        return self.__contains__(key)
143
144    def __getitem__(self, key):
145        raise NotImplementedError()
146
147    def __setitem__(self, key, value):
148        raise NotImplementedError()
149
150    def set_value(self, key, value, expiretime=None):
151        """Sets a value in this :class:`.NamespaceManager`.
152
153        This is the same as ``__setitem__()``, but
154        also allows an expiration time to be passed
155        at the same time.
156
157        """
158        self[key] = value
159
160    def __contains__(self, key):
161        raise NotImplementedError()
162
163    def __delitem__(self, key):
164        raise NotImplementedError()
165
166    def keys(self):
167        """Return the list of all keys.
168
169        This method may not be supported by all
170        :class:`.NamespaceManager` implementations.
171
172        """
173        raise NotImplementedError()
174
175    def remove(self):
176        """Remove the entire contents of this
177        :class:`.NamespaceManager`.
178
179        e.g. for a file-based namespace, this would remove
180        all the files.
181        """
182        self.do_remove()
183
184
185class OpenResourceNamespaceManager(NamespaceManager):
186    """A NamespaceManager where read/write operations require opening/
187    closing of a resource which is possibly mutexed.
188
189    """
190    def __init__(self, namespace):
191        NamespaceManager.__init__(self, namespace)
192        self.access_lock = self.get_access_lock()
193        self.openers = 0
194        self.mutex = _threading.Lock()
195
196    def get_access_lock(self):
197        raise NotImplementedError()
198
199    def do_open(self, flags, replace):
200        raise NotImplementedError()
201
202    def do_close(self):
203        raise NotImplementedError()
204
205    def acquire_read_lock(self):
206        self.access_lock.acquire_read_lock()
207        try:
208            self.open('r', checkcount=True)
209        except:
210            self.access_lock.release_read_lock()
211            raise
212
213    def release_read_lock(self):
214        try:
215            self.close(checkcount=True)
216        finally:
217            self.access_lock.release_read_lock()
218
219    def acquire_write_lock(self, wait=True, replace=False):
220        r = self.access_lock.acquire_write_lock(wait)
221        try:
222            if (wait or r):
223                self.open('c', checkcount=True, replace=replace)
224            return r
225        except:
226            self.access_lock.release_write_lock()
227            raise
228
229    def release_write_lock(self):
230        try:
231            self.close(checkcount=True)
232        finally:
233            self.access_lock.release_write_lock()
234
235    def open(self, flags, checkcount=False, replace=False):
236        self.mutex.acquire()
237        try:
238            if checkcount:
239                if self.openers == 0:
240                    self.do_open(flags, replace)
241                self.openers += 1
242            else:
243                self.do_open(flags, replace)
244                self.openers = 1
245        finally:
246            self.mutex.release()
247
248    def close(self, checkcount=False):
249        self.mutex.acquire()
250        try:
251            if checkcount:
252                self.openers -= 1
253                if self.openers == 0:
254                    self.do_close()
255            else:
256                if self.openers > 0:
257                    self.do_close()
258                self.openers = 0
259        finally:
260            self.mutex.release()
261
262    def remove(self):
263        self.access_lock.acquire_write_lock()
264        try:
265            self.close(checkcount=False)
266            self.do_remove()
267        finally:
268            self.access_lock.release_write_lock()
269
270
271class Value(object):
272    """Implements synchronization, expiration, and value-creation logic
273    for a single value stored in a :class:`.NamespaceManager`.
274
275    """
276
277    __slots__ = 'key', 'createfunc', 'expiretime', 'expire_argument', 'starttime', 'storedtime',\
278                'namespace'
279
280    def __init__(self, key, namespace, createfunc=None, expiretime=None, starttime=None):
281        self.key = key
282        self.createfunc = createfunc
283        self.expire_argument = expiretime
284        self.starttime = starttime
285        self.storedtime = -1
286        self.namespace = namespace
287
288    def has_value(self):
289        """return true if the container has a value stored.
290
291        This is regardless of it being expired or not.
292
293        """
294        self.namespace.acquire_read_lock()
295        try:
296            return self.key in self.namespace
297        finally:
298            self.namespace.release_read_lock()
299
300    def can_have_value(self):
301        return self.has_current_value() or self.createfunc is not None
302
303    def has_current_value(self):
304        self.namespace.acquire_read_lock()
305        try:
306            has_value = self.key in self.namespace
307            if has_value:
308                try:
309                    stored, expired, value = self._get_value()
310                    return not self._is_expired(stored, expired)
311                except KeyError:
312                    pass
313            return False
314        finally:
315            self.namespace.release_read_lock()
316
317    def _is_expired(self, storedtime, expiretime):
318        """Return true if this container's value is expired."""
319        return (
320            (
321                self.starttime is not None and
322                storedtime < self.starttime
323            )
324            or
325            (
326                expiretime is not None and
327                time.time() >= expiretime + storedtime
328            )
329        )
330
331    def get_value(self):
332        self.namespace.acquire_read_lock()
333        try:
334            has_value = self.has_value()
335            if has_value:
336                try:
337                    stored, expired, value = self._get_value()
338                    if not self._is_expired(stored, expired):
339                        return value
340                except KeyError:
341                    # guard against un-mutexed backends raising KeyError
342                    has_value = False
343
344            if not self.createfunc:
345                raise KeyError(self.key)
346        finally:
347            self.namespace.release_read_lock()
348
349        has_createlock = False
350        creation_lock = self.namespace.get_creation_lock(self.key)
351        if has_value:
352            if not creation_lock.acquire(wait=False):
353                debug("get_value returning old value while new one is created")
354                return value
355            else:
356                debug("lock_creatfunc (didnt wait)")
357                has_createlock = True
358
359        if not has_createlock:
360            debug("lock_createfunc (waiting)")
361            creation_lock.acquire()
362            debug("lock_createfunc (waited)")
363
364        try:
365            # see if someone created the value already
366            self.namespace.acquire_read_lock()
367            try:
368                if self.has_value():
369                    try:
370                        stored, expired, value = self._get_value()
371                        if not self._is_expired(stored, expired):
372                            return value
373                    except KeyError:
374                        # guard against un-mutexed backends raising KeyError
375                        pass
376            finally:
377                self.namespace.release_read_lock()
378
379            debug("get_value creating new value")
380            v = self.createfunc()
381            self.set_value(v)
382            return v
383        finally:
384            creation_lock.release()
385            debug("released create lock")
386
387    def _get_value(self):
388        value = self.namespace[self.key]
389        try:
390            stored, expired, value = value
391        except ValueError:
392            if not len(value) == 2:
393                raise
394            # Old format: upgrade
395            stored, value = value
396            expired = self.expire_argument
397            debug("get_value upgrading time %r expire time %r", stored, self.expire_argument)
398            self.namespace.release_read_lock()
399            self.set_value(value, stored)
400            self.namespace.acquire_read_lock()
401        except TypeError:
402            # occurs when the value is None.  memcached
403            # may yank the rug from under us in which case
404            # that's the result
405            raise KeyError(self.key)
406        return stored, expired, value
407
408    def set_value(self, value, storedtime=None):
409        self.namespace.acquire_write_lock()
410        try:
411            if storedtime is None:
412                storedtime = time.time()
413            debug("set_value stored time %r expire time %r", storedtime, self.expire_argument)
414            self.namespace.set_value(self.key, (storedtime, self.expire_argument, value),
415                                     expiretime=self.expire_argument)
416        finally:
417            self.namespace.release_write_lock()
418
419    def clear_value(self):
420        self.namespace.acquire_write_lock()
421        try:
422            debug("clear_value")
423            if self.key in self.namespace:
424                try:
425                    del self.namespace[self.key]
426                except KeyError:
427                    # guard against un-mutexed backends raising KeyError
428                    pass
429            self.storedtime = -1
430        finally:
431            self.namespace.release_write_lock()
432
433
434class AbstractDictionaryNSManager(NamespaceManager):
435    """A subclassable NamespaceManager that places data in a dictionary.
436
437    Subclasses should provide a "dictionary" attribute or descriptor
438    which returns a dict-like object.   The dictionary will store keys
439    that are local to the "namespace" attribute of this manager, so
440    ensure that the dictionary will not be used by any other namespace.
441
442    e.g.::
443
444        import collections
445        cached_data = collections.defaultdict(dict)
446
447        class MyDictionaryManager(AbstractDictionaryNSManager):
448            def __init__(self, namespace):
449                AbstractDictionaryNSManager.__init__(self, namespace)
450                self.dictionary = cached_data[self.namespace]
451
452    The above stores data in a global dictionary called "cached_data",
453    which is structured as a dictionary of dictionaries, keyed
454    first on namespace name to a sub-dictionary, then on actual
455    cache key to value.
456
457    """
458
459    def get_creation_lock(self, key):
460        return NameLock(
461            identifier="memorynamespace/funclock/%s/%s" %
462                        (self.namespace, key),
463            reentrant=True
464        )
465
466    def __getitem__(self, key):
467        return self.dictionary[key]
468
469    def __contains__(self, key):
470        return self.dictionary.__contains__(key)
471
472    def has_key(self, key):
473        return self.dictionary.__contains__(key)
474
475    def __setitem__(self, key, value):
476        self.dictionary[key] = value
477
478    def __delitem__(self, key):
479        del self.dictionary[key]
480
481    def do_remove(self):
482        self.dictionary.clear()
483
484    def keys(self):
485        return self.dictionary.keys()
486
487
488class MemoryNamespaceManager(AbstractDictionaryNSManager):
489    """:class:`.NamespaceManager` that uses a Python dictionary for storage."""
490
491    namespaces = util.SyncDict()
492
493    def __init__(self, namespace, **kwargs):
494        AbstractDictionaryNSManager.__init__(self, namespace)
495        self.dictionary = MemoryNamespaceManager.\
496                                namespaces.get(self.namespace, dict)
497
498
499class DBMNamespaceManager(OpenResourceNamespaceManager):
500    """:class:`.NamespaceManager` that uses ``dbm`` files for storage."""
501
502    def __init__(self, namespace, dbmmodule=None, data_dir=None,
503            dbm_dir=None, lock_dir=None,
504            digest_filenames=True, **kwargs):
505        self.digest_filenames = digest_filenames
506
507        if not dbm_dir and not data_dir:
508            raise MissingCacheParameter("data_dir or dbm_dir is required")
509        elif dbm_dir:
510            self.dbm_dir = dbm_dir
511        else:
512            self.dbm_dir = data_dir + "/container_dbm"
513        util.verify_directory(self.dbm_dir)
514
515        if not lock_dir and not data_dir:
516            raise MissingCacheParameter("data_dir or lock_dir is required")
517        elif lock_dir:
518            self.lock_dir = lock_dir
519        else:
520            self.lock_dir = data_dir + "/container_dbm_lock"
521        util.verify_directory(self.lock_dir)
522
523        self.dbmmodule = dbmmodule or anydbm
524
525        self.dbm = None
526        OpenResourceNamespaceManager.__init__(self, namespace)
527
528        self.file = util.encoded_path(root=self.dbm_dir,
529                                      identifiers=[self.namespace],
530                                      extension='.dbm',
531                                      digest_filenames=self.digest_filenames)
532
533        debug("data file %s", self.file)
534        self._checkfile()
535
536    def get_access_lock(self):
537        return file_synchronizer(identifier=self.namespace,
538                                 lock_dir=self.lock_dir)
539
540    def get_creation_lock(self, key):
541        return file_synchronizer(
542                    identifier="dbmcontainer/funclock/%s/%s" % (
543                        self.namespace, key
544                    ),
545                    lock_dir=self.lock_dir
546                )
547
548    def file_exists(self, file):
549        if os.access(file, os.F_OK):
550            return True
551        else:
552            for ext in ('db', 'dat', 'pag', 'dir'):
553                if os.access(file + os.extsep + ext, os.F_OK):
554                    return True
555
556        return False
557
558    def _ensuredir(self, filename):
559        dirname = os.path.dirname(filename)
560        if not os.path.exists(dirname):
561            util.verify_directory(dirname)
562
563    def _checkfile(self):
564        if not self.file_exists(self.file):
565            self._ensuredir(self.file)
566            g = self.dbmmodule.open(self.file, 'c')
567            g.close()
568
569    def get_filenames(self):
570        list = []
571        if os.access(self.file, os.F_OK):
572            list.append(self.file)
573
574        for ext in ('pag', 'dir', 'db', 'dat'):
575            if os.access(self.file + os.extsep + ext, os.F_OK):
576                list.append(self.file + os.extsep + ext)
577        return list
578
579    def do_open(self, flags, replace):
580        debug("opening dbm file %s", self.file)
581        try:
582            self.dbm = self.dbmmodule.open(self.file, flags)
583        except:
584            self._checkfile()
585            self.dbm = self.dbmmodule.open(self.file, flags)
586
587    def do_close(self):
588        if self.dbm is not None:
589            debug("closing dbm file %s", self.file)
590            self.dbm.close()
591
592    def do_remove(self):
593        for f in self.get_filenames():
594            os.remove(f)
595
596    def __getitem__(self, key):
597        return pickle.loads(self.dbm[key])
598
599    def __contains__(self, key):
600        if PYVER == (3, 2):
601            # Looks like this is a bug that got solved in PY3.3 and PY3.4
602            # http://bugs.python.org/issue19288
603            if isinstance(key, unicode_text):
604                key = key.encode('UTF-8')
605        return key in self.dbm
606
607    def __setitem__(self, key, value):
608        self.dbm[key] = pickle.dumps(value)
609
610    def __delitem__(self, key):
611        del self.dbm[key]
612
613    def keys(self):
614        return self.dbm.keys()
615
616
617class FileNamespaceManager(OpenResourceNamespaceManager):
618    """:class:`.NamespaceManager` that uses binary files for storage.
619
620    Each namespace is implemented as a single file storing a
621    dictionary of key/value pairs, serialized using the Python
622    ``pickle`` module.
623
624    """
625    def __init__(self, namespace, data_dir=None, file_dir=None, lock_dir=None,
626                 digest_filenames=True, **kwargs):
627        self.digest_filenames = digest_filenames
628
629        if not file_dir and not data_dir:
630            raise MissingCacheParameter("data_dir or file_dir is required")
631        elif file_dir:
632            self.file_dir = file_dir
633        else:
634            self.file_dir = data_dir + "/container_file"
635        util.verify_directory(self.file_dir)
636
637        if not lock_dir and not data_dir:
638            raise MissingCacheParameter("data_dir or lock_dir is required")
639        elif lock_dir:
640            self.lock_dir = lock_dir
641        else:
642            self.lock_dir = data_dir + "/container_file_lock"
643        util.verify_directory(self.lock_dir)
644        OpenResourceNamespaceManager.__init__(self, namespace)
645
646        self.file = util.encoded_path(root=self.file_dir,
647                                      identifiers=[self.namespace],
648                                      extension='.cache',
649                                      digest_filenames=self.digest_filenames)
650        self.hash = {}
651
652        debug("data file %s", self.file)
653
654    def get_access_lock(self):
655        return file_synchronizer(identifier=self.namespace,
656                                 lock_dir=self.lock_dir)
657
658    def get_creation_lock(self, key):
659        return file_synchronizer(
660                identifier="dbmcontainer/funclock/%s/%s" % (
661                    self.namespace, key
662                ),
663                lock_dir=self.lock_dir
664                )
665
666    def file_exists(self, file):
667        return os.access(file, os.F_OK)
668
669    def do_open(self, flags, replace):
670        if not replace and self.file_exists(self.file):
671            try:
672                with open(self.file, 'rb') as fh:
673                    self.hash = pickle.load(fh)
674            except IOError as e:
675                # Ignore EACCES and ENOENT as it just means we are no longer
676                # able to access the file or that it no longer exists
677                if e.errno not in [errno.EACCES, errno.ENOENT]:
678                    raise
679
680        self.flags = flags
681
682    def do_close(self):
683        if self.flags == 'c' or self.flags == 'w':
684            pickled = pickle.dumps(self.hash)
685            util.safe_write(self.file, pickled)
686
687        self.hash = {}
688        self.flags = None
689
690    def do_remove(self):
691        try:
692            os.remove(self.file)
693        except OSError:
694            # for instance, because we haven't yet used this cache,
695            # but client code has asked for a clear() operation...
696            pass
697        self.hash = {}
698
699    def __getitem__(self, key):
700        return self.hash[key]
701
702    def __contains__(self, key):
703        return key in self.hash
704
705    def __setitem__(self, key, value):
706        self.hash[key] = value
707
708    def __delitem__(self, key):
709        del self.hash[key]
710
711    def keys(self):
712        return self.hash.keys()
713
714
715#### legacy stuff to support the old "Container" class interface
716
717namespace_classes = {}
718
719ContainerContext = dict
720
721
722class ContainerMeta(type):
723    def __init__(cls, classname, bases, dict_):
724        namespace_classes[cls] = cls.namespace_class
725        return type.__init__(cls, classname, bases, dict_)
726
727    def __call__(self, key, context, namespace, createfunc=None,
728                 expiretime=None, starttime=None, **kwargs):
729        if namespace in context:
730            ns = context[namespace]
731        else:
732            nscls = namespace_classes[self]
733            context[namespace] = ns = nscls(namespace, **kwargs)
734        return Value(key, ns, createfunc=createfunc,
735                     expiretime=expiretime, starttime=starttime)
736
737@add_metaclass(ContainerMeta)
738class Container(object):
739    """Implements synchronization and value-creation logic
740    for a 'value' stored in a :class:`.NamespaceManager`.
741
742    :class:`.Container` and its subclasses are deprecated.   The
743    :class:`.Value` class is now used for this purpose.
744
745    """
746    namespace_class = NamespaceManager
747
748
749class FileContainer(Container):
750    namespace_class = FileNamespaceManager
751
752
753class MemoryContainer(Container):
754    namespace_class = MemoryNamespaceManager
755
756
757class DBMContainer(Container):
758    namespace_class = DBMNamespaceManager
759
760DbmContainer = DBMContainer
761