1"""Container and Namespace classes""" 2import errno 3 4from ._compat import pickle, anydbm, add_metaclass, PYVER, unicode_text 5 6import beaker.util as util 7import logging 8import os 9import time 10 11from beaker.exceptions import CreationAbortedError, MissingCacheParameter 12from beaker.synchronization import _threading, file_synchronizer, \ 13 mutex_synchronizer, NameLock, null_synchronizer 14 15__all__ = ['Value', 'Container', 'ContainerContext', 16 'MemoryContainer', 'DBMContainer', 'NamespaceManager', 17 'MemoryNamespaceManager', 'DBMNamespaceManager', 'FileContainer', 18 'OpenResourceNamespaceManager', 19 'FileNamespaceManager', 'CreationAbortedError'] 20 21 22logger = logging.getLogger('beaker.container') 23if logger.isEnabledFor(logging.DEBUG): 24 debug = logger.debug 25else: 26 def debug(message, *args): 27 pass 28 29 30class NamespaceManager(object): 31 """Handles dictionary operations and locking for a namespace of 32 values. 33 34 :class:`.NamespaceManager` provides a dictionary-like interface, 35 implementing ``__getitem__()``, ``__setitem__()``, and 36 ``__contains__()``, as well as functions related to lock 37 acquisition. 38 39 The implementation for setting and retrieving the namespace data is 40 handled by subclasses. 41 42 NamespaceManager may be used alone, or may be accessed by 43 one or more :class:`.Value` objects. :class:`.Value` objects provide per-key 44 services like expiration times and automatic recreation of values. 45 46 Multiple NamespaceManagers created with a particular name will all 47 share access to the same underlying datasource and will attempt to 48 synchronize against a common mutex object. The scope of this 49 sharing may be within a single process or across multiple 50 processes, depending on the type of NamespaceManager used. 51 52 The NamespaceManager itself is generally threadsafe, except in the 53 case of the DBMNamespaceManager in conjunction with the gdbm dbm 54 implementation. 55 56 """ 57 58 @classmethod 59 def _init_dependencies(cls): 60 """Initialize module-level dependent libraries required 61 by this :class:`.NamespaceManager`.""" 62 63 def __init__(self, namespace): 64 self._init_dependencies() 65 self.namespace = namespace 66 67 def get_creation_lock(self, key): 68 """Return a locking object that is used to synchronize 69 multiple threads or processes which wish to generate a new 70 cache value. 71 72 This function is typically an instance of 73 :class:`.FileSynchronizer`, :class:`.ConditionSynchronizer`, 74 or :class:`.null_synchronizer`. 75 76 The creation lock is only used when a requested value 77 does not exist, or has been expired, and is only used 78 by the :class:`.Value` key-management object in conjunction 79 with a "createfunc" value-creation function. 80 81 """ 82 raise NotImplementedError() 83 84 def do_remove(self): 85 """Implement removal of the entire contents of this 86 :class:`.NamespaceManager`. 87 88 e.g. for a file-based namespace, this would remove 89 all the files. 90 91 The front-end to this method is the 92 :meth:`.NamespaceManager.remove` method. 93 94 """ 95 raise NotImplementedError() 96 97 def acquire_read_lock(self): 98 """Establish a read lock. 99 100 This operation is called before a key is read. By 101 default the function does nothing. 102 103 """ 104 105 def release_read_lock(self): 106 """Release a read lock. 107 108 This operation is called after a key is read. By 109 default the function does nothing. 110 111 """ 112 113 def acquire_write_lock(self, wait=True, replace=False): 114 """Establish a write lock. 115 116 This operation is called before a key is written. 117 A return value of ``True`` indicates the lock has 118 been acquired. 119 120 By default the function returns ``True`` unconditionally. 121 122 'replace' is a hint indicating the full contents 123 of the namespace may be safely discarded. Some backends 124 may implement this (i.e. file backend won't unpickle the 125 current contents). 126 127 """ 128 return True 129 130 def release_write_lock(self): 131 """Release a write lock. 132 133 This operation is called after a new value is written. 134 By default this function does nothing. 135 136 """ 137 138 def has_key(self, key): 139 """Return ``True`` if the given key is present in this 140 :class:`.Namespace`. 141 """ 142 return self.__contains__(key) 143 144 def __getitem__(self, key): 145 raise NotImplementedError() 146 147 def __setitem__(self, key, value): 148 raise NotImplementedError() 149 150 def set_value(self, key, value, expiretime=None): 151 """Sets a value in this :class:`.NamespaceManager`. 152 153 This is the same as ``__setitem__()``, but 154 also allows an expiration time to be passed 155 at the same time. 156 157 """ 158 self[key] = value 159 160 def __contains__(self, key): 161 raise NotImplementedError() 162 163 def __delitem__(self, key): 164 raise NotImplementedError() 165 166 def keys(self): 167 """Return the list of all keys. 168 169 This method may not be supported by all 170 :class:`.NamespaceManager` implementations. 171 172 """ 173 raise NotImplementedError() 174 175 def remove(self): 176 """Remove the entire contents of this 177 :class:`.NamespaceManager`. 178 179 e.g. for a file-based namespace, this would remove 180 all the files. 181 """ 182 self.do_remove() 183 184 185class OpenResourceNamespaceManager(NamespaceManager): 186 """A NamespaceManager where read/write operations require opening/ 187 closing of a resource which is possibly mutexed. 188 189 """ 190 def __init__(self, namespace): 191 NamespaceManager.__init__(self, namespace) 192 self.access_lock = self.get_access_lock() 193 self.openers = 0 194 self.mutex = _threading.Lock() 195 196 def get_access_lock(self): 197 raise NotImplementedError() 198 199 def do_open(self, flags, replace): 200 raise NotImplementedError() 201 202 def do_close(self): 203 raise NotImplementedError() 204 205 def acquire_read_lock(self): 206 self.access_lock.acquire_read_lock() 207 try: 208 self.open('r', checkcount=True) 209 except: 210 self.access_lock.release_read_lock() 211 raise 212 213 def release_read_lock(self): 214 try: 215 self.close(checkcount=True) 216 finally: 217 self.access_lock.release_read_lock() 218 219 def acquire_write_lock(self, wait=True, replace=False): 220 r = self.access_lock.acquire_write_lock(wait) 221 try: 222 if (wait or r): 223 self.open('c', checkcount=True, replace=replace) 224 return r 225 except: 226 self.access_lock.release_write_lock() 227 raise 228 229 def release_write_lock(self): 230 try: 231 self.close(checkcount=True) 232 finally: 233 self.access_lock.release_write_lock() 234 235 def open(self, flags, checkcount=False, replace=False): 236 self.mutex.acquire() 237 try: 238 if checkcount: 239 if self.openers == 0: 240 self.do_open(flags, replace) 241 self.openers += 1 242 else: 243 self.do_open(flags, replace) 244 self.openers = 1 245 finally: 246 self.mutex.release() 247 248 def close(self, checkcount=False): 249 self.mutex.acquire() 250 try: 251 if checkcount: 252 self.openers -= 1 253 if self.openers == 0: 254 self.do_close() 255 else: 256 if self.openers > 0: 257 self.do_close() 258 self.openers = 0 259 finally: 260 self.mutex.release() 261 262 def remove(self): 263 self.access_lock.acquire_write_lock() 264 try: 265 self.close(checkcount=False) 266 self.do_remove() 267 finally: 268 self.access_lock.release_write_lock() 269 270 271class Value(object): 272 """Implements synchronization, expiration, and value-creation logic 273 for a single value stored in a :class:`.NamespaceManager`. 274 275 """ 276 277 __slots__ = 'key', 'createfunc', 'expiretime', 'expire_argument', 'starttime', 'storedtime',\ 278 'namespace' 279 280 def __init__(self, key, namespace, createfunc=None, expiretime=None, starttime=None): 281 self.key = key 282 self.createfunc = createfunc 283 self.expire_argument = expiretime 284 self.starttime = starttime 285 self.storedtime = -1 286 self.namespace = namespace 287 288 def has_value(self): 289 """return true if the container has a value stored. 290 291 This is regardless of it being expired or not. 292 293 """ 294 self.namespace.acquire_read_lock() 295 try: 296 return self.key in self.namespace 297 finally: 298 self.namespace.release_read_lock() 299 300 def can_have_value(self): 301 return self.has_current_value() or self.createfunc is not None 302 303 def has_current_value(self): 304 self.namespace.acquire_read_lock() 305 try: 306 has_value = self.key in self.namespace 307 if has_value: 308 try: 309 stored, expired, value = self._get_value() 310 return not self._is_expired(stored, expired) 311 except KeyError: 312 pass 313 return False 314 finally: 315 self.namespace.release_read_lock() 316 317 def _is_expired(self, storedtime, expiretime): 318 """Return true if this container's value is expired.""" 319 return ( 320 ( 321 self.starttime is not None and 322 storedtime < self.starttime 323 ) 324 or 325 ( 326 expiretime is not None and 327 time.time() >= expiretime + storedtime 328 ) 329 ) 330 331 def get_value(self): 332 self.namespace.acquire_read_lock() 333 try: 334 has_value = self.has_value() 335 if has_value: 336 try: 337 stored, expired, value = self._get_value() 338 if not self._is_expired(stored, expired): 339 return value 340 except KeyError: 341 # guard against un-mutexed backends raising KeyError 342 has_value = False 343 344 if not self.createfunc: 345 raise KeyError(self.key) 346 finally: 347 self.namespace.release_read_lock() 348 349 has_createlock = False 350 creation_lock = self.namespace.get_creation_lock(self.key) 351 if has_value: 352 if not creation_lock.acquire(wait=False): 353 debug("get_value returning old value while new one is created") 354 return value 355 else: 356 debug("lock_creatfunc (didnt wait)") 357 has_createlock = True 358 359 if not has_createlock: 360 debug("lock_createfunc (waiting)") 361 creation_lock.acquire() 362 debug("lock_createfunc (waited)") 363 364 try: 365 # see if someone created the value already 366 self.namespace.acquire_read_lock() 367 try: 368 if self.has_value(): 369 try: 370 stored, expired, value = self._get_value() 371 if not self._is_expired(stored, expired): 372 return value 373 except KeyError: 374 # guard against un-mutexed backends raising KeyError 375 pass 376 finally: 377 self.namespace.release_read_lock() 378 379 debug("get_value creating new value") 380 v = self.createfunc() 381 self.set_value(v) 382 return v 383 finally: 384 creation_lock.release() 385 debug("released create lock") 386 387 def _get_value(self): 388 value = self.namespace[self.key] 389 try: 390 stored, expired, value = value 391 except ValueError: 392 if not len(value) == 2: 393 raise 394 # Old format: upgrade 395 stored, value = value 396 expired = self.expire_argument 397 debug("get_value upgrading time %r expire time %r", stored, self.expire_argument) 398 self.namespace.release_read_lock() 399 self.set_value(value, stored) 400 self.namespace.acquire_read_lock() 401 except TypeError: 402 # occurs when the value is None. memcached 403 # may yank the rug from under us in which case 404 # that's the result 405 raise KeyError(self.key) 406 return stored, expired, value 407 408 def set_value(self, value, storedtime=None): 409 self.namespace.acquire_write_lock() 410 try: 411 if storedtime is None: 412 storedtime = time.time() 413 debug("set_value stored time %r expire time %r", storedtime, self.expire_argument) 414 self.namespace.set_value(self.key, (storedtime, self.expire_argument, value), 415 expiretime=self.expire_argument) 416 finally: 417 self.namespace.release_write_lock() 418 419 def clear_value(self): 420 self.namespace.acquire_write_lock() 421 try: 422 debug("clear_value") 423 if self.key in self.namespace: 424 try: 425 del self.namespace[self.key] 426 except KeyError: 427 # guard against un-mutexed backends raising KeyError 428 pass 429 self.storedtime = -1 430 finally: 431 self.namespace.release_write_lock() 432 433 434class AbstractDictionaryNSManager(NamespaceManager): 435 """A subclassable NamespaceManager that places data in a dictionary. 436 437 Subclasses should provide a "dictionary" attribute or descriptor 438 which returns a dict-like object. The dictionary will store keys 439 that are local to the "namespace" attribute of this manager, so 440 ensure that the dictionary will not be used by any other namespace. 441 442 e.g.:: 443 444 import collections 445 cached_data = collections.defaultdict(dict) 446 447 class MyDictionaryManager(AbstractDictionaryNSManager): 448 def __init__(self, namespace): 449 AbstractDictionaryNSManager.__init__(self, namespace) 450 self.dictionary = cached_data[self.namespace] 451 452 The above stores data in a global dictionary called "cached_data", 453 which is structured as a dictionary of dictionaries, keyed 454 first on namespace name to a sub-dictionary, then on actual 455 cache key to value. 456 457 """ 458 459 def get_creation_lock(self, key): 460 return NameLock( 461 identifier="memorynamespace/funclock/%s/%s" % 462 (self.namespace, key), 463 reentrant=True 464 ) 465 466 def __getitem__(self, key): 467 return self.dictionary[key] 468 469 def __contains__(self, key): 470 return self.dictionary.__contains__(key) 471 472 def has_key(self, key): 473 return self.dictionary.__contains__(key) 474 475 def __setitem__(self, key, value): 476 self.dictionary[key] = value 477 478 def __delitem__(self, key): 479 del self.dictionary[key] 480 481 def do_remove(self): 482 self.dictionary.clear() 483 484 def keys(self): 485 return self.dictionary.keys() 486 487 488class MemoryNamespaceManager(AbstractDictionaryNSManager): 489 """:class:`.NamespaceManager` that uses a Python dictionary for storage.""" 490 491 namespaces = util.SyncDict() 492 493 def __init__(self, namespace, **kwargs): 494 AbstractDictionaryNSManager.__init__(self, namespace) 495 self.dictionary = MemoryNamespaceManager.\ 496 namespaces.get(self.namespace, dict) 497 498 499class DBMNamespaceManager(OpenResourceNamespaceManager): 500 """:class:`.NamespaceManager` that uses ``dbm`` files for storage.""" 501 502 def __init__(self, namespace, dbmmodule=None, data_dir=None, 503 dbm_dir=None, lock_dir=None, 504 digest_filenames=True, **kwargs): 505 self.digest_filenames = digest_filenames 506 507 if not dbm_dir and not data_dir: 508 raise MissingCacheParameter("data_dir or dbm_dir is required") 509 elif dbm_dir: 510 self.dbm_dir = dbm_dir 511 else: 512 self.dbm_dir = data_dir + "/container_dbm" 513 util.verify_directory(self.dbm_dir) 514 515 if not lock_dir and not data_dir: 516 raise MissingCacheParameter("data_dir or lock_dir is required") 517 elif lock_dir: 518 self.lock_dir = lock_dir 519 else: 520 self.lock_dir = data_dir + "/container_dbm_lock" 521 util.verify_directory(self.lock_dir) 522 523 self.dbmmodule = dbmmodule or anydbm 524 525 self.dbm = None 526 OpenResourceNamespaceManager.__init__(self, namespace) 527 528 self.file = util.encoded_path(root=self.dbm_dir, 529 identifiers=[self.namespace], 530 extension='.dbm', 531 digest_filenames=self.digest_filenames) 532 533 debug("data file %s", self.file) 534 self._checkfile() 535 536 def get_access_lock(self): 537 return file_synchronizer(identifier=self.namespace, 538 lock_dir=self.lock_dir) 539 540 def get_creation_lock(self, key): 541 return file_synchronizer( 542 identifier="dbmcontainer/funclock/%s/%s" % ( 543 self.namespace, key 544 ), 545 lock_dir=self.lock_dir 546 ) 547 548 def file_exists(self, file): 549 if os.access(file, os.F_OK): 550 return True 551 else: 552 for ext in ('db', 'dat', 'pag', 'dir'): 553 if os.access(file + os.extsep + ext, os.F_OK): 554 return True 555 556 return False 557 558 def _ensuredir(self, filename): 559 dirname = os.path.dirname(filename) 560 if not os.path.exists(dirname): 561 util.verify_directory(dirname) 562 563 def _checkfile(self): 564 if not self.file_exists(self.file): 565 self._ensuredir(self.file) 566 g = self.dbmmodule.open(self.file, 'c') 567 g.close() 568 569 def get_filenames(self): 570 list = [] 571 if os.access(self.file, os.F_OK): 572 list.append(self.file) 573 574 for ext in ('pag', 'dir', 'db', 'dat'): 575 if os.access(self.file + os.extsep + ext, os.F_OK): 576 list.append(self.file + os.extsep + ext) 577 return list 578 579 def do_open(self, flags, replace): 580 debug("opening dbm file %s", self.file) 581 try: 582 self.dbm = self.dbmmodule.open(self.file, flags) 583 except: 584 self._checkfile() 585 self.dbm = self.dbmmodule.open(self.file, flags) 586 587 def do_close(self): 588 if self.dbm is not None: 589 debug("closing dbm file %s", self.file) 590 self.dbm.close() 591 592 def do_remove(self): 593 for f in self.get_filenames(): 594 os.remove(f) 595 596 def __getitem__(self, key): 597 return pickle.loads(self.dbm[key]) 598 599 def __contains__(self, key): 600 if PYVER == (3, 2): 601 # Looks like this is a bug that got solved in PY3.3 and PY3.4 602 # http://bugs.python.org/issue19288 603 if isinstance(key, unicode_text): 604 key = key.encode('UTF-8') 605 return key in self.dbm 606 607 def __setitem__(self, key, value): 608 self.dbm[key] = pickle.dumps(value) 609 610 def __delitem__(self, key): 611 del self.dbm[key] 612 613 def keys(self): 614 return self.dbm.keys() 615 616 617class FileNamespaceManager(OpenResourceNamespaceManager): 618 """:class:`.NamespaceManager` that uses binary files for storage. 619 620 Each namespace is implemented as a single file storing a 621 dictionary of key/value pairs, serialized using the Python 622 ``pickle`` module. 623 624 """ 625 def __init__(self, namespace, data_dir=None, file_dir=None, lock_dir=None, 626 digest_filenames=True, **kwargs): 627 self.digest_filenames = digest_filenames 628 629 if not file_dir and not data_dir: 630 raise MissingCacheParameter("data_dir or file_dir is required") 631 elif file_dir: 632 self.file_dir = file_dir 633 else: 634 self.file_dir = data_dir + "/container_file" 635 util.verify_directory(self.file_dir) 636 637 if not lock_dir and not data_dir: 638 raise MissingCacheParameter("data_dir or lock_dir is required") 639 elif lock_dir: 640 self.lock_dir = lock_dir 641 else: 642 self.lock_dir = data_dir + "/container_file_lock" 643 util.verify_directory(self.lock_dir) 644 OpenResourceNamespaceManager.__init__(self, namespace) 645 646 self.file = util.encoded_path(root=self.file_dir, 647 identifiers=[self.namespace], 648 extension='.cache', 649 digest_filenames=self.digest_filenames) 650 self.hash = {} 651 652 debug("data file %s", self.file) 653 654 def get_access_lock(self): 655 return file_synchronizer(identifier=self.namespace, 656 lock_dir=self.lock_dir) 657 658 def get_creation_lock(self, key): 659 return file_synchronizer( 660 identifier="dbmcontainer/funclock/%s/%s" % ( 661 self.namespace, key 662 ), 663 lock_dir=self.lock_dir 664 ) 665 666 def file_exists(self, file): 667 return os.access(file, os.F_OK) 668 669 def do_open(self, flags, replace): 670 if not replace and self.file_exists(self.file): 671 try: 672 with open(self.file, 'rb') as fh: 673 self.hash = pickle.load(fh) 674 except IOError as e: 675 # Ignore EACCES and ENOENT as it just means we are no longer 676 # able to access the file or that it no longer exists 677 if e.errno not in [errno.EACCES, errno.ENOENT]: 678 raise 679 680 self.flags = flags 681 682 def do_close(self): 683 if self.flags == 'c' or self.flags == 'w': 684 pickled = pickle.dumps(self.hash) 685 util.safe_write(self.file, pickled) 686 687 self.hash = {} 688 self.flags = None 689 690 def do_remove(self): 691 try: 692 os.remove(self.file) 693 except OSError: 694 # for instance, because we haven't yet used this cache, 695 # but client code has asked for a clear() operation... 696 pass 697 self.hash = {} 698 699 def __getitem__(self, key): 700 return self.hash[key] 701 702 def __contains__(self, key): 703 return key in self.hash 704 705 def __setitem__(self, key, value): 706 self.hash[key] = value 707 708 def __delitem__(self, key): 709 del self.hash[key] 710 711 def keys(self): 712 return self.hash.keys() 713 714 715#### legacy stuff to support the old "Container" class interface 716 717namespace_classes = {} 718 719ContainerContext = dict 720 721 722class ContainerMeta(type): 723 def __init__(cls, classname, bases, dict_): 724 namespace_classes[cls] = cls.namespace_class 725 return type.__init__(cls, classname, bases, dict_) 726 727 def __call__(self, key, context, namespace, createfunc=None, 728 expiretime=None, starttime=None, **kwargs): 729 if namespace in context: 730 ns = context[namespace] 731 else: 732 nscls = namespace_classes[self] 733 context[namespace] = ns = nscls(namespace, **kwargs) 734 return Value(key, ns, createfunc=createfunc, 735 expiretime=expiretime, starttime=starttime) 736 737@add_metaclass(ContainerMeta) 738class Container(object): 739 """Implements synchronization and value-creation logic 740 for a 'value' stored in a :class:`.NamespaceManager`. 741 742 :class:`.Container` and its subclasses are deprecated. The 743 :class:`.Value` class is now used for this purpose. 744 745 """ 746 namespace_class = NamespaceManager 747 748 749class FileContainer(Container): 750 namespace_class = FileNamespaceManager 751 752 753class MemoryContainer(Container): 754 namespace_class = MemoryNamespaceManager 755 756 757class DBMContainer(Container): 758 namespace_class = DBMNamespaceManager 759 760DbmContainer = DBMContainer 761