1#---------------------------------------------------------------------- 2# Copyright (c) 1999-2001, Digital Creations, Fredericksburg, VA, USA 3# and Andrew Kuchling. All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: 8# 9# o Redistributions of source code must retain the above copyright 10# notice, this list of conditions, and the disclaimer that follows. 11# 12# o Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions, and the following disclaimer in 14# the documentation and/or other materials provided with the 15# distribution. 16# 17# o Neither the name of Digital Creations nor the names of its 18# contributors may be used to endorse or promote products derived 19# from this software without specific prior written permission. 20# 21# THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS AND CONTRIBUTORS *AS 22# IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL 25# CREATIONS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 28# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 29# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 30# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 31# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 32# DAMAGE. 33#---------------------------------------------------------------------- 34 35 36"""Support for Berkeley DB 4.3 through 5.3 with a simple interface. 37 38For the full featured object oriented interface use the berkeleydb.db module 39instead. It mirrors the Oracle Berkeley DB C API. 40""" 41 42import sys 43 44try: 45 from . import _berkeleydb 46 from berkeleydb.dbutils import DeadlockWrap as _DeadlockWrap 47except ImportError: 48 # Remove ourselves from sys.modules 49 import sys 50 del sys.modules[__name__] 51 raise 52 53# berkeleydb calls it db, but provide _db for backwards compatibility 54db = _db = _berkeleydb 55__version__ = db.__version__ 56 57error = db.DBError # So berkeleydb.error will mean something... 58 59#---------------------------------------------------------------------- 60 61import sys, os 62 63from collections.abc import MutableMapping 64 65class _iter_mixin(MutableMapping): 66 def _make_iter_cursor(self): 67 cur = _DeadlockWrap(self.db.cursor) 68 self._cursor_refs[id(cur)] = cur 69 return cur 70 71 def _free_cursor(self, cur): 72 _DeadlockWrap(cur.close) 73 del self._cursor_refs[id(cur)] 74 75 def __iter__(self): 76 self._kill_iteration = False 77 self._in_iter += 1 78 try: 79 try: 80 cur = self._make_iter_cursor() 81 82 # FIXME-20031102-greg: race condition. cursor could 83 # be closed by another thread before this call. 84 85 # since we're only returning keys, we call the cursor 86 # methods with flags=0, dlen=0, dofs=0 87 key = _DeadlockWrap(cur.first, 0, 0, 0)[0] 88 yield key 89 90 while True: 91 try: 92 key = _DeadlockWrap(cur.next, 0, 0, 0)[0] 93 yield key 94 except _db.DBCursorClosedError: 95 if self._kill_iteration: 96 raise RuntimeError('Database changed size ' 97 'during iteration.') 98 self._free_cursor(cur) 99 cur = self._make_iter_cursor() 100 # FIXME-20031101-greg: race condition. cursor could 101 # be closed by another thread before this call. 102 _DeadlockWrap(cur.set, key, 0, 0, 0) 103 except _db.DBNotFoundError: 104 pass 105 except _db.DBCursorClosedError: 106 # the database was modified during iteration. abort. 107 pass 108 finally: 109 self._free_cursor(cur) 110 finally : 111 self._in_iter -= 1 112 113 def iteritems(self): 114 if not self.db: 115 return 116 self._kill_iteration = False 117 self._in_iter += 1 118 try: 119 try: 120 cur = self._make_iter_cursor() 121 122 # FIXME-20031102-greg: race condition. cursor could 123 # be closed by another thread before this call. 124 125 kv = _DeadlockWrap(cur.first) 126 key = kv[0] 127 yield kv 128 129 while True: 130 try: 131 kv = _DeadlockWrap(cur.next) 132 key = kv[0] 133 yield kv 134 except _db.DBCursorClosedError: 135 if self._kill_iteration: 136 raise RuntimeError('Database changed size ' 137 'during iteration.') 138 self._free_cursor(cur) 139 cur = self._make_iter_cursor() 140 # FIXME-20031101-greg: race condition. cursor could 141 # be closed by another thread before this call. 142 _DeadlockWrap(cur.set, key, 0, 0, 0) 143 except _db.DBNotFoundError: 144 pass 145 except _db.DBCursorClosedError: 146 # the database was modified during iteration. abort. 147 pass 148 finally: 149 self._free_cursor(cur) 150 finally : 151 self._in_iter -= 1 152 153 154class _DBWithCursor(_iter_mixin): 155 """ 156 A simple wrapper around DB that makes it look like the berkeleydbobject in 157 the old module. It uses a cursor as needed to provide DB traversal. 158 """ 159 def __init__(self, db): 160 self.db = db 161 self.db.set_get_returns_none(0) 162 163 # FIXME-20031101-greg: I believe there is still the potential 164 # for deadlocks in a multithreaded environment if someone 165 # attempts to use the any of the cursor interfaces in one 166 # thread while doing a put or delete in another thread. The 167 # reason is that _checkCursor and _closeCursors are not atomic 168 # operations. Doing our own locking around self.dbc, 169 # self.saved_dbc_key and self._cursor_refs could prevent this. 170 # TODO: A test case demonstrating the problem needs to be written. 171 172 # self.dbc is a DBCursor object used to implement the 173 # first/next/previous/last/set_location methods. 174 self.dbc = None 175 self.saved_dbc_key = None 176 177 # a collection of all DBCursor objects currently allocated 178 # by the _iter_mixin interface. 179 self._cursor_refs = {} 180 self._in_iter = 0 181 self._kill_iteration = False 182 183 def __del__(self): 184 self.close() 185 186 def _checkCursor(self): 187 if self.dbc is None: 188 self.dbc = _DeadlockWrap(self.db.cursor) 189 if self.saved_dbc_key is not None: 190 _DeadlockWrap(self.dbc.set, self.saved_dbc_key) 191 self.saved_dbc_key = None 192 193 # This method is needed for all non-cursor DB calls to avoid 194 # Berkeley DB deadlocks (due to being opened with DB_INIT_LOCK 195 # and DB_THREAD to be thread safe) when intermixing database 196 # operations that use the cursor internally with those that don't. 197 def _closeCursors(self, save=1): 198 if self.dbc: 199 c = self.dbc 200 self.dbc = None 201 if save: 202 try: 203 self.saved_dbc_key = _DeadlockWrap(c.current, 0, 0, 0)[0] 204 except db.DBError: 205 pass 206 _DeadlockWrap(c.close) 207 del c 208 for cursor in list(self._cursor_refs.values()): 209 self._free_cursor(cursor) 210 211 def _checkOpen(self): 212 if self.db is None: 213 raise error("berkeleydb object has already been closed") 214 215 def isOpen(self): 216 return self.db is not None 217 218 def __len__(self): 219 self._checkOpen() 220 return _DeadlockWrap(lambda: len(self.db)) # len(self.db) 221 222 def __repr__(self) : 223 if self.isOpen() : 224 return repr(dict(_DeadlockWrap(self.db.items))) 225 return repr(dict()) 226 227 def __getitem__(self, key): 228 self._checkOpen() 229 return _DeadlockWrap(lambda: self.db[key]) # self.db[key] 230 231 def __setitem__(self, key, value): 232 self._checkOpen() 233 self._closeCursors() 234 if self._in_iter and key not in self: 235 self._kill_iteration = True 236 def wrapF(): 237 self.db[key] = value 238 _DeadlockWrap(wrapF) # self.db[key] = value 239 240 def __delitem__(self, key): 241 self._checkOpen() 242 self._closeCursors() 243 if self._in_iter and key in self: 244 self._kill_iteration = True 245 def wrapF(): 246 del self.db[key] 247 _DeadlockWrap(wrapF) # del self.db[key] 248 249 def close(self): 250 self._closeCursors(save=0) 251 if self.dbc is not None: 252 _DeadlockWrap(self.dbc.close) 253 v = 0 254 if self.db is not None: 255 v = _DeadlockWrap(self.db.close) 256 self.dbc = None 257 self.db = None 258 return v 259 260 def keys(self): 261 self._checkOpen() 262 return _DeadlockWrap(self.db.keys) 263 264 def has_key(self, key): 265 self._checkOpen() 266 return _DeadlockWrap(self.db.has_key, key) 267 268 def set_location(self, key): 269 self._checkOpen() 270 self._checkCursor() 271 return _DeadlockWrap(self.dbc.set_range, key) 272 273 def next(self): 274 self._checkOpen() 275 self._checkCursor() 276 rv = _DeadlockWrap(self.dbc.next) 277 return rv 278 279 def __next__(self): 280 return self.next() 281 282 def previous(self): 283 self._checkOpen() 284 self._checkCursor() 285 rv = _DeadlockWrap(self.dbc.prev) 286 return rv 287 288 def first(self): 289 self._checkOpen() 290 # fix 1725856: don't needlessly try to restore our cursor position 291 self.saved_dbc_key = None 292 self._checkCursor() 293 rv = _DeadlockWrap(self.dbc.first) 294 return rv 295 296 def last(self): 297 self._checkOpen() 298 # fix 1725856: don't needlessly try to restore our cursor position 299 self.saved_dbc_key = None 300 self._checkCursor() 301 rv = _DeadlockWrap(self.dbc.last) 302 return rv 303 304 def sync(self): 305 self._checkOpen() 306 return _DeadlockWrap(self.db.sync) 307 308 309#---------------------------------------------------------------------- 310# Compatibility object factory functions 311 312def hashopen(file, flag='c', mode=0o666, pgsize=None, ffactor=None, nelem=None, 313 cachesize=None, lorder=None, hflags=0): 314 315 flags = _checkflag(flag, file) 316 e = _openDBEnv(cachesize) 317 d = db.DB(e) 318 d.set_flags(hflags) 319 if pgsize is not None: d.set_pagesize(pgsize) 320 if lorder is not None: d.set_lorder(lorder) 321 if ffactor is not None: d.set_h_ffactor(ffactor) 322 if nelem is not None: d.set_h_nelem(nelem) 323 d.open(file, db.DB_HASH, flags, mode) 324 return _DBWithCursor(d) 325 326#---------------------------------------------------------------------- 327 328def btopen(file, flag='c', mode=0o666, 329 btflags=0, cachesize=None, maxkeypage=None, minkeypage=None, 330 pgsize=None, lorder=None): 331 332 flags = _checkflag(flag, file) 333 e = _openDBEnv(cachesize) 334 d = db.DB(e) 335 if pgsize is not None: d.set_pagesize(pgsize) 336 if lorder is not None: d.set_lorder(lorder) 337 d.set_flags(btflags) 338 if minkeypage is not None: d.set_bt_minkey(minkeypage) 339 if maxkeypage is not None: d.set_bt_maxkey(maxkeypage) 340 d.open(file, db.DB_BTREE, flags, mode) 341 return _DBWithCursor(d) 342 343#---------------------------------------------------------------------- 344 345 346def rnopen(file, flag='c', mode=0o666, 347 rnflags=0, cachesize=None, pgsize=None, lorder=None, 348 rlen=None, delim=None, source=None, pad=None): 349 350 flags = _checkflag(flag, file) 351 e = _openDBEnv(cachesize) 352 d = db.DB(e) 353 if pgsize is not None: d.set_pagesize(pgsize) 354 if lorder is not None: d.set_lorder(lorder) 355 d.set_flags(rnflags) 356 if delim is not None: d.set_re_delim(delim) 357 if rlen is not None: d.set_re_len(rlen) 358 if source is not None: d.set_re_source(source) 359 if pad is not None: d.set_re_pad(pad) 360 d.open(file, db.DB_RECNO, flags, mode) 361 return _DBWithCursor(d) 362 363#---------------------------------------------------------------------- 364 365def _openDBEnv(cachesize): 366 e = db.DBEnv() 367 if cachesize is not None: 368 if cachesize >= 20480: 369 e.set_cachesize(0, cachesize) 370 else: 371 raise error("cachesize must be >= 20480") 372 e.set_lk_detect(db.DB_LOCK_DEFAULT) 373 e.open('.', db.DB_PRIVATE | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_LOCK | db.DB_INIT_MPOOL) 374 return e 375 376def _checkflag(flag, file): 377 if flag == 'r': 378 flags = db.DB_RDONLY 379 elif flag == 'rw': 380 flags = 0 381 elif flag == 'w': 382 flags = db.DB_CREATE 383 elif flag == 'c': 384 flags = db.DB_CREATE 385 elif flag == 'n': 386 flags = db.DB_CREATE 387 #flags = db.DB_CREATE | db.DB_TRUNCATE 388 # we used db.DB_TRUNCATE flag for this before but Berkeley DB 389 # 4.2.52 changed to disallowed truncate with txn environments. 390 if file is not None and os.path.isfile(file): 391 os.unlink(file) 392 else: 393 raise error("flags should be one of 'r', 'w', 'c' or 'n'") 394 return flags | db.DB_THREAD 395 396#---------------------------------------------------------------------- 397 398 399# This is a silly little hack that allows apps to continue to use the 400# DB_THREAD flag even on systems without threads without freaking out 401# Berkeley DB. 402# 403# This assumes that if Python was built with thread support then 404# Berkeley DB was too. 405 406try: 407 import _thread as T 408 del T 409except ImportError: 410 db.DB_THREAD = 0 411 412#---------------------------------------------------------------------- 413