1 /* $Id: bdb_env.cpp 624724 2021-02-03 18:51:44Z ivanov $
2 * ===========================================================================
3 *
4 * PUBLIC DOMAIN NOTICE
5 * National Center for Biotechnology Information
6 *
7 * This software/database is a "United States Government Work" under the
8 * terms of the United States Copyright Act. It was written as part of
9 * the author's official duties as a United States Government employee and
10 * thus cannot be copyrighted. This software/database is freely available
11 * to the public for use. The National Library of Medicine and the U.S.
12 * Government have not placed any restriction on its use or reproduction.
13 *
14 * Although all reasonable efforts have been taken to ensure the accuracy
15 * and reliability of the software and data, the NLM and the U.S.
16 * Government do not and cannot warrant the performance or results that
17 * may be obtained by using this software or data. The NLM and the U.S.
18 * Government disclaim all warranties, express or implied, including
19 * warranties of performance, merchantability or fitness for any particular
20 * purpose.
21 *
22 * Please cite the author in any work or product based on this material.
23 *
24 * ===========================================================================
25 *
26 * Author: Anatoliy Kuznetsov
27 *
28 * File Description: BDB libarary environment class implementation.
29 *
30 */
31
32 #include <ncbi_pch.hpp>
33
34 #include <corelib/ncbifile.hpp>
35
36 #include <db/bdb/bdb_env.hpp>
37 #include <db/bdb/bdb_trans.hpp>
38 #include <db/bdb/bdb_checkpoint_thread.hpp>
39
40 #include <db/error_codes.hpp>
41
42 #include <connect/server_monitor.hpp>
43 #include <common/ncbi_sanitizers.h>
44
45 #include <db.h>
46
47
48 // Berkeley DB sometimes changes the inteface so some calls have to be adjusted
49 // depending on the version
50 #ifdef BDB_FULL_VERSION
51 #undef BDB_FULL_VERSION
52 #endif
53 #define BDB_FULL_VERSION DB_VERSION_PATCH + \
54 1000 * DB_VERSION_MINOR + \
55 1000 * 1000 * DB_VERSION_MAJOR
56
57
58
59 // Berkeley DB 4.4.x reworked and extended the mutex API.
60 #if DB_VERSION_MAJOR >= 4
61 #if DB_VERSION_MINOR >= 4 || DB_VERSION_MAJOR > 4
62 #define BDB_NEW_MUTEX_API
63 #endif
64 #if DB_VERSION_MINOR >= 5 || DB_VERSION_MAJOR > 4
65 #define BDB_NEW_MEM_STATS
66 #endif
67 #endif
68
69
70 #define NCBI_USE_ERRCODE_X Db_Bdb_Env
71
72
73 BEGIN_NCBI_SCOPE
74
CBDB_Env()75 CBDB_Env::CBDB_Env()
76 : m_Env(0),
77 m_Transactional(false),
78 m_ErrFile(0),
79 m_LogInMemory(false),
80 m_TransSync(CBDB_Transaction::eTransSync),
81 m_MaxLocks(0),
82 m_MaxLockers(0),
83 m_MaxLockObjects(0),
84 m_DirectDB(false),
85 m_DirectLOG(false),
86 m_CheckPointEnable(true),
87 m_CheckPointKB(0),
88 m_CheckPointMin(0),
89 m_DeadLockMode(eDeadLock_Disable),
90 m_Monitor(0),
91 m_StopThreadFlag(new bool(false))
92 {
93 int ret = db_env_create(&m_Env, 0);
94 BDB_CHECK(ret, "DB_ENV::create");
95 }
96
97
CBDB_Env(DB_ENV * env)98 CBDB_Env::CBDB_Env(DB_ENV* env)
99 : m_Env(env),
100 m_Transactional(false),
101 m_ErrFile(0),
102 m_LogInMemory(false),
103 m_TransSync(CBDB_Transaction::eTransSync),
104 m_MaxLocks(0),
105 m_MaxLockers(0),
106 m_MaxLockObjects(0),
107 m_DirectDB(false),
108 m_DirectLOG(false),
109 m_CheckPointEnable(true),
110 m_CheckPointKB(0),
111 m_CheckPointMin(0),
112 m_DeadLockMode(eDeadLock_Disable),
113 m_Monitor(0)
114 {
115 }
116
~CBDB_Env()117 CBDB_Env::~CBDB_Env()
118 {
119 try {
120 Close();
121 }
122 catch (CBDB_Exception&) {
123 }
124 if (m_ErrFile) {
125 fclose(m_ErrFile);
126 }
127 }
128
Close()129 void CBDB_Env::Close()
130 {
131 if (m_Env) {
132 int ret = m_Env->close(m_Env, 0);
133 m_Env = 0;
134 BDB_CHECK(ret, "DB_ENV::close");
135 }
136 }
137
SetTransactionSync(CBDB_Transaction::ETransSync sync)138 void CBDB_Env::SetTransactionSync(CBDB_Transaction::ETransSync sync)
139 {
140 if (sync == CBDB_Transaction::eEnvDefault)
141 m_TransSync = CBDB_Transaction::eTransSync;
142 else
143 m_TransSync = sync;
144
145 if (sync == CBDB_Transaction::eTransSync) {
146 int ret = m_Env->set_flags(m_Env, DB_DSYNC_DB, 1);
147 BDB_CHECK(ret, "DB_ENV::set_flags(DB_DSYNC_DB)");
148 }
149 }
150
151
SetCacheSize(Uint8 cache_size,int num_caches)152 void CBDB_Env::SetCacheSize(Uint8 cache_size,
153 int num_caches)
154 {
155 unsigned cache_g = (unsigned) (cache_size / (Uint8)1073741824); // gig
156 if (cache_g) {
157 cache_size = cache_size % 1073741824;
158 }
159 unsigned ncache = max(num_caches, 1);
160 int ret =
161 m_Env->set_cachesize(m_Env, cache_g, (unsigned)cache_size, ncache);
162 BDB_CHECK(ret, "DB_ENV::set_cachesize");
163 }
164
SetLogRegionMax(unsigned size)165 void CBDB_Env::SetLogRegionMax(unsigned size)
166 {
167 int ret = m_Env->set_lg_regionmax(m_Env, size);
168 BDB_CHECK(ret, "DB_ENV::set_lg_regionmax");
169 }
170
SetLogBSize(unsigned lg_bsize)171 void CBDB_Env::SetLogBSize(unsigned lg_bsize)
172 {
173 int ret = m_Env->set_lg_bsize(m_Env, lg_bsize);
174 BDB_CHECK(ret, "DB_ENV::set_lg_bsize");
175 }
176
Open(const string & db_home,int flags)177 void CBDB_Env::Open(const string& db_home, int flags)
178 {
179 int ret = x_Open(db_home.c_str(), flags);
180 BDB_CHECK(ret, "DB_ENV::open");
181
182 SetDirectDB(m_DirectDB);
183 SetDirectLog(m_DirectLOG);
184 }
185
x_Open(const char * db_home,int flags)186 int CBDB_Env::x_Open(const char* db_home, int flags)
187 {
188 int recover_requested = flags & DB_RECOVER;
189
190 // db_home == NULL lets to have data in memory
191 const char* db_home_param = db_home;
192 if (strlen(db_home) == 0)
193 db_home_param = NULL;
194
195 int ret = m_Env->open(m_Env, db_home_param, flags, 0664);
196 if (ret == DB_RUNRECOVERY) {
197 if (flags & DB_JOINENV) { // join do not attempt recovery
198 return ret;
199 }
200 int recover_flag = flags;
201
202 if (!recover_requested) {
203 if (flags & DB_INIT_TXN) { // recovery needs transaction
204 recover_flag = flags | DB_RECOVER | DB_CREATE;
205 } else {
206 goto fatal_recovery;
207 }
208 } else {
209 goto fatal_recovery;
210 }
211
212 ret = m_Env->open(m_Env, db_home_param, recover_flag, 0664);
213
214 if (!recover_requested) {
215 fatal_recovery:
216 ERR_POST_X(1, Warning << "BDB_Env: Trying fatal recovery.");
217 if ((ret == DB_RUNRECOVERY) && (flags & DB_INIT_TXN)) {
218 recover_flag = flags | DB_RECOVER_FATAL | DB_CREATE;
219 recover_flag &= ~DB_RECOVER;
220
221 ret = m_Env->open(m_Env, db_home_param, recover_flag, 0664);
222 if (ret) {
223 ERR_POST_X(2, Warning <<
224 "Fatal recovery returned error code=" << ret);
225 }
226 }
227 }
228 }
229 m_HomePath = db_home;
230 return ret;
231 }
232
OpenWithLocks(const string & db_home)233 void CBDB_Env::OpenWithLocks(const string& db_home)
234 {
235 Open(db_home, DB_CREATE/*|DB_RECOVER*/|DB_THREAD|DB_INIT_LOCK|DB_INIT_MPOOL);
236 }
237
OpenPrivate(const string & db_home)238 void CBDB_Env::OpenPrivate(const string& db_home)
239 {
240 int ret = x_Open(db_home.c_str(),
241 DB_CREATE|DB_THREAD|DB_PRIVATE|DB_INIT_MPOOL);
242 BDB_CHECK(ret, "DB_ENV::open_private");
243 }
244
245
OpenWithTrans(const string & db_home,TEnvOpenFlags opt)246 void CBDB_Env::OpenWithTrans(const string& db_home, TEnvOpenFlags opt)
247 {
248 int ret = m_Env->set_lk_detect(m_Env, DB_LOCK_DEFAULT);
249 BDB_CHECK(ret, "DB_ENV::set_lk_detect");
250
251 if (m_MaxLockObjects) {
252 this->SetMaxLockObjects(m_MaxLockObjects);
253 }
254 if (m_MaxLocks) {
255 this->SetMaxLocks(m_MaxLocks);
256 }
257 if (m_MaxLockers) {
258 this->SetMaxLockers(m_MaxLockers);
259 }
260
261 int flag = DB_INIT_TXN |
262 DB_CREATE | DB_INIT_LOCK | DB_INIT_MPOOL;
263
264 // for windows we try to init environment in system memory
265 // (not file based)
266 // so it is reclaimed by OS automatically even if the application crashes
267 # ifdef NCBI_OS_MSWIN
268 flag |= DB_SYSTEM_MEM;
269 # endif
270 if (opt & eThreaded) {
271 flag |= DB_THREAD;
272 }
273 if (opt & ePrivate) {
274 flag |= DB_PRIVATE;
275 }
276
277 // Run recovery procedure, reinitialize the environment
278
279 if (opt & eRunRecovery) {
280 // use private environment as prescribed by "db_recover" utility
281 int recover_flag = flag | DB_RECOVER | DB_CREATE;
282
283 if (!(recover_flag & DB_SYSTEM_MEM)) {
284 recover_flag |= DB_PRIVATE;
285 }
286
287 ret = x_Open(db_home.c_str(), recover_flag);
288 BDB_CHECK(ret, "DB_ENV::open");
289
290 // non-private recovery
291 if (!(recover_flag & DB_PRIVATE)) {
292 m_Transactional = true;
293 return;
294 }
295
296 // "Private" recovery requires reopening, to make
297 // it available for other programs
298
299 ret = m_Env->close(m_Env, 0);
300 BDB_CHECK(ret, "DB_ENV::close");
301
302 ret = db_env_create(&m_Env, 0);
303 BDB_CHECK(ret, "DB_ENV::create");
304 } else
305 if (opt & eRunRecoveryFatal) {
306 // use private environment as prescribed by "db_recover" utility
307 int recover_flag = flag | DB_RECOVER_FATAL | DB_CREATE;
308
309 if (!(recover_flag & DB_SYSTEM_MEM)) {
310 recover_flag |= DB_PRIVATE;
311 }
312
313 ret = x_Open(db_home.c_str(), recover_flag);
314 BDB_CHECK(ret, "DB_ENV::open");
315
316 // non-private recovery
317 if (!(recover_flag & DB_PRIVATE)) {
318 m_Transactional = true;
319 return;
320 }
321
322 // "Private" recovery requires reopening, to make
323 // it available for other programs
324
325 ret = m_Env->close(m_Env, 0);
326 BDB_CHECK(ret, "DB_ENV::close");
327
328 ret = db_env_create(&m_Env, 0);
329 BDB_CHECK(ret, "DB_ENV::create");
330 }
331
332 Open(db_home, flag);
333 m_Transactional = true;
334 }
335
OpenConcurrentDB(const string & db_home)336 void CBDB_Env::OpenConcurrentDB(const string& db_home)
337 {
338 int ret = m_Env->set_flags(m_Env, DB_CDB_ALLDB, 1);
339 BDB_CHECK(ret, "DB_ENV::set_flags(DB_CDB_ALLDB)");
340
341 Open(db_home, DB_CREATE | DB_THREAD | DB_INIT_CDB | DB_INIT_MPOOL);
342 }
343
JoinEnv(const string & db_home,TEnvOpenFlags opt,ETransactionDiscovery trans_test)344 void CBDB_Env::JoinEnv(const string& db_home,
345 TEnvOpenFlags opt,
346 ETransactionDiscovery trans_test)
347 {
348 int flag = DB_JOINENV;
349 if (opt & eThreaded) {
350 flag |= DB_THREAD;
351 }
352
353 Open(db_home, flag);
354
355
356 switch (trans_test) {
357 case eTestTransactions:
358 {{
359 // Check if we joined the transactional environment
360 // Try to create a fake transaction to test the environment
361 DB_TXN* txn = 0;
362 int ret = m_Env->txn_begin(m_Env, 0, &txn, 0);
363
364 if (ret == 0) {
365 m_Transactional = true;
366 ret = txn->abort(txn);
367 }
368 }}
369 break;
370
371 case eInspectTransactions:
372 {{
373 // Check if we joined the transactional environment
374 Uint4 flags = 0;
375 int ret = m_Env->get_open_flags(m_Env, &flags);
376 BDB_CHECK(ret, "DB_ENV::get_open_flags");
377 if (flags & DB_INIT_TXN) {
378 m_Transactional = true;
379 } else {
380 m_Transactional = false;
381 }
382 }}
383 break;
384 case eAssumeTransactions:
385 m_Transactional = true;
386 break;
387 case eAssumeNoTransactions:
388 m_Transactional = false;
389 break;
390 default:
391 _ASSERT(0);
392 }
393
394 // commented since it caused crash on windows trying to free
395 // txn_statp structure. (Why it happened remains unknown)
396
397 /*
398 DB_TXN_STAT *txn_statp = 0;
399 int ret = m_Env->txn_stat(m_Env, &txn_statp, 0);
400 if (ret == 0)
401 {
402 ::free(txn_statp);
403 txn_statp = 0;
404
405 // Try to create a fake transaction to test the environment
406 DB_TXN* txn = 0;
407 ret = m_Env->txn_begin(m_Env, 0, &txn, 0);
408
409 if (ret == 0) {
410 m_Transactional = true;
411 ret = txn->abort(txn);
412 }
413 }
414 */
415 }
416
Remove()417 bool CBDB_Env::Remove()
418 {
419 if (m_HomePath.empty()) {
420 return true;
421 }
422 Close();
423
424 // for db_env_create()
425 NCBI_LSAN_DISABLE_GUARD;
426 int ret = db_env_create(&m_Env, 0);
427 BDB_CHECK(ret, "DB_ENV::create");
428
429 ret = m_Env->remove(m_Env, m_HomePath.c_str(), 0);
430
431 if (ret == EBUSY) {
432 // Some versions (observed with 4.6.21.1) will leak the environment
433 // in this case, but others close it, so err on the side of leakage
434 // because double closing is unsafe and can cause crashes.
435 // m_Env->close(m_Env, 0);
436 m_Env = 0;
437 return false;
438 }
439
440 m_Env = 0;
441 BDB_CHECK(ret, "DB_ENV::remove");
442 return true;
443 }
444
ForceRemove()445 void CBDB_Env::ForceRemove()
446 {
447 _ASSERT(!m_HomePath.empty());
448 Close();
449
450 int ret = db_env_create(&m_Env, 0);
451 BDB_CHECK(ret, "DB_ENV::create");
452
453 ret = m_Env->remove(m_Env, m_HomePath.c_str(), DB_FORCE);
454 m_Env = 0;
455 BDB_CHECK(ret, "DB_ENV::remove");
456 }
457
CheckRemove()458 bool CBDB_Env::CheckRemove()
459 {
460 if (Remove()) {
461 // Remove returned OK, but BerkeleyDB has a bug(?)
462 // that it does not even try to remove environment
463 // when it cannot attach to it.
464 // In case of Windows and opening was made using DB_SYSTEM_MEM
465 // it does not remove files.
466
467 // so we check that and force the removal
468 CDir dir(m_HomePath);
469 CDir::TEntries fl = dir.GetEntries("__db.*", CDir::eIgnoreRecursive);
470 if (!fl.empty()) {
471 ForceRemove();
472 }
473 return true;
474 }
475 return false;
476 }
477
SetErrPrefix(const string & prefix)478 void CBDB_Env::SetErrPrefix(const string& prefix)
479 {
480 _ASSERT(m_Env);
481 m_ErrPrefix = prefix;
482 m_Env->set_errpfx(m_Env, m_ErrPrefix.c_str());
483 }
484
SetLogDir(const string & log_dir)485 void CBDB_Env::SetLogDir(const string& log_dir)
486 {
487 if (log_dir.empty()) {
488 return;
489 }
490 try {
491 CDir dir(log_dir);
492 if (!dir.Exists()) {
493 if (!dir.Create()) {
494 ERR_POST_X(3, "Cannot create transaction log directory:" << log_dir);
495 return;
496 }
497 }
498 int ret = m_Env->set_lg_dir(m_Env, log_dir.c_str());
499 BDB_CHECK(ret, "DB_ENV::set_lg_dir");
500 }
501 catch(exception& ex)
502 {
503 ERR_POST_X(4, "Cannot set transaction log directory:" << ex.what());
504 }
505 }
506
CreateTxn(DB_TXN * parent_txn,unsigned int flags)507 DB_TXN* CBDB_Env::CreateTxn(DB_TXN* parent_txn, unsigned int flags)
508 {
509 DB_TXN* txn = 0;
510 int ret = m_Env->txn_begin(m_Env, parent_txn, &txn, flags);
511 BDB_CHECK(ret, "DB_ENV::txn_begin");
512 return txn;
513 }
514
SetLogFileMax(unsigned int lg_max)515 void CBDB_Env::SetLogFileMax(unsigned int lg_max)
516 {
517 int ret = m_Env->set_lg_max(m_Env, lg_max);
518 BDB_CHECK(ret, "DB_ENV::set_lg_max");
519 }
520
SetMaxLocks(unsigned locks)521 void CBDB_Env::SetMaxLocks(unsigned locks)
522 {
523 m_MaxLocks = locks;
524 if (m_Env) {
525 int ret = m_Env->set_lk_max_locks(m_Env, locks);
526 BDB_CHECK(ret, "DB_ENV::set_lk_max_locks");
527 }
528 }
529
530
SetTransactionMax(unsigned tx_max)531 void CBDB_Env::SetTransactionMax(unsigned tx_max)
532 {
533 _ASSERT(tx_max);
534 int ret = m_Env->set_tx_max(m_Env, tx_max);
535 BDB_CHECK(ret, "DB_ENV::set_tx_max");
536 }
537
538
LsnReset(const char * file_name)539 void CBDB_Env::LsnReset(const char* file_name)
540 {
541 int ret = m_Env->lsn_reset(m_Env, const_cast<char*>(file_name), 0);
542 BDB_CHECK(ret, "DB_ENV::lsn_reset");
543 }
544
LsnResetForMemLog(const char * file_name)545 void CBDB_Env::LsnResetForMemLog(const char* file_name)
546 {
547 if (!m_LogInMemory) return;
548 int ret = m_Env->lsn_reset(m_Env, const_cast<char*>(file_name), 0);
549 BDB_CHECK(ret, "DB_ENV::lsn_reset");
550 }
551
GetMaxLocks()552 unsigned CBDB_Env::GetMaxLocks()
553 {
554 if (!m_Env)
555 return m_MaxLocks;
556
557 u_int32_t lk_max;
558 int ret = m_Env->get_lk_max_locks(m_Env, &lk_max);
559 BDB_CHECK(ret, "DB_ENV::get_lk_max_locks");
560 return lk_max;
561 }
562
SetMaxLockObjects(unsigned lock_obj_max)563 void CBDB_Env::SetMaxLockObjects(unsigned lock_obj_max)
564 {
565 m_MaxLockObjects = lock_obj_max;
566 if (m_Env) {
567 int ret = m_Env->set_lk_max_objects(m_Env, m_MaxLockObjects);
568 BDB_CHECK(ret, "DB_ENV::set_lk_max_objects");
569 }
570 }
571
SetMaxLockers(unsigned max_lockers)572 void CBDB_Env::SetMaxLockers(unsigned max_lockers)
573 {
574 m_MaxLockers = max_lockers;
575 if (m_Env) {
576 int ret = m_Env->set_lk_max_lockers(m_Env, max_lockers);
577 BDB_CHECK(ret, "DB_ENV::set_lk_max_lockers");
578 }
579 }
580
581
SetLogInMemory(bool on_off)582 void CBDB_Env::SetLogInMemory(bool on_off)
583 {
584 #if BDB_FULL_VERSION < 4007000
585 int ret = m_Env->set_flags(m_Env, DB_LOG_INMEMORY, int(on_off));
586 BDB_CHECK(ret, "DB_ENV::set_flags(DB_LOG_INMEMORY)");
587 #else
588 int ret = m_Env->log_set_config(m_Env, DB_LOG_IN_MEMORY, (int)on_off);
589 BDB_CHECK(ret, "DB_ENV::log_set_config(DB_LOG_IN_MEMORY)");
590 #endif
591 m_LogInMemory = on_off;
592 }
593
SetTasSpins(unsigned tas_spins)594 void CBDB_Env::SetTasSpins(unsigned tas_spins)
595 {
596 #ifdef BDB_NEW_MUTEX_API
597 int ret = m_Env->mutex_set_tas_spins(m_Env, tas_spins);
598 BDB_CHECK(ret, "DB_ENV::mutex_set_tas_spins");
599 #else
600 int ret = m_Env->set_tas_spins(m_Env, tas_spins);
601 BDB_CHECK(ret, "DB_ENV::set_tas_spins");
602 #endif
603 }
604
OpenErrFile(const string & file_name)605 void CBDB_Env::OpenErrFile(const string& file_name)
606 {
607 if (m_ErrFile) {
608 fclose(m_ErrFile);
609 m_ErrFile = 0;
610 }
611
612 if (file_name == "stderr") {
613 m_Env->set_errfile(m_Env, stderr);
614 return;
615 }
616 if (file_name == "stdout") {
617 m_Env->set_errfile(m_Env, stdout);
618 return;
619 }
620
621 m_ErrFile = fopen(file_name.c_str(), "a");
622 if (m_ErrFile) {
623 m_Env->set_errfile(m_Env, m_ErrFile);
624 }
625 }
626
SetDirectDB(bool on_off)627 void CBDB_Env::SetDirectDB(bool on_off)
628 {
629 m_DirectDB = on_off;
630 if (m_Env) {
631 // error checking commented (not all platforms support direct IO)
632 /*int ret = */ m_Env->set_flags(m_Env, DB_DIRECT_DB, (int)on_off);
633 // BDB_CHECK(ret, "DB_ENV::set_flags(DB_DIRECT_DB)");
634 }
635 }
636
SetDirectLog(bool on_off)637 void CBDB_Env::SetDirectLog(bool on_off)
638 {
639 m_DirectLOG = on_off;
640 if (m_Env) {
641 // error checking commented (not all platforms support direct IO)
642 #if BDB_FULL_VERSION < 4007000
643 /*int ret = */ m_Env->set_flags(m_Env, DB_DIRECT_LOG, (int)on_off);
644 // BDB_CHECK(ret, "DB_ENV::set_flags(DB_DIRECT_LOG)");
645 #else
646 /*int ret = */ m_Env->log_set_config(m_Env, DB_LOG_DIRECT, (int)on_off);
647 // BDB_CHECK(ret, "DB_ENV::log_set_config(DB_LOG_DIRECT)");
648 #endif
649 }
650 }
651
SetLogAutoRemove(bool on_off)652 void CBDB_Env::SetLogAutoRemove(bool on_off)
653 {
654 #if BDB_FULL_VERSION < 4007000
655 int ret = m_Env->set_flags(m_Env, DB_LOG_AUTOREMOVE, (int)on_off);
656 BDB_CHECK(ret, "DB_ENV::set_flags(DB_LOG_AUTOREMOVE)");
657 #else
658 int ret = m_Env->log_set_config(m_Env, DB_LOG_AUTO_REMOVE, (int)on_off);
659 BDB_CHECK(ret, "DB_ENV::log_set_config(DB_LOG_AUTO_REMOVE)");
660 #endif
661 }
662
663
TransactionCheckpoint()664 void CBDB_Env::TransactionCheckpoint()
665 {
666 if (m_CheckPointEnable && IsTransactional()) {
667 int ret =
668 m_Env->txn_checkpoint(m_Env, m_CheckPointKB, m_CheckPointMin, 0);
669 BDB_CHECK(ret, "DB_ENV::txn_checkpoint");
670 }
671 }
672
ForceTransactionCheckpoint()673 void CBDB_Env::ForceTransactionCheckpoint()
674 {
675 if (IsTransactional()) {
676 int ret =
677 m_Env->txn_checkpoint(m_Env, 0, 0, DB_FORCE);
678 BDB_CHECK(ret, "DB_ENV::txn_checkpoint");
679 }
680 }
681
682
IsTransactional() const683 bool CBDB_Env::IsTransactional() const
684 {
685 return m_Transactional;
686 }
687
688
LogFlush()689 void CBDB_Env::LogFlush()
690 {
691 BDB_CHECK(m_Env->log_flush(m_Env, 0), "DB_ENV::log_flush");
692 }
693
694
CleanLog()695 void CBDB_Env::CleanLog()
696 {
697 char **nm_list = 0;
698 int ret =
699 m_Env->log_archive(m_Env, &nm_list, DB_ARCH_ABS);
700 BDB_CHECK(ret, "DB_ENV::CleanLog()");
701
702 if (nm_list != NULL) {
703 for (char** file = nm_list; *file != NULL; ++file) {
704 LOG_POST_X(5, Info << "BDB_Env: Removing LOG file: " << *file);
705 CDirEntry de(*file);
706 de.Remove();
707 }
708 free(nm_list);
709 }
710
711 }
712
SetLockTimeout(unsigned timeout)713 void CBDB_Env::SetLockTimeout(unsigned timeout)
714 {
715 db_timeout_t to = timeout;
716 int ret = m_Env->set_timeout(m_Env, to, DB_SET_LOCK_TIMEOUT);
717 BDB_CHECK(ret, "DB_ENV::set_timeout");
718 }
719
SetTransactionTimeout(unsigned timeout)720 void CBDB_Env::SetTransactionTimeout(unsigned timeout)
721 {
722 db_timeout_t to = timeout;
723 int ret = m_Env->set_timeout(m_Env, to, DB_SET_TXN_TIMEOUT);
724 BDB_CHECK(ret, "DB_ENV::set_timeout");
725 }
726
MutexSetMax(unsigned max)727 void CBDB_Env::MutexSetMax(unsigned max)
728 {
729 #ifdef BDB_NEW_MUTEX_API
730 int ret = m_Env->mutex_set_max(m_Env, max);
731 BDB_CHECK(ret, "DB_ENV::mutex_set_max");
732 #endif
733 }
734
MutexGetMax()735 unsigned CBDB_Env::MutexGetMax()
736 {
737 #ifdef BDB_NEW_MUTEX_API
738 u_int32_t maxp;
739 int ret = m_Env->mutex_get_max(m_Env, &maxp);
740 BDB_CHECK(ret, "DB_ENV::mutex_get_max");
741 return maxp;
742 #else
743 return 0;
744 #endif
745 }
746
MutexSetIncrement(unsigned inc)747 void CBDB_Env::MutexSetIncrement(unsigned inc)
748 {
749 #ifdef BDB_NEW_MUTEX_API
750 int ret = m_Env->mutex_set_increment(m_Env, inc);
751 BDB_CHECK(ret, "DB_ENV::mutex_set_increment");
752 #endif
753 }
754
MutexGetIncrement()755 unsigned CBDB_Env::MutexGetIncrement()
756 {
757 #ifdef BDB_NEW_MUTEX_API
758 u_int32_t inc;
759 int ret = m_Env->mutex_get_increment(m_Env, &inc);
760 BDB_CHECK(ret, "DB_ENV::mutex_get_increment");
761 return inc;
762 #else
763 return 0;
764 #endif
765 }
766
MutexGetFree()767 unsigned CBDB_Env::MutexGetFree()
768 {
769 unsigned free_m = 0;
770 #ifdef BDB_NEW_MUTEX_API
771 DB_MUTEX_STAT* stp = 0;
772 try {
773 int ret = m_Env->mutex_stat(m_Env, &stp, 0);
774 BDB_CHECK(ret, "DB_ENV::mutex_stat");
775 free_m = stp->st_mutex_free;
776 }
777 catch (...)
778 {
779 if (stp) {
780 ::free(stp);
781 }
782 throw;
783 }
784
785 if (stp) {
786 ::free(stp);
787 }
788 #endif
789 return free_m;
790 }
791
792
PrintMutexStat(CNcbiOstream & out)793 void CBDB_Env::PrintMutexStat(CNcbiOstream & out)
794 {
795 #ifdef BDB_NEW_MUTEX_API
796 DB_MUTEX_STAT* stp = 0;
797 try {
798 int ret = m_Env->mutex_stat(m_Env, &stp, 0);
799 BDB_CHECK(ret, "DB_ENV::mutex_stat");
800
801 out << "st_mutex_align : " << stp->st_mutex_align << NcbiEndl
802 << "st_mutex_tas_spins : " << stp->st_mutex_tas_spins << NcbiEndl
803 << "st_mutex_free : " << stp->st_mutex_free << NcbiEndl
804 << "st_mutex_inuse : " << stp->st_mutex_inuse << NcbiEndl
805 << "st_mutex_inuse_max : " << stp->st_mutex_inuse_max << NcbiEndl
806 << "st_regsize : " << stp->st_regsize << NcbiEndl
807 << "st_region_wait : " << stp->st_region_wait << NcbiEndl
808 << "st_region_nowait : " << stp->st_region_nowait << NcbiEndl
809 ;
810 }
811 catch (...)
812 {
813 if (stp) {
814 ::free(stp);
815 }
816 throw;
817 }
818
819 if (stp) {
820 ::free(stp);
821 }
822 #endif
823 }
824
PrintLockStat(CNcbiOstream & out)825 void CBDB_Env::PrintLockStat(CNcbiOstream & out)
826 {
827 #ifdef BDB_NEW_MUTEX_API
828 DB_LOCK_STAT *stp = 0;
829 try {
830
831 int ret = m_Env->lock_stat(m_Env, &stp, 0);
832 BDB_CHECK(ret, "DB_ENV::lock_stat");
833
834 out << "st_id : " << stp->st_id << NcbiEndl
835 << "st_cur_maxid : " << stp->st_cur_maxid << NcbiEndl
836 << "st_nmodes : " << stp->st_nmodes << NcbiEndl
837 << "st_maxlocks : " << stp->st_maxlocks << NcbiEndl
838 << "st_maxlockers : " << stp->st_maxlockers << NcbiEndl
839 << "st_maxobjects : " << stp->st_maxobjects << NcbiEndl
840 << "st_nlocks : " << stp->st_nlocks << NcbiEndl
841 << "st_maxnlocks : " << stp->st_maxnlocks << NcbiEndl
842 << "st_nlockers : " << stp->st_nlockers << NcbiEndl
843 << "st_maxnlockers : " << stp->st_maxnlockers << NcbiEndl
844 << "st_nobjects : " << stp->st_nobjects << NcbiEndl
845 << "st_maxnobjects : " << stp->st_maxnobjects << NcbiEndl
846 << "st_nrequests : " << stp->st_nrequests << NcbiEndl
847 << "st_nreleases : " << stp->st_nreleases << NcbiEndl
848 << "st_nupgrade : " << stp->st_nupgrade << NcbiEndl
849 << "st_ndowngrade : " << stp->st_ndowngrade << NcbiEndl
850 << "st_lock_wait : " << stp->st_lock_wait << NcbiEndl
851 << "st_lock_nowait : " << stp->st_lock_nowait << NcbiEndl
852 << "st_ndeadlocks : " << stp->st_ndeadlocks << NcbiEndl
853 << "st_locktimeout : " << stp->st_locktimeout << NcbiEndl
854 << "st_nlocktimeouts: " << stp->st_nlocktimeouts << NcbiEndl
855 << "st_txntimeout : " << stp->st_txntimeout << NcbiEndl
856 << "st_ntxntimeouts : " << stp->st_ntxntimeouts << NcbiEndl
857 << "st_regsize : " << stp->st_regsize << NcbiEndl
858 << "st_region_wait : " << stp->st_region_wait << NcbiEndl
859 << "st_region_nowait: " << stp->st_region_nowait << NcbiEndl
860 ;
861 }
862 catch (...)
863 {
864 if (stp) {
865 ::free(stp);
866 }
867 throw;
868 }
869
870 if (stp) {
871 ::free(stp);
872 }
873
874 #endif
875 }
876
PrintMemStat(CNcbiOstream & out)877 void CBDB_Env::PrintMemStat(CNcbiOstream & out)
878 {
879 DB_MPOOL_STAT *stp = 0;
880 try {
881
882 int ret = m_Env->memp_stat(m_Env, &stp, 0, 0);
883 BDB_CHECK(ret, "DB_ENV::memp_stat");
884
885
886 out << "st_gbytes : " << stp->st_gbytes << NcbiEndl
887 << "st_bytes : " << stp->st_bytes << NcbiEndl
888 << "st_ncache : " << stp->st_ncache << NcbiEndl
889 << "st_regsize : " << stp->st_regsize << NcbiEndl
890 << "st_mmapsize : " << stp->st_mmapsize << NcbiEndl
891 << "st_maxopenfd : " << stp->st_maxopenfd << NcbiEndl
892 << "st_maxwrite : " << stp->st_maxwrite << NcbiEndl
893 << "st_maxwrite_sleep : " << stp->st_maxwrite_sleep << NcbiEndl
894 << "st_map : " << stp->st_map << NcbiEndl
895 << "st_cache_hit : " << stp->st_cache_hit << NcbiEndl
896 << "st_cache_miss : " << stp->st_cache_miss << NcbiEndl
897 << "st_page_create : " << stp->st_page_create << NcbiEndl
898 << "st_page_in : " << stp->st_page_in << NcbiEndl
899 << "st_page_out : " << stp->st_page_out << NcbiEndl
900 << "st_ro_evict : " << stp->st_ro_evict << NcbiEndl
901 << "st_rw_evict : " << stp->st_rw_evict << NcbiEndl
902 << "st_page_trickle : " << stp->st_page_trickle << NcbiEndl
903 << "st_pages : " << stp->st_pages << NcbiEndl
904 << "st_page_clean : " << stp->st_page_clean << NcbiEndl
905 << "st_page_dirty : " << stp->st_page_dirty << NcbiEndl
906 << "st_hash_buckets : " << stp->st_hash_buckets << NcbiEndl
907 << "st_hash_searches : " << stp->st_hash_searches << NcbiEndl
908 << "st_hash_longest : " << stp->st_hash_longest << NcbiEndl
909 << "st_hash_examined : " << stp->st_hash_examined << NcbiEndl
910 << "st_hash_nowait : " << stp->st_hash_nowait << NcbiEndl
911 << "st_hash_wait : " << stp->st_hash_wait << NcbiEndl
912 #ifdef BDB_NEW_MEM_STATS
913 << "st_hash_max_nowait : " << stp->st_hash_max_nowait << NcbiEndl
914 #endif
915 << "st_hash_max_wait : " << stp->st_hash_max_wait << NcbiEndl
916 << "st_region_wait : " << stp->st_region_wait << NcbiEndl
917 << "st_region_nowait : " << stp->st_region_nowait << NcbiEndl
918 #ifdef BDB_NEW_MEM_STATS
919 << "st_mvcc_frozen : " << stp->st_mvcc_frozen << NcbiEndl
920 << "st_mvcc_thawed : " << stp->st_mvcc_thawed << NcbiEndl
921 << "st_mvcc_freed : " << stp->st_mvcc_freed << NcbiEndl
922 #endif
923 << "st_alloc : " << stp->st_alloc << NcbiEndl
924 << "st_alloc_buckets : " << stp->st_alloc_buckets << NcbiEndl
925 << "st_alloc_max_buckets: " << stp->st_alloc_max_buckets << NcbiEndl
926 << "st_alloc_pages : " << stp->st_alloc_pages << NcbiEndl
927 << "st_alloc_max_pages : " << stp->st_alloc_max_pages << NcbiEndl
928 #ifdef BDB_NEW_MEM_STATS
929 << "st_io_wait : " << stp->st_io_wait << NcbiEndl
930 #endif
931 ;
932
933 int max_write;
934 #if (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 6) || DB_VERSION_MAJOR > 4
935 db_timeout_t max_write_sleep;
936 #else
937 int max_write_sleep;
938 #endif
939 ret =
940 m_Env->get_mp_max_write(m_Env, &max_write, &max_write_sleep);
941 BDB_CHECK(ret, "DB_ENV::get_mp_max_write");
942
943 out << "max_write (pages) : " << max_write << NcbiEndl
944 << "max_write_sleep (microsec) : " << max_write_sleep << NcbiEndl
945 ;
946
947 }
948 catch (...)
949 {
950 if (stp) {
951 ::free(stp);
952 }
953 throw;
954 }
955
956 if (stp) {
957 ::free(stp);
958 }
959 }
960
961
MempTrickle(int percent,int * nwrotep)962 void CBDB_Env::MempTrickle(int percent, int *nwrotep)
963 {
964 int nwr;
965 int ret = m_Env->memp_trickle(m_Env, percent, &nwr);
966 BDB_CHECK(ret, "DB_ENV::memp_trickle");
967 if (nwrotep) {
968 *nwrotep = nwr;
969 }
970 if (m_Monitor && m_Monitor->IsActive()) {
971 string msg = "BDB_ENV: memp_tricle ";
972 msg += NStr::IntToString(percent);
973 msg += "% written ";
974 msg += NStr::IntToString(nwr);
975 msg += " pages.";
976 m_Monitor->Send(msg);
977 }
978 }
979
MempSync()980 void CBDB_Env::MempSync()
981 {
982 int ret = m_Env->memp_sync(m_Env, 0);
983 BDB_CHECK(ret, "DB_ENV::memp_sync");
984 }
985
986
SetMpMaxWrite(int maxwrite,int maxwrite_sleep)987 void CBDB_Env::SetMpMaxWrite(int maxwrite, int maxwrite_sleep)
988 {
989 int ret = m_Env->set_mp_max_write(m_Env, maxwrite, maxwrite_sleep);
990 BDB_CHECK(ret, "DB_ENV::set_mp_max_write");
991 }
992
993
GetMpMmapSize()994 size_t CBDB_Env::GetMpMmapSize()
995 {
996 size_t map_size = 0;
997 int ret = m_Env->get_mp_mmapsize(m_Env, &map_size);
998 BDB_CHECK(ret, "DB_ENV::get_mp_mmapsize");
999 return map_size;
1000 }
1001
1002
SetMpMmapSize(size_t map_size)1003 void CBDB_Env::SetMpMmapSize(size_t map_size)
1004 {
1005 int ret = m_Env->set_mp_mmapsize(m_Env, map_size);
1006 BDB_CHECK(ret, "DB_ENV::set_mp_mmapsize");
1007 }
1008
x_GetDeadLockDetect(EDeadLockDetect detect_mode)1009 unsigned CBDB_Env::x_GetDeadLockDetect(EDeadLockDetect detect_mode)
1010 {
1011 u_int32_t detect = 0;
1012 switch (detect_mode)
1013 {
1014 case eDeadLock_Disable:
1015 return 0;
1016 case eDeadLock_Default:
1017 detect = DB_LOCK_DEFAULT;
1018 break;
1019 case eDeadLock_MaxLocks:
1020 detect = DB_LOCK_MAXLOCKS;
1021 break;
1022 case eDeadLock_MinWrite:
1023 detect = DB_LOCK_MINWRITE;
1024 break;
1025 case eDeadLock_Oldest:
1026 detect = DB_LOCK_OLDEST;
1027 break;
1028 case eDeadLock_Random:
1029 detect = DB_LOCK_RANDOM;
1030 break;
1031 case eDeadLock_Youngest:
1032 detect = DB_LOCK_YOUNGEST;
1033 default:
1034 _ASSERT(0);
1035 }
1036 return detect;
1037 }
1038
1039
SetLkDetect(EDeadLockDetect detect_mode)1040 void CBDB_Env::SetLkDetect(EDeadLockDetect detect_mode)
1041 {
1042 m_DeadLockMode = detect_mode;
1043 if (m_DeadLockMode == eDeadLock_Disable) {
1044 return;
1045 }
1046 u_int32_t detect = x_GetDeadLockDetect(detect_mode);
1047
1048 int ret = m_Env->set_lk_detect(m_Env, detect);
1049 BDB_CHECK(ret, "DB_ENV::set_lk_detect");
1050 }
1051
DeadLockDetect()1052 void CBDB_Env::DeadLockDetect()
1053 {
1054 if (m_DeadLockMode == eDeadLock_Disable) {
1055 return;
1056 }
1057 u_int32_t detect = x_GetDeadLockDetect(m_DeadLockMode);
1058 int aborted;
1059 int ret = m_Env->lock_detect(m_Env, 0, detect, &aborted);
1060 BDB_CHECK(ret, "lock_detect");
1061 }
1062
1063
1064 class CBDB_Env_OnAppExit
1065 {
1066 public:
AddOnExitCallback(CBDB_Env & bdb_env)1067 static void AddOnExitCallback(CBDB_Env& bdb_env)
1068 {
1069 CNcbiApplicationGuard app = CNcbiApplication::InstanceGuard();
1070 if ( app ) {
1071 app->AddOnExitAction(CBDB_Env_OnAppExit(bdb_env));
1072 }
1073 }
1074
CBDB_Env_OnAppExit(CBDB_Env & bdb_env)1075 CBDB_Env_OnAppExit(CBDB_Env& bdb_env)
1076 : m_Env(bdb_env),
1077 m_StopThreadFlag(bdb_env.m_StopThreadFlag)
1078 {
1079 }
1080
operator ()(void) const1081 void operator()(void) const
1082 {
1083 if (*m_StopThreadFlag) {
1084 m_Env.StopBackgroundWriterThread();
1085 }
1086 }
1087
1088 private:
1089 CBDB_Env& m_Env;
1090 shared_ptr<bool> m_StopThreadFlag;
1091 };
1092
1093
RunBackgroundWriter(TBackgroundFlags flags,unsigned thread_delay,int memp_trickle,unsigned err_max)1094 void CBDB_Env::RunBackgroundWriter(TBackgroundFlags flags,
1095 unsigned thread_delay,
1096 int memp_trickle,
1097 unsigned err_max)
1098 {
1099 # ifdef NCBI_THREADS
1100 LOG_POST_X(6, Info << "Starting BDB transaction checkpoint thread.");
1101 CBDB_Env_OnAppExit::AddOnExitCallback(*this);
1102 m_CheckThread.Reset(
1103 new CBDB_CheckPointThread(*this, memp_trickle, thread_delay, 5));
1104 m_CheckThread->SetMaxErrors(err_max);
1105 m_CheckThread->SetWorkFlag(flags);
1106 m_CheckThread->Run();
1107 *m_StopThreadFlag = true;
1108 # else
1109 ERR_POST_X(7, Warning <<
1110 "Cannot run BDB transaction checkpoint thread in non-MT configuration.");
1111 # endif
1112 }
1113
1114
RunCheckpointThread(unsigned thread_delay,int memp_trickle,unsigned err_max)1115 void CBDB_Env::RunCheckpointThread(unsigned thread_delay,
1116 int memp_trickle,
1117 unsigned err_max)
1118 {
1119 TBackgroundFlags flags = CBDB_Env::eBackground_MempTrickle |
1120 CBDB_Env::eBackground_Checkpoint |
1121 CBDB_Env::eBackground_DeadLockDetect;
1122
1123 RunBackgroundWriter(flags, thread_delay, memp_trickle, err_max);
1124 }
1125
StopBackgroundWriterThread()1126 void CBDB_Env::StopBackgroundWriterThread()
1127 {
1128 # ifdef NCBI_THREADS
1129 if (!m_CheckThread.Empty()) {
1130 LOG_POST_X(8, Info << "Stopping BDB transaction checkpoint thread...");
1131 *m_StopThreadFlag = false;
1132 m_CheckThread->RequestStop();
1133 m_CheckThread->Join();
1134 m_CheckThread.Reset();
1135 LOG_POST_X(9, Info << "BDB transaction checkpoint thread stopped.");
1136 }
1137 # endif
1138 }
1139
StopCheckpointThread()1140 void CBDB_Env::StopCheckpointThread()
1141 {
1142 StopBackgroundWriterThread();
1143 }
1144
BDB_RecoverEnv(const string & path,bool fatal_recover)1145 void BDB_RecoverEnv(const string& path,
1146 bool fatal_recover)
1147 {
1148 DB_ENV *dbenv;
1149 int ret;
1150 if ((ret = db_env_create(&dbenv, 0)) != 0) {
1151 string msg =
1152 "Cannot create environment " + string(db_strerror(ret));
1153 BDB_THROW(eInvalidOperation, msg);
1154 }
1155
1156 dbenv->set_errfile(dbenv, stderr);
1157 //if (verbose)
1158 // (void)dbenv->set_verbose(dbenv, DB_VERB_RECOVERY, 1);
1159
1160 u_int32_t flags = 0;
1161 flags |= DB_CREATE | DB_INIT_LOG |
1162 DB_INIT_MPOOL | DB_INIT_TXN | DB_USE_ENVIRON;
1163 flags |= fatal_recover ? DB_RECOVER_FATAL : DB_RECOVER;
1164 flags |= DB_PRIVATE;
1165
1166 if ((ret = dbenv->open(dbenv, path.c_str(), flags, 0)) != 0) {
1167 dbenv->close(dbenv, 0);
1168 string msg =
1169 "Cannot open environment " + string(db_strerror(ret));
1170 BDB_THROW(eInvalidOperation, msg);
1171 }
1172 ret = dbenv->close(dbenv, 0);
1173 }
1174
1175
1176
1177
1178 END_NCBI_SCOPE
1179