1 /* $Id: bdb_file.cpp 571928 2018-10-04 18:07:23Z ucko $
2 * ===========================================================================
3 *
4 * PUBLIC DOMAIN NOTICE
5 * National Center for Biotechnology Information
6 *
7 * This software/database is a "United States Government Work" under the
8 * terms of the United States Copyright Act. It was written as part of
9 * the author's official duties as a United States Government employee and
10 * thus cannot be copyrighted. This software/database is freely available
11 * to the public for use. The National Library of Medicine and the U.S.
12 * Government have not placed any restriction on its use or reproduction.
13 *
14 * Although all reasonable efforts have been taken to ensure the accuracy
15 * and reliability of the software and data, the NLM and the U.S.
16 * Government do not and cannot warrant the performance or results that
17 * may be obtained by using this software or data. The NLM and the U.S.
18 * Government disclaim all warranties, express or implied, including
19 * warranties of performance, merchantability or fitness for any particular
20 * purpose.
21 *
22 * Please cite the author in any work or product based on this material.
23 *
24 * ===========================================================================
25 *
26 * Author: Anatoliy Kuznetsov
27 *
28 * File Description: BDB libarary file implementations.
29 *
30 */
31
32 #include <ncbi_pch.hpp>
33 #include <db/bdb/bdb_file.hpp>
34 #include <db/bdb/bdb_env.hpp>
35 #include <db/bdb/bdb_trans.hpp>
36
37 #include <db/error_codes.hpp>
38
39 #include <db.h>
40
41 #include <vector>
42
43 #ifdef verify
44 #undef verify
45 #endif
46
47 #if DB_VERSION_MAJOR >= 4
48 #if DB_VERSION_MINOR >= 4 || DB_VERSION_MAJOR > 4
49 #define HAVE_GET_MPF
50 #endif
51 #endif
52
53
54 #define NCBI_USE_ERRCODE_X Db_Bdb_File
55
56 BEGIN_NCBI_SCOPE
57
58 const char CBDB_RawFile::kDefaultDatabase[] = "_table";
59 const int CBDB_RawFile::kOpenFileMask = 0664;
60
61
62
63 /// Auto-pointer style guard class for DB structure
64 ///
65 /// @internal
66 ///
67 class CDB_guard
68 {
69 public:
CDB_guard(DB ** db)70 CDB_guard(DB** db) : m_DB(db) {}
~CDB_guard()71 ~CDB_guard()
72 {
73 if (m_DB && *m_DB) {
74 (*m_DB)->close(*m_DB, 0);
75 *m_DB = 0;
76 }
77 }
release()78 void release() { m_DB = 0; }
79 private:
80 DB** m_DB;
81 };
82
83 /////////////////////////////////////////////////////////////////////////////
84 // CBDB_MultiRowBuffer::
85 //
86
87
CBDB_MultiRowBuffer(size_t buf_size)88 CBDB_MultiRowBuffer::CBDB_MultiRowBuffer(size_t buf_size)
89 : m_Data_DBT(new DBT)
90 , m_Buf(new unsigned char[buf_size])
91 , m_BufSize(buf_size)
92 , m_BufPtr(0)
93 , m_LastKey(0)
94 , m_LastData(0)
95 , m_LastKeyLen(0)
96 , m_LastDataLen(0)
97 {
98 }
99
~CBDB_MultiRowBuffer()100 CBDB_MultiRowBuffer::~CBDB_MultiRowBuffer()
101 {
102 delete [] (unsigned char*)(m_Buf);
103 delete m_Data_DBT;
104 }
105
InitDBT()106 void CBDB_MultiRowBuffer::InitDBT()
107 {
108 memset(m_Data_DBT, 0, sizeof(DBT));
109 m_Data_DBT->data = m_Buf;
110 m_Data_DBT->ulen = m_Data_DBT->size = (u_int32_t)m_BufSize;
111 m_Data_DBT->flags = DB_DBT_USERMEM;
112 }
113
MultipleInit()114 void CBDB_MultiRowBuffer::MultipleInit()
115 {
116 DB_MULTIPLE_INIT(m_BufPtr, m_Data_DBT);
117 }
118
119
120 /////////////////////////////////////////////////////////////////////////////
121 // CBDB_RawFile::
122 //
123
124
125
CBDB_RawFile(EDuplicateKeys dup_keys,EDBType db_type)126 CBDB_RawFile::CBDB_RawFile(EDuplicateKeys dup_keys, EDBType db_type)
127 : m_DB_Type(db_type),
128 m_DB(0),
129 m_DBT_Key(0),
130 m_DBT_Data(0),
131 m_Env(0),
132 m_Trans(0),
133 m_TransAssociation(CBDB_Transaction::eFullAssociation),
134 m_RecLen(0),
135 m_H_ffactor(0),
136 m_H_nelem(0),
137 m_BT_minkey(0),
138 m_Compressor(0),
139 m_DB_Attached(false),
140 m_ByteSwapped(false),
141 m_RevSplitOff(false),
142 m_CmpOverride(true),
143 m_PageSize(0),
144 m_CacheSize(256 * 1024),
145 m_DuplicateKeys(dup_keys),
146 m_OpenMode(eReadOnly)
147 {
148 if (m_DB_Type == eQueue)
149 {
150 dup_keys = eDuplicatesDisable;
151 }
152
153 try
154 {
155 m_DBT_Key = new DBT;
156 m_DBT_Data = new DBT;
157 }
158 catch (...)
159 {
160 delete m_DBT_Key;
161 delete m_DBT_Data;
162 throw;
163 }
164
165 ::memset(m_DBT_Key, 0, sizeof(DBT));
166 ::memset(m_DBT_Data, 0, sizeof(DBT));
167 }
168
169
~CBDB_RawFile()170 CBDB_RawFile::~CBDB_RawFile()
171 {
172 x_Close(eIgnoreError);
173 delete m_DBT_Key;
174 delete m_DBT_Data;
175
176 // It's illegal to close a file involved in active transactions
177
178 if ( m_Trans != 0 &&
179 (m_TransAssociation == (int) CBDB_Transaction::eFullAssociation) &&
180 (m_Trans->IsInProgress())) {
181 _ASSERT(0);
182
183 // If we are here we can try to communicate by throwing
184 // an exception. It's illegal, but situation is bad enough already
185 BDB_THROW(eTransInProgress,
186 "Cannot close the file while transaction is in progress.");
187 }
188 }
189
190
Close()191 void CBDB_RawFile::Close()
192 {
193 x_Close(eThrowOnError);
194 }
195
Attach(CBDB_RawFile & bdb_file)196 void CBDB_RawFile::Attach(CBDB_RawFile& bdb_file)
197 {
198 Close();
199 m_DB = bdb_file.m_DB;
200 m_DB_Attached = true;
201 }
202
SetEnv(CBDB_Env & env)203 void CBDB_RawFile::SetEnv(CBDB_Env& env)
204 {
205 m_Env = &env;
206 }
207
208
209 #ifdef HAVE_GETMPF
210
SetCachePriority(ECachePriority priority)211 void CBDB_RawFile::SetCachePriority(ECachePriority priority)
212 {
213 if (m_DB) {
214 DB_MPOOLFILE* mpf = m_DB->get_mpf(m_DB);
215 if (mpf) {
216 DB_CACHE_PRIORITY p = DB_PRIORITY_DEFAULT;
217 switch (priority) {
218 case eCache_Lowest:
219 p = DB_PRIORITY_VERY_LOW;
220 break;
221 case eCache_Low:
222 p = DB_PRIORITY_LOW;
223 break;
224
225 default:
226 case eCache_Default:
227 p = DB_PRIORITY_DEFAULT;
228 break;
229 case eCache_High:
230 p = DB_PRIORITY_HIGH;
231 break;
232 case eCache_Highest:
233 p = DB_PRIORITY_VERY_HIGH;
234 break;
235 }
236 mpf->set_priority(mpf, p);
237 }
238 }
239 }
240 #else
SetCachePriority(ECachePriority)241 void CBDB_RawFile::SetCachePriority(ECachePriority)
242 {
243 return;
244 }
245 #endif // HAVE_GETMPF
246
247
GetTxn()248 DB_TXN* CBDB_RawFile::GetTxn()
249 {
250 if (m_Trans)
251 return m_Trans->GetTxn();
252 return 0;
253 }
254
SetCompressor(ICompression * compressor,EOwnership own)255 void CBDB_RawFile::SetCompressor(ICompression* compressor, EOwnership own)
256 {
257 m_Compressor.reset(compressor, own);
258 }
259
x_Close(EIgnoreError close_mode)260 void CBDB_RawFile::x_Close(EIgnoreError close_mode)
261 {
262 if (m_FileName.empty())
263 return;
264 //LOG_POST_X(1, Info << "Closing: " << m_FileName);
265
266 if (m_DB_Attached) {
267 m_DB = 0;
268 m_DB_Attached = false;
269 }
270 else
271 if (m_DB) {
272 int ret = m_DB->close(m_DB, 0);
273 m_DB = 0;
274 if (close_mode == eThrowOnError) {
275 BDB_CHECK(ret, m_FileName.c_str());
276 if (m_Env)
277 m_Env->LsnResetForMemLog(m_FileName.c_str());
278 } else {
279 if (ret != 0) {
280 ERR_POST_X(2, "Error when closing " << m_FileName);
281 } else {
282 try {
283 if (m_Env)
284 m_Env->LsnResetForMemLog(m_FileName.c_str());
285 } catch (CBDB_Exception& ex) {
286 ERR_POST_X(3, "Error " << ex.what() << " resetting LSN for "
287 << m_FileName);
288 }
289 }
290 }
291 }
292
293 m_FileName.erase();
294 m_Database.erase();
295 }
296
297
Open(const string & filename,const string & database,EOpenMode open_mode,bool support_dirty_read,unsigned rec_len)298 void CBDB_RawFile::Open(const string& filename,
299 const string& database,
300 EOpenMode open_mode,
301 bool support_dirty_read,
302 unsigned rec_len)
303 {
304 if ( !m_FileName.empty() )
305 Close();
306
307 const char * db;
308 if (database.empty()) {
309 db = 0;
310 } else {
311 db = database.c_str();
312 }
313
314
315 x_Open(filename.c_str(), db, open_mode, support_dirty_read, rec_len);
316
317 m_FileName = filename;
318 if (db)
319 m_Database = database;
320 else
321 m_Database = "";
322 }
323
324
Reopen(EOpenMode open_mode,bool support_dirty_read,unsigned rec_len)325 void CBDB_RawFile::Reopen(EOpenMode open_mode,
326 bool support_dirty_read,
327 unsigned rec_len)
328 {
329 _ASSERT(!m_FileName.empty());
330
331 if (m_DB_Attached) {
332 BDB_THROW(eInvalidOperation, "Cannot reopen attached object");
333 }
334
335 int ret = m_DB->close(m_DB, 0);
336 m_DB = 0;
337
338 BDB_CHECK(ret, m_FileName.c_str());
339 x_Open(m_FileName.c_str(),
340 !m_Database.empty() ? m_Database.c_str() : 0,
341 open_mode, support_dirty_read, rec_len);
342 }
343
344
Rename(const string & file,const string & old_name,const string & new_name)345 void CBDB_RawFile::Rename(const string& file,
346 const string& old_name,
347 const string& new_name)
348 {
349 _ASSERT(m_DB);
350 if (IsOpen()) {
351 NCBI_THROW(CBDB_Exception, eUnknown,
352 "Cannot call rename on an opened database");
353 }
354 int ret = m_DB->rename(m_DB,
355 file.c_str(), old_name.c_str(), new_name.c_str(),
356 0);
357 BDB_CHECK(ret, file.c_str());
358 }
359
360
Remove(const string & filename,const string & database)361 void CBDB_RawFile::Remove(const string& filename, const string& database)
362 {
363 const char* db_name;
364 if (database.empty()) {
365 db_name = 0;
366 } else {
367 db_name = database.c_str();
368 }
369
370 if (m_DB_Attached) {
371 BDB_THROW(eInvalidOperation, "Cannot remove attached object");
372 }
373
374 // temporary DB is used here, because BDB remove call invalidates the
375 // DB argument redardless of the result.
376 DB* db = 0;
377 CDB_guard guard(&db);
378 int ret = db_create(&db, m_Env ? m_Env->GetEnv() : 0, 0);
379 BDB_CHECK(ret, 0);
380
381 ret = db->remove(db, filename.c_str(), db_name, 0);
382 guard.release();
383 if (ret == ENOENT || ret == EINVAL)
384 return; // Non existent table cannot be removed
385
386 BDB_CHECK(ret, filename.c_str());
387 }
388
389
Truncate()390 unsigned int CBDB_RawFile::Truncate()
391 {
392 _ASSERT(m_DB != 0);
393 u_int32_t count;
394 DB_TXN* txn = GetTxn();
395 int ret = m_DB->truncate(m_DB,
396 txn,
397 &count,
398 0);
399
400 BDB_CHECK(ret, FileName().c_str());
401 return count;
402 }
403
404 class DBT_ptr {
405 public:
DBT_ptr()406 DBT_ptr() {
407 memset(&m_DBT, 0, sizeof(m_DBT));
408 }
~DBT_ptr()409 ~DBT_ptr() {
410 if (m_DBT.data) free(m_DBT.data);
411 m_DBT.size = 0;
412 m_DBT.data = 0;
413 }
operator =(const DBT & dbt)414 DBT_ptr& operator=(const DBT& dbt) {
415 if (m_DBT.data) free(m_DBT.data);
416 m_DBT.size = dbt.size;
417 m_DBT.data = malloc(dbt.size);
418 memcpy(m_DBT.data, dbt.data, dbt.size);
419 return *this;
420 }
operator *()421 DBT* operator*() {
422 return &m_DBT;
423 }
424 private:
425 DBT m_DBT;
426 };
427
428
SafeTruncate()429 unsigned int CBDB_RawFile::SafeTruncate()
430 {
431 _ASSERT(m_DB != 0);
432 u_int32_t count = 0;
433
434 SetTransaction(NULL);
435 bool done = false;
436 const int k_bulk_init = 1000;
437 int bulk = k_bulk_init;
438 int bulk_age = 0;
439
440 vector<DBT_ptr> keys;
441 keys.resize(k_bulk_init);
442 while (!done) {
443 int nrec, ret;
444 DBC* dbcp = 0;
445 DBT key, data;
446 ret = m_DB->cursor(m_DB, NULL, &dbcp, 0);
447 BDB_CHECK(ret, FileName().c_str());
448 memset(&key, 0, sizeof(key));
449 memset(&data, 0, sizeof(data));
450 for (nrec = 0; nrec < bulk && (ret = dbcp->c_get(dbcp, &key, &data, DB_NEXT)) == 0;
451 ++nrec) {
452 keys[nrec] = key;
453 }
454 if (dbcp) dbcp->c_close(dbcp);
455 dbcp = 0;
456 if (ret != DB_NOTFOUND) {
457 BDB_CHECK(ret, FileName().c_str());
458 }
459 if (!nrec) break;
460 DB_TXN* txn = m_Env->CreateTxn();
461 try {
462 for (int n = 0; n < nrec; ++n) {
463 ret = m_DB->del(m_DB, txn, *keys[n], 0);
464 if (ret) break;
465 }
466 if (ret == 0) txn->commit(txn, 0);
467 else {
468 txn->abort(txn);
469 BDB_CHECK(ret, FileName().c_str());
470 }
471 done = nrec < bulk;
472 count += nrec;
473 if (! done && bulk < k_bulk_init && ++bulk_age > 3) {
474 bulk *=2;
475 if (bulk > k_bulk_init) bulk = k_bulk_init;
476 bulk_age = 0;
477 }
478 } catch (CBDB_ErrnoException& ex) {
479 if (! ex.IsNoMem()) throw;
480 bulk /= 2;
481 if (bulk == 0) throw;
482 bulk_age = 0;
483 done = false;
484 }
485 }
486 return count;
487 }
488
489
490 /// Compact the database. The target fill percent per page can be
491 /// supplied, to allow for known expansion
s_DefaultCompactCallback()492 static bool s_DefaultCompactCallback()
493 {
494 return true;
495 }
496
Compact(ECompact compact_type,int target_fill_pct)497 void CBDB_RawFile::Compact(ECompact compact_type,
498 int target_fill_pct)
499 {
500 CompactEx(s_DefaultCompactCallback, compact_type, target_fill_pct);
501 }
502
503
CompactEx(FContinueCompact compact_callback,ECompact compact_type,int target_fill_pct)504 void CBDB_RawFile::CompactEx(FContinueCompact compact_callback,
505 ECompact compact_type,
506 int target_fill_pct)
507 {
508 #ifdef DB_COMPACT_FLAGS
509 _ASSERT(m_DB != 0);
510
511 u_int32_t flags = 0;
512 if (compact_type == eCompactNoFree) {
513 /// default
514 } else if (compact_type == eCompactFreeExisting) {
515 flags = DB_FREELIST_ONLY;
516 } else if (compact_type == eCompactFreeAll) {
517 flags = DB_FREE_SPACE;
518 }
519
520 target_fill_pct = max(target_fill_pct, 0);
521 DB_TXN* txn = GetTxn();
522
523 unsigned int pages_examined = 0;
524 unsigned int pages_freed = 0;
525 unsigned int levels_removed = 0;
526 unsigned int pages_truncated = 0;
527 for (int i = 0; i < 2; ++i) {
528 DB_COMPACT compact;
529 memset(&compact, 0, sizeof(compact));
530 compact.compact_fillpercent = target_fill_pct;
531 compact.compact_timeout = 0;
532
533 int ret = m_DB->compact(m_DB, txn, NULL, NULL, &compact,
534 flags, NULL);
535 BDB_CHECK(ret, FileName().c_str());
536
537 pages_examined += compact.compact_pages_examine;
538 pages_freed += compact.compact_pages_free;
539 levels_removed += compact.compact_levels;
540 pages_truncated += compact.compact_pages_truncated;
541
542 LOG_POST_X(4, Info << "CBDB_RawFile::Compact(): "
543 << "round " << i + 1 << ": "
544 << compact.compact_pages_examine << " pages examined / "
545 << compact.compact_pages_free << " pages freed / "
546 << compact.compact_levels << " levels removed / "
547 << compact.compact_pages_truncated << " pages truncated");
548
549 if ( !compact_callback() ) {
550 break;
551 }
552 }
553
554 LOG_POST_X(5, Info << "CBDB_RawFile::Compact(): "
555 << pages_examined << " pages examined / "
556 << pages_freed << " pages freed / "
557 << levels_removed << " levels removed / "
558 << pages_truncated << " pages truncated");
559 #endif
560 }
561
562
SetCacheSize(unsigned int cache_size)563 void CBDB_RawFile::SetCacheSize(unsigned int cache_size)
564 {
565 m_CacheSize = cache_size;
566 if (m_DB) {
567 int ret = m_DB->set_cachesize(m_DB, 0, m_CacheSize, 1);
568 BDB_CHECK(ret, 0);
569 }
570 }
571
572
RevSplitOff()573 void CBDB_RawFile::RevSplitOff()
574 {
575 m_RevSplitOff = true;
576 }
577
x_SetTransaction(CBDB_Transaction * trans)578 void CBDB_RawFile::x_SetTransaction(CBDB_Transaction* trans)
579 {
580 if (m_Trans) {
581 if (m_TransAssociation == (int) CBDB_Transaction::eFullAssociation) {
582 m_Trans->Remove(this);
583 }
584 }
585
586 m_Trans = trans;
587 if (m_Trans) {
588 m_TransAssociation = m_Trans->GetAssociationMode();
589 if (m_TransAssociation == (int) CBDB_Transaction::eFullAssociation) {
590 m_Trans->Add(this);
591 }
592 }
593 }
594
x_RemoveTransaction(CBDB_Transaction * trans)595 void CBDB_RawFile::x_RemoveTransaction(CBDB_Transaction* trans)
596 {
597 if (trans == m_Trans) {
598 m_Trans = 0;
599 }
600 }
601
SetTransaction(ITransaction * trans)602 void CBDB_RawFile::SetTransaction(ITransaction* trans)
603 {
604 CBDB_Transaction* db_trans = CBDB_Transaction::CastTransaction(trans);
605 x_SetTransaction(db_trans);
606 }
607
RemoveTransaction(ITransaction * trans)608 void CBDB_RawFile::RemoveTransaction(ITransaction* trans)
609 {
610 CBDB_Transaction* db_trans = CBDB_Transaction::CastTransaction(trans);
611 x_RemoveTransaction(db_trans);
612 }
613
GetTransaction()614 ITransaction* CBDB_RawFile::GetTransaction()
615 {
616 return m_Trans;
617 }
618
619
x_CreateDB(unsigned rec_len)620 void CBDB_RawFile::x_CreateDB(unsigned rec_len)
621 {
622 _ASSERT(m_DB == 0);
623 _ASSERT(!m_DB_Attached);
624
625 CDB_guard guard(&m_DB);
626
627 int ret = db_create(&m_DB, m_Env ? m_Env->GetEnv() : 0, 0);
628 BDB_CHECK(ret, 0);
629
630 if (m_DB_Type == eBtree && m_CmpOverride) {
631 SetCmp(m_DB);
632 }
633 if (m_DB_Type == eHash && m_CmpOverride) {
634 SetHash(m_DB);
635 }
636
637 if ( m_PageSize ) {
638 ret = m_DB->set_pagesize(m_DB, m_PageSize);
639 BDB_CHECK(ret, 0);
640 }
641
642 if (!m_Env) {
643 ret = m_DB->set_cachesize(m_DB, 0, m_CacheSize, 1);
644 BDB_CHECK(ret, 0);
645 }
646
647 if (DuplicatesAllowed()) {
648 ret = m_DB->set_flags(m_DB, DB_DUP);
649 BDB_CHECK(ret, 0);
650 }
651
652 if (m_RevSplitOff) {
653 ret = m_DB->set_flags(m_DB, DB_REVSPLITOFF);
654 BDB_CHECK(ret, 0);
655 }
656
657 switch (m_DB_Type)
658 {
659 case eQueue:
660 _ASSERT(rec_len);
661 m_RecLen = rec_len;
662 ret = m_DB->set_re_len(m_DB, rec_len);
663 BDB_CHECK(ret, 0);
664 break;
665
666 case eHash:
667 if (m_H_ffactor) {
668 ret = m_DB->set_h_ffactor(m_DB, m_H_ffactor);
669 BDB_CHECK(ret, FileName().c_str());
670 }
671 if (m_H_nelem) {
672 ret = m_DB->set_h_nelem(m_DB, m_H_nelem);
673 BDB_CHECK(ret, FileName().c_str());
674 }
675 break;
676
677 case eBtree:
678 if (m_BT_minkey) {
679 ret = m_DB->set_bt_minkey(m_DB, m_BT_minkey);
680 BDB_CHECK(ret, FileName().c_str());
681 }
682 break;
683
684 default:
685 break;
686 }
687
688 guard.release();
689 }
690
691
x_Open(const char * filename,const char * database,EOpenMode open_mode,bool support_dirty_read,unsigned rec_len)692 void CBDB_RawFile::x_Open(const char* filename,
693 const char* database,
694 EOpenMode open_mode,
695 bool support_dirty_read,
696 unsigned rec_len)
697 {
698 int ret;
699 const char* open_filename_param = filename;
700 if (strlen(filename) == 0)
701 open_filename_param = NULL;
702
703 if (m_DB == 0) {
704 x_CreateDB(rec_len);
705 }
706
707 if (open_mode == eCreate) {
708 Remove(filename, database ? database : "");
709 x_Create(filename, database);
710 }
711 else {
712 u_int32_t open_flags;
713
714 switch (open_mode)
715 {
716 case eReadOnly:
717 open_flags = DB_RDONLY;
718 break;
719 case eCreate:
720 open_flags = DB_CREATE;
721 break;
722 default:
723 open_flags = 0;
724 break;
725 }
726
727 DB_TXN* txn = 0; // GetTxn();
728
729 if (m_Env) {
730 if (m_Env->IsTransactional()) {
731 open_flags |= DB_THREAD | DB_AUTO_COMMIT;
732 }
733 }
734
735 if (support_dirty_read) {
736 open_flags |= DB_DIRTY_READ;
737 }
738
739
740 DBTYPE db_type = DB_BTREE;
741 switch (m_DB_Type) {
742 case eBtree:
743 db_type = DB_BTREE;
744 break;
745 case eHash:
746 db_type = DB_HASH;
747 break;
748 case eQueue:
749 db_type = DB_QUEUE;
750 m_RecLen = rec_len;
751 break;
752 default:
753 _ASSERT(0);
754 }
755
756 ret = m_DB->open(m_DB,
757 txn,
758 open_filename_param,
759 database, // database name
760 db_type,
761 open_flags,
762 kOpenFileMask
763 );
764 if ( ret ) {
765 m_DB->close(m_DB, 0);
766 m_DB = 0;
767 if (open_mode == eCreate ||
768 open_mode == eReadWriteCreate)
769 {
770 x_CreateDB(rec_len);
771 x_Create(filename, database);
772 }
773 else {
774 BDB_CHECK(ret, filename);
775 }
776 } else {
777 // file opened succesfully, check if it needs
778 // a byte swapping (different byteorder)
779
780 int isswapped;
781 ret = m_DB->get_byteswapped(m_DB, &isswapped);
782 BDB_CHECK(ret, filename);
783
784 m_ByteSwapped = (isswapped!=0);
785 if (m_ByteSwapped) {
786 // re-open the file
787 m_DB->close(m_DB, 0);
788 m_DB = 0;
789
790 x_SetByteSwapped(m_ByteSwapped);
791 x_CreateDB(rec_len);
792
793 ret = m_DB->open(m_DB,
794 txn,
795 open_filename_param,
796 database, // database name
797 db_type,
798 open_flags,
799 kOpenFileMask);
800 BDB_CHECK(ret, filename);
801
802 }
803
804 }
805 } // else open_mode == Create
806
807 m_OpenMode = open_mode;
808 }
809
810
SetPageSize(unsigned int page_size)811 void CBDB_RawFile::SetPageSize(unsigned int page_size)
812 {
813 _ASSERT(m_DB == 0); // we can set page size only before opening the file
814 if (((page_size - 1) & page_size) != 0) {
815 BDB_THROW(eInvalidValue, "Page size must be power of 2");
816 }
817 m_PageSize = page_size;
818 }
819
GetPageSize()820 unsigned int CBDB_RawFile::GetPageSize()
821 {
822 if ( !m_PageSize && m_DB) {
823 int ret = m_DB->get_pagesize(m_DB, &m_PageSize);
824 BDB_CHECK(ret, 0);
825 }
826 return m_PageSize;
827 }
828
Sync()829 void CBDB_RawFile::Sync()
830 {
831 int ret = m_DB->sync(m_DB, 0);
832 BDB_CHECK(ret, FileName().c_str());
833 }
834
835
CountRecs(bool bFast)836 unsigned CBDB_RawFile::CountRecs(bool bFast)
837 {
838 Uint4 flags = 0;
839 if (bFast) {
840 flags = DB_FAST_STAT;
841 }
842 DB_BTREE_STAT* stp;
843 CBDB_Transaction* trans = GetBDBTransaction();
844 DB_TXN* txn = trans ? trans->GetTxn() : 0;
845 int ret = m_DB->stat(m_DB, txn, &stp, flags);
846
847 BDB_CHECK(ret, FileName().c_str());
848 u_int32_t rc = stp->bt_ndata;
849
850 ::free(stp);
851
852 return rc;
853 }
854
PrintStat(CNcbiOstream & out)855 void CBDB_RawFile::PrintStat(CNcbiOstream & out)
856 {
857 DB_BTREE_STAT* stp = 0;
858 CBDB_Transaction* trans = GetBDBTransaction();
859 DB_TXN* txn = trans ? trans->GetTxn() : 0;
860 int ret = m_DB->stat(m_DB, txn, &stp, 0);
861
862 BDB_CHECK(ret, FileName().c_str());
863
864 out << FileName().c_str() << NcbiEndl;
865 out << "bt_version : " << stp->bt_version << NcbiEndl
866 << "bt_nkeys : " << stp->bt_nkeys << NcbiEndl
867 << "bt_ndata : " << stp->bt_ndata << NcbiEndl
868 << "bt_pagesize : " << stp->bt_pagesize << NcbiEndl
869 << "bt_levels : " << stp->bt_levels << NcbiEndl
870 << "bt_int_pg : " << stp->bt_int_pg << NcbiEndl
871 << "bt_leaf_pg : " << stp->bt_leaf_pg << NcbiEndl
872 << "bt_dup_pg : " << stp->bt_dup_pg << NcbiEndl
873 << "bt_over_pg : " << stp->bt_over_pg << NcbiEndl
874 << "bt_empty_pg : " << stp->bt_empty_pg << NcbiEndl
875 << "bt_free : " << stp->bt_free << NcbiEndl
876 << "bt_int_pgfree : " << stp->bt_int_pgfree << NcbiEndl
877 << "bt_leaf_pgfree: " << stp->bt_leaf_pgfree<< NcbiEndl
878 << "bt_dup_pgfree : " << stp->bt_dup_pgfree << NcbiEndl
879 << "bt_over_pgfree: " << stp->bt_over_pgfree<< NcbiEndl
880 ;
881
882 if (stp)
883 ::free(stp);
884 }
885
886
x_Create(const char * filename,const char * database)887 void CBDB_RawFile::x_Create(const char* filename, const char* database)
888 {
889 _ASSERT(!m_DB_Attached);
890 u_int32_t open_flags = DB_CREATE;
891
892 DB_TXN* txn = 0; //GetTxn();
893
894 if (m_Env) {
895 if (m_Env->IsTransactional()) {
896 open_flags |= DB_THREAD | DB_AUTO_COMMIT;
897 }
898 }
899
900 DBTYPE db_type = DB_BTREE;
901 switch (m_DB_Type) {
902 case eBtree:
903 break;
904 case eQueue:
905 db_type = DB_QUEUE;
906 break;
907 case eHash:
908 db_type = DB_HASH;
909 break;
910 default:
911 _ASSERT(0);
912 }
913
914 const char* open_filename_param = filename;
915 if (strlen(filename) == 0)
916 open_filename_param = NULL;
917
918 int ret = m_DB->open(m_DB,
919 txn,
920 open_filename_param,
921 database, // database name
922 db_type,
923 open_flags,
924 kOpenFileMask);
925 if ( ret ) {
926 m_DB->close(m_DB, 0);
927 m_DB = 0;
928 BDB_CHECK(ret, filename);
929 }
930 }
931
932
CreateCursor(CBDB_Transaction * trans,unsigned int flags) const933 DBC* CBDB_RawFile::CreateCursor(CBDB_Transaction* trans,
934 unsigned int flags) const
935 {
936 DBC* cursor;
937
938 if (!m_DB) {
939 BDB_THROW(eInvalidValue, "Cannot create cursor for unopen file.");
940 }
941
942 DB_TXN* txn = 0; // GetTxn();
943 if (trans) {
944 txn = trans->GetTxn();
945 }
946
947 int ret = m_DB->cursor(m_DB,
948 txn,
949 &cursor,
950 flags);
951 BDB_CHECK(ret, FileName().c_str());
952 return cursor;
953 }
954
x_SetByteSwapped(bool bswp)955 void CBDB_RawFile::x_SetByteSwapped(bool bswp)
956 {
957 m_ByteSwapped = bswp;
958 }
959
960
SetHashFillFactor(unsigned h_ffactor)961 void CBDB_RawFile::SetHashFillFactor(unsigned h_ffactor)
962 {
963 _ASSERT(m_DB_Type == eHash);
964 m_H_ffactor = h_ffactor;
965 }
966
SetHashNelem(unsigned h_nelem)967 void CBDB_RawFile::SetHashNelem(unsigned h_nelem)
968 {
969 _ASSERT(m_DB_Type == eHash);
970 m_H_nelem = h_nelem;
971 }
972
SetHash(DB * db)973 void CBDB_RawFile::SetHash(DB* db)
974 {
975 _ASSERT(m_DB_Type == eHash);
976 int ret = db->set_h_hash(db, BDB_Hash);
977 BDB_CHECK(ret, 0);
978 }
979
SetBtreeMinKeysPerPage(unsigned int keys_per_page)980 void CBDB_RawFile::SetBtreeMinKeysPerPage(unsigned int keys_per_page)
981 {
982 _ASSERT(m_DB_Type == eBtree);
983 m_BT_minkey = max((unsigned int)2, keys_per_page);
984 }
985
GetBtreeMinKeysPerPage()986 unsigned int CBDB_RawFile::GetBtreeMinKeysPerPage()
987 {
988 _ASSERT(m_DB_Type == eBtree);
989 if ( !m_BT_minkey && m_DB) {
990 int ret = m_DB->get_bt_minkey(m_DB, &m_BT_minkey);
991 BDB_CHECK(ret, 0);
992 }
993 return m_BT_minkey;
994 }
995
x_FetchBufferDecompress(DBT * data,void * usr_data)996 int CBDB_RawFile::x_FetchBufferDecompress(DBT *data, void* usr_data)
997 {
998 data->data = usr_data;
999
1000 unsigned char* compressed = m_CompressBuffer.data();
1001 unsigned bytes_compressed;
1002 #ifdef HAVE_UNALIGNED_READS
1003 bytes_compressed = *((unsigned*)compressed);
1004 #else
1005 ::memcpy(&bytes_compressed, compressed, 4);
1006 #endif
1007
1008 if (bytes_compressed == 0) {
1009 data->size-=4;
1010 if (data->data) {
1011 ::memcpy(data->data, compressed + 4, data->size);
1012 }
1013 } else {
1014 if (data->ulen < bytes_compressed) {
1015 data->size = bytes_compressed;
1016 return DB_BUFFER_SMALL;
1017 }
1018 data->size-=4;
1019 if (data->data) {
1020 size_t dst_len;
1021 bool decomp_ok =
1022 m_Compressor->DecompressBuffer(compressed + 4,
1023 data->size,
1024 data->data,
1025 data->ulen,
1026 &dst_len);
1027 data->size = bytes_compressed;
1028 if (!decomp_ok) {
1029 BDB_THROW(eCompressorError,
1030 m_Compressor->GetErrorDescription());
1031 }
1032 _ASSERT(dst_len == bytes_compressed);
1033 }
1034 }
1035 return 0;
1036 }
1037
x_DB_Fetch(DBT * key,DBT * data,unsigned flags)1038 int CBDB_RawFile::x_DB_Fetch(DBT *key,
1039 DBT *data,
1040 unsigned flags)
1041 {
1042 _ASSERT(key);
1043 _ASSERT(data);
1044
1045 int ret;
1046 DB_TXN* txn = GetTxn();
1047
1048 if (m_Compressor.get()) {
1049 _ASSERT(flags == 0 || flags & DB_DBT_USERMEM);
1050 _ASSERT(data->doff == 0);
1051
1052 m_CompressBuffer.resize_mem(data->ulen + 4);
1053 void* usr_data = data->data;
1054 data->data = m_CompressBuffer.data();
1055
1056 ret = m_DB->get(m_DB, txn, key, data, flags);
1057 if (ret == 0) {
1058 ret = x_FetchBufferDecompress(data, usr_data);
1059 }
1060 } else {
1061 ret = m_DB->get(m_DB, txn, key, data, flags);
1062 }
1063 return ret;
1064 }
1065
1066
x_DBC_Fetch(DBC * dbc,DBT * key,DBT * data,unsigned flags)1067 int CBDB_RawFile::x_DBC_Fetch(DBC* dbc,
1068 DBT *key,
1069 DBT *data,
1070 unsigned flags)
1071 {
1072 int ret;
1073 if (m_Compressor.get()) {
1074 m_CompressBuffer.resize_mem(data->ulen + 4);
1075 void* usr_data = data->data;
1076 data->data = m_CompressBuffer.data();
1077
1078 ret = dbc->c_get(dbc, key, data, flags);
1079 if (ret == 0) {
1080 ret = x_FetchBufferDecompress(data, usr_data);
1081 }
1082 } else {
1083 ret = dbc->c_get(dbc, key, data, flags);
1084 }
1085 return ret;
1086 }
1087
1088 /// Record size cut off for compression
1089 ///
1090 const unsigned k_BDB_CompressionCutOff = 128;
1091
x_DB_Put(DBT * key,DBT * data,unsigned flags)1092 int CBDB_RawFile::x_DB_Put(DBT *key,
1093 DBT *data,
1094 unsigned flags)
1095 {
1096 int ret;
1097
1098 DB_TXN* txn = GetTxn();
1099 if (m_Compressor.get()) {
1100 // save original data fields
1101 void* usr_data = data->data;
1102 unsigned usr_size = data->size;
1103
1104 m_CompressBuffer.resize_mem(data->size + 4);
1105
1106 bool compressed = false;
1107
1108 if (data->size > k_BDB_CompressionCutOff) {
1109 m_CompressBuffer.resize_mem(data->size + 4);
1110
1111 unsigned *buf = (unsigned*)m_CompressBuffer.data();
1112 #ifdef HAVE_UNALIGNED_READS
1113 *buf = 0;
1114 #else
1115 ::memset(buf, 0, 4);
1116 #endif
1117 buf += 4;
1118 size_t dst_len;
1119
1120 compressed = m_Compressor->CompressBuffer(data->data, data->size,
1121 buf, data->size,
1122 &dst_len);
1123 if (compressed) {
1124 _ASSERT(dst_len <= data->size);
1125 buf = (unsigned*)m_CompressBuffer.data();
1126 #ifdef HAVE_UNALIGNED_READS
1127 *buf = (unsigned)dst_len;
1128 #else
1129 ::memcpy(buf, &dst_len, 4);
1130 #endif
1131 m_CompressBuffer.resize_mem(dst_len);
1132 }
1133 }
1134
1135 if (!compressed) { // store uncompressed data
1136 unsigned *buf = (unsigned*)m_CompressBuffer.data();
1137 #ifdef HAVE_UNALIGNED_READS
1138 *buf = 0;
1139 #else
1140 ::memset(buf, 0, 4);
1141 #endif
1142 buf += 4;
1143 ::memcpy(buf, data->data, data->size);
1144 }
1145
1146 // store the compress buffer
1147 data->data = m_CompressBuffer.data();
1148 data->size = (u_int32_t)m_CompressBuffer.size();
1149
1150 ret = m_DB->put(m_DB, txn, key, data, flags);
1151
1152 // restore buffers
1153 data->data = usr_data;
1154 data->size = usr_size;
1155
1156 } else {
1157 ret = m_DB->put(m_DB, txn, key, data, flags);
1158 }
1159 return ret;
1160 }
1161
1162
x_DB_CPut(DBC * dbc,DBT * key,DBT * data,unsigned flags)1163 int CBDB_RawFile::x_DB_CPut(DBC *dbc,
1164 DBT *key,
1165 DBT *data,
1166 unsigned flags)
1167 {
1168 int ret;
1169
1170 if (m_Compressor.get()) {
1171 // save original data fields
1172 void* usr_data = data->data;
1173 unsigned usr_size = data->size;
1174
1175 m_CompressBuffer.resize_mem(data->size + 4);
1176
1177 bool compressed = false;
1178
1179 if (data->size > k_BDB_CompressionCutOff) {
1180 m_CompressBuffer.resize_mem(data->size + 4);
1181
1182 unsigned *buf = (unsigned*)m_CompressBuffer.data();
1183 #ifdef HAVE_UNALIGNED_READS
1184 *buf = 0;
1185 #else
1186 ::memset(buf, 0, 4);
1187 #endif
1188 buf += 4;
1189 size_t dst_len;
1190 compressed = m_Compressor->CompressBuffer(data->data, data->size,
1191 buf, data->size,
1192 &dst_len);
1193 if (compressed) {
1194 _ASSERT(dst_len <= data->size);
1195 buf = (unsigned*)m_CompressBuffer.data();
1196 #ifdef HAVE_UNALIGNED_READS
1197 *buf = (unsigned)dst_len;
1198 #else
1199 ::memcpy(buf, &dst_len, 4);
1200 #endif
1201 m_CompressBuffer.resize_mem(dst_len);
1202 }
1203 }
1204
1205 if (!compressed) { // store uncompressed data
1206 unsigned *buf = (unsigned*)m_CompressBuffer.data();
1207 #ifdef HAVE_UNALIGNED_READS
1208 *buf = 0;
1209 #else
1210 ::memset(buf, 0, 4);
1211 #endif
1212 buf += 4;
1213 ::memcpy(buf, data->data, data->size);
1214 }
1215
1216 // store the compress buffer
1217 data->data = m_CompressBuffer.data();
1218 data->size = (u_int32_t)m_CompressBuffer.size();
1219
1220 ret = dbc->c_put(dbc, key, data, flags);
1221
1222 // restore buffers
1223 data->data = usr_data;
1224 data->size = usr_size;
1225
1226 } else {
1227 ret = dbc->c_put(dbc, key, data, flags);
1228 }
1229
1230 return ret;
1231
1232 }
1233
1234
1235 /////////////////////////////////////////////////////////////////////////////
1236 //
1237 // CBDB_File::
1238 //
1239
1240
CBDB_File(EDuplicateKeys dup_keys,EDBType db_type)1241 CBDB_File::CBDB_File(EDuplicateKeys dup_keys, EDBType db_type)
1242 : CBDB_RawFile(dup_keys, db_type),
1243 m_KeyBuf(new CBDB_BufferManager),
1244 m_BufsAttached(false),
1245 m_BufsCreated(false),
1246 m_DataBufDisabled(false),
1247 m_LegacyString(false),
1248 m_OwnFields(false),
1249 m_DisabledNull(false),
1250 m_PrefixCompress(false)
1251 {
1252 }
1253
SetFieldOwnership(bool own_fields)1254 void CBDB_File::SetFieldOwnership(bool own_fields)
1255 {
1256 m_OwnFields = own_fields;
1257
1258 m_KeyBuf->SetFieldOwnership(own_fields);
1259 if (m_DataBuf.get() != 0) {
1260 m_DataBuf->SetFieldOwnership(own_fields);
1261 }
1262 }
1263
x_ConstructKeyBuf()1264 void CBDB_File::x_ConstructKeyBuf()
1265 {
1266 m_KeyBuf.reset(new CBDB_BufferManager);
1267 m_KeyBuf->SetLegacyStringsCheck(m_LegacyString);
1268 m_KeyBuf->SetFieldOwnership(m_OwnFields);
1269 }
1270
x_ConstructDataBuf()1271 void CBDB_File::x_ConstructDataBuf()
1272 {
1273 m_DataBuf.reset(new CBDB_BufferManager);
1274 if (!m_DisabledNull) {
1275 m_DataBuf->SetNullable();
1276 }
1277 m_DataBuf->SetLegacyStringsCheck(m_LegacyString);
1278 m_DataBuf->SetFieldOwnership(m_OwnFields);
1279 }
1280
BindKey(const char * field_name,CBDB_Field * key_field,size_t buf_size)1281 void CBDB_File::BindKey(const char* field_name,
1282 CBDB_Field* key_field,
1283 size_t buf_size)
1284 {
1285 _ASSERT(!IsOpen());
1286 _ASSERT(m_KeyBuf.get());
1287 _ASSERT(key_field);
1288
1289 key_field->SetName(field_name);
1290 m_KeyBuf->Bind(key_field);
1291 if ( buf_size )
1292 key_field->SetDataSize(buf_size);
1293 }
1294
1295
BindData(const char * field_name,CBDB_Field * data_field,size_t buf_size,ENullable is_nullable)1296 void CBDB_File::BindData(const char* field_name,
1297 CBDB_Field* data_field,
1298 size_t buf_size,
1299 ENullable is_nullable)
1300 {
1301 _ASSERT(!IsOpen());
1302 _ASSERT(data_field);
1303
1304 data_field->SetName(field_name);
1305
1306 if (m_DataBuf.get() == 0) { // data buffer is not yet created
1307 x_ConstructDataBuf();
1308 }
1309
1310 m_DataBuf->Bind(data_field);
1311 if ( buf_size > 0) {
1312 data_field->SetDataSize(buf_size);
1313 }
1314 if (is_nullable == eNullable && !m_DisabledNull) {
1315 data_field->SetNullable();
1316 }
1317 }
1318
1319
Open(const string & filename,const string & database,EOpenMode open_mode,bool support_dirty_read,unsigned rec_len)1320 void CBDB_File::Open(const string& filename,
1321 const string& database,
1322 EOpenMode open_mode,
1323 bool support_dirty_read,
1324 unsigned rec_len)
1325 {
1326 if ( IsOpen() )
1327 Close();
1328 x_CheckConstructBuffers();
1329
1330 if (m_DB_Type == eQueue) {
1331 DisableDataPacking();
1332 if (m_DataBuf.get()) {
1333 rec_len = (unsigned)m_DataBuf->ComputeBufferSize();
1334 }
1335 }
1336
1337 CBDB_RawFile::Open(filename, database,
1338 open_mode, support_dirty_read, rec_len);
1339
1340 m_DB->app_private = (void*) m_KeyBuf.get();
1341
1342 }
1343
1344
Reopen(EOpenMode open_mode,bool support_dirty_read)1345 void CBDB_File::Reopen(EOpenMode open_mode, bool support_dirty_read)
1346 {
1347 unsigned rec_len = 0;
1348 if (m_DB_Type == eQueue) {
1349 if (m_DataBuf.get()) {
1350 rec_len = (unsigned)m_DataBuf->ComputeBufferSize();
1351 }
1352 }
1353 CBDB_RawFile::Reopen(open_mode, support_dirty_read, rec_len);
1354 m_DB->app_private = (void*) m_KeyBuf.get();
1355 if ( m_DataBuf.get() ) {
1356 m_DataBuf->SetAllNull();
1357 }
1358 bool byte_swapped = IsByteSwapped();
1359 m_KeyBuf->SetByteSwapped(byte_swapped);
1360 if (m_DataBuf.get()) {
1361 m_DataBuf->SetByteSwapped(byte_swapped);
1362 }
1363 }
1364
1365
Attach(CBDB_File & db_file)1366 void CBDB_File::Attach(CBDB_File& db_file)
1367 {
1368 CBDB_RawFile::Attach(db_file);
1369 x_CheckConstructBuffers();
1370 SetLegacyStringsCheck(db_file.m_LegacyString);
1371 }
1372
SetLegacyStringsCheck(bool value)1373 void CBDB_File::SetLegacyStringsCheck(bool value)
1374 {
1375 m_LegacyString = value;
1376 if (m_KeyBuf.get()) {
1377 m_KeyBuf->SetLegacyStringsCheck(value);
1378 }
1379 if (m_DataBuf.get()) {
1380 m_DataBuf->SetLegacyStringsCheck(value);
1381 }
1382 }
1383
x_SetByteSwapped(bool bswp)1384 void CBDB_File::x_SetByteSwapped(bool bswp)
1385 {
1386 CBDB_RawFile::x_SetByteSwapped(bswp);
1387 m_KeyBuf->SetByteSwapped(bswp);
1388 if (m_DataBuf.get()) {
1389 m_DataBuf->SetByteSwapped(bswp);
1390 }
1391 }
1392
Verify(const char * filename,const char * database,FILE * backup)1393 void CBDB_File::Verify(const char* filename,
1394 const char* database,
1395 FILE* backup)
1396 {
1397 if (m_DB == 0) {
1398 x_CreateDB(0);
1399 }
1400 x_CheckConstructBuffers();
1401 m_DB->app_private = (void*) m_KeyBuf.get();
1402
1403 /*int ret = */
1404 m_DB->verify(m_DB, filename, database, backup, backup ? DB_SALVAGE: 0);
1405 }
1406
1407
x_Fetch(unsigned int flags)1408 EBDB_ErrCode CBDB_File::x_Fetch(unsigned int flags)
1409 {
1410 x_StartRead();
1411
1412 int ret = x_DB_Fetch(m_DBT_Key,
1413 m_DBT_Data,
1414 flags);
1415
1416 if (ret == DB_NOTFOUND) {
1417 return eBDB_NotFound;
1418 }
1419
1420 // Disable error reporting for custom m_DBT_data management
1421
1422 if ((ret == ENOMEM || ret == DB_BUFFER_SMALL)
1423 && m_DataBufDisabled && m_DBT_Data->data == 0) {
1424 ret = 0;
1425 }
1426 BDB_CHECK(ret, FileName().c_str());
1427
1428 x_EndRead();
1429 return eBDB_Ok;
1430 }
1431
FetchForUpdate()1432 EBDB_ErrCode CBDB_File::FetchForUpdate()
1433 {
1434 return x_Fetch(DB_RMW);
1435 }
1436
1437
CloneDBT_Key()1438 DBT* CBDB_File::CloneDBT_Key()
1439 {
1440 x_StartRead();
1441 x_EndRead();
1442
1443 DBT* dbt = new DBT;
1444 ::memset(dbt, 0, sizeof(DBT));
1445
1446 // Clone the "data" area (needs to be properly deleted!)
1447 if (m_DBT_Key->ulen) {
1448 dbt->size = m_DBT_Key->size;
1449 dbt->ulen = m_DBT_Key->ulen;
1450 unsigned char* p = (unsigned char*)malloc(dbt->ulen);
1451 ::memcpy(p, m_DBT_Key->data, m_DBT_Key->size);
1452 dbt->data = p;
1453 dbt->flags = DB_DBT_USERMEM;
1454 }
1455 return dbt;
1456 }
1457
DestroyDBT_Clone(DBT * dbt)1458 void CBDB_File::DestroyDBT_Clone(DBT* dbt)
1459 {
1460 unsigned char* p = (unsigned char*)dbt->data;
1461 free(p); dbt->data = NULL;
1462 delete dbt;
1463 }
1464
1465
Insert(EAfterWrite write_flag)1466 EBDB_ErrCode CBDB_File::Insert(EAfterWrite write_flag)
1467 {
1468 CheckNullDataConstraint();
1469
1470 unsigned int flags;
1471 if (DuplicatesAllowed()) {
1472 flags = 0;
1473 } else {
1474 flags = /*DB_NODUPDATA |*/ DB_NOOVERWRITE;
1475 }
1476 return x_Write(flags, write_flag);
1477 }
1478
Append(EAfterWrite write_flag)1479 unsigned CBDB_File::Append(EAfterWrite write_flag)
1480 {
1481 unsigned int flags = DB_APPEND;
1482 x_Write(flags, write_flag);
1483 unsigned rec_id;
1484 memcpy(&rec_id, m_DBT_Key->data, sizeof(rec_id));
1485 return rec_id;
1486 }
1487
1488
UpdateInsert(EAfterWrite write_flag)1489 EBDB_ErrCode CBDB_File::UpdateInsert(EAfterWrite write_flag)
1490 {
1491 CheckNullDataConstraint();
1492 return x_Write(0, write_flag);
1493 }
1494
1495
Delete(EIgnoreError on_error)1496 EBDB_ErrCode CBDB_File::Delete(EIgnoreError on_error)
1497 {
1498 EBDB_ErrCode rcode = eBDB_Ok;
1499 m_KeyBuf->PrepareDBT_ForWrite(m_DBT_Key);
1500 DB_TXN* txn = GetTxn();
1501
1502 int ret = m_DB->del(m_DB,
1503 txn,
1504 m_DBT_Key,
1505 0);
1506 if (ret == DB_NOTFOUND) {
1507 ret = 0;
1508 rcode = eBDB_NotFound;
1509 }
1510 if (on_error != eIgnoreError) {
1511 BDB_CHECK(ret, FileName().c_str());
1512 }
1513 Discard();
1514 return rcode;
1515 }
1516
1517
Discard()1518 void CBDB_File::Discard()
1519 {
1520 m_KeyBuf->ArrangePtrsUnpacked();
1521 if ( m_DataBuf.get() ) {
1522 m_DataBuf->ArrangePtrsUnpacked();
1523 m_DataBuf->SetAllNull();
1524 }
1525 }
1526
1527
1528 /// @internal
1529 size_t
BDB_compare_prefix(DB *,const DBT * a,const DBT * b)1530 BDB_compare_prefix(DB* /*dbp*/, const DBT* a, const DBT* b)
1531 {
1532 size_t cnt, len;
1533 char* p1, *p2;
1534
1535 cnt = 1; len = a->size > b->size ? b->size : a->size;
1536 p1 = (char*)a->data, p2 = (char*)b->data;
1537 for (;len--; ++p1, ++p2, ++cnt) {
1538 if (*p1 != *p2) {
1539 return (cnt);
1540 }
1541 }
1542 if (a->size < b->size) return (a->size + 1);
1543 if (b->size < a->size) return (b->size + 1);
1544 return (b->size);
1545 }
1546
1547
1548 #if DB_VERSION_MAJOR >= 6
x_CompareShim(DB * db,const DBT * dbt1,const DBT * dbt2,size_t *)1549 int CBDB_File::x_CompareShim(DB* db, const DBT* dbt1, const DBT* dbt2, size_t*)
1550 {
1551 const CBDB_BufferManager* key_buf
1552 = static_cast<const CBDB_BufferManager*>(db->app_private);
1553 _ASSERT(key_buf);
1554 return (key_buf->GetCompareFunction())(db, dbt1, dbt2);
1555 }
1556 #endif
1557
1558
SetCmp(DB * db)1559 void CBDB_File::SetCmp(DB* db)
1560 {
1561 _ASSERT(m_DB_Type == eBtree);
1562 #if DB_VERSION_MAJOR >= 6
1563 int ret = db->set_bt_compare(db, x_CompareShim);
1564 #else
1565 BDB_CompareFunction func = m_KeyBuf->GetCompareFunction();
1566 _ASSERT(func);
1567 int ret = db->set_bt_compare(db, func);
1568 #endif
1569 BDB_CHECK(ret, 0);
1570
1571 if (m_PrefixCompress) {
1572 ret = m_DB->set_bt_prefix(m_DB, BDB_compare_prefix);
1573 BDB_CHECK(ret, 0);
1574 }
1575 }
1576
1577
1578
1579 CBDB_File::TUnifiedFieldIndex
GetFieldIdx(const string & name) const1580 CBDB_File::GetFieldIdx(const string& name) const
1581 {
1582 int fidx = 0;
1583 if (m_KeyBuf.get()) {
1584 fidx = m_KeyBuf->GetFieldIndex(name);
1585 if (fidx >= 0) { // field name found
1586 return BDB_GetUFieldIdx(fidx, true /*key*/);
1587 }
1588 }
1589
1590 if (m_DataBuf.get()) {
1591 fidx = m_DataBuf->GetFieldIndex(name);
1592 if (fidx >= 0) { // field name found
1593 return BDB_GetUFieldIdx(fidx, false /*non-key*/);
1594 }
1595 }
1596 return 0;
1597 }
1598
GetField(TUnifiedFieldIndex idx) const1599 const CBDB_Field& CBDB_File::GetField(TUnifiedFieldIndex idx) const
1600 {
1601 _ASSERT(idx != 0);
1602
1603 const CBDB_BufferManager* buffer;
1604
1605 if (idx < 0) { // key buffer
1606 idx = -idx;
1607 --idx;
1608 buffer = m_KeyBuf.get();
1609 } else { // data buffer
1610 --idx;
1611 buffer = m_DataBuf.get();
1612 }
1613
1614 _ASSERT(buffer);
1615
1616 const CBDB_Field& fld = buffer->GetField(idx);
1617 return fld;
1618 }
1619
1620
GetField(TUnifiedFieldIndex idx)1621 CBDB_Field& CBDB_File::GetField(TUnifiedFieldIndex idx)
1622 {
1623 _ASSERT(idx != 0);
1624
1625 CBDB_BufferManager* buffer;
1626
1627 if (idx < 0) { // key buffer
1628 idx = -idx;
1629 --idx;
1630 buffer = m_KeyBuf.get();
1631 } else { // data buffer
1632 --idx;
1633 buffer = m_DataBuf.get();
1634 }
1635 _ASSERT(buffer);
1636
1637 CBDB_Field& fld = buffer->GetField(idx);
1638 return fld;
1639 }
1640
DisableDataPacking()1641 void CBDB_File::DisableDataPacking()
1642 {
1643 if (m_DataBuf.get()) {
1644 m_DataBuf->SetPackable(false); // disable packing
1645 }
1646 }
1647
CopyFrom(const CBDB_File & dbf)1648 void CBDB_File::CopyFrom(const CBDB_File& dbf)
1649 {
1650 const CBDB_BufferManager* src_key = dbf.GetKeyBuffer();
1651 const CBDB_BufferManager* src_data = dbf.GetDataBuffer();
1652
1653 CBDB_BufferManager* key = GetKeyBuffer();
1654 CBDB_BufferManager* data = GetDataBuffer();
1655
1656 key->CopyFrom(*src_key);
1657 if (data) {
1658 data->CopyFrom(*src_data);
1659 }
1660 }
1661
DuplicateStructure(const CBDB_File & dbf)1662 void CBDB_File::DuplicateStructure(const CBDB_File& dbf)
1663 {
1664 const CBDB_BufferManager* src_key = dbf.GetKeyBuffer();
1665 const CBDB_BufferManager* src_data = dbf.GetDataBuffer();
1666
1667 _ASSERT(src_key);
1668
1669 x_ConstructKeyBuf();
1670 m_KeyBuf->DuplicateStructureFrom(*src_key);
1671
1672 if (src_data) {
1673 x_ConstructDataBuf();
1674 m_DataBuf->DuplicateStructureFrom(*src_data);
1675 } else {
1676 m_DataBuf.reset(0);
1677 }
1678 }
1679
ReadCursor(DBC * dbc,unsigned int bdb_flag)1680 EBDB_ErrCode CBDB_File::ReadCursor(DBC* dbc, unsigned int bdb_flag)
1681 {
1682 x_StartRead();
1683
1684 if (m_DataBufDisabled) {
1685 m_DBT_Data->size = 0;
1686 m_DBT_Data->flags = 0;
1687 m_DBT_Data->data = 0;
1688 }
1689
1690 int ret = x_DBC_Fetch(dbc, m_DBT_Key, m_DBT_Data, bdb_flag);
1691
1692 switch (ret) {
1693 case DB_NOTFOUND:
1694 return eBDB_NotFound;
1695 case DB_KEYEMPTY:
1696 // record has been deleted
1697 return eBDB_KeyEmpty;
1698 }
1699
1700 BDB_CHECK(ret, FileName().c_str());
1701 x_EndRead();
1702 return eBDB_Ok;
1703 }
1704
ReadCursor(DBC * dbc,unsigned int bdb_flag,TBuffer * buf)1705 EBDB_ErrCode CBDB_File::ReadCursor(DBC* dbc,
1706 unsigned int bdb_flag,
1707 TBuffer* buf)
1708 {
1709 _ASSERT(buf);
1710
1711 if (buf->size() == 0) {
1712 buf->resize(1024);
1713 }
1714 if (buf->size() < buf->capacity()) {
1715 buf->resize(buf->capacity());
1716 }
1717
1718 x_StartRead();
1719 m_DBT_Data->data = &((*buf)[0]);
1720 m_DBT_Data->ulen = (u_int32_t)buf->size();
1721 m_DBT_Data->size = 0;
1722 m_DBT_Data->flags = DB_DBT_USERMEM;
1723
1724 int ret = x_DBC_Fetch(dbc, m_DBT_Key, m_DBT_Data, bdb_flag);
1725
1726 switch (ret) {
1727 case DB_NOTFOUND:
1728 buf->resize_mem(0);
1729 return eBDB_NotFound;
1730 case DB_KEYEMPTY:
1731 // record has been deleted
1732 buf->resize_mem(0);
1733 return eBDB_KeyEmpty;
1734 case DB_BUFFER_SMALL:
1735 buf->resize_mem(m_DBT_Data->size);
1736 return this->ReadCursor(dbc, bdb_flag, buf);
1737 break;
1738 } // switch
1739
1740 BDB_CHECK(ret, FileName().c_str());
1741 buf->resize(m_DBT_Data->size);
1742 x_EndRead();
1743
1744 return eBDB_Ok;
1745 }
1746
ReadCursor(DBC * dbc,unsigned int bdb_flag,void ** buf,size_t buf_size,EReallocMode allow_realloc)1747 EBDB_ErrCode CBDB_File::ReadCursor(DBC* dbc,
1748 unsigned int bdb_flag,
1749 void** buf,
1750 size_t buf_size,
1751 EReallocMode allow_realloc)
1752 {
1753 x_StartRead();
1754 m_DBT_Data->data = buf ? *buf : 0;
1755 m_DBT_Data->ulen = (unsigned)buf_size;
1756 m_DBT_Data->size = 0;
1757
1758 if (allow_realloc == eReallocForbidden) {
1759 m_DBT_Data->flags = DB_DBT_USERMEM;
1760 } else {
1761 if (m_DBT_Data->data == 0) {
1762 m_DBT_Data->flags = DB_DBT_MALLOC;
1763 } else {
1764 // compressor does not support re-alloc mode
1765 _ASSERT(m_Compressor.get() == 0);
1766
1767 if (m_Compressor.get()) {
1768 BDB_THROW(eCompressorError,
1769 "Use of dynamic reallocation on compressed file - not implemented");
1770 }
1771
1772 m_DBT_Data->flags = DB_DBT_REALLOC;
1773 }
1774 }
1775
1776 int ret = x_DBC_Fetch(dbc, m_DBT_Key, m_DBT_Data, bdb_flag);
1777
1778 if ( buf )
1779 *buf = m_DBT_Data->data;
1780
1781 switch (ret) {
1782 case DB_NOTFOUND:
1783 return eBDB_NotFound;
1784 case DB_KEYEMPTY:
1785 // record has been deleted
1786 return eBDB_KeyEmpty;
1787 }
1788
1789 BDB_CHECK(ret, FileName().c_str());
1790 x_EndRead();
1791 return eBDB_Ok;
1792 }
1793
ReadCursor(DBC * dbc,unsigned int bdb_flag,CBDB_MultiRowBuffer * multirow_buf,bool multirow_only)1794 EBDB_ErrCode CBDB_File::ReadCursor(DBC* dbc,
1795 unsigned int bdb_flag,
1796 CBDB_MultiRowBuffer* multirow_buf,
1797 bool multirow_only)
1798 {
1799 int ret;
1800 db_recno_t recno = 0;
1801 if (multirow_buf == 0) {
1802 return ReadCursor(dbc, bdb_flag);
1803 }
1804
1805 // Something sits in the memory buffer already, get the next record
1806 //
1807 if (multirow_buf->m_BufPtr != 0) {
1808 switch (m_DB_Type) {
1809 case eBtree:
1810 case eHash:
1811 {{
1812 DB_MULTIPLE_KEY_NEXT(multirow_buf->m_BufPtr,
1813 multirow_buf->m_Data_DBT,
1814 multirow_buf->m_LastKey,
1815 multirow_buf->m_LastKeyLen,
1816 multirow_buf->m_LastData,
1817 multirow_buf->m_LastDataLen);
1818 }}
1819 break;
1820
1821 case eQueue:
1822 {{
1823 DB_MULTIPLE_RECNO_NEXT(multirow_buf->m_BufPtr,
1824 multirow_buf->m_Data_DBT,
1825 recno,
1826 multirow_buf->m_LastData,
1827 multirow_buf->m_LastDataLen);
1828 multirow_buf->m_LastKey = &recno;
1829 multirow_buf->m_LastKeyLen = sizeof(recno);
1830 }}
1831 break;
1832
1833 default:
1834 _ASSERT(0);
1835 NCBI_THROW(CException, eUnknown, "invalid multifetch cursor type");
1836 }
1837
1838 if (multirow_buf->m_BufPtr != 0) {
1839 goto read_epilog;
1840 }
1841 if (multirow_only) {
1842 return eBDB_MultiRowEnd;
1843 }
1844 } else {
1845 if (multirow_only) {
1846 return eBDB_MultiRowEnd;
1847 }
1848 }
1849
1850
1851 // read prolog actions
1852 m_KeyBuf->Pack();
1853 m_KeyBuf->PrepareDBT_ForRead(m_DBT_Key);
1854
1855 multirow_buf->InitDBT();
1856
1857 // Cursor read
1858 ret = dbc->c_get(dbc,
1859 m_DBT_Key,
1860 multirow_buf->m_Data_DBT,
1861 bdb_flag | DB_MULTIPLE_KEY);
1862 switch (ret) {
1863 case DB_NOTFOUND:
1864 return eBDB_NotFound;
1865 case DB_KEYEMPTY:
1866 // record has been deleted
1867 return eBDB_KeyEmpty;
1868 }
1869
1870 BDB_CHECK(ret, FileName().c_str());
1871
1872
1873 // Get the first record out of the fetching buffer
1874 //
1875 multirow_buf->MultipleInit();
1876
1877 switch (m_DB_Type) {
1878 case eBtree:
1879 case eHash:
1880 {{
1881 DB_MULTIPLE_KEY_NEXT(multirow_buf->m_BufPtr,
1882 multirow_buf->m_Data_DBT,
1883 multirow_buf->m_LastKey,
1884 multirow_buf->m_LastKeyLen,
1885 multirow_buf->m_LastData,
1886 multirow_buf->m_LastDataLen);
1887 }}
1888 break;
1889
1890 case eQueue:
1891 {{
1892 DB_MULTIPLE_RECNO_NEXT(multirow_buf->m_BufPtr,
1893 multirow_buf->m_Data_DBT,
1894 recno,
1895 multirow_buf->m_LastData,
1896 multirow_buf->m_LastDataLen);
1897 multirow_buf->m_LastKey = &recno;
1898 multirow_buf->m_LastKeyLen = sizeof(recno);
1899 }}
1900 break;
1901
1902 default:
1903 _ASSERT(0);
1904 NCBI_THROW(CException, eUnknown, "invalid multifetch cursor type");
1905 }
1906
1907 if (multirow_buf->m_BufPtr == 0) {
1908 return eBDB_NotFound;
1909 }
1910
1911 // Read epilog (copy things into field buffer)
1912 //
1913 read_epilog:
1914 m_KeyBuf->CopyPackedFrom(multirow_buf->m_LastKey,
1915 multirow_buf->m_LastKeyLen);
1916 if ( m_DataBuf.get() ) {
1917 if (m_Compressor.get()) {
1918 // first 4 bytes in the buffer encode length
1919 unsigned char* uncompressed_data =
1920 (unsigned char*) multirow_buf->m_LastData;
1921 unsigned bytes_compressed;
1922 #ifdef HAVE_UNALIGNED_READS
1923 bytes_compressed = *((unsigned*)multirow_buf->m_LastData);
1924 #else
1925 ::memcpy(&bytes_compressed, multirow_buf->m_LastData, 4);
1926 #endif
1927 uncompressed_data += 4;
1928
1929 if (bytes_compressed == 0) {
1930 m_DataBuf->CopyPackedFrom(uncompressed_data,
1931 multirow_buf->m_LastDataLen-4);
1932 } else {
1933 m_CompressBuffer.resize_mem(bytes_compressed);
1934 size_t dst_len;
1935 bool decomp_ok =
1936 m_Compressor->DecompressBuffer(uncompressed_data,
1937 multirow_buf->m_LastDataLen-4,
1938 m_CompressBuffer.data(),
1939 m_CompressBuffer.size(),
1940 &dst_len);
1941 if (!decomp_ok) {
1942 BDB_THROW(eCompressorError,
1943 m_Compressor->GetErrorDescription());
1944 }
1945 _ASSERT(dst_len == m_CompressBuffer.size());
1946 m_DataBuf->CopyPackedFrom(m_CompressBuffer.data(),
1947 dst_len);
1948
1949 }
1950
1951 } else {
1952 m_DataBuf->CopyPackedFrom(multirow_buf->m_LastData,
1953 multirow_buf->m_LastDataLen);
1954 }
1955 }
1956 return eBDB_Ok;
1957 }
1958
1959
1960
WriteCursor(DBC * dbc,unsigned int bdb_flag,EAfterWrite write_flag)1961 EBDB_ErrCode CBDB_File::WriteCursor(DBC* dbc, unsigned int bdb_flag,
1962 EAfterWrite write_flag)
1963 {
1964 CheckNullDataConstraint();
1965 return x_Write(bdb_flag, write_flag, dbc);
1966 }
1967
WriteCursor(const void * data,size_t size,DBC * dbc,unsigned int bdb_flag,EAfterWrite write_flag)1968 EBDB_ErrCode CBDB_File::WriteCursor(const void* data,
1969 size_t size,
1970 DBC* dbc,
1971 unsigned int bdb_flag,
1972 EAfterWrite write_flag)
1973 {
1974 if (!m_DataBufDisabled) {
1975 BDB_THROW(eInvalidOperation, "BLOB operation on non BLOB table");
1976 }
1977
1978 m_DBT_Data->data = const_cast<void*> (data);
1979 m_DBT_Data->size = m_DBT_Data->ulen = (unsigned)size;
1980
1981 return x_Write(bdb_flag, write_flag, dbc);
1982 }
1983
1984
DeleteCursor(DBC * dbc,EIgnoreError on_error)1985 EBDB_ErrCode CBDB_File::DeleteCursor(DBC* dbc, EIgnoreError on_error)
1986 {
1987 int ret = dbc->c_del(dbc, 0);
1988
1989 if (on_error != CBDB_File::eIgnoreError) {
1990 BDB_CHECK(ret, FileName().c_str());
1991 }
1992
1993 return eBDB_Ok;
1994 }
1995
x_CheckConstructBuffers()1996 void CBDB_File::x_CheckConstructBuffers()
1997 {
1998 if (!m_BufsAttached && !m_BufsCreated) {
1999 if (m_KeyBuf->FieldCount() == 0) {
2000 BDB_THROW(eInvalidValue, "Empty BDB key (no fields defined).");
2001 }
2002
2003 m_KeyBuf->Construct();
2004 if ( m_DataBuf.get() ) {
2005 m_DataBuf->Construct();
2006 m_DataBuf->SetAllNull();
2007 }
2008 m_BufsCreated = 1;
2009 }
2010 }
2011
x_StartRead()2012 void CBDB_File::x_StartRead()
2013 {
2014 m_KeyBuf->Pack();
2015 m_KeyBuf->PrepareDBT_ForRead(m_DBT_Key);
2016
2017 if (!m_DataBufDisabled) {
2018 if ( m_DataBuf.get() ) {
2019 m_DataBuf->PrepareDBT_ForRead(m_DBT_Data);
2020 }
2021 else {
2022 m_DBT_Data->size = 0;
2023 m_DBT_Data->flags = 0;
2024 m_DBT_Data->data = 0;
2025 }
2026 }
2027 }
2028
2029
x_EndRead()2030 void CBDB_File::x_EndRead()
2031 {
2032 m_KeyBuf->SetDBT_Size(m_DBT_Key->size);
2033 m_KeyBuf->ArrangePtrsPacked();
2034 if ( m_DataBuf.get() ) {
2035 m_DataBuf->SetDBT_Size(m_DBT_Data->size);
2036 m_DataBuf->ArrangePtrsPacked();
2037 }
2038 }
2039
2040
x_Write(unsigned int flags,EAfterWrite write_flag,DBC * dbc)2041 EBDB_ErrCode CBDB_File::x_Write(unsigned int flags,
2042 EAfterWrite write_flag,
2043 DBC * dbc)
2044 {
2045 m_KeyBuf->PrepareDBT_ForWrite(m_DBT_Key);
2046
2047 if (!m_DataBufDisabled) {
2048 if ( m_DataBuf.get() ) {
2049 m_DataBuf->PrepareDBT_ForWrite(m_DBT_Data);
2050 }
2051 }
2052
2053 int ret=0;
2054 if (dbc) {
2055 ret = x_DB_CPut(dbc, m_DBT_Key, m_DBT_Data, flags);
2056 } else {
2057 ret = x_DB_Put(m_DBT_Key, m_DBT_Data, flags);
2058 }
2059 if (ret == DB_KEYEXIST)
2060 return eBDB_KeyDup;
2061
2062 BDB_CHECK(ret, FileName().c_str());
2063
2064 if (write_flag == eDiscardData) {
2065 Discard();
2066 }
2067
2068 return eBDB_Ok;
2069 }
2070
2071
2072
2073 /////////////////////////////////////////////////////////////////////////////
2074 //
2075 // CBDB_IdFile::
2076 //
2077
CBDB_IdFile()2078 CBDB_IdFile::CBDB_IdFile()
2079 : CBDB_File()
2080 {
2081 BindKey("id", &IdKey);
2082 }
2083
2084 #if DB_VERSION_MAJOR >= 6
2085 extern "C" {
2086 typedef int (*BDB_CompareFunction_V6)(DB*, const DBT*, const DBT*,
2087 size_t*);
BDB_Int4Compare_V6(DB * db,const DBT * dbt1,const DBT * dbt2,size_t *)2088 int BDB_Int4Compare_V6(DB* db, const DBT* dbt1, const DBT* dbt2, size_t*)
2089 { return BDB_Int4Compare(db, dbt1, dbt2); }
BDB_ByteSwap_Int4Compare_V6(DB * db,const DBT * dbt1,const DBT * dbt2,size_t *)2090 int BDB_ByteSwap_Int4Compare_V6(DB* db, const DBT* dbt1, const DBT* dbt2,
2091 size_t*)
2092 { return BDB_ByteSwap_Int4Compare(db, dbt1, dbt2); }
2093 }
2094 #endif
2095
SetCmp(DB *)2096 void CBDB_IdFile::SetCmp(DB* /* db */)
2097 {
2098 #if DB_VERSION_MAJOR >= 6
2099 BDB_CompareFunction_V6 func = BDB_Int4Compare_V6;
2100 if (IsByteSwapped()) {
2101 func = BDB_ByteSwap_Int4Compare_V6;
2102 }
2103 #else
2104 BDB_CompareFunction func = BDB_Int4Compare;
2105 if (IsByteSwapped()) {
2106 func = BDB_ByteSwap_Int4Compare;
2107 }
2108 #endif
2109
2110 _ASSERT(func);
2111 int ret = m_DB->set_bt_compare(m_DB, func);
2112 BDB_CHECK(ret, 0);
2113 }
2114
2115
2116 END_NCBI_SCOPE
2117