1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2015 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5
6 #include "db.h"
7
8 #include "addrman.h"
9 #include "hash.h"
10 #include "protocol.h"
11 #include "util.h"
12 #include "utilstrencodings.h"
13
14 #include <stdint.h>
15
16 #ifndef WIN32
17 #include <sys/stat.h>
18 #endif
19
20 #include <boost/filesystem.hpp>
21 #include <boost/thread.hpp>
22 #include <boost/version.hpp>
23
24 using namespace std;
25
26
27 unsigned int nWalletDBUpdated;
28
29
30 //
31 // CDB
32 //
33
34 CDBEnv bitdb;
35
EnvShutdown()36 void CDBEnv::EnvShutdown()
37 {
38 if (!fDbEnvInit)
39 return;
40
41 fDbEnvInit = false;
42 int ret = dbenv->close(0);
43 if (ret != 0)
44 LogPrintf("CDBEnv::EnvShutdown: Error %d shutting down database environment: %s\n", ret, DbEnv::strerror(ret));
45 if (!fMockDb)
46 DbEnv((u_int32_t)0).remove(strPath.c_str(), 0);
47 }
48
Reset()49 void CDBEnv::Reset()
50 {
51 delete dbenv;
52 dbenv = new DbEnv(DB_CXX_NO_EXCEPTIONS);
53 fDbEnvInit = false;
54 fMockDb = false;
55 }
56
CDBEnv()57 CDBEnv::CDBEnv() : dbenv(NULL)
58 {
59 Reset();
60 }
61
~CDBEnv()62 CDBEnv::~CDBEnv()
63 {
64 EnvShutdown();
65 delete dbenv;
66 dbenv = NULL;
67 }
68
Close()69 void CDBEnv::Close()
70 {
71 EnvShutdown();
72 }
73
Open(const boost::filesystem::path & pathIn)74 bool CDBEnv::Open(const boost::filesystem::path& pathIn)
75 {
76 if (fDbEnvInit)
77 return true;
78
79 boost::this_thread::interruption_point();
80
81 strPath = pathIn.string();
82 boost::filesystem::path pathLogDir = pathIn / "database";
83 TryCreateDirectory(pathLogDir);
84 boost::filesystem::path pathErrorFile = pathIn / "db.log";
85 LogPrintf("CDBEnv::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string());
86
87 unsigned int nEnvFlags = 0;
88 if (GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB))
89 nEnvFlags |= DB_PRIVATE;
90
91 dbenv->set_lg_dir(pathLogDir.string().c_str());
92 dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
93 dbenv->set_lg_bsize(0x10000);
94 dbenv->set_lg_max(1048576);
95 dbenv->set_lk_max_locks(40000);
96 dbenv->set_lk_max_objects(40000);
97 dbenv->set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug
98 dbenv->set_flags(DB_AUTO_COMMIT, 1);
99 dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
100 dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
101 int ret = dbenv->open(strPath.c_str(),
102 DB_CREATE |
103 DB_INIT_LOCK |
104 DB_INIT_LOG |
105 DB_INIT_MPOOL |
106 DB_INIT_TXN |
107 DB_THREAD |
108 DB_RECOVER |
109 nEnvFlags,
110 S_IRUSR | S_IWUSR);
111 if (ret != 0)
112 return error("CDBEnv::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret));
113
114 fDbEnvInit = true;
115 fMockDb = false;
116 return true;
117 }
118
MakeMock()119 void CDBEnv::MakeMock()
120 {
121 if (fDbEnvInit)
122 throw runtime_error("CDBEnv::MakeMock: Already initialized");
123
124 boost::this_thread::interruption_point();
125
126 LogPrint("db", "CDBEnv::MakeMock\n");
127
128 dbenv->set_cachesize(1, 0, 1);
129 dbenv->set_lg_bsize(10485760 * 4);
130 dbenv->set_lg_max(10485760);
131 dbenv->set_lk_max_locks(10000);
132 dbenv->set_lk_max_objects(10000);
133 dbenv->set_flags(DB_AUTO_COMMIT, 1);
134 dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
135 int ret = dbenv->open(NULL,
136 DB_CREATE |
137 DB_INIT_LOCK |
138 DB_INIT_LOG |
139 DB_INIT_MPOOL |
140 DB_INIT_TXN |
141 DB_THREAD |
142 DB_PRIVATE,
143 S_IRUSR | S_IWUSR);
144 if (ret > 0)
145 throw runtime_error(strprintf("CDBEnv::MakeMock: Error %d opening database environment.", ret));
146
147 fDbEnvInit = true;
148 fMockDb = true;
149 }
150
Verify(const std::string & strFile,bool (* recoverFunc)(CDBEnv & dbenv,const std::string & strFile))151 CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFunc)(CDBEnv& dbenv, const std::string& strFile))
152 {
153 LOCK(cs_db);
154 assert(mapFileUseCount.count(strFile) == 0);
155
156 Db db(dbenv, 0);
157 int result = db.verify(strFile.c_str(), NULL, NULL, 0);
158 if (result == 0)
159 return VERIFY_OK;
160 else if (recoverFunc == NULL)
161 return RECOVER_FAIL;
162
163 // Try to recover:
164 bool fRecovered = (*recoverFunc)(*this, strFile);
165 return (fRecovered ? RECOVER_OK : RECOVER_FAIL);
166 }
167
168 /* End of headers, beginning of key/value data */
169 static const char *HEADER_END = "HEADER=END";
170 /* End of key/value data */
171 static const char *DATA_END = "DATA=END";
172
Salvage(const std::string & strFile,bool fAggressive,std::vector<CDBEnv::KeyValPair> & vResult)173 bool CDBEnv::Salvage(const std::string& strFile, bool fAggressive, std::vector<CDBEnv::KeyValPair>& vResult)
174 {
175 LOCK(cs_db);
176 assert(mapFileUseCount.count(strFile) == 0);
177
178 u_int32_t flags = DB_SALVAGE;
179 if (fAggressive)
180 flags |= DB_AGGRESSIVE;
181
182 stringstream strDump;
183
184 Db db(dbenv, 0);
185 int result = db.verify(strFile.c_str(), NULL, &strDump, flags);
186 if (result == DB_VERIFY_BAD) {
187 LogPrintf("CDBEnv::Salvage: Database salvage found errors, all data may not be recoverable.\n");
188 if (!fAggressive) {
189 LogPrintf("CDBEnv::Salvage: Rerun with aggressive mode to ignore errors and continue.\n");
190 return false;
191 }
192 }
193 if (result != 0 && result != DB_VERIFY_BAD) {
194 LogPrintf("CDBEnv::Salvage: Database salvage failed with result %d.\n", result);
195 return false;
196 }
197
198 // Format of bdb dump is ascii lines:
199 // header lines...
200 // HEADER=END
201 // hexadecimal key
202 // hexadecimal value
203 // ... repeated
204 // DATA=END
205
206 string strLine;
207 while (!strDump.eof() && strLine != HEADER_END)
208 getline(strDump, strLine); // Skip past header
209
210 std::string keyHex, valueHex;
211 while (!strDump.eof() && keyHex != DATA_END) {
212 getline(strDump, keyHex);
213 if (keyHex != DATA_END) {
214 if (strDump.eof())
215 break;
216 getline(strDump, valueHex);
217 if (valueHex == DATA_END) {
218 LogPrintf("CDBEnv::Salvage: WARNING: Number of keys in data does not match number of values.\n");
219 break;
220 }
221 vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex)));
222 }
223 }
224
225 if (keyHex != DATA_END) {
226 LogPrintf("CDBEnv::Salvage: WARNING: Unexpected end of file while reading salvage output.\n");
227 return false;
228 }
229
230 return (result == 0);
231 }
232
233
CheckpointLSN(const std::string & strFile)234 void CDBEnv::CheckpointLSN(const std::string& strFile)
235 {
236 dbenv->txn_checkpoint(0, 0, 0);
237 if (fMockDb)
238 return;
239 dbenv->lsn_reset(strFile.c_str(), 0);
240 }
241
242
CDB(const std::string & strFilename,const char * pszMode,bool fFlushOnCloseIn)243 CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL)
244 {
245 int ret;
246 fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
247 fFlushOnClose = fFlushOnCloseIn;
248 if (strFilename.empty())
249 return;
250
251 bool fCreate = strchr(pszMode, 'c') != NULL;
252 unsigned int nFlags = DB_THREAD;
253 if (fCreate)
254 nFlags |= DB_CREATE;
255
256 {
257 LOCK(bitdb.cs_db);
258 if (!bitdb.Open(GetDataDir()))
259 throw runtime_error("CDB: Failed to open database environment.");
260
261 strFile = strFilename;
262 ++bitdb.mapFileUseCount[strFile];
263 pdb = bitdb.mapDb[strFile];
264 if (pdb == NULL) {
265 pdb = new Db(bitdb.dbenv, 0);
266
267 bool fMockDb = bitdb.IsMock();
268 if (fMockDb) {
269 DbMpoolFile* mpf = pdb->get_mpf();
270 ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
271 if (ret != 0)
272 throw runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile));
273 }
274
275 ret = pdb->open(NULL, // Txn pointer
276 fMockDb ? NULL : strFile.c_str(), // Filename
277 fMockDb ? strFile.c_str() : "main", // Logical db name
278 DB_BTREE, // Database type
279 nFlags, // Flags
280 0);
281
282 if (ret != 0) {
283 delete pdb;
284 pdb = NULL;
285 --bitdb.mapFileUseCount[strFile];
286 strFile = "";
287 throw runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename));
288 }
289
290 if (fCreate && !Exists(string("version"))) {
291 bool fTmp = fReadOnly;
292 fReadOnly = false;
293 WriteVersion(CLIENT_VERSION);
294 fReadOnly = fTmp;
295 }
296
297 bitdb.mapDb[strFile] = pdb;
298 }
299 }
300 }
301
Flush()302 void CDB::Flush()
303 {
304 if (activeTxn)
305 return;
306
307 // Flush database activity from memory pool to disk log
308 unsigned int nMinutes = 0;
309 if (fReadOnly)
310 nMinutes = 1;
311
312 bitdb.dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
313 }
314
Close()315 void CDB::Close()
316 {
317 if (!pdb)
318 return;
319 if (activeTxn)
320 activeTxn->abort();
321 activeTxn = NULL;
322 pdb = NULL;
323
324 if (fFlushOnClose)
325 Flush();
326
327 {
328 LOCK(bitdb.cs_db);
329 --bitdb.mapFileUseCount[strFile];
330 }
331 }
332
CloseDb(const string & strFile)333 void CDBEnv::CloseDb(const string& strFile)
334 {
335 {
336 LOCK(cs_db);
337 if (mapDb[strFile] != NULL) {
338 // Close the database handle
339 Db* pdb = mapDb[strFile];
340 pdb->close(0);
341 delete pdb;
342 mapDb[strFile] = NULL;
343 }
344 }
345 }
346
RemoveDb(const string & strFile)347 bool CDBEnv::RemoveDb(const string& strFile)
348 {
349 this->CloseDb(strFile);
350
351 LOCK(cs_db);
352 int rc = dbenv->dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT);
353 return (rc == 0);
354 }
355
Rewrite(const string & strFile,const char * pszSkip)356 bool CDB::Rewrite(const string& strFile, const char* pszSkip)
357 {
358 while (true) {
359 {
360 LOCK(bitdb.cs_db);
361 if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0) {
362 // Flush log data to the dat file
363 bitdb.CloseDb(strFile);
364 bitdb.CheckpointLSN(strFile);
365 bitdb.mapFileUseCount.erase(strFile);
366
367 bool fSuccess = true;
368 LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile);
369 string strFileRes = strFile + ".rewrite";
370 { // surround usage of db with extra {}
371 CDB db(strFile.c_str(), "r");
372 Db* pdbCopy = new Db(bitdb.dbenv, 0);
373
374 int ret = pdbCopy->open(NULL, // Txn pointer
375 strFileRes.c_str(), // Filename
376 "main", // Logical db name
377 DB_BTREE, // Database type
378 DB_CREATE, // Flags
379 0);
380 if (ret > 0) {
381 LogPrintf("CDB::Rewrite: Can't create database file %s\n", strFileRes);
382 fSuccess = false;
383 }
384
385 Dbc* pcursor = db.GetCursor();
386 if (pcursor)
387 while (fSuccess) {
388 CDataStream ssKey(SER_DISK, CLIENT_VERSION);
389 CDataStream ssValue(SER_DISK, CLIENT_VERSION);
390 int ret = db.ReadAtCursor(pcursor, ssKey, ssValue, DB_NEXT);
391 if (ret == DB_NOTFOUND) {
392 pcursor->close();
393 break;
394 } else if (ret != 0) {
395 pcursor->close();
396 fSuccess = false;
397 break;
398 }
399 if (pszSkip &&
400 strncmp(&ssKey[0], pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
401 continue;
402 if (strncmp(&ssKey[0], "\x07version", 8) == 0) {
403 // Update version:
404 ssValue.clear();
405 ssValue << CLIENT_VERSION;
406 }
407 Dbt datKey(&ssKey[0], ssKey.size());
408 Dbt datValue(&ssValue[0], ssValue.size());
409 int ret2 = pdbCopy->put(NULL, &datKey, &datValue, DB_NOOVERWRITE);
410 if (ret2 > 0)
411 fSuccess = false;
412 }
413 if (fSuccess) {
414 db.Close();
415 bitdb.CloseDb(strFile);
416 if (pdbCopy->close(0))
417 fSuccess = false;
418 delete pdbCopy;
419 }
420 }
421 if (fSuccess) {
422 Db dbA(bitdb.dbenv, 0);
423 if (dbA.remove(strFile.c_str(), NULL, 0))
424 fSuccess = false;
425 Db dbB(bitdb.dbenv, 0);
426 if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0))
427 fSuccess = false;
428 }
429 if (!fSuccess)
430 LogPrintf("CDB::Rewrite: Failed to rewrite database file %s\n", strFileRes);
431 return fSuccess;
432 }
433 }
434 MilliSleep(100);
435 }
436 return false;
437 }
438
439
Flush(bool fShutdown)440 void CDBEnv::Flush(bool fShutdown)
441 {
442 int64_t nStart = GetTimeMillis();
443 // Flush log data to the actual data file on all files that are not in use
444 LogPrint("db", "CDBEnv::Flush: Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
445 if (!fDbEnvInit)
446 return;
447 {
448 LOCK(cs_db);
449 map<string, int>::iterator mi = mapFileUseCount.begin();
450 while (mi != mapFileUseCount.end()) {
451 string strFile = (*mi).first;
452 int nRefCount = (*mi).second;
453 LogPrint("db", "CDBEnv::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
454 if (nRefCount == 0) {
455 // Move log data to the dat file
456 CloseDb(strFile);
457 LogPrint("db", "CDBEnv::Flush: %s checkpoint\n", strFile);
458 dbenv->txn_checkpoint(0, 0, 0);
459 LogPrint("db", "CDBEnv::Flush: %s detach\n", strFile);
460 if (!fMockDb)
461 dbenv->lsn_reset(strFile.c_str(), 0);
462 LogPrint("db", "CDBEnv::Flush: %s closed\n", strFile);
463 mapFileUseCount.erase(mi++);
464 } else
465 mi++;
466 }
467 LogPrint("db", "CDBEnv::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart);
468 if (fShutdown) {
469 char** listp;
470 if (mapFileUseCount.empty()) {
471 dbenv->log_archive(&listp, DB_ARCH_REMOVE);
472 Close();
473 if (!fMockDb)
474 boost::filesystem::remove_all(boost::filesystem::path(strPath) / "database");
475 }
476 }
477 }
478 }
479