1 // Copyright (c) 2012-2018 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <dbwrapper.h>
6
7 #include <memory>
8 #include <random.h>
9
10 #include <leveldb/cache.h>
11 #include <leveldb/env.h>
12 #include <leveldb/filter_policy.h>
13 #include <memenv.h>
14 #include <stdint.h>
15 #include <algorithm>
16
17 class CBitcoinLevelDBLogger : public leveldb::Logger {
18 public:
19 // This code is adapted from posix_logger.h, which is why it is using vsprintf.
20 // Please do not do this in normal code
Logv(const char * format,va_list ap)21 void Logv(const char * format, va_list ap) override {
22 if (!LogAcceptCategory(BCLog::LEVELDB)) {
23 return;
24 }
25 char buffer[500];
26 for (int iter = 0; iter < 2; iter++) {
27 char* base;
28 int bufsize;
29 if (iter == 0) {
30 bufsize = sizeof(buffer);
31 base = buffer;
32 }
33 else {
34 bufsize = 30000;
35 base = new char[bufsize];
36 }
37 char* p = base;
38 char* limit = base + bufsize;
39
40 // Print the message
41 if (p < limit) {
42 va_list backup_ap;
43 va_copy(backup_ap, ap);
44 // Do not use vsnprintf elsewhere in bitcoin source code, see above.
45 p += vsnprintf(p, limit - p, format, backup_ap);
46 va_end(backup_ap);
47 }
48
49 // Truncate to available space if necessary
50 if (p >= limit) {
51 if (iter == 0) {
52 continue; // Try again with larger buffer
53 }
54 else {
55 p = limit - 1;
56 }
57 }
58
59 // Add newline if necessary
60 if (p == base || p[-1] != '\n') {
61 *p++ = '\n';
62 }
63
64 assert(p <= limit);
65 base[std::min(bufsize - 1, (int)(p - base))] = '\0';
66 LogPrintf("leveldb: %s", base); /* Continued */
67 if (base != buffer) {
68 delete[] base;
69 }
70 break;
71 }
72 }
73 };
74
SetMaxOpenFiles(leveldb::Options * options)75 static void SetMaxOpenFiles(leveldb::Options *options) {
76 // On most platforms the default setting of max_open_files (which is 1000)
77 // is optimal. On Windows using a large file count is OK because the handles
78 // do not interfere with select() loops. On 64-bit Unix hosts this value is
79 // also OK, because up to that amount LevelDB will use an mmap
80 // implementation that does not use extra file descriptors (the fds are
81 // closed after being mmap'ed).
82 //
83 // Increasing the value beyond the default is dangerous because LevelDB will
84 // fall back to a non-mmap implementation when the file count is too large.
85 // On 32-bit Unix host we should decrease the value because the handles use
86 // up real fds, and we want to avoid fd exhaustion issues.
87 //
88 // See PR #12495 for further discussion.
89
90 int default_open_files = options->max_open_files;
91 #ifndef WIN32
92 if (sizeof(void*) < 8) {
93 options->max_open_files = 64;
94 }
95 #endif
96 LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
97 options->max_open_files, default_open_files);
98 }
99
GetOptions(size_t nCacheSize)100 static leveldb::Options GetOptions(size_t nCacheSize)
101 {
102 leveldb::Options options;
103 options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
104 options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
105 options.filter_policy = leveldb::NewBloomFilterPolicy(10);
106 options.compression = leveldb::kNoCompression;
107 options.info_log = new CBitcoinLevelDBLogger();
108 if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
109 // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
110 // on corruption in later versions.
111 options.paranoid_checks = true;
112 }
113 SetMaxOpenFiles(&options);
114 return options;
115 }
116
CDBWrapper(const fs::path & path,size_t nCacheSize,bool fMemory,bool fWipe,bool obfuscate)117 CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
118 : m_name(fs::basename(path))
119 {
120 penv = nullptr;
121 readoptions.verify_checksums = true;
122 iteroptions.verify_checksums = true;
123 iteroptions.fill_cache = false;
124 syncoptions.sync = true;
125 options = GetOptions(nCacheSize);
126 options.create_if_missing = true;
127 if (fMemory) {
128 penv = leveldb::NewMemEnv(leveldb::Env::Default());
129 options.env = penv;
130 } else {
131 if (fWipe) {
132 LogPrintf("Wiping LevelDB in %s\n", path.string());
133 leveldb::Status result = leveldb::DestroyDB(path.string(), options);
134 dbwrapper_private::HandleError(result);
135 }
136 TryCreateDirectories(path);
137 LogPrintf("Opening LevelDB in %s\n", path.string());
138 }
139 leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb);
140 dbwrapper_private::HandleError(status);
141 LogPrintf("Opened LevelDB successfully\n");
142
143 if (gArgs.GetBoolArg("-forcecompactdb", false)) {
144 LogPrintf("Starting database compaction of %s\n", path.string());
145 pdb->CompactRange(nullptr, nullptr);
146 LogPrintf("Finished database compaction of %s\n", path.string());
147 }
148
149 // The base-case obfuscation key, which is a noop.
150 obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
151
152 bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
153
154 if (!key_exists && obfuscate && IsEmpty()) {
155 // Initialize non-degenerate obfuscation if it won't upset
156 // existing, non-obfuscated data.
157 std::vector<unsigned char> new_key = CreateObfuscateKey();
158
159 // Write `new_key` so we don't obfuscate the key with itself
160 Write(OBFUSCATE_KEY_KEY, new_key);
161 obfuscate_key = new_key;
162
163 LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), HexStr(obfuscate_key));
164 }
165
166 LogPrintf("Using obfuscation key for %s: %s\n", path.string(), HexStr(obfuscate_key));
167 }
168
~CDBWrapper()169 CDBWrapper::~CDBWrapper()
170 {
171 delete pdb;
172 pdb = nullptr;
173 delete options.filter_policy;
174 options.filter_policy = nullptr;
175 delete options.info_log;
176 options.info_log = nullptr;
177 delete options.block_cache;
178 options.block_cache = nullptr;
179 delete penv;
180 options.env = nullptr;
181 }
182
WriteBatch(CDBBatch & batch,bool fSync)183 bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
184 {
185 const bool log_memory = LogAcceptCategory(BCLog::LEVELDB);
186 double mem_before = 0;
187 if (log_memory) {
188 mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
189 }
190 leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
191 dbwrapper_private::HandleError(status);
192 if (log_memory) {
193 double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
194 LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
195 m_name, mem_before, mem_after);
196 }
197 return true;
198 }
199
DynamicMemoryUsage() const200 size_t CDBWrapper::DynamicMemoryUsage() const {
201 std::string memory;
202 if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory)) {
203 LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
204 return 0;
205 }
206 return stoul(memory);
207 }
208
209 // Prefixed with null character to avoid collisions with other keys
210 //
211 // We must use a string constructor which specifies length so that we copy
212 // past the null-terminator.
213 const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
214
215 const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
216
217 /**
218 * Returns a string (consisting of 8 random bytes) suitable for use as an
219 * obfuscating XOR key.
220 */
CreateObfuscateKey() const221 std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
222 {
223 unsigned char buff[OBFUSCATE_KEY_NUM_BYTES];
224 GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES);
225 return std::vector<unsigned char>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]);
226
227 }
228
IsEmpty()229 bool CDBWrapper::IsEmpty()
230 {
231 std::unique_ptr<CDBIterator> it(NewIterator());
232 it->SeekToFirst();
233 return !(it->Valid());
234 }
235
~CDBIterator()236 CDBIterator::~CDBIterator() { delete piter; }
Valid() const237 bool CDBIterator::Valid() const { return piter->Valid(); }
SeekToFirst()238 void CDBIterator::SeekToFirst() { piter->SeekToFirst(); }
Next()239 void CDBIterator::Next() { piter->Next(); }
240
241 namespace dbwrapper_private {
242
HandleError(const leveldb::Status & status)243 void HandleError(const leveldb::Status& status)
244 {
245 if (status.ok())
246 return;
247 const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
248 LogPrintf("%s\n", errmsg);
249 LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
250 throw dbwrapper_error(errmsg);
251 }
252
GetObfuscateKey(const CDBWrapper & w)253 const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
254 {
255 return w.obfuscate_key;
256 }
257
258 } // namespace dbwrapper_private
259