1 // Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5
6 #include "rocksdb/options.h"
7 #ifndef ROCKSDB_LITE
8
9 #include <algorithm>
10 #include <string>
11 #include <vector>
12
13 #include "db/db_impl/db_impl.h"
14 #include "db/db_test_util.h"
15 #include "file/file_util.h"
16 #include "rocksdb/comparator.h"
17 #include "rocksdb/db.h"
18 #include "rocksdb/transaction_log.h"
19 #include "util/string_util.h"
20
21 namespace ROCKSDB_NAMESPACE {
22
23 #ifndef ROCKSDB_LITE
24 class RepairTest : public DBTestBase {
25 public:
RepairTest()26 RepairTest() : DBTestBase("repair_test", /*env_do_fsync=*/true) {}
27
GetFirstSstPath(std::string * first_sst_path)28 Status GetFirstSstPath(std::string* first_sst_path) {
29 assert(first_sst_path != nullptr);
30 first_sst_path->clear();
31 uint64_t manifest_size;
32 std::vector<std::string> files;
33 Status s = db_->GetLiveFiles(files, &manifest_size);
34 if (s.ok()) {
35 auto sst_iter =
36 std::find_if(files.begin(), files.end(), [](const std::string& file) {
37 uint64_t number;
38 FileType type;
39 bool ok = ParseFileName(file, &number, &type);
40 return ok && type == kTableFile;
41 });
42 *first_sst_path = sst_iter == files.end() ? "" : dbname_ + *sst_iter;
43 }
44 return s;
45 }
46 };
47
TEST_F(RepairTest,LostManifest)48 TEST_F(RepairTest, LostManifest) {
49 // Add a couple SST files, delete the manifest, and verify RepairDB() saves
50 // the day.
51 ASSERT_OK(Put("key", "val"));
52 ASSERT_OK(Flush());
53 ASSERT_OK(Put("key2", "val2"));
54 ASSERT_OK(Flush());
55 // Need to get path before Close() deletes db_, but delete it after Close() to
56 // ensure Close() didn't change the manifest.
57 std::string manifest_path =
58 DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
59
60 Close();
61 ASSERT_OK(env_->FileExists(manifest_path));
62 ASSERT_OK(env_->DeleteFile(manifest_path));
63 ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
64 Reopen(CurrentOptions());
65
66 ASSERT_EQ(Get("key"), "val");
67 ASSERT_EQ(Get("key2"), "val2");
68 }
69
TEST_F(RepairTest,LostManifestMoreDbFeatures)70 TEST_F(RepairTest, LostManifestMoreDbFeatures) {
71 // Add a couple SST files, delete the manifest, and verify RepairDB() saves
72 // the day.
73 ASSERT_OK(Put("key", "val"));
74 ASSERT_OK(Put("key2", "val2"));
75 ASSERT_OK(Put("key3", "val3"));
76 ASSERT_OK(Put("key4", "val4"));
77 ASSERT_OK(Flush());
78 // Test an SST file containing only a range tombstone
79 ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "key2",
80 "key3z"));
81 ASSERT_OK(Flush());
82 // Need to get path before Close() deletes db_, but delete it after Close() to
83 // ensure Close() didn't change the manifest.
84 std::string manifest_path =
85 DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
86
87 Close();
88 ASSERT_OK(env_->FileExists(manifest_path));
89 ASSERT_OK(env_->DeleteFile(manifest_path));
90 ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
91 Reopen(CurrentOptions());
92
93 ASSERT_EQ(Get("key"), "val");
94 ASSERT_EQ(Get("key2"), "NOT_FOUND");
95 ASSERT_EQ(Get("key3"), "NOT_FOUND");
96 ASSERT_EQ(Get("key4"), "val4");
97 }
98
TEST_F(RepairTest,CorruptManifest)99 TEST_F(RepairTest, CorruptManifest) {
100 // Manifest is in an invalid format. Expect a full recovery.
101 ASSERT_OK(Put("key", "val"));
102 ASSERT_OK(Flush());
103 ASSERT_OK(Put("key2", "val2"));
104 ASSERT_OK(Flush());
105 // Need to get path before Close() deletes db_, but overwrite it after Close()
106 // to ensure Close() didn't change the manifest.
107 std::string manifest_path =
108 DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
109
110 Close();
111 ASSERT_OK(env_->FileExists(manifest_path));
112
113 ASSERT_OK(CreateFile(env_->GetFileSystem(), manifest_path, "blah",
114 false /* use_fsync */));
115 ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
116 Reopen(CurrentOptions());
117
118 ASSERT_EQ(Get("key"), "val");
119 ASSERT_EQ(Get("key2"), "val2");
120 }
121
TEST_F(RepairTest,IncompleteManifest)122 TEST_F(RepairTest, IncompleteManifest) {
123 // In this case, the manifest is valid but does not reference all of the SST
124 // files. Expect a full recovery.
125 ASSERT_OK(Put("key", "val"));
126 ASSERT_OK(Flush());
127 std::string orig_manifest_path =
128 DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
129 CopyFile(orig_manifest_path, orig_manifest_path + ".tmp");
130 ASSERT_OK(Put("key2", "val2"));
131 ASSERT_OK(Flush());
132 // Need to get path before Close() deletes db_, but overwrite it after Close()
133 // to ensure Close() didn't change the manifest.
134 std::string new_manifest_path =
135 DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
136
137 Close();
138 ASSERT_OK(env_->FileExists(new_manifest_path));
139 // Replace the manifest with one that is only aware of the first SST file.
140 CopyFile(orig_manifest_path + ".tmp", new_manifest_path);
141 ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
142 Reopen(CurrentOptions());
143
144 ASSERT_EQ(Get("key"), "val");
145 ASSERT_EQ(Get("key2"), "val2");
146 }
147
TEST_F(RepairTest,PostRepairSstFileNumbering)148 TEST_F(RepairTest, PostRepairSstFileNumbering) {
149 // Verify after a DB is repaired, new files will be assigned higher numbers
150 // than old files.
151 ASSERT_OK(Put("key", "val"));
152 ASSERT_OK(Flush());
153 ASSERT_OK(Put("key2", "val2"));
154 ASSERT_OK(Flush());
155 uint64_t pre_repair_file_num = dbfull()->TEST_Current_Next_FileNo();
156 Close();
157
158 ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
159
160 Reopen(CurrentOptions());
161 uint64_t post_repair_file_num = dbfull()->TEST_Current_Next_FileNo();
162 ASSERT_GE(post_repair_file_num, pre_repair_file_num);
163 }
164
TEST_F(RepairTest,LostSst)165 TEST_F(RepairTest, LostSst) {
166 // Delete one of the SST files but preserve the manifest that refers to it,
167 // then verify the DB is still usable for the intact SST.
168 ASSERT_OK(Put("key", "val"));
169 ASSERT_OK(Flush());
170 ASSERT_OK(Put("key2", "val2"));
171 ASSERT_OK(Flush());
172 std::string sst_path;
173 ASSERT_OK(GetFirstSstPath(&sst_path));
174 ASSERT_FALSE(sst_path.empty());
175 ASSERT_OK(env_->DeleteFile(sst_path));
176
177 Close();
178 ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
179 Reopen(CurrentOptions());
180
181 // Exactly one of the key-value pairs should be in the DB now.
182 ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
183 }
184
TEST_F(RepairTest,CorruptSst)185 TEST_F(RepairTest, CorruptSst) {
186 // Corrupt one of the SST files but preserve the manifest that refers to it,
187 // then verify the DB is still usable for the intact SST.
188 ASSERT_OK(Put("key", "val"));
189 ASSERT_OK(Flush());
190 ASSERT_OK(Put("key2", "val2"));
191 ASSERT_OK(Flush());
192 std::string sst_path;
193 ASSERT_OK(GetFirstSstPath(&sst_path));
194 ASSERT_FALSE(sst_path.empty());
195
196 ASSERT_OK(CreateFile(env_->GetFileSystem(), sst_path, "blah",
197 false /* use_fsync */));
198
199 Close();
200 ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
201 Reopen(CurrentOptions());
202
203 // Exactly one of the key-value pairs should be in the DB now.
204 ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
205 }
206
TEST_F(RepairTest,UnflushedSst)207 TEST_F(RepairTest, UnflushedSst) {
208 // This test case invokes repair while some data is unflushed, then verifies
209 // that data is in the db.
210 ASSERT_OK(Put("key", "val"));
211 VectorLogPtr wal_files;
212 ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
213 ASSERT_EQ(wal_files.size(), 1);
214 {
215 uint64_t total_ssts_size;
216 std::unordered_map<std::string, uint64_t> sst_files;
217 ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
218 ASSERT_EQ(total_ssts_size, 0);
219 }
220 // Need to get path before Close() deletes db_, but delete it after Close() to
221 // ensure Close() didn't change the manifest.
222 std::string manifest_path =
223 DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
224
225 Close();
226 ASSERT_OK(env_->FileExists(manifest_path));
227 ASSERT_OK(env_->DeleteFile(manifest_path));
228 ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
229 Reopen(CurrentOptions());
230
231 ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
232 ASSERT_EQ(wal_files.size(), 0);
233 {
234 uint64_t total_ssts_size;
235 std::unordered_map<std::string, uint64_t> sst_files;
236 ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
237 ASSERT_GT(total_ssts_size, 0);
238 }
239 ASSERT_EQ(Get("key"), "val");
240 }
241
TEST_F(RepairTest,SeparateWalDir)242 TEST_F(RepairTest, SeparateWalDir) {
243 do {
244 Options options = CurrentOptions();
245 DestroyAndReopen(options);
246 ASSERT_OK(Put("key", "val"));
247 ASSERT_OK(Put("foo", "bar"));
248 VectorLogPtr wal_files;
249 ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
250 ASSERT_EQ(wal_files.size(), 1);
251 {
252 uint64_t total_ssts_size;
253 std::unordered_map<std::string, uint64_t> sst_files;
254 ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
255 ASSERT_EQ(total_ssts_size, 0);
256 }
257 std::string manifest_path =
258 DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
259
260 Close();
261 ASSERT_OK(env_->FileExists(manifest_path));
262 ASSERT_OK(env_->DeleteFile(manifest_path));
263 ASSERT_OK(RepairDB(dbname_, options));
264
265 // make sure that all WALs are converted to SSTables.
266 options.wal_dir = "";
267
268 Reopen(options);
269 ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
270 ASSERT_EQ(wal_files.size(), 0);
271 {
272 uint64_t total_ssts_size;
273 std::unordered_map<std::string, uint64_t> sst_files;
274 ASSERT_OK(GetAllDataFiles(kTableFile, &sst_files, &total_ssts_size));
275 ASSERT_GT(total_ssts_size, 0);
276 }
277 ASSERT_EQ(Get("key"), "val");
278 ASSERT_EQ(Get("foo"), "bar");
279
280 } while(ChangeWalOptions());
281 }
282
TEST_F(RepairTest,RepairMultipleColumnFamilies)283 TEST_F(RepairTest, RepairMultipleColumnFamilies) {
284 // Verify repair logic associates SST files with their original column
285 // families.
286 const int kNumCfs = 3;
287 const int kEntriesPerCf = 2;
288 DestroyAndReopen(CurrentOptions());
289 CreateAndReopenWithCF({"pikachu1", "pikachu2"}, CurrentOptions());
290 for (int i = 0; i < kNumCfs; ++i) {
291 for (int j = 0; j < kEntriesPerCf; ++j) {
292 ASSERT_OK(Put(i, "key" + ToString(j), "val" + ToString(j)));
293 if (j == kEntriesPerCf - 1 && i == kNumCfs - 1) {
294 // Leave one unflushed so we can verify WAL entries are properly
295 // associated with column families.
296 continue;
297 }
298 ASSERT_OK(Flush(i));
299 }
300 }
301
302 // Need to get path before Close() deletes db_, but delete it after Close() to
303 // ensure Close() doesn't re-create the manifest.
304 std::string manifest_path =
305 DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
306 Close();
307 ASSERT_OK(env_->FileExists(manifest_path));
308 ASSERT_OK(env_->DeleteFile(manifest_path));
309
310 ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
311
312 ReopenWithColumnFamilies({"default", "pikachu1", "pikachu2"},
313 CurrentOptions());
314 for (int i = 0; i < kNumCfs; ++i) {
315 for (int j = 0; j < kEntriesPerCf; ++j) {
316 ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
317 }
318 }
319 }
320
TEST_F(RepairTest,RepairColumnFamilyOptions)321 TEST_F(RepairTest, RepairColumnFamilyOptions) {
322 // Verify repair logic uses correct ColumnFamilyOptions when repairing a
323 // database with different options for column families.
324 const int kNumCfs = 2;
325 const int kEntriesPerCf = 2;
326
327 Options opts(CurrentOptions()), rev_opts(CurrentOptions());
328 opts.comparator = BytewiseComparator();
329 rev_opts.comparator = ReverseBytewiseComparator();
330
331 DestroyAndReopen(opts);
332 CreateColumnFamilies({"reverse"}, rev_opts);
333 ReopenWithColumnFamilies({"default", "reverse"},
334 std::vector<Options>{opts, rev_opts});
335 for (int i = 0; i < kNumCfs; ++i) {
336 for (int j = 0; j < kEntriesPerCf; ++j) {
337 ASSERT_OK(Put(i, "key" + ToString(j), "val" + ToString(j)));
338 if (i == kNumCfs - 1 && j == kEntriesPerCf - 1) {
339 // Leave one unflushed so we can verify RepairDB's flush logic
340 continue;
341 }
342 ASSERT_OK(Flush(i));
343 }
344 }
345 Close();
346
347 // RepairDB() records the comparator in the manifest, and DB::Open would fail
348 // if a different comparator were used.
349 ASSERT_OK(RepairDB(dbname_, opts, {{"default", opts}, {"reverse", rev_opts}},
350 opts /* unknown_cf_opts */));
351 ASSERT_OK(TryReopenWithColumnFamilies({"default", "reverse"},
352 std::vector<Options>{opts, rev_opts}));
353 for (int i = 0; i < kNumCfs; ++i) {
354 for (int j = 0; j < kEntriesPerCf; ++j) {
355 ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
356 }
357 }
358
359 // Examine table properties to verify RepairDB() used the right options when
360 // converting WAL->SST
361 TablePropertiesCollection fname_to_props;
362 ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[1], &fname_to_props));
363 ASSERT_EQ(fname_to_props.size(), 2U);
364 for (const auto& fname_and_props : fname_to_props) {
365 std::string comparator_name (
366 InternalKeyComparator(rev_opts.comparator).Name());
367 comparator_name = comparator_name.substr(comparator_name.find(':') + 1);
368 ASSERT_EQ(comparator_name,
369 fname_and_props.second->comparator_name);
370 }
371 Close();
372
373 // Also check comparator when it's provided via "unknown" CF options
374 ASSERT_OK(RepairDB(dbname_, opts, {{"default", opts}},
375 rev_opts /* unknown_cf_opts */));
376 ASSERT_OK(TryReopenWithColumnFamilies({"default", "reverse"},
377 std::vector<Options>{opts, rev_opts}));
378 for (int i = 0; i < kNumCfs; ++i) {
379 for (int j = 0; j < kEntriesPerCf; ++j) {
380 ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
381 }
382 }
383 }
384
TEST_F(RepairTest,DbNameContainsTrailingSlash)385 TEST_F(RepairTest, DbNameContainsTrailingSlash) {
386 {
387 bool tmp;
388 if (env_->AreFilesSame("", "", &tmp).IsNotSupported()) {
389 fprintf(stderr,
390 "skipping RepairTest.DbNameContainsTrailingSlash due to "
391 "unsupported Env::AreFilesSame\n");
392 return;
393 }
394 }
395
396 ASSERT_OK(Put("key", "val"));
397 ASSERT_OK(Flush());
398 Close();
399
400 ASSERT_OK(RepairDB(dbname_ + "/", CurrentOptions()));
401 Reopen(CurrentOptions());
402 ASSERT_EQ(Get("key"), "val");
403 }
404 #endif // ROCKSDB_LITE
405 } // namespace ROCKSDB_NAMESPACE
406
main(int argc,char ** argv)407 int main(int argc, char** argv) {
408 ::testing::InitGoogleTest(&argc, argv);
409 return RUN_ALL_TESTS();
410 }
411
412 #else
413 #include <stdio.h>
414
main(int,char **)415 int main(int /*argc*/, char** /*argv*/) {
416 fprintf(stderr, "SKIPPED as RepairDB is not supported in ROCKSDB_LITE\n");
417 return 0;
418 }
419
420 #endif // ROCKSDB_LITE
421