1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #include "rocksdb/utilities/option_change_migration.h"
11
12 #include <set>
13
14 #include "db/db_test_util.h"
15 #include "port/stack_trace.h"
16 #include "util/random.h"
17
18 namespace ROCKSDB_NAMESPACE {
19
20 class DBOptionChangeMigrationTests
21 : public DBTestBase,
22 public testing::WithParamInterface<
23 std::tuple<int, int, bool, int, int, bool>> {
24 public:
DBOptionChangeMigrationTests()25 DBOptionChangeMigrationTests()
26 : DBTestBase("/db_option_change_migration_test", /*env_do_fsync=*/true) {
27 level1_ = std::get<0>(GetParam());
28 compaction_style1_ = std::get<1>(GetParam());
29 is_dynamic1_ = std::get<2>(GetParam());
30
31 level2_ = std::get<3>(GetParam());
32 compaction_style2_ = std::get<4>(GetParam());
33 is_dynamic2_ = std::get<5>(GetParam());
34 }
35
36 // Required if inheriting from testing::WithParamInterface<>
SetUpTestCase()37 static void SetUpTestCase() {}
TearDownTestCase()38 static void TearDownTestCase() {}
39
40 int level1_;
41 int compaction_style1_;
42 bool is_dynamic1_;
43
44 int level2_;
45 int compaction_style2_;
46 bool is_dynamic2_;
47 };
48
49 #ifndef ROCKSDB_LITE
TEST_P(DBOptionChangeMigrationTests,Migrate1)50 TEST_P(DBOptionChangeMigrationTests, Migrate1) {
51 Options old_options = CurrentOptions();
52 old_options.compaction_style =
53 static_cast<CompactionStyle>(compaction_style1_);
54 if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
55 old_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
56 }
57 if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
58 old_options.max_open_files = -1;
59 }
60 old_options.level0_file_num_compaction_trigger = 3;
61 old_options.write_buffer_size = 64 * 1024;
62 old_options.target_file_size_base = 128 * 1024;
63 // Make level target of L1, L2 to be 200KB and 600KB
64 old_options.num_levels = level1_;
65 old_options.max_bytes_for_level_multiplier = 3;
66 old_options.max_bytes_for_level_base = 200 * 1024;
67
68 Reopen(old_options);
69
70 Random rnd(301);
71 int key_idx = 0;
72
73 // Generate at least 2MB of data
74 for (int num = 0; num < 20; num++) {
75 GenerateNewFile(&rnd, &key_idx);
76 }
77 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
78 ASSERT_OK(dbfull()->TEST_WaitForCompact());
79
80 // Will make sure exactly those keys are in the DB after migration.
81 std::set<std::string> keys;
82 {
83 std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
84 it->SeekToFirst();
85 for (; it->Valid(); it->Next()) {
86 keys.insert(it->key().ToString());
87 }
88 }
89 Close();
90
91 Options new_options = old_options;
92 new_options.compaction_style =
93 static_cast<CompactionStyle>(compaction_style2_);
94 if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
95 new_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
96 }
97 if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
98 new_options.max_open_files = -1;
99 }
100 new_options.target_file_size_base = 256 * 1024;
101 new_options.num_levels = level2_;
102 new_options.max_bytes_for_level_base = 150 * 1024;
103 new_options.max_bytes_for_level_multiplier = 4;
104 ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
105 Reopen(new_options);
106
107 // Wait for compaction to finish and make sure it can reopen
108 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
109 ASSERT_OK(dbfull()->TEST_WaitForCompact());
110 Reopen(new_options);
111
112 {
113 std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
114 it->SeekToFirst();
115 for (std::string key : keys) {
116 ASSERT_TRUE(it->Valid());
117 ASSERT_EQ(key, it->key().ToString());
118 it->Next();
119 }
120 ASSERT_TRUE(!it->Valid());
121 }
122 }
123
TEST_P(DBOptionChangeMigrationTests,Migrate2)124 TEST_P(DBOptionChangeMigrationTests, Migrate2) {
125 Options old_options = CurrentOptions();
126 old_options.compaction_style =
127 static_cast<CompactionStyle>(compaction_style2_);
128 if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
129 old_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
130 }
131 if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
132 old_options.max_open_files = -1;
133 }
134 old_options.level0_file_num_compaction_trigger = 3;
135 old_options.write_buffer_size = 64 * 1024;
136 old_options.target_file_size_base = 128 * 1024;
137 // Make level target of L1, L2 to be 200KB and 600KB
138 old_options.num_levels = level2_;
139 old_options.max_bytes_for_level_multiplier = 3;
140 old_options.max_bytes_for_level_base = 200 * 1024;
141
142 Reopen(old_options);
143
144 Random rnd(301);
145 int key_idx = 0;
146
147 // Generate at least 2MB of data
148 for (int num = 0; num < 20; num++) {
149 GenerateNewFile(&rnd, &key_idx);
150 }
151 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
152 ASSERT_OK(dbfull()->TEST_WaitForCompact());
153
154 // Will make sure exactly those keys are in the DB after migration.
155 std::set<std::string> keys;
156 {
157 std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
158 it->SeekToFirst();
159 for (; it->Valid(); it->Next()) {
160 keys.insert(it->key().ToString());
161 }
162 }
163
164 Close();
165
166 Options new_options = old_options;
167 new_options.compaction_style =
168 static_cast<CompactionStyle>(compaction_style1_);
169 if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
170 new_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
171 }
172 if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
173 new_options.max_open_files = -1;
174 }
175 new_options.target_file_size_base = 256 * 1024;
176 new_options.num_levels = level1_;
177 new_options.max_bytes_for_level_base = 150 * 1024;
178 new_options.max_bytes_for_level_multiplier = 4;
179 ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
180 Reopen(new_options);
181 // Wait for compaction to finish and make sure it can reopen
182 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
183 ASSERT_OK(dbfull()->TEST_WaitForCompact());
184 Reopen(new_options);
185
186 {
187 std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
188 it->SeekToFirst();
189 for (std::string key : keys) {
190 ASSERT_TRUE(it->Valid());
191 ASSERT_EQ(key, it->key().ToString());
192 it->Next();
193 }
194 ASSERT_TRUE(!it->Valid());
195 }
196 }
197
TEST_P(DBOptionChangeMigrationTests,Migrate3)198 TEST_P(DBOptionChangeMigrationTests, Migrate3) {
199 Options old_options = CurrentOptions();
200 old_options.compaction_style =
201 static_cast<CompactionStyle>(compaction_style1_);
202 if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
203 old_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
204 }
205 if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
206 old_options.max_open_files = -1;
207 }
208 old_options.level0_file_num_compaction_trigger = 3;
209 old_options.write_buffer_size = 64 * 1024;
210 old_options.target_file_size_base = 128 * 1024;
211 // Make level target of L1, L2 to be 200KB and 600KB
212 old_options.num_levels = level1_;
213 old_options.max_bytes_for_level_multiplier = 3;
214 old_options.max_bytes_for_level_base = 200 * 1024;
215
216 Reopen(old_options);
217 Random rnd(301);
218 for (int num = 0; num < 20; num++) {
219 for (int i = 0; i < 50; i++) {
220 ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900)));
221 }
222 Flush();
223 ASSERT_OK(dbfull()->TEST_WaitForCompact());
224 if (num == 9) {
225 // Issue a full compaction to generate some zero-out files
226 CompactRangeOptions cro;
227 cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
228 ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
229 }
230 }
231 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
232 ASSERT_OK(dbfull()->TEST_WaitForCompact());
233
234 // Will make sure exactly those keys are in the DB after migration.
235 std::set<std::string> keys;
236 {
237 std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
238 it->SeekToFirst();
239 for (; it->Valid(); it->Next()) {
240 keys.insert(it->key().ToString());
241 }
242 }
243 Close();
244
245 Options new_options = old_options;
246 new_options.compaction_style =
247 static_cast<CompactionStyle>(compaction_style2_);
248 if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
249 new_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
250 }
251 if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
252 new_options.max_open_files = -1;
253 }
254 new_options.target_file_size_base = 256 * 1024;
255 new_options.num_levels = level2_;
256 new_options.max_bytes_for_level_base = 150 * 1024;
257 new_options.max_bytes_for_level_multiplier = 4;
258 ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
259 Reopen(new_options);
260
261 // Wait for compaction to finish and make sure it can reopen
262 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
263 ASSERT_OK(dbfull()->TEST_WaitForCompact());
264 Reopen(new_options);
265
266 {
267 std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
268 it->SeekToFirst();
269 for (std::string key : keys) {
270 ASSERT_TRUE(it->Valid());
271 ASSERT_EQ(key, it->key().ToString());
272 it->Next();
273 }
274 ASSERT_TRUE(!it->Valid());
275 }
276 }
277
TEST_P(DBOptionChangeMigrationTests,Migrate4)278 TEST_P(DBOptionChangeMigrationTests, Migrate4) {
279 Options old_options = CurrentOptions();
280 old_options.compaction_style =
281 static_cast<CompactionStyle>(compaction_style2_);
282 if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
283 old_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
284 }
285 if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
286 old_options.max_open_files = -1;
287 }
288 old_options.level0_file_num_compaction_trigger = 3;
289 old_options.write_buffer_size = 64 * 1024;
290 old_options.target_file_size_base = 128 * 1024;
291 // Make level target of L1, L2 to be 200KB and 600KB
292 old_options.num_levels = level2_;
293 old_options.max_bytes_for_level_multiplier = 3;
294 old_options.max_bytes_for_level_base = 200 * 1024;
295
296 Reopen(old_options);
297 Random rnd(301);
298 for (int num = 0; num < 20; num++) {
299 for (int i = 0; i < 50; i++) {
300 ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900)));
301 }
302 Flush();
303 ASSERT_OK(dbfull()->TEST_WaitForCompact());
304 if (num == 9) {
305 // Issue a full compaction to generate some zero-out files
306 CompactRangeOptions cro;
307 cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
308 ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
309 }
310 }
311 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
312 ASSERT_OK(dbfull()->TEST_WaitForCompact());
313
314 // Will make sure exactly those keys are in the DB after migration.
315 std::set<std::string> keys;
316 {
317 std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
318 it->SeekToFirst();
319 for (; it->Valid(); it->Next()) {
320 keys.insert(it->key().ToString());
321 }
322 }
323
324 Close();
325
326 Options new_options = old_options;
327 new_options.compaction_style =
328 static_cast<CompactionStyle>(compaction_style1_);
329 if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
330 new_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
331 }
332 if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
333 new_options.max_open_files = -1;
334 }
335 new_options.target_file_size_base = 256 * 1024;
336 new_options.num_levels = level1_;
337 new_options.max_bytes_for_level_base = 150 * 1024;
338 new_options.max_bytes_for_level_multiplier = 4;
339 ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
340 Reopen(new_options);
341 // Wait for compaction to finish and make sure it can reopen
342 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
343 ASSERT_OK(dbfull()->TEST_WaitForCompact());
344 Reopen(new_options);
345
346 {
347 std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
348 it->SeekToFirst();
349 for (std::string key : keys) {
350 ASSERT_TRUE(it->Valid());
351 ASSERT_EQ(key, it->key().ToString());
352 it->Next();
353 }
354 ASSERT_TRUE(!it->Valid());
355 }
356 }
357
358 INSTANTIATE_TEST_CASE_P(
359 DBOptionChangeMigrationTests, DBOptionChangeMigrationTests,
360 ::testing::Values(std::make_tuple(3, 0, false, 4, 0, false),
361 std::make_tuple(3, 0, true, 4, 0, true),
362 std::make_tuple(3, 0, true, 4, 0, false),
363 std::make_tuple(3, 0, false, 4, 0, true),
364 std::make_tuple(3, 1, false, 4, 1, false),
365 std::make_tuple(1, 1, false, 4, 1, false),
366 std::make_tuple(3, 0, false, 4, 1, false),
367 std::make_tuple(3, 0, false, 1, 1, false),
368 std::make_tuple(3, 0, true, 4, 1, false),
369 std::make_tuple(3, 0, true, 1, 1, false),
370 std::make_tuple(1, 1, false, 4, 0, false),
371 std::make_tuple(4, 0, false, 1, 2, false),
372 std::make_tuple(3, 0, true, 2, 2, false),
373 std::make_tuple(3, 1, false, 3, 2, false),
374 std::make_tuple(1, 1, false, 4, 2, false)));
375
376 class DBOptionChangeMigrationTest : public DBTestBase {
377 public:
DBOptionChangeMigrationTest()378 DBOptionChangeMigrationTest()
379 : DBTestBase("/db_option_change_migration_test2", /*env_do_fsync=*/true) {
380 }
381 };
382
TEST_F(DBOptionChangeMigrationTest,CompactedSrcToUniversal)383 TEST_F(DBOptionChangeMigrationTest, CompactedSrcToUniversal) {
384 Options old_options = CurrentOptions();
385 old_options.compaction_style = CompactionStyle::kCompactionStyleLevel;
386 old_options.max_compaction_bytes = 200 * 1024;
387 old_options.level_compaction_dynamic_level_bytes = false;
388 old_options.level0_file_num_compaction_trigger = 3;
389 old_options.write_buffer_size = 64 * 1024;
390 old_options.target_file_size_base = 128 * 1024;
391 // Make level target of L1, L2 to be 200KB and 600KB
392 old_options.num_levels = 4;
393 old_options.max_bytes_for_level_multiplier = 3;
394 old_options.max_bytes_for_level_base = 200 * 1024;
395
396 Reopen(old_options);
397 Random rnd(301);
398 for (int num = 0; num < 20; num++) {
399 for (int i = 0; i < 50; i++) {
400 ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900)));
401 }
402 }
403 Flush();
404 CompactRangeOptions cro;
405 cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
406 ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
407
408 // Will make sure exactly those keys are in the DB after migration.
409 std::set<std::string> keys;
410 {
411 std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
412 it->SeekToFirst();
413 for (; it->Valid(); it->Next()) {
414 keys.insert(it->key().ToString());
415 }
416 }
417
418 Close();
419
420 Options new_options = old_options;
421 new_options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
422 new_options.target_file_size_base = 256 * 1024;
423 new_options.num_levels = 1;
424 new_options.max_bytes_for_level_base = 150 * 1024;
425 new_options.max_bytes_for_level_multiplier = 4;
426 ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
427 Reopen(new_options);
428 // Wait for compaction to finish and make sure it can reopen
429 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
430 ASSERT_OK(dbfull()->TEST_WaitForCompact());
431 Reopen(new_options);
432
433 {
434 std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
435 it->SeekToFirst();
436 for (std::string key : keys) {
437 ASSERT_TRUE(it->Valid());
438 ASSERT_EQ(key, it->key().ToString());
439 it->Next();
440 }
441 ASSERT_TRUE(!it->Valid());
442 ASSERT_OK(it->status());
443 }
444 }
445
446 #endif // ROCKSDB_LITE
447 } // namespace ROCKSDB_NAMESPACE
448
main(int argc,char ** argv)449 int main(int argc, char** argv) {
450 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
451 ::testing::InitGoogleTest(&argc, argv);
452 return RUN_ALL_TESTS();
453 }
454