1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2 //  This source code is licensed under both the GPLv2 (found in the
3 //  COPYING file in the root directory) and Apache 2.0 License
4 //  (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 
10 #include "rocksdb/utilities/option_change_migration.h"
11 #include <set>
12 #include "db/db_test_util.h"
13 #include "port/stack_trace.h"
14 namespace ROCKSDB_NAMESPACE {
15 
16 class DBOptionChangeMigrationTests
17     : public DBTestBase,
18       public testing::WithParamInterface<
19           std::tuple<int, int, bool, int, int, bool>> {
20  public:
DBOptionChangeMigrationTests()21   DBOptionChangeMigrationTests()
22       : DBTestBase("/db_option_change_migration_test") {
23     level1_ = std::get<0>(GetParam());
24     compaction_style1_ = std::get<1>(GetParam());
25     is_dynamic1_ = std::get<2>(GetParam());
26 
27     level2_ = std::get<3>(GetParam());
28     compaction_style2_ = std::get<4>(GetParam());
29     is_dynamic2_ = std::get<5>(GetParam());
30   }
31 
32   // Required if inheriting from testing::WithParamInterface<>
SetUpTestCase()33   static void SetUpTestCase() {}
TearDownTestCase()34   static void TearDownTestCase() {}
35 
36   int level1_;
37   int compaction_style1_;
38   bool is_dynamic1_;
39 
40   int level2_;
41   int compaction_style2_;
42   bool is_dynamic2_;
43 };
44 
45 #ifndef ROCKSDB_LITE
TEST_P(DBOptionChangeMigrationTests,Migrate1)46 TEST_P(DBOptionChangeMigrationTests, Migrate1) {
47   Options old_options = CurrentOptions();
48   old_options.compaction_style =
49       static_cast<CompactionStyle>(compaction_style1_);
50   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
51     old_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
52   }
53 
54   old_options.level0_file_num_compaction_trigger = 3;
55   old_options.write_buffer_size = 64 * 1024;
56   old_options.target_file_size_base = 128 * 1024;
57   // Make level target of L1, L2 to be 200KB and 600KB
58   old_options.num_levels = level1_;
59   old_options.max_bytes_for_level_multiplier = 3;
60   old_options.max_bytes_for_level_base = 200 * 1024;
61 
62   Reopen(old_options);
63 
64   Random rnd(301);
65   int key_idx = 0;
66 
67   // Generate at least 2MB of data
68   for (int num = 0; num < 20; num++) {
69     GenerateNewFile(&rnd, &key_idx);
70   }
71   dbfull()->TEST_WaitForFlushMemTable();
72   dbfull()->TEST_WaitForCompact();
73 
74   // Will make sure exactly those keys are in the DB after migration.
75   std::set<std::string> keys;
76   {
77     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
78     it->SeekToFirst();
79     for (; it->Valid(); it->Next()) {
80       keys.insert(it->key().ToString());
81     }
82   }
83   Close();
84 
85   Options new_options = old_options;
86   new_options.compaction_style =
87       static_cast<CompactionStyle>(compaction_style2_);
88   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
89     new_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
90   }
91   new_options.target_file_size_base = 256 * 1024;
92   new_options.num_levels = level2_;
93   new_options.max_bytes_for_level_base = 150 * 1024;
94   new_options.max_bytes_for_level_multiplier = 4;
95   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
96   Reopen(new_options);
97 
98   // Wait for compaction to finish and make sure it can reopen
99   dbfull()->TEST_WaitForFlushMemTable();
100   dbfull()->TEST_WaitForCompact();
101   Reopen(new_options);
102 
103   {
104     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
105     it->SeekToFirst();
106     for (std::string key : keys) {
107       ASSERT_TRUE(it->Valid());
108       ASSERT_EQ(key, it->key().ToString());
109       it->Next();
110     }
111     ASSERT_TRUE(!it->Valid());
112   }
113 }
114 
TEST_P(DBOptionChangeMigrationTests,Migrate2)115 TEST_P(DBOptionChangeMigrationTests, Migrate2) {
116   Options old_options = CurrentOptions();
117   old_options.compaction_style =
118       static_cast<CompactionStyle>(compaction_style2_);
119   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
120     old_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
121   }
122   old_options.level0_file_num_compaction_trigger = 3;
123   old_options.write_buffer_size = 64 * 1024;
124   old_options.target_file_size_base = 128 * 1024;
125   // Make level target of L1, L2 to be 200KB and 600KB
126   old_options.num_levels = level2_;
127   old_options.max_bytes_for_level_multiplier = 3;
128   old_options.max_bytes_for_level_base = 200 * 1024;
129 
130   Reopen(old_options);
131 
132   Random rnd(301);
133   int key_idx = 0;
134 
135   // Generate at least 2MB of data
136   for (int num = 0; num < 20; num++) {
137     GenerateNewFile(&rnd, &key_idx);
138   }
139   dbfull()->TEST_WaitForFlushMemTable();
140   dbfull()->TEST_WaitForCompact();
141 
142   // Will make sure exactly those keys are in the DB after migration.
143   std::set<std::string> keys;
144   {
145     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
146     it->SeekToFirst();
147     for (; it->Valid(); it->Next()) {
148       keys.insert(it->key().ToString());
149     }
150   }
151 
152   Close();
153 
154   Options new_options = old_options;
155   new_options.compaction_style =
156       static_cast<CompactionStyle>(compaction_style1_);
157   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
158     new_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
159   }
160   new_options.target_file_size_base = 256 * 1024;
161   new_options.num_levels = level1_;
162   new_options.max_bytes_for_level_base = 150 * 1024;
163   new_options.max_bytes_for_level_multiplier = 4;
164   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
165   Reopen(new_options);
166   // Wait for compaction to finish and make sure it can reopen
167   dbfull()->TEST_WaitForFlushMemTable();
168   dbfull()->TEST_WaitForCompact();
169   Reopen(new_options);
170 
171   {
172     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
173     it->SeekToFirst();
174     for (std::string key : keys) {
175       ASSERT_TRUE(it->Valid());
176       ASSERT_EQ(key, it->key().ToString());
177       it->Next();
178     }
179     ASSERT_TRUE(!it->Valid());
180   }
181 }
182 
TEST_P(DBOptionChangeMigrationTests,Migrate3)183 TEST_P(DBOptionChangeMigrationTests, Migrate3) {
184   Options old_options = CurrentOptions();
185   old_options.compaction_style =
186       static_cast<CompactionStyle>(compaction_style1_);
187   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
188     old_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
189   }
190 
191   old_options.level0_file_num_compaction_trigger = 3;
192   old_options.write_buffer_size = 64 * 1024;
193   old_options.target_file_size_base = 128 * 1024;
194   // Make level target of L1, L2 to be 200KB and 600KB
195   old_options.num_levels = level1_;
196   old_options.max_bytes_for_level_multiplier = 3;
197   old_options.max_bytes_for_level_base = 200 * 1024;
198 
199   Reopen(old_options);
200   Random rnd(301);
201   for (int num = 0; num < 20; num++) {
202     for (int i = 0; i < 50; i++) {
203       ASSERT_OK(Put(Key(num * 100 + i), RandomString(&rnd, 900)));
204     }
205     Flush();
206     dbfull()->TEST_WaitForCompact();
207     if (num == 9) {
208       // Issue a full compaction to generate some zero-out files
209       CompactRangeOptions cro;
210       cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
211       dbfull()->CompactRange(cro, nullptr, nullptr);
212     }
213   }
214   dbfull()->TEST_WaitForFlushMemTable();
215   dbfull()->TEST_WaitForCompact();
216 
217   // Will make sure exactly those keys are in the DB after migration.
218   std::set<std::string> keys;
219   {
220     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
221     it->SeekToFirst();
222     for (; it->Valid(); it->Next()) {
223       keys.insert(it->key().ToString());
224     }
225   }
226   Close();
227 
228   Options new_options = old_options;
229   new_options.compaction_style =
230       static_cast<CompactionStyle>(compaction_style2_);
231   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
232     new_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
233   }
234   new_options.target_file_size_base = 256 * 1024;
235   new_options.num_levels = level2_;
236   new_options.max_bytes_for_level_base = 150 * 1024;
237   new_options.max_bytes_for_level_multiplier = 4;
238   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
239   Reopen(new_options);
240 
241   // Wait for compaction to finish and make sure it can reopen
242   dbfull()->TEST_WaitForFlushMemTable();
243   dbfull()->TEST_WaitForCompact();
244   Reopen(new_options);
245 
246   {
247     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
248     it->SeekToFirst();
249     for (std::string key : keys) {
250       ASSERT_TRUE(it->Valid());
251       ASSERT_EQ(key, it->key().ToString());
252       it->Next();
253     }
254     ASSERT_TRUE(!it->Valid());
255   }
256 }
257 
TEST_P(DBOptionChangeMigrationTests,Migrate4)258 TEST_P(DBOptionChangeMigrationTests, Migrate4) {
259   Options old_options = CurrentOptions();
260   old_options.compaction_style =
261       static_cast<CompactionStyle>(compaction_style2_);
262   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
263     old_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
264   }
265   old_options.level0_file_num_compaction_trigger = 3;
266   old_options.write_buffer_size = 64 * 1024;
267   old_options.target_file_size_base = 128 * 1024;
268   // Make level target of L1, L2 to be 200KB and 600KB
269   old_options.num_levels = level2_;
270   old_options.max_bytes_for_level_multiplier = 3;
271   old_options.max_bytes_for_level_base = 200 * 1024;
272 
273   Reopen(old_options);
274   Random rnd(301);
275   for (int num = 0; num < 20; num++) {
276     for (int i = 0; i < 50; i++) {
277       ASSERT_OK(Put(Key(num * 100 + i), RandomString(&rnd, 900)));
278     }
279     Flush();
280     dbfull()->TEST_WaitForCompact();
281     if (num == 9) {
282       // Issue a full compaction to generate some zero-out files
283       CompactRangeOptions cro;
284       cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
285       dbfull()->CompactRange(cro, nullptr, nullptr);
286     }
287   }
288   dbfull()->TEST_WaitForFlushMemTable();
289   dbfull()->TEST_WaitForCompact();
290 
291   // Will make sure exactly those keys are in the DB after migration.
292   std::set<std::string> keys;
293   {
294     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
295     it->SeekToFirst();
296     for (; it->Valid(); it->Next()) {
297       keys.insert(it->key().ToString());
298     }
299   }
300 
301   Close();
302 
303   Options new_options = old_options;
304   new_options.compaction_style =
305       static_cast<CompactionStyle>(compaction_style1_);
306   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
307     new_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
308   }
309   new_options.target_file_size_base = 256 * 1024;
310   new_options.num_levels = level1_;
311   new_options.max_bytes_for_level_base = 150 * 1024;
312   new_options.max_bytes_for_level_multiplier = 4;
313   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
314   Reopen(new_options);
315   // Wait for compaction to finish and make sure it can reopen
316   dbfull()->TEST_WaitForFlushMemTable();
317   dbfull()->TEST_WaitForCompact();
318   Reopen(new_options);
319 
320   {
321     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
322     it->SeekToFirst();
323     for (std::string key : keys) {
324       ASSERT_TRUE(it->Valid());
325       ASSERT_EQ(key, it->key().ToString());
326       it->Next();
327     }
328     ASSERT_TRUE(!it->Valid());
329   }
330 }
331 
332 INSTANTIATE_TEST_CASE_P(
333     DBOptionChangeMigrationTests, DBOptionChangeMigrationTests,
334     ::testing::Values(std::make_tuple(3, 0, false, 4, 0, false),
335                       std::make_tuple(3, 0, true, 4, 0, true),
336                       std::make_tuple(3, 0, true, 4, 0, false),
337                       std::make_tuple(3, 0, false, 4, 0, true),
338                       std::make_tuple(3, 1, false, 4, 1, false),
339                       std::make_tuple(1, 1, false, 4, 1, false),
340                       std::make_tuple(3, 0, false, 4, 1, false),
341                       std::make_tuple(3, 0, false, 1, 1, false),
342                       std::make_tuple(3, 0, true, 4, 1, false),
343                       std::make_tuple(3, 0, true, 1, 1, false),
344                       std::make_tuple(1, 1, false, 4, 0, false),
345                       std::make_tuple(4, 0, false, 1, 2, false),
346                       std::make_tuple(3, 0, true, 2, 2, false),
347                       std::make_tuple(3, 1, false, 3, 2, false),
348                       std::make_tuple(1, 1, false, 4, 2, false)));
349 
350 class DBOptionChangeMigrationTest : public DBTestBase {
351  public:
DBOptionChangeMigrationTest()352   DBOptionChangeMigrationTest()
353       : DBTestBase("/db_option_change_migration_test2") {}
354 };
355 
TEST_F(DBOptionChangeMigrationTest,CompactedSrcToUniversal)356 TEST_F(DBOptionChangeMigrationTest, CompactedSrcToUniversal) {
357   Options old_options = CurrentOptions();
358   old_options.compaction_style = CompactionStyle::kCompactionStyleLevel;
359   old_options.max_compaction_bytes = 200 * 1024;
360   old_options.level_compaction_dynamic_level_bytes = false;
361   old_options.level0_file_num_compaction_trigger = 3;
362   old_options.write_buffer_size = 64 * 1024;
363   old_options.target_file_size_base = 128 * 1024;
364   // Make level target of L1, L2 to be 200KB and 600KB
365   old_options.num_levels = 4;
366   old_options.max_bytes_for_level_multiplier = 3;
367   old_options.max_bytes_for_level_base = 200 * 1024;
368 
369   Reopen(old_options);
370   Random rnd(301);
371   for (int num = 0; num < 20; num++) {
372     for (int i = 0; i < 50; i++) {
373       ASSERT_OK(Put(Key(num * 100 + i), RandomString(&rnd, 900)));
374     }
375   }
376   Flush();
377   CompactRangeOptions cro;
378   cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
379   dbfull()->CompactRange(cro, nullptr, nullptr);
380 
381   // Will make sure exactly those keys are in the DB after migration.
382   std::set<std::string> keys;
383   {
384     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
385     it->SeekToFirst();
386     for (; it->Valid(); it->Next()) {
387       keys.insert(it->key().ToString());
388     }
389   }
390 
391   Close();
392 
393   Options new_options = old_options;
394   new_options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
395   new_options.target_file_size_base = 256 * 1024;
396   new_options.num_levels = 1;
397   new_options.max_bytes_for_level_base = 150 * 1024;
398   new_options.max_bytes_for_level_multiplier = 4;
399   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
400   Reopen(new_options);
401   // Wait for compaction to finish and make sure it can reopen
402   dbfull()->TEST_WaitForFlushMemTable();
403   dbfull()->TEST_WaitForCompact();
404   Reopen(new_options);
405 
406   {
407     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
408     it->SeekToFirst();
409     for (std::string key : keys) {
410       ASSERT_TRUE(it->Valid());
411       ASSERT_EQ(key, it->key().ToString());
412       it->Next();
413     }
414     ASSERT_TRUE(!it->Valid());
415   }
416 }
417 
418 #endif  // ROCKSDB_LITE
419 }  // namespace ROCKSDB_NAMESPACE
420 
main(int argc,char ** argv)421 int main(int argc, char** argv) {
422   ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
423   ::testing::InitGoogleTest(&argc, argv);
424   return RUN_ALL_TESTS();
425 }
426