1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2 //  This source code is licensed under both the GPLv2 (found in the
3 //  COPYING file in the root directory) and Apache 2.0 License
4 //  (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 
10 #include "rocksdb/utilities/option_change_migration.h"
11 
12 #include <set>
13 
14 #include "db/db_test_util.h"
15 #include "port/stack_trace.h"
16 #include "util/random.h"
17 
18 namespace ROCKSDB_NAMESPACE {
19 
20 class DBOptionChangeMigrationTests
21     : public DBTestBase,
22       public testing::WithParamInterface<
23           std::tuple<int, int, bool, int, int, bool>> {
24  public:
DBOptionChangeMigrationTests()25   DBOptionChangeMigrationTests()
26       : DBTestBase("/db_option_change_migration_test", /*env_do_fsync=*/true) {
27     level1_ = std::get<0>(GetParam());
28     compaction_style1_ = std::get<1>(GetParam());
29     is_dynamic1_ = std::get<2>(GetParam());
30 
31     level2_ = std::get<3>(GetParam());
32     compaction_style2_ = std::get<4>(GetParam());
33     is_dynamic2_ = std::get<5>(GetParam());
34   }
35 
36   // Required if inheriting from testing::WithParamInterface<>
SetUpTestCase()37   static void SetUpTestCase() {}
TearDownTestCase()38   static void TearDownTestCase() {}
39 
40   int level1_;
41   int compaction_style1_;
42   bool is_dynamic1_;
43 
44   int level2_;
45   int compaction_style2_;
46   bool is_dynamic2_;
47 };
48 
49 #ifndef ROCKSDB_LITE
TEST_P(DBOptionChangeMigrationTests,Migrate1)50 TEST_P(DBOptionChangeMigrationTests, Migrate1) {
51   Options old_options = CurrentOptions();
52   old_options.compaction_style =
53       static_cast<CompactionStyle>(compaction_style1_);
54   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
55     old_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
56   }
57   if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
58     old_options.max_open_files = -1;
59   }
60   old_options.level0_file_num_compaction_trigger = 3;
61   old_options.write_buffer_size = 64 * 1024;
62   old_options.target_file_size_base = 128 * 1024;
63   // Make level target of L1, L2 to be 200KB and 600KB
64   old_options.num_levels = level1_;
65   old_options.max_bytes_for_level_multiplier = 3;
66   old_options.max_bytes_for_level_base = 200 * 1024;
67 
68   Reopen(old_options);
69 
70   Random rnd(301);
71   int key_idx = 0;
72 
73   // Generate at least 2MB of data
74   for (int num = 0; num < 20; num++) {
75     GenerateNewFile(&rnd, &key_idx);
76   }
77   ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
78   ASSERT_OK(dbfull()->TEST_WaitForCompact());
79 
80   // Will make sure exactly those keys are in the DB after migration.
81   std::set<std::string> keys;
82   {
83     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
84     it->SeekToFirst();
85     for (; it->Valid(); it->Next()) {
86       keys.insert(it->key().ToString());
87     }
88   }
89   Close();
90 
91   Options new_options = old_options;
92   new_options.compaction_style =
93       static_cast<CompactionStyle>(compaction_style2_);
94   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
95     new_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
96   }
97   if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
98     new_options.max_open_files = -1;
99   }
100   new_options.target_file_size_base = 256 * 1024;
101   new_options.num_levels = level2_;
102   new_options.max_bytes_for_level_base = 150 * 1024;
103   new_options.max_bytes_for_level_multiplier = 4;
104   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
105   Reopen(new_options);
106 
107   // Wait for compaction to finish and make sure it can reopen
108   ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
109   ASSERT_OK(dbfull()->TEST_WaitForCompact());
110   Reopen(new_options);
111 
112   {
113     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
114     it->SeekToFirst();
115     for (std::string key : keys) {
116       ASSERT_TRUE(it->Valid());
117       ASSERT_EQ(key, it->key().ToString());
118       it->Next();
119     }
120     ASSERT_TRUE(!it->Valid());
121   }
122 }
123 
TEST_P(DBOptionChangeMigrationTests,Migrate2)124 TEST_P(DBOptionChangeMigrationTests, Migrate2) {
125   Options old_options = CurrentOptions();
126   old_options.compaction_style =
127       static_cast<CompactionStyle>(compaction_style2_);
128   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
129     old_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
130   }
131   if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
132     old_options.max_open_files = -1;
133   }
134   old_options.level0_file_num_compaction_trigger = 3;
135   old_options.write_buffer_size = 64 * 1024;
136   old_options.target_file_size_base = 128 * 1024;
137   // Make level target of L1, L2 to be 200KB and 600KB
138   old_options.num_levels = level2_;
139   old_options.max_bytes_for_level_multiplier = 3;
140   old_options.max_bytes_for_level_base = 200 * 1024;
141 
142   Reopen(old_options);
143 
144   Random rnd(301);
145   int key_idx = 0;
146 
147   // Generate at least 2MB of data
148   for (int num = 0; num < 20; num++) {
149     GenerateNewFile(&rnd, &key_idx);
150   }
151   ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
152   ASSERT_OK(dbfull()->TEST_WaitForCompact());
153 
154   // Will make sure exactly those keys are in the DB after migration.
155   std::set<std::string> keys;
156   {
157     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
158     it->SeekToFirst();
159     for (; it->Valid(); it->Next()) {
160       keys.insert(it->key().ToString());
161     }
162   }
163 
164   Close();
165 
166   Options new_options = old_options;
167   new_options.compaction_style =
168       static_cast<CompactionStyle>(compaction_style1_);
169   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
170     new_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
171   }
172   if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
173     new_options.max_open_files = -1;
174   }
175   new_options.target_file_size_base = 256 * 1024;
176   new_options.num_levels = level1_;
177   new_options.max_bytes_for_level_base = 150 * 1024;
178   new_options.max_bytes_for_level_multiplier = 4;
179   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
180   Reopen(new_options);
181   // Wait for compaction to finish and make sure it can reopen
182   ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
183   ASSERT_OK(dbfull()->TEST_WaitForCompact());
184   Reopen(new_options);
185 
186   {
187     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
188     it->SeekToFirst();
189     for (std::string key : keys) {
190       ASSERT_TRUE(it->Valid());
191       ASSERT_EQ(key, it->key().ToString());
192       it->Next();
193     }
194     ASSERT_TRUE(!it->Valid());
195   }
196 }
197 
TEST_P(DBOptionChangeMigrationTests,Migrate3)198 TEST_P(DBOptionChangeMigrationTests, Migrate3) {
199   Options old_options = CurrentOptions();
200   old_options.compaction_style =
201       static_cast<CompactionStyle>(compaction_style1_);
202   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
203     old_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
204   }
205   if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
206     old_options.max_open_files = -1;
207   }
208   old_options.level0_file_num_compaction_trigger = 3;
209   old_options.write_buffer_size = 64 * 1024;
210   old_options.target_file_size_base = 128 * 1024;
211   // Make level target of L1, L2 to be 200KB and 600KB
212   old_options.num_levels = level1_;
213   old_options.max_bytes_for_level_multiplier = 3;
214   old_options.max_bytes_for_level_base = 200 * 1024;
215 
216   Reopen(old_options);
217   Random rnd(301);
218   for (int num = 0; num < 20; num++) {
219     for (int i = 0; i < 50; i++) {
220       ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900)));
221     }
222     Flush();
223     ASSERT_OK(dbfull()->TEST_WaitForCompact());
224     if (num == 9) {
225       // Issue a full compaction to generate some zero-out files
226       CompactRangeOptions cro;
227       cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
228       ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
229     }
230   }
231   ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
232   ASSERT_OK(dbfull()->TEST_WaitForCompact());
233 
234   // Will make sure exactly those keys are in the DB after migration.
235   std::set<std::string> keys;
236   {
237     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
238     it->SeekToFirst();
239     for (; it->Valid(); it->Next()) {
240       keys.insert(it->key().ToString());
241     }
242   }
243   Close();
244 
245   Options new_options = old_options;
246   new_options.compaction_style =
247       static_cast<CompactionStyle>(compaction_style2_);
248   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
249     new_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
250   }
251   if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
252     new_options.max_open_files = -1;
253   }
254   new_options.target_file_size_base = 256 * 1024;
255   new_options.num_levels = level2_;
256   new_options.max_bytes_for_level_base = 150 * 1024;
257   new_options.max_bytes_for_level_multiplier = 4;
258   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
259   Reopen(new_options);
260 
261   // Wait for compaction to finish and make sure it can reopen
262   ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
263   ASSERT_OK(dbfull()->TEST_WaitForCompact());
264   Reopen(new_options);
265 
266   {
267     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
268     it->SeekToFirst();
269     for (std::string key : keys) {
270       ASSERT_TRUE(it->Valid());
271       ASSERT_EQ(key, it->key().ToString());
272       it->Next();
273     }
274     ASSERT_TRUE(!it->Valid());
275   }
276 }
277 
TEST_P(DBOptionChangeMigrationTests,Migrate4)278 TEST_P(DBOptionChangeMigrationTests, Migrate4) {
279   Options old_options = CurrentOptions();
280   old_options.compaction_style =
281       static_cast<CompactionStyle>(compaction_style2_);
282   if (old_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
283     old_options.level_compaction_dynamic_level_bytes = is_dynamic2_;
284   }
285   if (old_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
286     old_options.max_open_files = -1;
287   }
288   old_options.level0_file_num_compaction_trigger = 3;
289   old_options.write_buffer_size = 64 * 1024;
290   old_options.target_file_size_base = 128 * 1024;
291   // Make level target of L1, L2 to be 200KB and 600KB
292   old_options.num_levels = level2_;
293   old_options.max_bytes_for_level_multiplier = 3;
294   old_options.max_bytes_for_level_base = 200 * 1024;
295 
296   Reopen(old_options);
297   Random rnd(301);
298   for (int num = 0; num < 20; num++) {
299     for (int i = 0; i < 50; i++) {
300       ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900)));
301     }
302     Flush();
303     ASSERT_OK(dbfull()->TEST_WaitForCompact());
304     if (num == 9) {
305       // Issue a full compaction to generate some zero-out files
306       CompactRangeOptions cro;
307       cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
308       ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
309     }
310   }
311   ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
312   ASSERT_OK(dbfull()->TEST_WaitForCompact());
313 
314   // Will make sure exactly those keys are in the DB after migration.
315   std::set<std::string> keys;
316   {
317     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
318     it->SeekToFirst();
319     for (; it->Valid(); it->Next()) {
320       keys.insert(it->key().ToString());
321     }
322   }
323 
324   Close();
325 
326   Options new_options = old_options;
327   new_options.compaction_style =
328       static_cast<CompactionStyle>(compaction_style1_);
329   if (new_options.compaction_style == CompactionStyle::kCompactionStyleLevel) {
330     new_options.level_compaction_dynamic_level_bytes = is_dynamic1_;
331   }
332   if (new_options.compaction_style == CompactionStyle::kCompactionStyleFIFO) {
333     new_options.max_open_files = -1;
334   }
335   new_options.target_file_size_base = 256 * 1024;
336   new_options.num_levels = level1_;
337   new_options.max_bytes_for_level_base = 150 * 1024;
338   new_options.max_bytes_for_level_multiplier = 4;
339   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
340   Reopen(new_options);
341   // Wait for compaction to finish and make sure it can reopen
342   ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
343   ASSERT_OK(dbfull()->TEST_WaitForCompact());
344   Reopen(new_options);
345 
346   {
347     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
348     it->SeekToFirst();
349     for (std::string key : keys) {
350       ASSERT_TRUE(it->Valid());
351       ASSERT_EQ(key, it->key().ToString());
352       it->Next();
353     }
354     ASSERT_TRUE(!it->Valid());
355   }
356 }
357 
358 INSTANTIATE_TEST_CASE_P(
359     DBOptionChangeMigrationTests, DBOptionChangeMigrationTests,
360     ::testing::Values(std::make_tuple(3, 0, false, 4, 0, false),
361                       std::make_tuple(3, 0, true, 4, 0, true),
362                       std::make_tuple(3, 0, true, 4, 0, false),
363                       std::make_tuple(3, 0, false, 4, 0, true),
364                       std::make_tuple(3, 1, false, 4, 1, false),
365                       std::make_tuple(1, 1, false, 4, 1, false),
366                       std::make_tuple(3, 0, false, 4, 1, false),
367                       std::make_tuple(3, 0, false, 1, 1, false),
368                       std::make_tuple(3, 0, true, 4, 1, false),
369                       std::make_tuple(3, 0, true, 1, 1, false),
370                       std::make_tuple(1, 1, false, 4, 0, false),
371                       std::make_tuple(4, 0, false, 1, 2, false),
372                       std::make_tuple(3, 0, true, 2, 2, false),
373                       std::make_tuple(3, 1, false, 3, 2, false),
374                       std::make_tuple(1, 1, false, 4, 2, false)));
375 
376 class DBOptionChangeMigrationTest : public DBTestBase {
377  public:
DBOptionChangeMigrationTest()378   DBOptionChangeMigrationTest()
379       : DBTestBase("/db_option_change_migration_test2", /*env_do_fsync=*/true) {
380   }
381 };
382 
TEST_F(DBOptionChangeMigrationTest,CompactedSrcToUniversal)383 TEST_F(DBOptionChangeMigrationTest, CompactedSrcToUniversal) {
384   Options old_options = CurrentOptions();
385   old_options.compaction_style = CompactionStyle::kCompactionStyleLevel;
386   old_options.max_compaction_bytes = 200 * 1024;
387   old_options.level_compaction_dynamic_level_bytes = false;
388   old_options.level0_file_num_compaction_trigger = 3;
389   old_options.write_buffer_size = 64 * 1024;
390   old_options.target_file_size_base = 128 * 1024;
391   // Make level target of L1, L2 to be 200KB and 600KB
392   old_options.num_levels = 4;
393   old_options.max_bytes_for_level_multiplier = 3;
394   old_options.max_bytes_for_level_base = 200 * 1024;
395 
396   Reopen(old_options);
397   Random rnd(301);
398   for (int num = 0; num < 20; num++) {
399     for (int i = 0; i < 50; i++) {
400       ASSERT_OK(Put(Key(num * 100 + i), rnd.RandomString(900)));
401     }
402   }
403   Flush();
404   CompactRangeOptions cro;
405   cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
406   ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
407 
408   // Will make sure exactly those keys are in the DB after migration.
409   std::set<std::string> keys;
410   {
411     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
412     it->SeekToFirst();
413     for (; it->Valid(); it->Next()) {
414       keys.insert(it->key().ToString());
415     }
416   }
417 
418   Close();
419 
420   Options new_options = old_options;
421   new_options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
422   new_options.target_file_size_base = 256 * 1024;
423   new_options.num_levels = 1;
424   new_options.max_bytes_for_level_base = 150 * 1024;
425   new_options.max_bytes_for_level_multiplier = 4;
426   ASSERT_OK(OptionChangeMigration(dbname_, old_options, new_options));
427   Reopen(new_options);
428   // Wait for compaction to finish and make sure it can reopen
429   ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
430   ASSERT_OK(dbfull()->TEST_WaitForCompact());
431   Reopen(new_options);
432 
433   {
434     std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
435     it->SeekToFirst();
436     for (std::string key : keys) {
437       ASSERT_TRUE(it->Valid());
438       ASSERT_EQ(key, it->key().ToString());
439       it->Next();
440     }
441     ASSERT_TRUE(!it->Valid());
442     ASSERT_OK(it->status());
443   }
444 }
445 
446 #endif  // ROCKSDB_LITE
447 }  // namespace ROCKSDB_NAMESPACE
448 
main(int argc,char ** argv)449 int main(int argc, char** argv) {
450   ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
451   ::testing::InitGoogleTest(&argc, argv);
452   return RUN_ALL_TESTS();
453 }
454