1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2 //  This source code is licensed under both the GPLv2 (found in the
3 //  COPYING file in the root directory) and Apache 2.0 License
4 //  (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 
10 // Introduction of SyncPoint effectively disabled building and running this test
11 // in Release build.
12 // which is a pity, it is a good test
13 #if !defined(ROCKSDB_LITE)
14 
15 #include "db/db_test_util.h"
16 #include "env/mock_env.h"
17 #include "port/port.h"
18 #include "port/stack_trace.h"
19 #include "util/random.h"
20 
21 namespace ROCKSDB_NAMESPACE {
22 class DBTestDynamicLevel : public DBTestBase {
23  public:
DBTestDynamicLevel()24   DBTestDynamicLevel()
25       : DBTestBase("/db_dynamic_level_test", /*env_do_fsync=*/true) {}
26 };
27 
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesBase)28 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
29   if (!Snappy_Supported() || !LZ4_Supported()) {
30     return;
31   }
32   // Use InMemoryEnv, or it would be too slow.
33   std::unique_ptr<Env> env(new MockEnv(env_));
34 
35   const int kNKeys = 1000;
36   int keys[kNKeys];
37 
38   auto verify_func = [&]() {
39     for (int i = 0; i < kNKeys; i++) {
40       ASSERT_NE("NOT_FOUND", Get(Key(i)));
41       ASSERT_NE("NOT_FOUND", Get(Key(kNKeys * 2 + i)));
42       if (i < kNKeys / 10) {
43         ASSERT_EQ("NOT_FOUND", Get(Key(kNKeys + keys[i])));
44       } else {
45         ASSERT_NE("NOT_FOUND", Get(Key(kNKeys + keys[i])));
46       }
47     }
48   };
49 
50   Random rnd(301);
51   for (int ordered_insert = 0; ordered_insert <= 1; ordered_insert++) {
52     for (int i = 0; i < kNKeys; i++) {
53       keys[i] = i;
54     }
55     if (ordered_insert == 0) {
56       RandomShuffle(std::begin(keys), std::end(keys), rnd.Next());
57     }
58     for (int max_background_compactions = 1; max_background_compactions < 4;
59          max_background_compactions += 2) {
60       Options options;
61       options.env = env.get();
62       options.create_if_missing = true;
63       options.write_buffer_size = 2048;
64       options.max_write_buffer_number = 2;
65       options.level0_file_num_compaction_trigger = 2;
66       options.level0_slowdown_writes_trigger = 2;
67       options.level0_stop_writes_trigger = 2;
68       options.target_file_size_base = 2048;
69       options.level_compaction_dynamic_level_bytes = true;
70       options.max_bytes_for_level_base = 10240;
71       options.max_bytes_for_level_multiplier = 4;
72       options.soft_rate_limit = 1.1;
73       options.max_background_compactions = max_background_compactions;
74       options.num_levels = 5;
75 
76       options.compression_per_level.resize(3);
77       options.compression_per_level[0] = kNoCompression;
78       options.compression_per_level[1] = kLZ4Compression;
79       options.compression_per_level[2] = kSnappyCompression;
80       options.env = env_;
81 
82       DestroyAndReopen(options);
83 
84       for (int i = 0; i < kNKeys; i++) {
85         int key = keys[i];
86         ASSERT_OK(Put(Key(kNKeys + key), rnd.RandomString(102)));
87         ASSERT_OK(Put(Key(key), rnd.RandomString(102)));
88         ASSERT_OK(Put(Key(kNKeys * 2 + key), rnd.RandomString(102)));
89         ASSERT_OK(Delete(Key(kNKeys + keys[i / 10])));
90         env_->SleepForMicroseconds(5000);
91       }
92 
93       uint64_t int_prop;
94       ASSERT_TRUE(db_->GetIntProperty("rocksdb.background-errors", &int_prop));
95       ASSERT_EQ(0U, int_prop);
96 
97       // Verify DB
98       for (int j = 0; j < 2; j++) {
99         verify_func();
100         if (j == 0) {
101           Reopen(options);
102         }
103       }
104 
105       // Test compact range works
106       ASSERT_OK(
107           dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
108       // All data should be in the last level.
109       ColumnFamilyMetaData cf_meta;
110       db_->GetColumnFamilyMetaData(&cf_meta);
111       ASSERT_EQ(5U, cf_meta.levels.size());
112       for (int i = 0; i < 4; i++) {
113         ASSERT_EQ(0U, cf_meta.levels[i].files.size());
114       }
115       ASSERT_GT(cf_meta.levels[4U].files.size(), 0U);
116       verify_func();
117 
118       Close();
119     }
120   }
121 
122   env_->SetBackgroundThreads(1, Env::LOW);
123   env_->SetBackgroundThreads(1, Env::HIGH);
124 }
125 
126 // Test specific cases in dynamic max bytes
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesBase2)127 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
128   Random rnd(301);
129   int kMaxKey = 1000000;
130 
131   Options options = CurrentOptions();
132   options.compression = kNoCompression;
133   options.create_if_missing = true;
134   options.write_buffer_size = 20480;
135   options.max_write_buffer_number = 2;
136   options.level0_file_num_compaction_trigger = 2;
137   options.level0_slowdown_writes_trigger = 9999;
138   options.level0_stop_writes_trigger = 9999;
139   options.target_file_size_base = 9102;
140   options.level_compaction_dynamic_level_bytes = true;
141   options.max_bytes_for_level_base = 40960;
142   options.max_bytes_for_level_multiplier = 4;
143   options.max_background_compactions = 2;
144   options.num_levels = 5;
145   options.max_compaction_bytes = 0;  // Force not expanding in compactions
146   options.db_host_id = "";  // Setting this messes up the file size calculation
147   BlockBasedTableOptions table_options;
148   table_options.block_size = 1024;
149   options.table_factory.reset(NewBlockBasedTableFactory(table_options));
150 
151   DestroyAndReopen(options);
152   ASSERT_OK(dbfull()->SetOptions({
153       {"disable_auto_compactions", "true"},
154   }));
155 
156   uint64_t int_prop;
157   std::string str_prop;
158 
159   // Initial base level is the last level
160   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
161   ASSERT_EQ(4U, int_prop);
162 
163   // Put about 28K to L0
164   for (int i = 0; i < 70; i++) {
165     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
166                   rnd.RandomString(380)));
167   }
168   ASSERT_OK(dbfull()->SetOptions({
169       {"disable_auto_compactions", "false"},
170   }));
171   ASSERT_OK(Flush());
172   ASSERT_OK(dbfull()->TEST_WaitForCompact());
173   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
174   ASSERT_EQ(4U, int_prop);
175 
176   // Insert extra about 28K to L0. After they are compacted to L4, the base
177   // level should be changed to L3.
178   ASSERT_OK(dbfull()->SetOptions({
179       {"disable_auto_compactions", "true"},
180   }));
181   for (int i = 0; i < 70; i++) {
182     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
183                   rnd.RandomString(380)));
184   }
185 
186   ASSERT_OK(dbfull()->SetOptions({
187       {"disable_auto_compactions", "false"},
188   }));
189   ASSERT_OK(Flush());
190   ASSERT_OK(dbfull()->TEST_WaitForCompact());
191   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
192   ASSERT_EQ(3U, int_prop);
193   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level1", &str_prop));
194   ASSERT_EQ("0", str_prop);
195   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level2", &str_prop));
196   ASSERT_EQ("0", str_prop);
197 
198   // Write even more data while leaving the base level at L3.
199   ASSERT_OK(dbfull()->SetOptions({
200       {"disable_auto_compactions", "true"},
201   }));
202   // Write about 40K more
203   for (int i = 0; i < 100; i++) {
204     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
205                   rnd.RandomString(380)));
206   }
207   ASSERT_OK(dbfull()->SetOptions({
208       {"disable_auto_compactions", "false"},
209   }));
210   ASSERT_OK(Flush());
211   ASSERT_OK(dbfull()->TEST_WaitForCompact());
212   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
213   ASSERT_EQ(3U, int_prop);
214 
215   // Fill up L0, and then run an (auto) L0->Lmax compaction to raise the base
216   // level to 2.
217   ASSERT_OK(dbfull()->SetOptions({
218       {"disable_auto_compactions", "true"},
219   }));
220   // Write about 650K more.
221   // Each file is about 11KB, with 9KB of data.
222   for (int i = 0; i < 1300; i++) {
223     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
224                   rnd.RandomString(380)));
225   }
226 
227   // Make sure that the compaction starts before the last bit of data is
228   // flushed, so that the base level isn't raised to L1.
229   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
230       {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:0"},
231   });
232   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
233 
234   ASSERT_OK(dbfull()->SetOptions({
235       {"disable_auto_compactions", "false"},
236   }));
237 
238   TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:0");
239   ASSERT_OK(Flush());
240   ASSERT_OK(dbfull()->TEST_WaitForCompact());
241   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
242   ASSERT_EQ(2U, int_prop);
243   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
244   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
245 
246   // Write more data until the base level changes to L1. There will be
247   // a manual compaction going on at the same time.
248   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
249       {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:1"},
250       {"DynamicLevelMaxBytesBase2:2", "CompactionJob::Run():End"},
251       {"DynamicLevelMaxBytesBase2:compact_range_finish",
252        "FlushJob::WriteLevel0Table"},
253   });
254   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
255 
256   ROCKSDB_NAMESPACE::port::Thread thread([this] {
257     TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_start");
258     ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
259     TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_finish");
260   });
261 
262   TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:1");
263   for (int i = 0; i < 2; i++) {
264     ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
265                   rnd.RandomString(380)));
266   }
267   TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:2");
268 
269   ASSERT_OK(Flush());
270 
271   thread.join();
272 
273   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
274   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
275 
276   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
277   ASSERT_EQ(1U, int_prop);
278 }
279 
280 // Test specific cases in dynamic max bytes
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesCompactRange)281 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesCompactRange) {
282   Random rnd(301);
283   int kMaxKey = 1000000;
284 
285   Options options = CurrentOptions();
286   options.create_if_missing = true;
287   options.write_buffer_size = 2048;
288   options.max_write_buffer_number = 2;
289   options.level0_file_num_compaction_trigger = 2;
290   options.level0_slowdown_writes_trigger = 9999;
291   options.level0_stop_writes_trigger = 9999;
292   options.target_file_size_base = 2;
293   options.level_compaction_dynamic_level_bytes = true;
294   options.max_bytes_for_level_base = 10240;
295   options.max_bytes_for_level_multiplier = 4;
296   options.max_background_compactions = 1;
297   const int kNumLevels = 5;
298   options.num_levels = kNumLevels;
299   options.max_compaction_bytes = 1;  // Force not expanding in compactions
300   BlockBasedTableOptions table_options;
301   table_options.block_size = 1024;
302   options.table_factory.reset(NewBlockBasedTableFactory(table_options));
303 
304   DestroyAndReopen(options);
305 
306   // Compact against empty DB
307   ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
308 
309   uint64_t int_prop;
310   std::string str_prop;
311 
312   // Initial base level is the last level
313   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
314   ASSERT_EQ(4U, int_prop);
315 
316   // Put about 7K to L0
317   for (int i = 0; i < 140; i++) {
318     ASSERT_OK(
319         Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))), rnd.RandomString(80)));
320   }
321   ASSERT_OK(Flush());
322   ASSERT_OK(dbfull()->TEST_WaitForCompact());
323   if (NumTableFilesAtLevel(0) == 0) {
324     // Make sure level 0 is not empty
325     ASSERT_OK(
326         Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))), rnd.RandomString(80)));
327     ASSERT_OK(Flush());
328   }
329 
330   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
331   ASSERT_EQ(3U, int_prop);
332   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level1", &str_prop));
333   ASSERT_EQ("0", str_prop);
334   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level2", &str_prop));
335   ASSERT_EQ("0", str_prop);
336 
337   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
338   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
339 
340   std::set<int> output_levels;
341   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
342       "CompactionPicker::CompactRange:Return", [&](void* arg) {
343         Compaction* compaction = reinterpret_cast<Compaction*>(arg);
344         output_levels.insert(compaction->output_level());
345       });
346   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
347 
348   ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
349   ASSERT_EQ(output_levels.size(), 2);
350   ASSERT_TRUE(output_levels.find(3) != output_levels.end());
351   ASSERT_TRUE(output_levels.find(4) != output_levels.end());
352   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level0", &str_prop));
353   ASSERT_EQ("0", str_prop);
354   ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level3", &str_prop));
355   ASSERT_EQ("0", str_prop);
356   // Base level is still level 3.
357   ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
358   ASSERT_EQ(3U, int_prop);
359 }
360 
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesBaseInc)361 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
362   Options options = CurrentOptions();
363   options.create_if_missing = true;
364   options.write_buffer_size = 2048;
365   options.max_write_buffer_number = 2;
366   options.level0_file_num_compaction_trigger = 2;
367   options.level0_slowdown_writes_trigger = 2;
368   options.level0_stop_writes_trigger = 2;
369   options.target_file_size_base = 2048;
370   options.level_compaction_dynamic_level_bytes = true;
371   options.max_bytes_for_level_base = 10240;
372   options.max_bytes_for_level_multiplier = 4;
373   options.soft_rate_limit = 1.1;
374   options.max_background_compactions = 2;
375   options.num_levels = 5;
376   options.max_compaction_bytes = 100000000;
377 
378   DestroyAndReopen(options);
379 
380   int non_trivial = 0;
381   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
382       "DBImpl::BackgroundCompaction:NonTrivial",
383       [&](void* /*arg*/) { non_trivial++; });
384   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
385 
386   Random rnd(301);
387   const int total_keys = 3000;
388   const int random_part_size = 100;
389   for (int i = 0; i < total_keys; i++) {
390     std::string value = rnd.RandomString(random_part_size);
391     PutFixed32(&value, static_cast<uint32_t>(i));
392     ASSERT_OK(Put(Key(i), value));
393   }
394   ASSERT_OK(Flush());
395   ASSERT_OK(dbfull()->TEST_WaitForCompact());
396   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
397 
398   ASSERT_EQ(non_trivial, 0);
399 
400   for (int i = 0; i < total_keys; i++) {
401     std::string value = Get(Key(i));
402     ASSERT_EQ(DecodeFixed32(value.c_str() + random_part_size),
403               static_cast<uint32_t>(i));
404   }
405 
406   env_->SetBackgroundThreads(1, Env::LOW);
407   env_->SetBackgroundThreads(1, Env::HIGH);
408 }
409 
TEST_F(DBTestDynamicLevel,DISABLED_MigrateToDynamicLevelMaxBytesBase)410 TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
411   Random rnd(301);
412   const int kMaxKey = 2000;
413 
414   Options options;
415   options.create_if_missing = true;
416   options.write_buffer_size = 2048;
417   options.max_write_buffer_number = 8;
418   options.level0_file_num_compaction_trigger = 4;
419   options.level0_slowdown_writes_trigger = 4;
420   options.level0_stop_writes_trigger = 8;
421   options.target_file_size_base = 2048;
422   options.level_compaction_dynamic_level_bytes = false;
423   options.max_bytes_for_level_base = 10240;
424   options.max_bytes_for_level_multiplier = 4;
425   options.soft_rate_limit = 1.1;
426   options.num_levels = 8;
427 
428   DestroyAndReopen(options);
429 
430   auto verify_func = [&](int num_keys, bool if_sleep) {
431     for (int i = 0; i < num_keys; i++) {
432       ASSERT_NE("NOT_FOUND", Get(Key(kMaxKey + i)));
433       if (i < num_keys / 10) {
434         ASSERT_EQ("NOT_FOUND", Get(Key(i)));
435       } else {
436         ASSERT_NE("NOT_FOUND", Get(Key(i)));
437       }
438       if (if_sleep && i % 1000 == 0) {
439         // Without it, valgrind may choose not to give another
440         // thread a chance to run before finishing the function,
441         // causing the test to be extremely slow.
442         env_->SleepForMicroseconds(1);
443       }
444     }
445   };
446 
447   int total_keys = 1000;
448   for (int i = 0; i < total_keys; i++) {
449     ASSERT_OK(Put(Key(i), rnd.RandomString(102)));
450     ASSERT_OK(Put(Key(kMaxKey + i), rnd.RandomString(102)));
451     ASSERT_OK(Delete(Key(i / 10)));
452   }
453   verify_func(total_keys, false);
454   ASSERT_OK(dbfull()->TEST_WaitForCompact());
455 
456   options.level_compaction_dynamic_level_bytes = true;
457   options.disable_auto_compactions = true;
458   Reopen(options);
459   verify_func(total_keys, false);
460 
461   std::atomic_bool compaction_finished;
462   compaction_finished = false;
463   // Issue manual compaction in one thread and still verify DB state
464   // in main thread.
465   ROCKSDB_NAMESPACE::port::Thread t([&]() {
466     CompactRangeOptions compact_options;
467     compact_options.change_level = true;
468     compact_options.target_level = options.num_levels - 1;
469     ASSERT_OK(dbfull()->CompactRange(compact_options, nullptr, nullptr));
470     compaction_finished.store(true);
471   });
472   do {
473     verify_func(total_keys, true);
474   } while (!compaction_finished.load());
475   t.join();
476 
477   ASSERT_OK(dbfull()->SetOptions({
478       {"disable_auto_compactions", "false"},
479   }));
480 
481   int total_keys2 = 2000;
482   for (int i = total_keys; i < total_keys2; i++) {
483     ASSERT_OK(Put(Key(i), rnd.RandomString(102)));
484     ASSERT_OK(Put(Key(kMaxKey + i), rnd.RandomString(102)));
485     ASSERT_OK(Delete(Key(i / 10)));
486   }
487 
488   verify_func(total_keys2, false);
489   ASSERT_OK(dbfull()->TEST_WaitForCompact());
490   verify_func(total_keys2, false);
491 
492   // Base level is not level 1
493   ASSERT_EQ(NumTableFilesAtLevel(1), 0);
494   ASSERT_EQ(NumTableFilesAtLevel(2), 0);
495 }
496 }  // namespace ROCKSDB_NAMESPACE
497 
498 #endif  // !defined(ROCKSDB_LITE)
499 
main(int argc,char ** argv)500 int main(int argc, char** argv) {
501 #if !defined(ROCKSDB_LITE)
502   ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
503   ::testing::InitGoogleTest(&argc, argv);
504   return RUN_ALL_TESTS();
505 #else
506   (void) argc;
507   (void) argv;
508   return 0;
509 #endif
510 }
511