1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 // Introduction of SyncPoint effectively disabled building and running this test
11 // in Release build.
12 // which is a pity, it is a good test
13 #if !defined(ROCKSDB_LITE)
14
15 #include "db/db_test_util.h"
16 #include "env/mock_env.h"
17 #include "port/port.h"
18 #include "port/stack_trace.h"
19 #include "util/random.h"
20
21 namespace ROCKSDB_NAMESPACE {
22 class DBTestDynamicLevel : public DBTestBase {
23 public:
DBTestDynamicLevel()24 DBTestDynamicLevel()
25 : DBTestBase("/db_dynamic_level_test", /*env_do_fsync=*/true) {}
26 };
27
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesBase)28 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
29 if (!Snappy_Supported() || !LZ4_Supported()) {
30 return;
31 }
32 // Use InMemoryEnv, or it would be too slow.
33 std::unique_ptr<Env> env(new MockEnv(env_));
34
35 const int kNKeys = 1000;
36 int keys[kNKeys];
37
38 auto verify_func = [&]() {
39 for (int i = 0; i < kNKeys; i++) {
40 ASSERT_NE("NOT_FOUND", Get(Key(i)));
41 ASSERT_NE("NOT_FOUND", Get(Key(kNKeys * 2 + i)));
42 if (i < kNKeys / 10) {
43 ASSERT_EQ("NOT_FOUND", Get(Key(kNKeys + keys[i])));
44 } else {
45 ASSERT_NE("NOT_FOUND", Get(Key(kNKeys + keys[i])));
46 }
47 }
48 };
49
50 Random rnd(301);
51 for (int ordered_insert = 0; ordered_insert <= 1; ordered_insert++) {
52 for (int i = 0; i < kNKeys; i++) {
53 keys[i] = i;
54 }
55 if (ordered_insert == 0) {
56 RandomShuffle(std::begin(keys), std::end(keys), rnd.Next());
57 }
58 for (int max_background_compactions = 1; max_background_compactions < 4;
59 max_background_compactions += 2) {
60 Options options;
61 options.env = env.get();
62 options.create_if_missing = true;
63 options.write_buffer_size = 2048;
64 options.max_write_buffer_number = 2;
65 options.level0_file_num_compaction_trigger = 2;
66 options.level0_slowdown_writes_trigger = 2;
67 options.level0_stop_writes_trigger = 2;
68 options.target_file_size_base = 2048;
69 options.level_compaction_dynamic_level_bytes = true;
70 options.max_bytes_for_level_base = 10240;
71 options.max_bytes_for_level_multiplier = 4;
72 options.soft_rate_limit = 1.1;
73 options.max_background_compactions = max_background_compactions;
74 options.num_levels = 5;
75
76 options.compression_per_level.resize(3);
77 options.compression_per_level[0] = kNoCompression;
78 options.compression_per_level[1] = kLZ4Compression;
79 options.compression_per_level[2] = kSnappyCompression;
80 options.env = env_;
81
82 DestroyAndReopen(options);
83
84 for (int i = 0; i < kNKeys; i++) {
85 int key = keys[i];
86 ASSERT_OK(Put(Key(kNKeys + key), rnd.RandomString(102)));
87 ASSERT_OK(Put(Key(key), rnd.RandomString(102)));
88 ASSERT_OK(Put(Key(kNKeys * 2 + key), rnd.RandomString(102)));
89 ASSERT_OK(Delete(Key(kNKeys + keys[i / 10])));
90 env_->SleepForMicroseconds(5000);
91 }
92
93 uint64_t int_prop;
94 ASSERT_TRUE(db_->GetIntProperty("rocksdb.background-errors", &int_prop));
95 ASSERT_EQ(0U, int_prop);
96
97 // Verify DB
98 for (int j = 0; j < 2; j++) {
99 verify_func();
100 if (j == 0) {
101 Reopen(options);
102 }
103 }
104
105 // Test compact range works
106 ASSERT_OK(
107 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
108 // All data should be in the last level.
109 ColumnFamilyMetaData cf_meta;
110 db_->GetColumnFamilyMetaData(&cf_meta);
111 ASSERT_EQ(5U, cf_meta.levels.size());
112 for (int i = 0; i < 4; i++) {
113 ASSERT_EQ(0U, cf_meta.levels[i].files.size());
114 }
115 ASSERT_GT(cf_meta.levels[4U].files.size(), 0U);
116 verify_func();
117
118 Close();
119 }
120 }
121
122 env_->SetBackgroundThreads(1, Env::LOW);
123 env_->SetBackgroundThreads(1, Env::HIGH);
124 }
125
126 // Test specific cases in dynamic max bytes
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesBase2)127 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
128 Random rnd(301);
129 int kMaxKey = 1000000;
130
131 Options options = CurrentOptions();
132 options.compression = kNoCompression;
133 options.create_if_missing = true;
134 options.write_buffer_size = 20480;
135 options.max_write_buffer_number = 2;
136 options.level0_file_num_compaction_trigger = 2;
137 options.level0_slowdown_writes_trigger = 9999;
138 options.level0_stop_writes_trigger = 9999;
139 options.target_file_size_base = 9102;
140 options.level_compaction_dynamic_level_bytes = true;
141 options.max_bytes_for_level_base = 40960;
142 options.max_bytes_for_level_multiplier = 4;
143 options.max_background_compactions = 2;
144 options.num_levels = 5;
145 options.max_compaction_bytes = 0; // Force not expanding in compactions
146 options.db_host_id = ""; // Setting this messes up the file size calculation
147 BlockBasedTableOptions table_options;
148 table_options.block_size = 1024;
149 options.table_factory.reset(NewBlockBasedTableFactory(table_options));
150
151 DestroyAndReopen(options);
152 ASSERT_OK(dbfull()->SetOptions({
153 {"disable_auto_compactions", "true"},
154 }));
155
156 uint64_t int_prop;
157 std::string str_prop;
158
159 // Initial base level is the last level
160 ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
161 ASSERT_EQ(4U, int_prop);
162
163 // Put about 28K to L0
164 for (int i = 0; i < 70; i++) {
165 ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
166 rnd.RandomString(380)));
167 }
168 ASSERT_OK(dbfull()->SetOptions({
169 {"disable_auto_compactions", "false"},
170 }));
171 ASSERT_OK(Flush());
172 ASSERT_OK(dbfull()->TEST_WaitForCompact());
173 ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
174 ASSERT_EQ(4U, int_prop);
175
176 // Insert extra about 28K to L0. After they are compacted to L4, the base
177 // level should be changed to L3.
178 ASSERT_OK(dbfull()->SetOptions({
179 {"disable_auto_compactions", "true"},
180 }));
181 for (int i = 0; i < 70; i++) {
182 ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
183 rnd.RandomString(380)));
184 }
185
186 ASSERT_OK(dbfull()->SetOptions({
187 {"disable_auto_compactions", "false"},
188 }));
189 ASSERT_OK(Flush());
190 ASSERT_OK(dbfull()->TEST_WaitForCompact());
191 ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
192 ASSERT_EQ(3U, int_prop);
193 ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level1", &str_prop));
194 ASSERT_EQ("0", str_prop);
195 ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level2", &str_prop));
196 ASSERT_EQ("0", str_prop);
197
198 // Write even more data while leaving the base level at L3.
199 ASSERT_OK(dbfull()->SetOptions({
200 {"disable_auto_compactions", "true"},
201 }));
202 // Write about 40K more
203 for (int i = 0; i < 100; i++) {
204 ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
205 rnd.RandomString(380)));
206 }
207 ASSERT_OK(dbfull()->SetOptions({
208 {"disable_auto_compactions", "false"},
209 }));
210 ASSERT_OK(Flush());
211 ASSERT_OK(dbfull()->TEST_WaitForCompact());
212 ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
213 ASSERT_EQ(3U, int_prop);
214
215 // Fill up L0, and then run an (auto) L0->Lmax compaction to raise the base
216 // level to 2.
217 ASSERT_OK(dbfull()->SetOptions({
218 {"disable_auto_compactions", "true"},
219 }));
220 // Write about 650K more.
221 // Each file is about 11KB, with 9KB of data.
222 for (int i = 0; i < 1300; i++) {
223 ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
224 rnd.RandomString(380)));
225 }
226
227 // Make sure that the compaction starts before the last bit of data is
228 // flushed, so that the base level isn't raised to L1.
229 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
230 {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:0"},
231 });
232 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
233
234 ASSERT_OK(dbfull()->SetOptions({
235 {"disable_auto_compactions", "false"},
236 }));
237
238 TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:0");
239 ASSERT_OK(Flush());
240 ASSERT_OK(dbfull()->TEST_WaitForCompact());
241 ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
242 ASSERT_EQ(2U, int_prop);
243 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
244 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
245
246 // Write more data until the base level changes to L1. There will be
247 // a manual compaction going on at the same time.
248 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
249 {"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:1"},
250 {"DynamicLevelMaxBytesBase2:2", "CompactionJob::Run():End"},
251 {"DynamicLevelMaxBytesBase2:compact_range_finish",
252 "FlushJob::WriteLevel0Table"},
253 });
254 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
255
256 ROCKSDB_NAMESPACE::port::Thread thread([this] {
257 TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_start");
258 ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
259 TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_finish");
260 });
261
262 TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:1");
263 for (int i = 0; i < 2; i++) {
264 ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
265 rnd.RandomString(380)));
266 }
267 TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:2");
268
269 ASSERT_OK(Flush());
270
271 thread.join();
272
273 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
274 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
275
276 ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
277 ASSERT_EQ(1U, int_prop);
278 }
279
280 // Test specific cases in dynamic max bytes
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesCompactRange)281 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesCompactRange) {
282 Random rnd(301);
283 int kMaxKey = 1000000;
284
285 Options options = CurrentOptions();
286 options.create_if_missing = true;
287 options.write_buffer_size = 2048;
288 options.max_write_buffer_number = 2;
289 options.level0_file_num_compaction_trigger = 2;
290 options.level0_slowdown_writes_trigger = 9999;
291 options.level0_stop_writes_trigger = 9999;
292 options.target_file_size_base = 2;
293 options.level_compaction_dynamic_level_bytes = true;
294 options.max_bytes_for_level_base = 10240;
295 options.max_bytes_for_level_multiplier = 4;
296 options.max_background_compactions = 1;
297 const int kNumLevels = 5;
298 options.num_levels = kNumLevels;
299 options.max_compaction_bytes = 1; // Force not expanding in compactions
300 BlockBasedTableOptions table_options;
301 table_options.block_size = 1024;
302 options.table_factory.reset(NewBlockBasedTableFactory(table_options));
303
304 DestroyAndReopen(options);
305
306 // Compact against empty DB
307 ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
308
309 uint64_t int_prop;
310 std::string str_prop;
311
312 // Initial base level is the last level
313 ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
314 ASSERT_EQ(4U, int_prop);
315
316 // Put about 7K to L0
317 for (int i = 0; i < 140; i++) {
318 ASSERT_OK(
319 Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))), rnd.RandomString(80)));
320 }
321 ASSERT_OK(Flush());
322 ASSERT_OK(dbfull()->TEST_WaitForCompact());
323 if (NumTableFilesAtLevel(0) == 0) {
324 // Make sure level 0 is not empty
325 ASSERT_OK(
326 Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))), rnd.RandomString(80)));
327 ASSERT_OK(Flush());
328 }
329
330 ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
331 ASSERT_EQ(3U, int_prop);
332 ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level1", &str_prop));
333 ASSERT_EQ("0", str_prop);
334 ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level2", &str_prop));
335 ASSERT_EQ("0", str_prop);
336
337 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
338 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
339
340 std::set<int> output_levels;
341 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
342 "CompactionPicker::CompactRange:Return", [&](void* arg) {
343 Compaction* compaction = reinterpret_cast<Compaction*>(arg);
344 output_levels.insert(compaction->output_level());
345 });
346 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
347
348 ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
349 ASSERT_EQ(output_levels.size(), 2);
350 ASSERT_TRUE(output_levels.find(3) != output_levels.end());
351 ASSERT_TRUE(output_levels.find(4) != output_levels.end());
352 ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level0", &str_prop));
353 ASSERT_EQ("0", str_prop);
354 ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level3", &str_prop));
355 ASSERT_EQ("0", str_prop);
356 // Base level is still level 3.
357 ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
358 ASSERT_EQ(3U, int_prop);
359 }
360
TEST_F(DBTestDynamicLevel,DynamicLevelMaxBytesBaseInc)361 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
362 Options options = CurrentOptions();
363 options.create_if_missing = true;
364 options.write_buffer_size = 2048;
365 options.max_write_buffer_number = 2;
366 options.level0_file_num_compaction_trigger = 2;
367 options.level0_slowdown_writes_trigger = 2;
368 options.level0_stop_writes_trigger = 2;
369 options.target_file_size_base = 2048;
370 options.level_compaction_dynamic_level_bytes = true;
371 options.max_bytes_for_level_base = 10240;
372 options.max_bytes_for_level_multiplier = 4;
373 options.soft_rate_limit = 1.1;
374 options.max_background_compactions = 2;
375 options.num_levels = 5;
376 options.max_compaction_bytes = 100000000;
377
378 DestroyAndReopen(options);
379
380 int non_trivial = 0;
381 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
382 "DBImpl::BackgroundCompaction:NonTrivial",
383 [&](void* /*arg*/) { non_trivial++; });
384 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
385
386 Random rnd(301);
387 const int total_keys = 3000;
388 const int random_part_size = 100;
389 for (int i = 0; i < total_keys; i++) {
390 std::string value = rnd.RandomString(random_part_size);
391 PutFixed32(&value, static_cast<uint32_t>(i));
392 ASSERT_OK(Put(Key(i), value));
393 }
394 ASSERT_OK(Flush());
395 ASSERT_OK(dbfull()->TEST_WaitForCompact());
396 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
397
398 ASSERT_EQ(non_trivial, 0);
399
400 for (int i = 0; i < total_keys; i++) {
401 std::string value = Get(Key(i));
402 ASSERT_EQ(DecodeFixed32(value.c_str() + random_part_size),
403 static_cast<uint32_t>(i));
404 }
405
406 env_->SetBackgroundThreads(1, Env::LOW);
407 env_->SetBackgroundThreads(1, Env::HIGH);
408 }
409
TEST_F(DBTestDynamicLevel,DISABLED_MigrateToDynamicLevelMaxBytesBase)410 TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
411 Random rnd(301);
412 const int kMaxKey = 2000;
413
414 Options options;
415 options.create_if_missing = true;
416 options.write_buffer_size = 2048;
417 options.max_write_buffer_number = 8;
418 options.level0_file_num_compaction_trigger = 4;
419 options.level0_slowdown_writes_trigger = 4;
420 options.level0_stop_writes_trigger = 8;
421 options.target_file_size_base = 2048;
422 options.level_compaction_dynamic_level_bytes = false;
423 options.max_bytes_for_level_base = 10240;
424 options.max_bytes_for_level_multiplier = 4;
425 options.soft_rate_limit = 1.1;
426 options.num_levels = 8;
427
428 DestroyAndReopen(options);
429
430 auto verify_func = [&](int num_keys, bool if_sleep) {
431 for (int i = 0; i < num_keys; i++) {
432 ASSERT_NE("NOT_FOUND", Get(Key(kMaxKey + i)));
433 if (i < num_keys / 10) {
434 ASSERT_EQ("NOT_FOUND", Get(Key(i)));
435 } else {
436 ASSERT_NE("NOT_FOUND", Get(Key(i)));
437 }
438 if (if_sleep && i % 1000 == 0) {
439 // Without it, valgrind may choose not to give another
440 // thread a chance to run before finishing the function,
441 // causing the test to be extremely slow.
442 env_->SleepForMicroseconds(1);
443 }
444 }
445 };
446
447 int total_keys = 1000;
448 for (int i = 0; i < total_keys; i++) {
449 ASSERT_OK(Put(Key(i), rnd.RandomString(102)));
450 ASSERT_OK(Put(Key(kMaxKey + i), rnd.RandomString(102)));
451 ASSERT_OK(Delete(Key(i / 10)));
452 }
453 verify_func(total_keys, false);
454 ASSERT_OK(dbfull()->TEST_WaitForCompact());
455
456 options.level_compaction_dynamic_level_bytes = true;
457 options.disable_auto_compactions = true;
458 Reopen(options);
459 verify_func(total_keys, false);
460
461 std::atomic_bool compaction_finished;
462 compaction_finished = false;
463 // Issue manual compaction in one thread and still verify DB state
464 // in main thread.
465 ROCKSDB_NAMESPACE::port::Thread t([&]() {
466 CompactRangeOptions compact_options;
467 compact_options.change_level = true;
468 compact_options.target_level = options.num_levels - 1;
469 ASSERT_OK(dbfull()->CompactRange(compact_options, nullptr, nullptr));
470 compaction_finished.store(true);
471 });
472 do {
473 verify_func(total_keys, true);
474 } while (!compaction_finished.load());
475 t.join();
476
477 ASSERT_OK(dbfull()->SetOptions({
478 {"disable_auto_compactions", "false"},
479 }));
480
481 int total_keys2 = 2000;
482 for (int i = total_keys; i < total_keys2; i++) {
483 ASSERT_OK(Put(Key(i), rnd.RandomString(102)));
484 ASSERT_OK(Put(Key(kMaxKey + i), rnd.RandomString(102)));
485 ASSERT_OK(Delete(Key(i / 10)));
486 }
487
488 verify_func(total_keys2, false);
489 ASSERT_OK(dbfull()->TEST_WaitForCompact());
490 verify_func(total_keys2, false);
491
492 // Base level is not level 1
493 ASSERT_EQ(NumTableFilesAtLevel(1), 0);
494 ASSERT_EQ(NumTableFilesAtLevel(2), 0);
495 }
496 } // namespace ROCKSDB_NAMESPACE
497
498 #endif // !defined(ROCKSDB_LITE)
499
main(int argc,char ** argv)500 int main(int argc, char** argv) {
501 #if !defined(ROCKSDB_LITE)
502 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
503 ::testing::InitGoogleTest(&argc, argv);
504 return RUN_ALL_TESTS();
505 #else
506 (void) argc;
507 (void) argv;
508 return 0;
509 #endif
510 }
511