1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 #include <limits>
10 #include <string>
11 #include <unordered_map>
12
13 #include "db/column_family.h"
14 #include "db/db_impl/db_impl.h"
15 #include "db/db_test_util.h"
16 #include "options/options_helper.h"
17 #include "port/stack_trace.h"
18 #include "rocksdb/cache.h"
19 #include "rocksdb/convenience.h"
20 #include "rocksdb/rate_limiter.h"
21 #include "rocksdb/stats_history.h"
22 #include "test_util/sync_point.h"
23 #include "test_util/testutil.h"
24 #include "util/random.h"
25
26 namespace ROCKSDB_NAMESPACE {
27
28 class DBOptionsTest : public DBTestBase {
29 public:
DBOptionsTest()30 DBOptionsTest() : DBTestBase("/db_options_test") {}
31
32 #ifndef ROCKSDB_LITE
GetMutableDBOptionsMap(const DBOptions & options)33 std::unordered_map<std::string, std::string> GetMutableDBOptionsMap(
34 const DBOptions& options) {
35 std::string options_str;
36 GetStringFromDBOptions(&options_str, options);
37 std::unordered_map<std::string, std::string> options_map;
38 StringToMap(options_str, &options_map);
39 std::unordered_map<std::string, std::string> mutable_map;
40 for (const auto opt : db_options_type_info) {
41 if (opt.second.is_mutable &&
42 opt.second.verification != OptionVerificationType::kDeprecated) {
43 mutable_map[opt.first] = options_map[opt.first];
44 }
45 }
46 return mutable_map;
47 }
48
GetMutableCFOptionsMap(const ColumnFamilyOptions & options)49 std::unordered_map<std::string, std::string> GetMutableCFOptionsMap(
50 const ColumnFamilyOptions& options) {
51 std::string options_str;
52 GetStringFromColumnFamilyOptions(&options_str, options);
53 std::unordered_map<std::string, std::string> options_map;
54 StringToMap(options_str, &options_map);
55 std::unordered_map<std::string, std::string> mutable_map;
56 for (const auto opt : cf_options_type_info) {
57 if (opt.second.is_mutable &&
58 opt.second.verification != OptionVerificationType::kDeprecated) {
59 mutable_map[opt.first] = options_map[opt.first];
60 }
61 }
62 return mutable_map;
63 }
64
GetRandomizedMutableCFOptionsMap(Random * rnd)65 std::unordered_map<std::string, std::string> GetRandomizedMutableCFOptionsMap(
66 Random* rnd) {
67 Options options = CurrentOptions();
68 options.env = env_;
69 ImmutableDBOptions db_options(options);
70 test::RandomInitCFOptions(&options, options, rnd);
71 auto sanitized_options = SanitizeOptions(db_options, options);
72 auto opt_map = GetMutableCFOptionsMap(sanitized_options);
73 delete options.compaction_filter;
74 return opt_map;
75 }
76
GetRandomizedMutableDBOptionsMap(Random * rnd)77 std::unordered_map<std::string, std::string> GetRandomizedMutableDBOptionsMap(
78 Random* rnd) {
79 DBOptions db_options;
80 test::RandomInitDBOptions(&db_options, rnd);
81 auto sanitized_options = SanitizeOptions(dbname_, db_options);
82 return GetMutableDBOptionsMap(sanitized_options);
83 }
84 #endif // ROCKSDB_LITE
85 };
86
87 // RocksDB lite don't support dynamic options.
88 #ifndef ROCKSDB_LITE
89
TEST_F(DBOptionsTest,GetLatestDBOptions)90 TEST_F(DBOptionsTest, GetLatestDBOptions) {
91 // GetOptions should be able to get latest option changed by SetOptions.
92 Options options;
93 options.create_if_missing = true;
94 options.env = env_;
95 Random rnd(228);
96 Reopen(options);
97 auto new_options = GetRandomizedMutableDBOptionsMap(&rnd);
98 ASSERT_OK(dbfull()->SetDBOptions(new_options));
99 ASSERT_EQ(new_options, GetMutableDBOptionsMap(dbfull()->GetDBOptions()));
100 }
101
TEST_F(DBOptionsTest,GetLatestCFOptions)102 TEST_F(DBOptionsTest, GetLatestCFOptions) {
103 // GetOptions should be able to get latest option changed by SetOptions.
104 Options options;
105 options.create_if_missing = true;
106 options.env = env_;
107 Random rnd(228);
108 Reopen(options);
109 CreateColumnFamilies({"foo"}, options);
110 ReopenWithColumnFamilies({"default", "foo"}, options);
111 auto options_default = GetRandomizedMutableCFOptionsMap(&rnd);
112 auto options_foo = GetRandomizedMutableCFOptionsMap(&rnd);
113 ASSERT_OK(dbfull()->SetOptions(handles_[0], options_default));
114 ASSERT_OK(dbfull()->SetOptions(handles_[1], options_foo));
115 ASSERT_EQ(options_default,
116 GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[0])));
117 ASSERT_EQ(options_foo,
118 GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[1])));
119 }
120
TEST_F(DBOptionsTest,SetBytesPerSync)121 TEST_F(DBOptionsTest, SetBytesPerSync) {
122 const size_t kValueSize = 1024 * 1024; // 1MB
123 Options options;
124 options.create_if_missing = true;
125 options.bytes_per_sync = 1024 * 1024;
126 options.use_direct_reads = false;
127 options.write_buffer_size = 400 * kValueSize;
128 options.disable_auto_compactions = true;
129 options.compression = kNoCompression;
130 options.env = env_;
131 Reopen(options);
132 int counter = 0;
133 int low_bytes_per_sync = 0;
134 int i = 0;
135 const std::string kValue(kValueSize, 'v');
136 ASSERT_EQ(options.bytes_per_sync, dbfull()->GetDBOptions().bytes_per_sync);
137 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
138 "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) { counter++; });
139
140 WriteOptions write_opts;
141 // should sync approximately 40MB/1MB ~= 40 times.
142 for (i = 0; i < 40; i++) {
143 Put(Key(i), kValue, write_opts);
144 }
145 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
146 ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
147 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
148 low_bytes_per_sync = counter;
149 ASSERT_GT(low_bytes_per_sync, 35);
150 ASSERT_LT(low_bytes_per_sync, 45);
151
152 counter = 0;
153 // 8388608 = 8 * 1024 * 1024
154 ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "8388608"}}));
155 ASSERT_EQ(8388608, dbfull()->GetDBOptions().bytes_per_sync);
156 // should sync approximately 40MB*2/8MB ~= 10 times.
157 // data will be 40*2MB because of previous Puts too.
158 for (i = 0; i < 40; i++) {
159 Put(Key(i), kValue, write_opts);
160 }
161 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
162 ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
163 ASSERT_GT(counter, 5);
164 ASSERT_LT(counter, 15);
165
166 // Redundant assert. But leaving it here just to get the point across that
167 // low_bytes_per_sync > counter.
168 ASSERT_GT(low_bytes_per_sync, counter);
169 }
170
TEST_F(DBOptionsTest,SetWalBytesPerSync)171 TEST_F(DBOptionsTest, SetWalBytesPerSync) {
172 const size_t kValueSize = 1024 * 1024 * 3;
173 Options options;
174 options.create_if_missing = true;
175 options.wal_bytes_per_sync = 512;
176 options.write_buffer_size = 100 * kValueSize;
177 options.disable_auto_compactions = true;
178 options.compression = kNoCompression;
179 options.env = env_;
180 Reopen(options);
181 ASSERT_EQ(512, dbfull()->GetDBOptions().wal_bytes_per_sync);
182 int counter = 0;
183 int low_bytes_per_sync = 0;
184 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
185 "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) { counter++; });
186 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
187 const std::string kValue(kValueSize, 'v');
188 int i = 0;
189 for (; i < 10; i++) {
190 Put(Key(i), kValue);
191 }
192 // Do not flush. If we flush here, SwitchWAL will reuse old WAL file since its
193 // empty and will not get the new wal_bytes_per_sync value.
194 low_bytes_per_sync = counter;
195 //5242880 = 1024 * 1024 * 5
196 ASSERT_OK(dbfull()->SetDBOptions({{"wal_bytes_per_sync", "5242880"}}));
197 ASSERT_EQ(5242880, dbfull()->GetDBOptions().wal_bytes_per_sync);
198 counter = 0;
199 i = 0;
200 for (; i < 10; i++) {
201 Put(Key(i), kValue);
202 }
203 ASSERT_GT(counter, 0);
204 ASSERT_GT(low_bytes_per_sync, 0);
205 ASSERT_GT(low_bytes_per_sync, counter);
206 }
207
TEST_F(DBOptionsTest,WritableFileMaxBufferSize)208 TEST_F(DBOptionsTest, WritableFileMaxBufferSize) {
209 Options options;
210 options.create_if_missing = true;
211 options.writable_file_max_buffer_size = 1024 * 1024;
212 options.level0_file_num_compaction_trigger = 3;
213 options.max_manifest_file_size = 1;
214 options.env = env_;
215 int buffer_size = 1024 * 1024;
216 Reopen(options);
217 ASSERT_EQ(buffer_size,
218 dbfull()->GetDBOptions().writable_file_max_buffer_size);
219
220 std::atomic<int> match_cnt(0);
221 std::atomic<int> unmatch_cnt(0);
222 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
223 "WritableFileWriter::WritableFileWriter:0", [&](void* arg) {
224 int value = static_cast<int>(reinterpret_cast<uintptr_t>(arg));
225 if (value == buffer_size) {
226 match_cnt++;
227 } else {
228 unmatch_cnt++;
229 }
230 });
231 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
232 int i = 0;
233 for (; i < 3; i++) {
234 ASSERT_OK(Put("foo", ToString(i)));
235 ASSERT_OK(Put("bar", ToString(i)));
236 Flush();
237 }
238 dbfull()->TEST_WaitForCompact();
239 ASSERT_EQ(unmatch_cnt, 0);
240 ASSERT_GE(match_cnt, 11);
241
242 ASSERT_OK(
243 dbfull()->SetDBOptions({{"writable_file_max_buffer_size", "524288"}}));
244 buffer_size = 512 * 1024;
245 match_cnt = 0;
246 unmatch_cnt = 0; // SetDBOptions() will create a WriteableFileWriter
247
248 ASSERT_EQ(buffer_size,
249 dbfull()->GetDBOptions().writable_file_max_buffer_size);
250 i = 0;
251 for (; i < 3; i++) {
252 ASSERT_OK(Put("foo", ToString(i)));
253 ASSERT_OK(Put("bar", ToString(i)));
254 Flush();
255 }
256 dbfull()->TEST_WaitForCompact();
257 ASSERT_EQ(unmatch_cnt, 0);
258 ASSERT_GE(match_cnt, 11);
259 }
260
TEST_F(DBOptionsTest,SetOptionsAndReopen)261 TEST_F(DBOptionsTest, SetOptionsAndReopen) {
262 Random rnd(1044);
263 auto rand_opts = GetRandomizedMutableCFOptionsMap(&rnd);
264 ASSERT_OK(dbfull()->SetOptions(rand_opts));
265 // Verify if DB can be reopen after setting options.
266 Options options;
267 options.env = env_;
268 ASSERT_OK(TryReopen(options));
269 }
270
TEST_F(DBOptionsTest,EnableAutoCompactionAndTriggerStall)271 TEST_F(DBOptionsTest, EnableAutoCompactionAndTriggerStall) {
272 const std::string kValue(1024, 'v');
273 for (int method_type = 0; method_type < 2; method_type++) {
274 for (int option_type = 0; option_type < 4; option_type++) {
275 Options options;
276 options.create_if_missing = true;
277 options.disable_auto_compactions = true;
278 options.write_buffer_size = 1024 * 1024 * 10;
279 options.compression = CompressionType::kNoCompression;
280 options.level0_file_num_compaction_trigger = 1;
281 options.level0_stop_writes_trigger = std::numeric_limits<int>::max();
282 options.level0_slowdown_writes_trigger = std::numeric_limits<int>::max();
283 options.hard_pending_compaction_bytes_limit =
284 std::numeric_limits<uint64_t>::max();
285 options.soft_pending_compaction_bytes_limit =
286 std::numeric_limits<uint64_t>::max();
287 options.env = env_;
288
289 DestroyAndReopen(options);
290 int i = 0;
291 for (; i < 1024; i++) {
292 Put(Key(i), kValue);
293 }
294 Flush();
295 for (; i < 1024 * 2; i++) {
296 Put(Key(i), kValue);
297 }
298 Flush();
299 dbfull()->TEST_WaitForFlushMemTable();
300 ASSERT_EQ(2, NumTableFilesAtLevel(0));
301 uint64_t l0_size = SizeAtLevel(0);
302
303 switch (option_type) {
304 case 0:
305 // test with level0_stop_writes_trigger
306 options.level0_stop_writes_trigger = 2;
307 options.level0_slowdown_writes_trigger = 2;
308 break;
309 case 1:
310 options.level0_slowdown_writes_trigger = 2;
311 break;
312 case 2:
313 options.hard_pending_compaction_bytes_limit = l0_size;
314 options.soft_pending_compaction_bytes_limit = l0_size;
315 break;
316 case 3:
317 options.soft_pending_compaction_bytes_limit = l0_size;
318 break;
319 }
320 Reopen(options);
321 dbfull()->TEST_WaitForCompact();
322 ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped());
323 ASSERT_FALSE(dbfull()->TEST_write_controler().NeedsDelay());
324
325 SyncPoint::GetInstance()->LoadDependency(
326 {{"DBOptionsTest::EnableAutoCompactionAndTriggerStall:1",
327 "BackgroundCallCompaction:0"},
328 {"DBImpl::BackgroundCompaction():BeforePickCompaction",
329 "DBOptionsTest::EnableAutoCompactionAndTriggerStall:2"},
330 {"DBOptionsTest::EnableAutoCompactionAndTriggerStall:3",
331 "DBImpl::BackgroundCompaction():AfterPickCompaction"}});
332 // Block background compaction.
333 SyncPoint::GetInstance()->EnableProcessing();
334
335 switch (method_type) {
336 case 0:
337 ASSERT_OK(
338 dbfull()->SetOptions({{"disable_auto_compactions", "false"}}));
339 break;
340 case 1:
341 ASSERT_OK(dbfull()->EnableAutoCompaction(
342 {dbfull()->DefaultColumnFamily()}));
343 break;
344 }
345 TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:1");
346 // Wait for stall condition recalculate.
347 TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:2");
348
349 switch (option_type) {
350 case 0:
351 ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped());
352 break;
353 case 1:
354 ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped());
355 ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
356 break;
357 case 2:
358 ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped());
359 break;
360 case 3:
361 ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped());
362 ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
363 break;
364 }
365 TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:3");
366
367 // Background compaction executed.
368 dbfull()->TEST_WaitForCompact();
369 ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped());
370 ASSERT_FALSE(dbfull()->TEST_write_controler().NeedsDelay());
371 }
372 }
373 }
374
TEST_F(DBOptionsTest,SetOptionsMayTriggerCompaction)375 TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) {
376 Options options;
377 options.create_if_missing = true;
378 options.level0_file_num_compaction_trigger = 1000;
379 options.env = env_;
380 Reopen(options);
381 for (int i = 0; i < 3; i++) {
382 // Need to insert two keys to avoid trivial move.
383 ASSERT_OK(Put("foo", ToString(i)));
384 ASSERT_OK(Put("bar", ToString(i)));
385 Flush();
386 }
387 ASSERT_EQ("3", FilesPerLevel());
388 ASSERT_OK(
389 dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "3"}}));
390 dbfull()->TEST_WaitForCompact();
391 ASSERT_EQ("0,1", FilesPerLevel());
392 }
393
TEST_F(DBOptionsTest,SetBackgroundCompactionThreads)394 TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) {
395 Options options;
396 options.create_if_missing = true;
397 options.max_background_compactions = 1; // default value
398 options.env = env_;
399 Reopen(options);
400 ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
401 ASSERT_OK(dbfull()->SetDBOptions({{"max_background_compactions", "3"}}));
402 ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
403 auto stop_token = dbfull()->TEST_write_controler().GetStopToken();
404 ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed());
405 }
406
TEST_F(DBOptionsTest,SetBackgroundJobs)407 TEST_F(DBOptionsTest, SetBackgroundJobs) {
408 Options options;
409 options.create_if_missing = true;
410 options.max_background_jobs = 8;
411 options.env = env_;
412 Reopen(options);
413
414 for (int i = 0; i < 2; ++i) {
415 if (i > 0) {
416 options.max_background_jobs = 12;
417 ASSERT_OK(dbfull()->SetDBOptions(
418 {{"max_background_jobs",
419 std::to_string(options.max_background_jobs)}}));
420 }
421
422 const int expected_max_flushes = options.max_background_jobs / 4;
423
424 ASSERT_EQ(expected_max_flushes, dbfull()->TEST_BGFlushesAllowed());
425 ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
426
427 auto stop_token = dbfull()->TEST_write_controler().GetStopToken();
428
429 const int expected_max_compactions = 3 * expected_max_flushes;
430
431 ASSERT_EQ(expected_max_flushes, dbfull()->TEST_BGFlushesAllowed());
432 ASSERT_EQ(expected_max_compactions, dbfull()->TEST_BGCompactionsAllowed());
433
434 ASSERT_EQ(expected_max_flushes,
435 env_->GetBackgroundThreads(Env::Priority::HIGH));
436 ASSERT_EQ(expected_max_compactions,
437 env_->GetBackgroundThreads(Env::Priority::LOW));
438 }
439 }
440
TEST_F(DBOptionsTest,AvoidFlushDuringShutdown)441 TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) {
442 Options options;
443 options.create_if_missing = true;
444 options.disable_auto_compactions = true;
445 options.env = env_;
446 WriteOptions write_without_wal;
447 write_without_wal.disableWAL = true;
448
449 ASSERT_FALSE(options.avoid_flush_during_shutdown);
450 DestroyAndReopen(options);
451 ASSERT_OK(Put("foo", "v1", write_without_wal));
452 Reopen(options);
453 ASSERT_EQ("v1", Get("foo"));
454 ASSERT_EQ("1", FilesPerLevel());
455
456 DestroyAndReopen(options);
457 ASSERT_OK(Put("foo", "v2", write_without_wal));
458 ASSERT_OK(dbfull()->SetDBOptions({{"avoid_flush_during_shutdown", "true"}}));
459 Reopen(options);
460 ASSERT_EQ("NOT_FOUND", Get("foo"));
461 ASSERT_EQ("", FilesPerLevel());
462 }
463
TEST_F(DBOptionsTest,SetDelayedWriteRateOption)464 TEST_F(DBOptionsTest, SetDelayedWriteRateOption) {
465 Options options;
466 options.create_if_missing = true;
467 options.delayed_write_rate = 2 * 1024U * 1024U;
468 options.env = env_;
469 Reopen(options);
470 ASSERT_EQ(2 * 1024U * 1024U, dbfull()->TEST_write_controler().max_delayed_write_rate());
471
472 ASSERT_OK(dbfull()->SetDBOptions({{"delayed_write_rate", "20000"}}));
473 ASSERT_EQ(20000, dbfull()->TEST_write_controler().max_delayed_write_rate());
474 }
475
TEST_F(DBOptionsTest,MaxTotalWalSizeChange)476 TEST_F(DBOptionsTest, MaxTotalWalSizeChange) {
477 Random rnd(1044);
478 const auto value_size = size_t(1024);
479 std::string value;
480 test::RandomString(&rnd, value_size, &value);
481
482 Options options;
483 options.create_if_missing = true;
484 options.env = env_;
485 CreateColumnFamilies({"1", "2", "3"}, options);
486 ReopenWithColumnFamilies({"default", "1", "2", "3"}, options);
487
488 WriteOptions write_options;
489
490 const int key_count = 100;
491 for (int i = 0; i < key_count; ++i) {
492 for (size_t cf = 0; cf < handles_.size(); ++cf) {
493 ASSERT_OK(Put(static_cast<int>(cf), Key(i), value));
494 }
495 }
496 ASSERT_OK(dbfull()->SetDBOptions({{"max_total_wal_size", "10"}}));
497
498 for (size_t cf = 0; cf < handles_.size(); ++cf) {
499 dbfull()->TEST_WaitForFlushMemTable(handles_[cf]);
500 ASSERT_EQ("1", FilesPerLevel(static_cast<int>(cf)));
501 }
502 }
503
TEST_F(DBOptionsTest,SetStatsDumpPeriodSec)504 TEST_F(DBOptionsTest, SetStatsDumpPeriodSec) {
505 Options options;
506 options.create_if_missing = true;
507 options.stats_dump_period_sec = 5;
508 options.env = env_;
509 Reopen(options);
510 ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec);
511
512 for (int i = 0; i < 20; i++) {
513 unsigned int num = rand() % 5000 + 1;
514 ASSERT_OK(
515 dbfull()->SetDBOptions({{"stats_dump_period_sec", ToString(num)}}));
516 ASSERT_EQ(num, dbfull()->GetDBOptions().stats_dump_period_sec);
517 }
518 Close();
519 }
520
TEST_F(DBOptionsTest,SetOptionsStatsPersistPeriodSec)521 TEST_F(DBOptionsTest, SetOptionsStatsPersistPeriodSec) {
522 Options options;
523 options.create_if_missing = true;
524 options.stats_persist_period_sec = 5;
525 options.env = env_;
526 Reopen(options);
527 ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec);
528
529 ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "12345"}}));
530 ASSERT_EQ(12345u, dbfull()->GetDBOptions().stats_persist_period_sec);
531 ASSERT_NOK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "abcde"}}));
532 ASSERT_EQ(12345u, dbfull()->GetDBOptions().stats_persist_period_sec);
533 }
534
assert_candidate_files_empty(DBImpl * dbfull,const bool empty)535 static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) {
536 dbfull->TEST_LockMutex();
537 JobContext job_context(0);
538 dbfull->FindObsoleteFiles(&job_context, false);
539 ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty());
540 dbfull->TEST_UnlockMutex();
541 if (job_context.HaveSomethingToDelete()) {
542 // fulfill the contract of FindObsoleteFiles by calling PurgeObsoleteFiles
543 // afterwards; otherwise the test may hang on shutdown
544 dbfull->PurgeObsoleteFiles(job_context);
545 }
546 job_context.Clean();
547 }
548
TEST_F(DBOptionsTest,DeleteObsoleteFilesPeriodChange)549 TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
550 SpecialEnv env(env_);
551 env.time_elapse_only_sleep_ = true;
552 Options options;
553 options.env = &env;
554 options.create_if_missing = true;
555 ASSERT_OK(TryReopen(options));
556
557 // Verify that candidate files set is empty when no full scan requested.
558 assert_candidate_files_empty(dbfull(), true);
559
560 ASSERT_OK(
561 dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}}));
562
563 // After delete_obsolete_files_period_micros updated to 0, the next call
564 // to FindObsoleteFiles should make a full scan
565 assert_candidate_files_empty(dbfull(), false);
566
567 ASSERT_OK(
568 dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}}));
569
570 assert_candidate_files_empty(dbfull(), true);
571
572 env.addon_time_.store(20);
573 assert_candidate_files_empty(dbfull(), true);
574
575 env.addon_time_.store(21);
576 assert_candidate_files_empty(dbfull(), false);
577
578 Close();
579 }
580
TEST_F(DBOptionsTest,MaxOpenFilesChange)581 TEST_F(DBOptionsTest, MaxOpenFilesChange) {
582 SpecialEnv env(env_);
583 Options options;
584 options.env = CurrentOptions().env;
585 options.max_open_files = -1;
586
587 Reopen(options);
588
589 Cache* tc = dbfull()->TEST_table_cache();
590
591 ASSERT_EQ(-1, dbfull()->GetDBOptions().max_open_files);
592 ASSERT_LT(2000, tc->GetCapacity());
593 ASSERT_OK(dbfull()->SetDBOptions({{"max_open_files", "1024"}}));
594 ASSERT_EQ(1024, dbfull()->GetDBOptions().max_open_files);
595 // examine the table cache (actual size should be 1014)
596 ASSERT_GT(1500, tc->GetCapacity());
597 Close();
598 }
599
TEST_F(DBOptionsTest,SanitizeDelayedWriteRate)600 TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) {
601 Options options;
602 options.delayed_write_rate = 0;
603 Reopen(options);
604 ASSERT_EQ(16 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
605
606 options.rate_limiter.reset(NewGenericRateLimiter(31 * 1024 * 1024));
607 Reopen(options);
608 ASSERT_EQ(31 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
609 }
610
TEST_F(DBOptionsTest,SanitizeUniversalTTLCompaction)611 TEST_F(DBOptionsTest, SanitizeUniversalTTLCompaction) {
612 Options options;
613 options.compaction_style = kCompactionStyleUniversal;
614
615 options.ttl = 0;
616 options.periodic_compaction_seconds = 0;
617 Reopen(options);
618 ASSERT_EQ(0, dbfull()->GetOptions().ttl);
619 ASSERT_EQ(0, dbfull()->GetOptions().periodic_compaction_seconds);
620
621 options.ttl = 0;
622 options.periodic_compaction_seconds = 100;
623 Reopen(options);
624 ASSERT_EQ(0, dbfull()->GetOptions().ttl);
625 ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds);
626
627 options.ttl = 100;
628 options.periodic_compaction_seconds = 0;
629 Reopen(options);
630 ASSERT_EQ(100, dbfull()->GetOptions().ttl);
631 ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds);
632
633 options.ttl = 100;
634 options.periodic_compaction_seconds = 500;
635 Reopen(options);
636 ASSERT_EQ(100, dbfull()->GetOptions().ttl);
637 ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds);
638 }
639
TEST_F(DBOptionsTest,SanitizeTtlDefault)640 TEST_F(DBOptionsTest, SanitizeTtlDefault) {
641 Options options;
642 Reopen(options);
643 ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl);
644
645 options.compaction_style = kCompactionStyleLevel;
646 options.ttl = 0;
647 Reopen(options);
648 ASSERT_EQ(0, dbfull()->GetOptions().ttl);
649
650 options.ttl = 100;
651 Reopen(options);
652 ASSERT_EQ(100, dbfull()->GetOptions().ttl);
653 }
654
TEST_F(DBOptionsTest,SanitizeFIFOPeriodicCompaction)655 TEST_F(DBOptionsTest, SanitizeFIFOPeriodicCompaction) {
656 Options options;
657 options.compaction_style = kCompactionStyleFIFO;
658 options.ttl = 0;
659 Reopen(options);
660 ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl);
661
662 options.ttl = 100;
663 Reopen(options);
664 ASSERT_EQ(100, dbfull()->GetOptions().ttl);
665
666 options.ttl = 100 * 24 * 60 * 60;
667 Reopen(options);
668 ASSERT_EQ(100 * 24 * 60 * 60, dbfull()->GetOptions().ttl);
669
670 options.ttl = 200;
671 options.periodic_compaction_seconds = 300;
672 Reopen(options);
673 ASSERT_EQ(200, dbfull()->GetOptions().ttl);
674
675 options.ttl = 500;
676 options.periodic_compaction_seconds = 300;
677 Reopen(options);
678 ASSERT_EQ(300, dbfull()->GetOptions().ttl);
679 }
680
TEST_F(DBOptionsTest,SetFIFOCompactionOptions)681 TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
682 Options options;
683 options.compaction_style = kCompactionStyleFIFO;
684 options.write_buffer_size = 10 << 10; // 10KB
685 options.arena_block_size = 4096;
686 options.compression = kNoCompression;
687 options.create_if_missing = true;
688 options.compaction_options_fifo.allow_compaction = false;
689 env_->time_elapse_only_sleep_ = false;
690 options.env = env_;
691
692 // Test dynamically changing ttl.
693 env_->addon_time_.store(0);
694 options.ttl = 1 * 60 * 60; // 1 hour
695 ASSERT_OK(TryReopen(options));
696
697 Random rnd(301);
698 for (int i = 0; i < 10; i++) {
699 // Generate and flush a file about 10KB.
700 for (int j = 0; j < 10; j++) {
701 ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
702 }
703 Flush();
704 }
705 ASSERT_OK(dbfull()->TEST_WaitForCompact());
706 ASSERT_EQ(NumTableFilesAtLevel(0), 10);
707
708 // Add 61 seconds to the time.
709 env_->addon_time_.fetch_add(61);
710
711 // No files should be compacted as ttl is set to 1 hour.
712 ASSERT_EQ(dbfull()->GetOptions().ttl, 3600);
713 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
714 ASSERT_EQ(NumTableFilesAtLevel(0), 10);
715
716 // Set ttl to 1 minute. So all files should get deleted.
717 ASSERT_OK(dbfull()->SetOptions({{"ttl", "60"}}));
718 ASSERT_EQ(dbfull()->GetOptions().ttl, 60);
719 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
720 ASSERT_OK(dbfull()->TEST_WaitForCompact());
721 ASSERT_EQ(NumTableFilesAtLevel(0), 0);
722
723 // Test dynamically changing compaction_options_fifo.max_table_files_size
724 env_->addon_time_.store(0);
725 options.compaction_options_fifo.max_table_files_size = 500 << 10; // 00KB
726 options.ttl = 0;
727 DestroyAndReopen(options);
728
729 for (int i = 0; i < 10; i++) {
730 // Generate and flush a file about 10KB.
731 for (int j = 0; j < 10; j++) {
732 ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
733 }
734 Flush();
735 }
736 ASSERT_OK(dbfull()->TEST_WaitForCompact());
737 ASSERT_EQ(NumTableFilesAtLevel(0), 10);
738
739 // No files should be compacted as max_table_files_size is set to 500 KB.
740 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
741 500 << 10);
742 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
743 ASSERT_EQ(NumTableFilesAtLevel(0), 10);
744
745 // Set max_table_files_size to 12 KB. So only 1 file should remain now.
746 ASSERT_OK(dbfull()->SetOptions(
747 {{"compaction_options_fifo", "{max_table_files_size=12288;}"}}));
748 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
749 12 << 10);
750 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
751 ASSERT_OK(dbfull()->TEST_WaitForCompact());
752 ASSERT_EQ(NumTableFilesAtLevel(0), 1);
753
754 // Test dynamically changing compaction_options_fifo.allow_compaction
755 options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB
756 options.ttl = 0;
757 options.compaction_options_fifo.allow_compaction = false;
758 options.level0_file_num_compaction_trigger = 6;
759 DestroyAndReopen(options);
760
761 for (int i = 0; i < 10; i++) {
762 // Generate and flush a file about 10KB.
763 for (int j = 0; j < 10; j++) {
764 ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
765 }
766 Flush();
767 }
768 ASSERT_OK(dbfull()->TEST_WaitForCompact());
769 ASSERT_EQ(NumTableFilesAtLevel(0), 10);
770
771 // No files should be compacted as max_table_files_size is set to 500 KB and
772 // allow_compaction is false
773 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
774 false);
775 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
776 ASSERT_EQ(NumTableFilesAtLevel(0), 10);
777
778 // Set allow_compaction to true. So number of files should be between 1 and 5.
779 ASSERT_OK(dbfull()->SetOptions(
780 {{"compaction_options_fifo", "{allow_compaction=true;}"}}));
781 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
782 true);
783 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
784 ASSERT_OK(dbfull()->TEST_WaitForCompact());
785 ASSERT_GE(NumTableFilesAtLevel(0), 1);
786 ASSERT_LE(NumTableFilesAtLevel(0), 5);
787 }
788
TEST_F(DBOptionsTest,CompactionReadaheadSizeChange)789 TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) {
790 SpecialEnv env(env_);
791 Options options;
792 options.env = &env;
793
794 options.compaction_readahead_size = 0;
795 options.new_table_reader_for_compaction_inputs = true;
796 options.level0_file_num_compaction_trigger = 2;
797 const std::string kValue(1024, 'v');
798 Reopen(options);
799
800 ASSERT_EQ(0, dbfull()->GetDBOptions().compaction_readahead_size);
801 ASSERT_OK(dbfull()->SetDBOptions({{"compaction_readahead_size", "256"}}));
802 ASSERT_EQ(256, dbfull()->GetDBOptions().compaction_readahead_size);
803 for (int i = 0; i < 1024; i++) {
804 Put(Key(i), kValue);
805 }
806 Flush();
807 for (int i = 0; i < 1024 * 2; i++) {
808 Put(Key(i), kValue);
809 }
810 Flush();
811 dbfull()->TEST_WaitForCompact();
812 ASSERT_EQ(256, env_->compaction_readahead_size_);
813 Close();
814 }
815
TEST_F(DBOptionsTest,FIFOTtlBackwardCompatible)816 TEST_F(DBOptionsTest, FIFOTtlBackwardCompatible) {
817 Options options;
818 options.compaction_style = kCompactionStyleFIFO;
819 options.write_buffer_size = 10 << 10; // 10KB
820 options.create_if_missing = true;
821
822 ASSERT_OK(TryReopen(options));
823
824 Random rnd(301);
825 for (int i = 0; i < 10; i++) {
826 // Generate and flush a file about 10KB.
827 for (int j = 0; j < 10; j++) {
828 ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
829 }
830 Flush();
831 }
832 ASSERT_OK(dbfull()->TEST_WaitForCompact());
833 ASSERT_EQ(NumTableFilesAtLevel(0), 10);
834
835 // In release 6.0, ttl was promoted from a secondary level option under
836 // compaction_options_fifo to a top level option under ColumnFamilyOptions.
837 // We still need to handle old SetOptions calls but should ignore
838 // ttl under compaction_options_fifo.
839 ASSERT_OK(dbfull()->SetOptions(
840 {{"compaction_options_fifo",
841 "{allow_compaction=true;max_table_files_size=1024;ttl=731;}"},
842 {"ttl", "60"}}));
843 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
844 true);
845 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
846 1024);
847 ASSERT_EQ(dbfull()->GetOptions().ttl, 60);
848
849 // Put ttl as the first option inside compaction_options_fifo. That works as
850 // it doesn't overwrite any other option.
851 ASSERT_OK(dbfull()->SetOptions(
852 {{"compaction_options_fifo",
853 "{ttl=985;allow_compaction=true;max_table_files_size=1024;}"},
854 {"ttl", "191"}}));
855 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
856 true);
857 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size,
858 1024);
859 ASSERT_EQ(dbfull()->GetOptions().ttl, 191);
860 }
861
862 #endif // ROCKSDB_LITE
863
864 } // namespace ROCKSDB_NAMESPACE
865
main(int argc,char ** argv)866 int main(int argc, char** argv) {
867 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
868 ::testing::InitGoogleTest(&argc, argv);
869 return RUN_ALL_TESTS();
870 }
871