1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 // This source code is licensed under both the GPLv2 (found in the 3 // COPYING file in the root directory) and Apache 2.0 License 4 // (found in the LICENSE.Apache file in the root directory). 5 // 6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 // Use of this source code is governed by a BSD-style license that can be 8 // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 #include <limits> 10 #include <string> 11 #include <unordered_map> 12 13 #include "db/column_family.h" 14 #include "db/db_impl/db_impl.h" 15 #include "db/db_test_util.h" 16 #include "options/options_helper.h" 17 #include "port/stack_trace.h" 18 #include "rocksdb/cache.h" 19 #include "rocksdb/convenience.h" 20 #include "rocksdb/rate_limiter.h" 21 #include "rocksdb/stats_history.h" 22 #include "test_util/sync_point.h" 23 #include "test_util/testutil.h" 24 #include "util/random.h" 25 26 namespace ROCKSDB_NAMESPACE { 27 28 class DBOptionsTest : public DBTestBase { 29 public: 30 DBOptionsTest() : DBTestBase("/db_options_test") {} 31 32 #ifndef ROCKSDB_LITE 33 std::unordered_map<std::string, std::string> GetMutableDBOptionsMap( 34 const DBOptions& options) { 35 std::string options_str; 36 GetStringFromDBOptions(&options_str, options); 37 std::unordered_map<std::string, std::string> options_map; 38 StringToMap(options_str, &options_map); 39 std::unordered_map<std::string, std::string> mutable_map; 40 for (const auto opt : db_options_type_info) { 41 if (opt.second.is_mutable && 42 opt.second.verification != OptionVerificationType::kDeprecated) { 43 mutable_map[opt.first] = options_map[opt.first]; 44 } 45 } 46 return mutable_map; 47 } 48 49 std::unordered_map<std::string, std::string> GetMutableCFOptionsMap( 50 const ColumnFamilyOptions& options) { 51 std::string options_str; 52 GetStringFromColumnFamilyOptions(&options_str, options); 53 std::unordered_map<std::string, std::string> options_map; 54 StringToMap(options_str, &options_map); 55 std::unordered_map<std::string, std::string> mutable_map; 56 for (const auto opt : cf_options_type_info) { 57 if (opt.second.is_mutable && 58 opt.second.verification != OptionVerificationType::kDeprecated) { 59 mutable_map[opt.first] = options_map[opt.first]; 60 } 61 } 62 return mutable_map; 63 } 64 65 std::unordered_map<std::string, std::string> GetRandomizedMutableCFOptionsMap( 66 Random* rnd) { 67 Options options = CurrentOptions(); 68 options.env = env_; 69 ImmutableDBOptions db_options(options); 70 test::RandomInitCFOptions(&options, options, rnd); 71 auto sanitized_options = SanitizeOptions(db_options, options); 72 auto opt_map = GetMutableCFOptionsMap(sanitized_options); 73 delete options.compaction_filter; 74 return opt_map; 75 } 76 77 std::unordered_map<std::string, std::string> GetRandomizedMutableDBOptionsMap( 78 Random* rnd) { 79 DBOptions db_options; 80 test::RandomInitDBOptions(&db_options, rnd); 81 auto sanitized_options = SanitizeOptions(dbname_, db_options); 82 return GetMutableDBOptionsMap(sanitized_options); 83 } 84 #endif // ROCKSDB_LITE 85 }; 86 87 // RocksDB lite don't support dynamic options. 88 #ifndef ROCKSDB_LITE 89 90 TEST_F(DBOptionsTest, GetLatestDBOptions) { 91 // GetOptions should be able to get latest option changed by SetOptions. 92 Options options; 93 options.create_if_missing = true; 94 options.env = env_; 95 Random rnd(228); 96 Reopen(options); 97 auto new_options = GetRandomizedMutableDBOptionsMap(&rnd); 98 ASSERT_OK(dbfull()->SetDBOptions(new_options)); 99 ASSERT_EQ(new_options, GetMutableDBOptionsMap(dbfull()->GetDBOptions())); 100 } 101 102 TEST_F(DBOptionsTest, GetLatestCFOptions) { 103 // GetOptions should be able to get latest option changed by SetOptions. 104 Options options; 105 options.create_if_missing = true; 106 options.env = env_; 107 Random rnd(228); 108 Reopen(options); 109 CreateColumnFamilies({"foo"}, options); 110 ReopenWithColumnFamilies({"default", "foo"}, options); 111 auto options_default = GetRandomizedMutableCFOptionsMap(&rnd); 112 auto options_foo = GetRandomizedMutableCFOptionsMap(&rnd); 113 ASSERT_OK(dbfull()->SetOptions(handles_[0], options_default)); 114 ASSERT_OK(dbfull()->SetOptions(handles_[1], options_foo)); 115 ASSERT_EQ(options_default, 116 GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[0]))); 117 ASSERT_EQ(options_foo, 118 GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[1]))); 119 } 120 121 TEST_F(DBOptionsTest, SetBytesPerSync) { 122 const size_t kValueSize = 1024 * 1024; // 1MB 123 Options options; 124 options.create_if_missing = true; 125 options.bytes_per_sync = 1024 * 1024; 126 options.use_direct_reads = false; 127 options.write_buffer_size = 400 * kValueSize; 128 options.disable_auto_compactions = true; 129 options.compression = kNoCompression; 130 options.env = env_; 131 Reopen(options); 132 int counter = 0; 133 int low_bytes_per_sync = 0; 134 int i = 0; 135 const std::string kValue(kValueSize, 'v'); 136 ASSERT_EQ(options.bytes_per_sync, dbfull()->GetDBOptions().bytes_per_sync); 137 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( 138 "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) { counter++; }); 139 140 WriteOptions write_opts; 141 // should sync approximately 40MB/1MB ~= 40 times. 142 for (i = 0; i < 40; i++) { 143 Put(Key(i), kValue, write_opts); 144 } 145 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); 146 ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); 147 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); 148 low_bytes_per_sync = counter; 149 ASSERT_GT(low_bytes_per_sync, 35); 150 ASSERT_LT(low_bytes_per_sync, 45); 151 152 counter = 0; 153 // 8388608 = 8 * 1024 * 1024 154 ASSERT_OK(dbfull()->SetDBOptions({{"bytes_per_sync", "8388608"}})); 155 ASSERT_EQ(8388608, dbfull()->GetDBOptions().bytes_per_sync); 156 // should sync approximately 40MB*2/8MB ~= 10 times. 157 // data will be 40*2MB because of previous Puts too. 158 for (i = 0; i < 40; i++) { 159 Put(Key(i), kValue, write_opts); 160 } 161 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); 162 ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); 163 ASSERT_GT(counter, 5); 164 ASSERT_LT(counter, 15); 165 166 // Redundant assert. But leaving it here just to get the point across that 167 // low_bytes_per_sync > counter. 168 ASSERT_GT(low_bytes_per_sync, counter); 169 } 170 171 TEST_F(DBOptionsTest, SetWalBytesPerSync) { 172 const size_t kValueSize = 1024 * 1024 * 3; 173 Options options; 174 options.create_if_missing = true; 175 options.wal_bytes_per_sync = 512; 176 options.write_buffer_size = 100 * kValueSize; 177 options.disable_auto_compactions = true; 178 options.compression = kNoCompression; 179 options.env = env_; 180 Reopen(options); 181 ASSERT_EQ(512, dbfull()->GetDBOptions().wal_bytes_per_sync); 182 int counter = 0; 183 int low_bytes_per_sync = 0; 184 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( 185 "WritableFileWriter::RangeSync:0", [&](void* /*arg*/) { counter++; }); 186 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); 187 const std::string kValue(kValueSize, 'v'); 188 int i = 0; 189 for (; i < 10; i++) { 190 Put(Key(i), kValue); 191 } 192 // Do not flush. If we flush here, SwitchWAL will reuse old WAL file since its 193 // empty and will not get the new wal_bytes_per_sync value. 194 low_bytes_per_sync = counter; 195 //5242880 = 1024 * 1024 * 5 196 ASSERT_OK(dbfull()->SetDBOptions({{"wal_bytes_per_sync", "5242880"}})); 197 ASSERT_EQ(5242880, dbfull()->GetDBOptions().wal_bytes_per_sync); 198 counter = 0; 199 i = 0; 200 for (; i < 10; i++) { 201 Put(Key(i), kValue); 202 } 203 ASSERT_GT(counter, 0); 204 ASSERT_GT(low_bytes_per_sync, 0); 205 ASSERT_GT(low_bytes_per_sync, counter); 206 } 207 208 TEST_F(DBOptionsTest, WritableFileMaxBufferSize) { 209 Options options; 210 options.create_if_missing = true; 211 options.writable_file_max_buffer_size = 1024 * 1024; 212 options.level0_file_num_compaction_trigger = 3; 213 options.max_manifest_file_size = 1; 214 options.env = env_; 215 int buffer_size = 1024 * 1024; 216 Reopen(options); 217 ASSERT_EQ(buffer_size, 218 dbfull()->GetDBOptions().writable_file_max_buffer_size); 219 220 std::atomic<int> match_cnt(0); 221 std::atomic<int> unmatch_cnt(0); 222 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( 223 "WritableFileWriter::WritableFileWriter:0", [&](void* arg) { 224 int value = static_cast<int>(reinterpret_cast<uintptr_t>(arg)); 225 if (value == buffer_size) { 226 match_cnt++; 227 } else { 228 unmatch_cnt++; 229 } 230 }); 231 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); 232 int i = 0; 233 for (; i < 3; i++) { 234 ASSERT_OK(Put("foo", ToString(i))); 235 ASSERT_OK(Put("bar", ToString(i))); 236 Flush(); 237 } 238 dbfull()->TEST_WaitForCompact(); 239 ASSERT_EQ(unmatch_cnt, 0); 240 ASSERT_GE(match_cnt, 11); 241 242 ASSERT_OK( 243 dbfull()->SetDBOptions({{"writable_file_max_buffer_size", "524288"}})); 244 buffer_size = 512 * 1024; 245 match_cnt = 0; 246 unmatch_cnt = 0; // SetDBOptions() will create a WriteableFileWriter 247 248 ASSERT_EQ(buffer_size, 249 dbfull()->GetDBOptions().writable_file_max_buffer_size); 250 i = 0; 251 for (; i < 3; i++) { 252 ASSERT_OK(Put("foo", ToString(i))); 253 ASSERT_OK(Put("bar", ToString(i))); 254 Flush(); 255 } 256 dbfull()->TEST_WaitForCompact(); 257 ASSERT_EQ(unmatch_cnt, 0); 258 ASSERT_GE(match_cnt, 11); 259 } 260 261 TEST_F(DBOptionsTest, SetOptionsAndReopen) { 262 Random rnd(1044); 263 auto rand_opts = GetRandomizedMutableCFOptionsMap(&rnd); 264 ASSERT_OK(dbfull()->SetOptions(rand_opts)); 265 // Verify if DB can be reopen after setting options. 266 Options options; 267 options.env = env_; 268 ASSERT_OK(TryReopen(options)); 269 } 270 271 TEST_F(DBOptionsTest, EnableAutoCompactionAndTriggerStall) { 272 const std::string kValue(1024, 'v'); 273 for (int method_type = 0; method_type < 2; method_type++) { 274 for (int option_type = 0; option_type < 4; option_type++) { 275 Options options; 276 options.create_if_missing = true; 277 options.disable_auto_compactions = true; 278 options.write_buffer_size = 1024 * 1024 * 10; 279 options.compression = CompressionType::kNoCompression; 280 options.level0_file_num_compaction_trigger = 1; 281 options.level0_stop_writes_trigger = std::numeric_limits<int>::max(); 282 options.level0_slowdown_writes_trigger = std::numeric_limits<int>::max(); 283 options.hard_pending_compaction_bytes_limit = 284 std::numeric_limits<uint64_t>::max(); 285 options.soft_pending_compaction_bytes_limit = 286 std::numeric_limits<uint64_t>::max(); 287 options.env = env_; 288 289 DestroyAndReopen(options); 290 int i = 0; 291 for (; i < 1024; i++) { 292 Put(Key(i), kValue); 293 } 294 Flush(); 295 for (; i < 1024 * 2; i++) { 296 Put(Key(i), kValue); 297 } 298 Flush(); 299 dbfull()->TEST_WaitForFlushMemTable(); 300 ASSERT_EQ(2, NumTableFilesAtLevel(0)); 301 uint64_t l0_size = SizeAtLevel(0); 302 303 switch (option_type) { 304 case 0: 305 // test with level0_stop_writes_trigger 306 options.level0_stop_writes_trigger = 2; 307 options.level0_slowdown_writes_trigger = 2; 308 break; 309 case 1: 310 options.level0_slowdown_writes_trigger = 2; 311 break; 312 case 2: 313 options.hard_pending_compaction_bytes_limit = l0_size; 314 options.soft_pending_compaction_bytes_limit = l0_size; 315 break; 316 case 3: 317 options.soft_pending_compaction_bytes_limit = l0_size; 318 break; 319 } 320 Reopen(options); 321 dbfull()->TEST_WaitForCompact(); 322 ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); 323 ASSERT_FALSE(dbfull()->TEST_write_controler().NeedsDelay()); 324 325 SyncPoint::GetInstance()->LoadDependency( 326 {{"DBOptionsTest::EnableAutoCompactionAndTriggerStall:1", 327 "BackgroundCallCompaction:0"}, 328 {"DBImpl::BackgroundCompaction():BeforePickCompaction", 329 "DBOptionsTest::EnableAutoCompactionAndTriggerStall:2"}, 330 {"DBOptionsTest::EnableAutoCompactionAndTriggerStall:3", 331 "DBImpl::BackgroundCompaction():AfterPickCompaction"}}); 332 // Block background compaction. 333 SyncPoint::GetInstance()->EnableProcessing(); 334 335 switch (method_type) { 336 case 0: 337 ASSERT_OK( 338 dbfull()->SetOptions({{"disable_auto_compactions", "false"}})); 339 break; 340 case 1: 341 ASSERT_OK(dbfull()->EnableAutoCompaction( 342 {dbfull()->DefaultColumnFamily()})); 343 break; 344 } 345 TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:1"); 346 // Wait for stall condition recalculate. 347 TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:2"); 348 349 switch (option_type) { 350 case 0: 351 ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped()); 352 break; 353 case 1: 354 ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); 355 ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); 356 break; 357 case 2: 358 ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped()); 359 break; 360 case 3: 361 ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); 362 ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); 363 break; 364 } 365 TEST_SYNC_POINT("DBOptionsTest::EnableAutoCompactionAndTriggerStall:3"); 366 367 // Background compaction executed. 368 dbfull()->TEST_WaitForCompact(); 369 ASSERT_FALSE(dbfull()->TEST_write_controler().IsStopped()); 370 ASSERT_FALSE(dbfull()->TEST_write_controler().NeedsDelay()); 371 } 372 } 373 } 374 375 TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) { 376 Options options; 377 options.create_if_missing = true; 378 options.level0_file_num_compaction_trigger = 1000; 379 options.env = env_; 380 Reopen(options); 381 for (int i = 0; i < 3; i++) { 382 // Need to insert two keys to avoid trivial move. 383 ASSERT_OK(Put("foo", ToString(i))); 384 ASSERT_OK(Put("bar", ToString(i))); 385 Flush(); 386 } 387 ASSERT_EQ("3", FilesPerLevel()); 388 ASSERT_OK( 389 dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "3"}})); 390 dbfull()->TEST_WaitForCompact(); 391 ASSERT_EQ("0,1", FilesPerLevel()); 392 } 393 394 TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) { 395 Options options; 396 options.create_if_missing = true; 397 options.max_background_compactions = 1; // default value 398 options.env = env_; 399 Reopen(options); 400 ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); 401 ASSERT_OK(dbfull()->SetDBOptions({{"max_background_compactions", "3"}})); 402 ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); 403 auto stop_token = dbfull()->TEST_write_controler().GetStopToken(); 404 ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed()); 405 } 406 407 TEST_F(DBOptionsTest, SetBackgroundJobs) { 408 Options options; 409 options.create_if_missing = true; 410 options.max_background_jobs = 8; 411 options.env = env_; 412 Reopen(options); 413 414 for (int i = 0; i < 2; ++i) { 415 if (i > 0) { 416 options.max_background_jobs = 12; 417 ASSERT_OK(dbfull()->SetDBOptions( 418 {{"max_background_jobs", 419 std::to_string(options.max_background_jobs)}})); 420 } 421 422 const int expected_max_flushes = options.max_background_jobs / 4; 423 424 ASSERT_EQ(expected_max_flushes, dbfull()->TEST_BGFlushesAllowed()); 425 ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); 426 427 auto stop_token = dbfull()->TEST_write_controler().GetStopToken(); 428 429 const int expected_max_compactions = 3 * expected_max_flushes; 430 431 ASSERT_EQ(expected_max_flushes, dbfull()->TEST_BGFlushesAllowed()); 432 ASSERT_EQ(expected_max_compactions, dbfull()->TEST_BGCompactionsAllowed()); 433 434 ASSERT_EQ(expected_max_flushes, 435 env_->GetBackgroundThreads(Env::Priority::HIGH)); 436 ASSERT_EQ(expected_max_compactions, 437 env_->GetBackgroundThreads(Env::Priority::LOW)); 438 } 439 } 440 441 TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) { 442 Options options; 443 options.create_if_missing = true; 444 options.disable_auto_compactions = true; 445 options.env = env_; 446 WriteOptions write_without_wal; 447 write_without_wal.disableWAL = true; 448 449 ASSERT_FALSE(options.avoid_flush_during_shutdown); 450 DestroyAndReopen(options); 451 ASSERT_OK(Put("foo", "v1", write_without_wal)); 452 Reopen(options); 453 ASSERT_EQ("v1", Get("foo")); 454 ASSERT_EQ("1", FilesPerLevel()); 455 456 DestroyAndReopen(options); 457 ASSERT_OK(Put("foo", "v2", write_without_wal)); 458 ASSERT_OK(dbfull()->SetDBOptions({{"avoid_flush_during_shutdown", "true"}})); 459 Reopen(options); 460 ASSERT_EQ("NOT_FOUND", Get("foo")); 461 ASSERT_EQ("", FilesPerLevel()); 462 } 463 464 TEST_F(DBOptionsTest, SetDelayedWriteRateOption) { 465 Options options; 466 options.create_if_missing = true; 467 options.delayed_write_rate = 2 * 1024U * 1024U; 468 options.env = env_; 469 Reopen(options); 470 ASSERT_EQ(2 * 1024U * 1024U, dbfull()->TEST_write_controler().max_delayed_write_rate()); 471 472 ASSERT_OK(dbfull()->SetDBOptions({{"delayed_write_rate", "20000"}})); 473 ASSERT_EQ(20000, dbfull()->TEST_write_controler().max_delayed_write_rate()); 474 } 475 476 TEST_F(DBOptionsTest, MaxTotalWalSizeChange) { 477 Random rnd(1044); 478 const auto value_size = size_t(1024); 479 std::string value; 480 test::RandomString(&rnd, value_size, &value); 481 482 Options options; 483 options.create_if_missing = true; 484 options.env = env_; 485 CreateColumnFamilies({"1", "2", "3"}, options); 486 ReopenWithColumnFamilies({"default", "1", "2", "3"}, options); 487 488 WriteOptions write_options; 489 490 const int key_count = 100; 491 for (int i = 0; i < key_count; ++i) { 492 for (size_t cf = 0; cf < handles_.size(); ++cf) { 493 ASSERT_OK(Put(static_cast<int>(cf), Key(i), value)); 494 } 495 } 496 ASSERT_OK(dbfull()->SetDBOptions({{"max_total_wal_size", "10"}})); 497 498 for (size_t cf = 0; cf < handles_.size(); ++cf) { 499 dbfull()->TEST_WaitForFlushMemTable(handles_[cf]); 500 ASSERT_EQ("1", FilesPerLevel(static_cast<int>(cf))); 501 } 502 } 503 504 TEST_F(DBOptionsTest, SetStatsDumpPeriodSec) { 505 Options options; 506 options.create_if_missing = true; 507 options.stats_dump_period_sec = 5; 508 options.env = env_; 509 Reopen(options); 510 ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec); 511 512 for (int i = 0; i < 20; i++) { 513 unsigned int num = rand() % 5000 + 1; 514 ASSERT_OK( 515 dbfull()->SetDBOptions({{"stats_dump_period_sec", ToString(num)}})); 516 ASSERT_EQ(num, dbfull()->GetDBOptions().stats_dump_period_sec); 517 } 518 Close(); 519 } 520 521 TEST_F(DBOptionsTest, SetOptionsStatsPersistPeriodSec) { 522 Options options; 523 options.create_if_missing = true; 524 options.stats_persist_period_sec = 5; 525 options.env = env_; 526 Reopen(options); 527 ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec); 528 529 ASSERT_OK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "12345"}})); 530 ASSERT_EQ(12345u, dbfull()->GetDBOptions().stats_persist_period_sec); 531 ASSERT_NOK(dbfull()->SetDBOptions({{"stats_persist_period_sec", "abcde"}})); 532 ASSERT_EQ(12345u, dbfull()->GetDBOptions().stats_persist_period_sec); 533 } 534 535 static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) { 536 dbfull->TEST_LockMutex(); 537 JobContext job_context(0); 538 dbfull->FindObsoleteFiles(&job_context, false); 539 ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty()); 540 dbfull->TEST_UnlockMutex(); 541 if (job_context.HaveSomethingToDelete()) { 542 // fulfill the contract of FindObsoleteFiles by calling PurgeObsoleteFiles 543 // afterwards; otherwise the test may hang on shutdown 544 dbfull->PurgeObsoleteFiles(job_context); 545 } 546 job_context.Clean(); 547 } 548 549 TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) { 550 SpecialEnv env(env_); 551 env.time_elapse_only_sleep_ = true; 552 Options options; 553 options.env = &env; 554 options.create_if_missing = true; 555 ASSERT_OK(TryReopen(options)); 556 557 // Verify that candidate files set is empty when no full scan requested. 558 assert_candidate_files_empty(dbfull(), true); 559 560 ASSERT_OK( 561 dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}})); 562 563 // After delete_obsolete_files_period_micros updated to 0, the next call 564 // to FindObsoleteFiles should make a full scan 565 assert_candidate_files_empty(dbfull(), false); 566 567 ASSERT_OK( 568 dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}})); 569 570 assert_candidate_files_empty(dbfull(), true); 571 572 env.addon_time_.store(20); 573 assert_candidate_files_empty(dbfull(), true); 574 575 env.addon_time_.store(21); 576 assert_candidate_files_empty(dbfull(), false); 577 578 Close(); 579 } 580 581 TEST_F(DBOptionsTest, MaxOpenFilesChange) { 582 SpecialEnv env(env_); 583 Options options; 584 options.env = CurrentOptions().env; 585 options.max_open_files = -1; 586 587 Reopen(options); 588 589 Cache* tc = dbfull()->TEST_table_cache(); 590 591 ASSERT_EQ(-1, dbfull()->GetDBOptions().max_open_files); 592 ASSERT_LT(2000, tc->GetCapacity()); 593 ASSERT_OK(dbfull()->SetDBOptions({{"max_open_files", "1024"}})); 594 ASSERT_EQ(1024, dbfull()->GetDBOptions().max_open_files); 595 // examine the table cache (actual size should be 1014) 596 ASSERT_GT(1500, tc->GetCapacity()); 597 Close(); 598 } 599 600 TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) { 601 Options options; 602 options.delayed_write_rate = 0; 603 Reopen(options); 604 ASSERT_EQ(16 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate); 605 606 options.rate_limiter.reset(NewGenericRateLimiter(31 * 1024 * 1024)); 607 Reopen(options); 608 ASSERT_EQ(31 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate); 609 } 610 611 TEST_F(DBOptionsTest, SanitizeUniversalTTLCompaction) { 612 Options options; 613 options.compaction_style = kCompactionStyleUniversal; 614 615 options.ttl = 0; 616 options.periodic_compaction_seconds = 0; 617 Reopen(options); 618 ASSERT_EQ(0, dbfull()->GetOptions().ttl); 619 ASSERT_EQ(0, dbfull()->GetOptions().periodic_compaction_seconds); 620 621 options.ttl = 0; 622 options.periodic_compaction_seconds = 100; 623 Reopen(options); 624 ASSERT_EQ(0, dbfull()->GetOptions().ttl); 625 ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds); 626 627 options.ttl = 100; 628 options.periodic_compaction_seconds = 0; 629 Reopen(options); 630 ASSERT_EQ(100, dbfull()->GetOptions().ttl); 631 ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds); 632 633 options.ttl = 100; 634 options.periodic_compaction_seconds = 500; 635 Reopen(options); 636 ASSERT_EQ(100, dbfull()->GetOptions().ttl); 637 ASSERT_EQ(100, dbfull()->GetOptions().periodic_compaction_seconds); 638 } 639 640 TEST_F(DBOptionsTest, SanitizeTtlDefault) { 641 Options options; 642 Reopen(options); 643 ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl); 644 645 options.compaction_style = kCompactionStyleLevel; 646 options.ttl = 0; 647 Reopen(options); 648 ASSERT_EQ(0, dbfull()->GetOptions().ttl); 649 650 options.ttl = 100; 651 Reopen(options); 652 ASSERT_EQ(100, dbfull()->GetOptions().ttl); 653 } 654 655 TEST_F(DBOptionsTest, SanitizeFIFOPeriodicCompaction) { 656 Options options; 657 options.compaction_style = kCompactionStyleFIFO; 658 options.ttl = 0; 659 Reopen(options); 660 ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl); 661 662 options.ttl = 100; 663 Reopen(options); 664 ASSERT_EQ(100, dbfull()->GetOptions().ttl); 665 666 options.ttl = 100 * 24 * 60 * 60; 667 Reopen(options); 668 ASSERT_EQ(100 * 24 * 60 * 60, dbfull()->GetOptions().ttl); 669 670 options.ttl = 200; 671 options.periodic_compaction_seconds = 300; 672 Reopen(options); 673 ASSERT_EQ(200, dbfull()->GetOptions().ttl); 674 675 options.ttl = 500; 676 options.periodic_compaction_seconds = 300; 677 Reopen(options); 678 ASSERT_EQ(300, dbfull()->GetOptions().ttl); 679 } 680 681 TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { 682 Options options; 683 options.compaction_style = kCompactionStyleFIFO; 684 options.write_buffer_size = 10 << 10; // 10KB 685 options.arena_block_size = 4096; 686 options.compression = kNoCompression; 687 options.create_if_missing = true; 688 options.compaction_options_fifo.allow_compaction = false; 689 env_->time_elapse_only_sleep_ = false; 690 options.env = env_; 691 692 // Test dynamically changing ttl. 693 env_->addon_time_.store(0); 694 options.ttl = 1 * 60 * 60; // 1 hour 695 ASSERT_OK(TryReopen(options)); 696 697 Random rnd(301); 698 for (int i = 0; i < 10; i++) { 699 // Generate and flush a file about 10KB. 700 for (int j = 0; j < 10; j++) { 701 ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980))); 702 } 703 Flush(); 704 } 705 ASSERT_OK(dbfull()->TEST_WaitForCompact()); 706 ASSERT_EQ(NumTableFilesAtLevel(0), 10); 707 708 // Add 61 seconds to the time. 709 env_->addon_time_.fetch_add(61); 710 711 // No files should be compacted as ttl is set to 1 hour. 712 ASSERT_EQ(dbfull()->GetOptions().ttl, 3600); 713 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); 714 ASSERT_EQ(NumTableFilesAtLevel(0), 10); 715 716 // Set ttl to 1 minute. So all files should get deleted. 717 ASSERT_OK(dbfull()->SetOptions({{"ttl", "60"}})); 718 ASSERT_EQ(dbfull()->GetOptions().ttl, 60); 719 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); 720 ASSERT_OK(dbfull()->TEST_WaitForCompact()); 721 ASSERT_EQ(NumTableFilesAtLevel(0), 0); 722 723 // Test dynamically changing compaction_options_fifo.max_table_files_size 724 env_->addon_time_.store(0); 725 options.compaction_options_fifo.max_table_files_size = 500 << 10; // 00KB 726 options.ttl = 0; 727 DestroyAndReopen(options); 728 729 for (int i = 0; i < 10; i++) { 730 // Generate and flush a file about 10KB. 731 for (int j = 0; j < 10; j++) { 732 ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980))); 733 } 734 Flush(); 735 } 736 ASSERT_OK(dbfull()->TEST_WaitForCompact()); 737 ASSERT_EQ(NumTableFilesAtLevel(0), 10); 738 739 // No files should be compacted as max_table_files_size is set to 500 KB. 740 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, 741 500 << 10); 742 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); 743 ASSERT_EQ(NumTableFilesAtLevel(0), 10); 744 745 // Set max_table_files_size to 12 KB. So only 1 file should remain now. 746 ASSERT_OK(dbfull()->SetOptions( 747 {{"compaction_options_fifo", "{max_table_files_size=12288;}"}})); 748 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, 749 12 << 10); 750 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); 751 ASSERT_OK(dbfull()->TEST_WaitForCompact()); 752 ASSERT_EQ(NumTableFilesAtLevel(0), 1); 753 754 // Test dynamically changing compaction_options_fifo.allow_compaction 755 options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB 756 options.ttl = 0; 757 options.compaction_options_fifo.allow_compaction = false; 758 options.level0_file_num_compaction_trigger = 6; 759 DestroyAndReopen(options); 760 761 for (int i = 0; i < 10; i++) { 762 // Generate and flush a file about 10KB. 763 for (int j = 0; j < 10; j++) { 764 ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980))); 765 } 766 Flush(); 767 } 768 ASSERT_OK(dbfull()->TEST_WaitForCompact()); 769 ASSERT_EQ(NumTableFilesAtLevel(0), 10); 770 771 // No files should be compacted as max_table_files_size is set to 500 KB and 772 // allow_compaction is false 773 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, 774 false); 775 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); 776 ASSERT_EQ(NumTableFilesAtLevel(0), 10); 777 778 // Set allow_compaction to true. So number of files should be between 1 and 5. 779 ASSERT_OK(dbfull()->SetOptions( 780 {{"compaction_options_fifo", "{allow_compaction=true;}"}})); 781 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, 782 true); 783 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); 784 ASSERT_OK(dbfull()->TEST_WaitForCompact()); 785 ASSERT_GE(NumTableFilesAtLevel(0), 1); 786 ASSERT_LE(NumTableFilesAtLevel(0), 5); 787 } 788 789 TEST_F(DBOptionsTest, CompactionReadaheadSizeChange) { 790 SpecialEnv env(env_); 791 Options options; 792 options.env = &env; 793 794 options.compaction_readahead_size = 0; 795 options.new_table_reader_for_compaction_inputs = true; 796 options.level0_file_num_compaction_trigger = 2; 797 const std::string kValue(1024, 'v'); 798 Reopen(options); 799 800 ASSERT_EQ(0, dbfull()->GetDBOptions().compaction_readahead_size); 801 ASSERT_OK(dbfull()->SetDBOptions({{"compaction_readahead_size", "256"}})); 802 ASSERT_EQ(256, dbfull()->GetDBOptions().compaction_readahead_size); 803 for (int i = 0; i < 1024; i++) { 804 Put(Key(i), kValue); 805 } 806 Flush(); 807 for (int i = 0; i < 1024 * 2; i++) { 808 Put(Key(i), kValue); 809 } 810 Flush(); 811 dbfull()->TEST_WaitForCompact(); 812 ASSERT_EQ(256, env_->compaction_readahead_size_); 813 Close(); 814 } 815 816 TEST_F(DBOptionsTest, FIFOTtlBackwardCompatible) { 817 Options options; 818 options.compaction_style = kCompactionStyleFIFO; 819 options.write_buffer_size = 10 << 10; // 10KB 820 options.create_if_missing = true; 821 822 ASSERT_OK(TryReopen(options)); 823 824 Random rnd(301); 825 for (int i = 0; i < 10; i++) { 826 // Generate and flush a file about 10KB. 827 for (int j = 0; j < 10; j++) { 828 ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980))); 829 } 830 Flush(); 831 } 832 ASSERT_OK(dbfull()->TEST_WaitForCompact()); 833 ASSERT_EQ(NumTableFilesAtLevel(0), 10); 834 835 // In release 6.0, ttl was promoted from a secondary level option under 836 // compaction_options_fifo to a top level option under ColumnFamilyOptions. 837 // We still need to handle old SetOptions calls but should ignore 838 // ttl under compaction_options_fifo. 839 ASSERT_OK(dbfull()->SetOptions( 840 {{"compaction_options_fifo", 841 "{allow_compaction=true;max_table_files_size=1024;ttl=731;}"}, 842 {"ttl", "60"}})); 843 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, 844 true); 845 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, 846 1024); 847 ASSERT_EQ(dbfull()->GetOptions().ttl, 60); 848 849 // Put ttl as the first option inside compaction_options_fifo. That works as 850 // it doesn't overwrite any other option. 851 ASSERT_OK(dbfull()->SetOptions( 852 {{"compaction_options_fifo", 853 "{ttl=985;allow_compaction=true;max_table_files_size=1024;}"}, 854 {"ttl", "191"}})); 855 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, 856 true); 857 ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, 858 1024); 859 ASSERT_EQ(dbfull()->GetOptions().ttl, 191); 860 } 861 862 #endif // ROCKSDB_LITE 863 864 } // namespace ROCKSDB_NAMESPACE 865 866 int main(int argc, char** argv) { 867 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); 868 ::testing::InitGoogleTest(&argc, argv); 869 return RUN_ALL_TESTS(); 870 } 871