1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #include <stdio.h>
11
12 #include <algorithm>
13 #include <string>
14
15 #include "db/db_test_util.h"
16 #include "port/stack_trace.h"
17 #include "rocksdb/listener.h"
18 #include "rocksdb/options.h"
19 #include "rocksdb/perf_context.h"
20 #include "rocksdb/perf_level.h"
21 #include "rocksdb/table.h"
22 #include "util/random.h"
23 #include "util/string_util.h"
24
25 namespace ROCKSDB_NAMESPACE {
26
27 class DBPropertiesTest : public DBTestBase {
28 public:
DBPropertiesTest()29 DBPropertiesTest()
30 : DBTestBase("/db_properties_test", /*env_do_fsync=*/false) {}
31 };
32
33 #ifndef ROCKSDB_LITE
TEST_F(DBPropertiesTest,Empty)34 TEST_F(DBPropertiesTest, Empty) {
35 do {
36 Options options;
37 options.env = env_;
38 options.write_buffer_size = 100000; // Small write buffer
39 options.allow_concurrent_memtable_write = false;
40 options = CurrentOptions(options);
41 CreateAndReopenWithCF({"pikachu"}, options);
42
43 std::string num;
44 ASSERT_TRUE(dbfull()->GetProperty(
45 handles_[1], "rocksdb.num-entries-active-mem-table", &num));
46 ASSERT_EQ("0", num);
47
48 ASSERT_OK(Put(1, "foo", "v1"));
49 ASSERT_EQ("v1", Get(1, "foo"));
50 ASSERT_TRUE(dbfull()->GetProperty(
51 handles_[1], "rocksdb.num-entries-active-mem-table", &num));
52 ASSERT_EQ("1", num);
53
54 // Block sync calls
55 env_->delay_sstable_sync_.store(true, std::memory_order_release);
56 ASSERT_OK(Put(1, "k1", std::string(100000, 'x'))); // Fill memtable
57 ASSERT_TRUE(dbfull()->GetProperty(
58 handles_[1], "rocksdb.num-entries-active-mem-table", &num));
59 ASSERT_EQ("2", num);
60
61 ASSERT_OK(Put(1, "k2", std::string(100000, 'y'))); // Trigger compaction
62 ASSERT_TRUE(dbfull()->GetProperty(
63 handles_[1], "rocksdb.num-entries-active-mem-table", &num));
64 ASSERT_EQ("1", num);
65
66 ASSERT_EQ("v1", Get(1, "foo"));
67 // Release sync calls
68 env_->delay_sstable_sync_.store(false, std::memory_order_release);
69
70 ASSERT_OK(db_->DisableFileDeletions());
71 ASSERT_TRUE(
72 dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
73 ASSERT_EQ("0", num);
74
75 ASSERT_OK(db_->DisableFileDeletions());
76 ASSERT_TRUE(
77 dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
78 ASSERT_EQ("0", num);
79
80 ASSERT_OK(db_->DisableFileDeletions());
81 ASSERT_TRUE(
82 dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
83 ASSERT_EQ("0", num);
84
85 ASSERT_OK(db_->EnableFileDeletions(false));
86 ASSERT_TRUE(
87 dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
88 ASSERT_EQ("0", num);
89
90 ASSERT_OK(db_->EnableFileDeletions());
91 ASSERT_TRUE(
92 dbfull()->GetProperty("rocksdb.is-file-deletions-enabled", &num));
93 ASSERT_EQ("1", num);
94 } while (ChangeOptions());
95 }
96
TEST_F(DBPropertiesTest,CurrentVersionNumber)97 TEST_F(DBPropertiesTest, CurrentVersionNumber) {
98 uint64_t v1, v2, v3;
99 ASSERT_TRUE(
100 dbfull()->GetIntProperty("rocksdb.current-super-version-number", &v1));
101 ASSERT_OK(Put("12345678", ""));
102 ASSERT_TRUE(
103 dbfull()->GetIntProperty("rocksdb.current-super-version-number", &v2));
104 ASSERT_OK(Flush());
105 ASSERT_TRUE(
106 dbfull()->GetIntProperty("rocksdb.current-super-version-number", &v3));
107
108 ASSERT_EQ(v1, v2);
109 ASSERT_GT(v3, v2);
110 }
111
TEST_F(DBPropertiesTest,GetAggregatedIntPropertyTest)112 TEST_F(DBPropertiesTest, GetAggregatedIntPropertyTest) {
113 const int kKeySize = 100;
114 const int kValueSize = 500;
115 const int kKeyNum = 100;
116
117 Options options;
118 options.env = env_;
119 options.create_if_missing = true;
120 options.write_buffer_size = (kKeySize + kValueSize) * kKeyNum / 10;
121 // Make them never flush
122 options.min_write_buffer_number_to_merge = 1000;
123 options.max_write_buffer_number = 1000;
124 options = CurrentOptions(options);
125 CreateAndReopenWithCF({"one", "two", "three", "four"}, options);
126
127 Random rnd(301);
128 for (auto* handle : handles_) {
129 for (int i = 0; i < kKeyNum; ++i) {
130 ASSERT_OK(db_->Put(WriteOptions(), handle, rnd.RandomString(kKeySize),
131 rnd.RandomString(kValueSize)));
132 }
133 }
134
135 uint64_t manual_sum = 0;
136 uint64_t api_sum = 0;
137 uint64_t value = 0;
138 for (auto* handle : handles_) {
139 ASSERT_TRUE(
140 db_->GetIntProperty(handle, DB::Properties::kSizeAllMemTables, &value));
141 manual_sum += value;
142 }
143 ASSERT_TRUE(db_->GetAggregatedIntProperty(DB::Properties::kSizeAllMemTables,
144 &api_sum));
145 ASSERT_GT(manual_sum, 0);
146 ASSERT_EQ(manual_sum, api_sum);
147
148 ASSERT_FALSE(db_->GetAggregatedIntProperty(DB::Properties::kDBStats, &value));
149
150 uint64_t before_flush_trm;
151 uint64_t after_flush_trm;
152 for (auto* handle : handles_) {
153 ASSERT_TRUE(db_->GetAggregatedIntProperty(
154 DB::Properties::kEstimateTableReadersMem, &before_flush_trm));
155
156 // Issue flush and expect larger memory usage of table readers.
157 ASSERT_OK(db_->Flush(FlushOptions(), handle));
158
159 ASSERT_TRUE(db_->GetAggregatedIntProperty(
160 DB::Properties::kEstimateTableReadersMem, &after_flush_trm));
161 ASSERT_GT(after_flush_trm, before_flush_trm);
162 }
163 }
164
165 namespace {
ResetTableProperties(TableProperties * tp)166 void ResetTableProperties(TableProperties* tp) {
167 tp->data_size = 0;
168 tp->index_size = 0;
169 tp->filter_size = 0;
170 tp->raw_key_size = 0;
171 tp->raw_value_size = 0;
172 tp->num_data_blocks = 0;
173 tp->num_entries = 0;
174 tp->num_deletions = 0;
175 tp->num_merge_operands = 0;
176 tp->num_range_deletions = 0;
177 }
178
ParseTablePropertiesString(std::string tp_string,TableProperties * tp)179 void ParseTablePropertiesString(std::string tp_string, TableProperties* tp) {
180 double dummy_double;
181 std::replace(tp_string.begin(), tp_string.end(), ';', ' ');
182 std::replace(tp_string.begin(), tp_string.end(), '=', ' ');
183 ResetTableProperties(tp);
184 sscanf(tp_string.c_str(),
185 "# data blocks %" SCNu64 " # entries %" SCNu64 " # deletions %" SCNu64
186 " # merge operands %" SCNu64 " # range deletions %" SCNu64
187 " raw key size %" SCNu64
188 " raw average key size %lf "
189 " raw value size %" SCNu64
190 " raw average value size %lf "
191 " data block size %" SCNu64 " index block size (user-key? %" SCNu64
192 ", delta-value? %" SCNu64 ") %" SCNu64 " filter block size %" SCNu64,
193 &tp->num_data_blocks, &tp->num_entries, &tp->num_deletions,
194 &tp->num_merge_operands, &tp->num_range_deletions, &tp->raw_key_size,
195 &dummy_double, &tp->raw_value_size, &dummy_double, &tp->data_size,
196 &tp->index_key_is_user_key, &tp->index_value_is_delta_encoded,
197 &tp->index_size, &tp->filter_size);
198 }
199
VerifySimilar(uint64_t a,uint64_t b,double bias)200 void VerifySimilar(uint64_t a, uint64_t b, double bias) {
201 ASSERT_EQ(a == 0U, b == 0U);
202 if (a == 0) {
203 return;
204 }
205 double dbl_a = static_cast<double>(a);
206 double dbl_b = static_cast<double>(b);
207 if (dbl_a > dbl_b) {
208 ASSERT_LT(static_cast<double>(dbl_a - dbl_b) / (dbl_a + dbl_b), bias);
209 } else {
210 ASSERT_LT(static_cast<double>(dbl_b - dbl_a) / (dbl_a + dbl_b), bias);
211 }
212 }
213
VerifyTableProperties(const TableProperties & base_tp,const TableProperties & new_tp,double filter_size_bias=CACHE_LINE_SIZE>=256?0.15:0.1,double index_size_bias=0.1,double data_size_bias=0.1,double num_data_blocks_bias=0.05)214 void VerifyTableProperties(
215 const TableProperties& base_tp, const TableProperties& new_tp,
216 double filter_size_bias = CACHE_LINE_SIZE >= 256 ? 0.15 : 0.1,
217 double index_size_bias = 0.1, double data_size_bias = 0.1,
218 double num_data_blocks_bias = 0.05) {
219 VerifySimilar(base_tp.data_size, new_tp.data_size, data_size_bias);
220 VerifySimilar(base_tp.index_size, new_tp.index_size, index_size_bias);
221 VerifySimilar(base_tp.filter_size, new_tp.filter_size, filter_size_bias);
222 VerifySimilar(base_tp.num_data_blocks, new_tp.num_data_blocks,
223 num_data_blocks_bias);
224
225 ASSERT_EQ(base_tp.raw_key_size, new_tp.raw_key_size);
226 ASSERT_EQ(base_tp.raw_value_size, new_tp.raw_value_size);
227 ASSERT_EQ(base_tp.num_entries, new_tp.num_entries);
228 ASSERT_EQ(base_tp.num_deletions, new_tp.num_deletions);
229 ASSERT_EQ(base_tp.num_range_deletions, new_tp.num_range_deletions);
230
231 // Merge operands may become Puts, so we only have an upper bound the exact
232 // number of merge operands.
233 ASSERT_GE(base_tp.num_merge_operands, new_tp.num_merge_operands);
234 }
235
GetExpectedTableProperties(TableProperties * expected_tp,const int kKeySize,const int kValueSize,const int kPutsPerTable,const int kDeletionsPerTable,const int kMergeOperandsPerTable,const int kRangeDeletionsPerTable,const int kTableCount,const int kBloomBitsPerKey,const size_t kBlockSize,const bool index_key_is_user_key,const bool value_delta_encoding)236 void GetExpectedTableProperties(
237 TableProperties* expected_tp, const int kKeySize, const int kValueSize,
238 const int kPutsPerTable, const int kDeletionsPerTable,
239 const int kMergeOperandsPerTable, const int kRangeDeletionsPerTable,
240 const int kTableCount, const int kBloomBitsPerKey, const size_t kBlockSize,
241 const bool index_key_is_user_key, const bool value_delta_encoding) {
242 const int kKeysPerTable =
243 kPutsPerTable + kDeletionsPerTable + kMergeOperandsPerTable;
244 const int kPutCount = kTableCount * kPutsPerTable;
245 const int kDeletionCount = kTableCount * kDeletionsPerTable;
246 const int kMergeCount = kTableCount * kMergeOperandsPerTable;
247 const int kRangeDeletionCount = kTableCount * kRangeDeletionsPerTable;
248 const int kKeyCount = kPutCount + kDeletionCount + kMergeCount + kRangeDeletionCount;
249 const int kAvgSuccessorSize = kKeySize / 5;
250 const int kEncodingSavePerKey = kKeySize / 4;
251 expected_tp->raw_key_size = kKeyCount * (kKeySize + 8);
252 expected_tp->raw_value_size =
253 (kPutCount + kMergeCount + kRangeDeletionCount) * kValueSize;
254 expected_tp->num_entries = kKeyCount;
255 expected_tp->num_deletions = kDeletionCount + kRangeDeletionCount;
256 expected_tp->num_merge_operands = kMergeCount;
257 expected_tp->num_range_deletions = kRangeDeletionCount;
258 expected_tp->num_data_blocks =
259 kTableCount * (kKeysPerTable * (kKeySize - kEncodingSavePerKey + kValueSize)) /
260 kBlockSize;
261 expected_tp->data_size =
262 kTableCount * (kKeysPerTable * (kKeySize + 8 + kValueSize));
263 expected_tp->index_size =
264 expected_tp->num_data_blocks *
265 (kAvgSuccessorSize + (index_key_is_user_key ? 0 : 8) -
266 // discount 1 byte as value size is not encoded in value delta encoding
267 (value_delta_encoding ? 1 : 0));
268 expected_tp->filter_size =
269 kTableCount * ((kKeysPerTable * kBloomBitsPerKey + 7) / 8 +
270 /*average-ish overhead*/ CACHE_LINE_SIZE / 2);
271 }
272 } // anonymous namespace
273
TEST_F(DBPropertiesTest,ValidatePropertyInfo)274 TEST_F(DBPropertiesTest, ValidatePropertyInfo) {
275 for (const auto& ppt_name_and_info : InternalStats::ppt_name_to_info) {
276 // If C++ gets a std::string_literal, this would be better to check at
277 // compile-time using static_assert.
278 ASSERT_TRUE(ppt_name_and_info.first.empty() ||
279 !isdigit(ppt_name_and_info.first.back()));
280
281 int count = 0;
282 count += (ppt_name_and_info.second.handle_string == nullptr) ? 0 : 1;
283 count += (ppt_name_and_info.second.handle_int == nullptr) ? 0 : 1;
284 count += (ppt_name_and_info.second.handle_string_dbimpl == nullptr) ? 0 : 1;
285 ASSERT_TRUE(count == 1);
286 }
287 }
288
TEST_F(DBPropertiesTest,ValidateSampleNumber)289 TEST_F(DBPropertiesTest, ValidateSampleNumber) {
290 // When "max_open_files" is -1, we read all the files for
291 // "rocksdb.estimate-num-keys" computation, which is the ground truth.
292 // Otherwise, we sample 20 newest files to make an estimation.
293 // Formula: lastest_20_files_active_key_ratio * total_files
294 Options options = CurrentOptions();
295 options.disable_auto_compactions = true;
296 options.level0_stop_writes_trigger = 1000;
297 DestroyAndReopen(options);
298 int key = 0;
299 for (int files = 20; files >= 10; files -= 10) {
300 for (int i = 0; i < files; i++) {
301 int rows = files / 10;
302 for (int j = 0; j < rows; j++) {
303 ASSERT_OK(db_->Put(WriteOptions(), std::to_string(++key), "foo"));
304 }
305 ASSERT_OK(db_->Flush(FlushOptions()));
306 }
307 }
308 std::string num;
309 Reopen(options);
310 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
311 ASSERT_EQ("45", num);
312 options.max_open_files = -1;
313 Reopen(options);
314 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
315 ASSERT_EQ("50", num);
316 }
317
TEST_F(DBPropertiesTest,AggregatedTableProperties)318 TEST_F(DBPropertiesTest, AggregatedTableProperties) {
319 for (int kTableCount = 40; kTableCount <= 100; kTableCount += 30) {
320 const int kDeletionsPerTable = 5;
321 const int kMergeOperandsPerTable = 15;
322 const int kRangeDeletionsPerTable = 5;
323 const int kPutsPerTable = 100;
324 const int kKeySize = 80;
325 const int kValueSize = 200;
326 const int kBloomBitsPerKey = 20;
327
328 Options options = CurrentOptions();
329 options.level0_file_num_compaction_trigger = 8;
330 options.compression = kNoCompression;
331 options.create_if_missing = true;
332 options.preserve_deletes = true;
333 options.merge_operator.reset(new TestPutOperator());
334
335 BlockBasedTableOptions table_options;
336 table_options.filter_policy.reset(
337 NewBloomFilterPolicy(kBloomBitsPerKey, false));
338 table_options.block_size = 1024;
339 options.table_factory.reset(NewBlockBasedTableFactory(table_options));
340
341 DestroyAndReopen(options);
342
343 // Hold open a snapshot to prevent range tombstones from being compacted
344 // away.
345 ManagedSnapshot snapshot(db_);
346
347 Random rnd(5632);
348 for (int table = 1; table <= kTableCount; ++table) {
349 for (int i = 0; i < kPutsPerTable; ++i) {
350 ASSERT_OK(db_->Put(WriteOptions(), rnd.RandomString(kKeySize),
351 rnd.RandomString(kValueSize)));
352 }
353 for (int i = 0; i < kDeletionsPerTable; i++) {
354 ASSERT_OK(db_->Delete(WriteOptions(), rnd.RandomString(kKeySize)));
355 }
356 for (int i = 0; i < kMergeOperandsPerTable; i++) {
357 ASSERT_OK(db_->Merge(WriteOptions(), rnd.RandomString(kKeySize),
358 rnd.RandomString(kValueSize)));
359 }
360 for (int i = 0; i < kRangeDeletionsPerTable; i++) {
361 std::string start = rnd.RandomString(kKeySize);
362 std::string end = start;
363 end.resize(kValueSize);
364 ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
365 start, end));
366 }
367 ASSERT_OK(db_->Flush(FlushOptions()));
368 }
369 std::string property;
370 db_->GetProperty(DB::Properties::kAggregatedTableProperties, &property);
371 TableProperties output_tp;
372 ParseTablePropertiesString(property, &output_tp);
373 bool index_key_is_user_key = output_tp.index_key_is_user_key > 0;
374 bool value_is_delta_encoded = output_tp.index_value_is_delta_encoded > 0;
375
376 TableProperties expected_tp;
377 GetExpectedTableProperties(
378 &expected_tp, kKeySize, kValueSize, kPutsPerTable, kDeletionsPerTable,
379 kMergeOperandsPerTable, kRangeDeletionsPerTable, kTableCount,
380 kBloomBitsPerKey, table_options.block_size, index_key_is_user_key,
381 value_is_delta_encoded);
382
383 VerifyTableProperties(expected_tp, output_tp);
384 }
385 }
386
TEST_F(DBPropertiesTest,ReadLatencyHistogramByLevel)387 TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) {
388 Options options = CurrentOptions();
389 options.write_buffer_size = 110 << 10;
390 options.level0_file_num_compaction_trigger = 6;
391 options.num_levels = 4;
392 options.compression = kNoCompression;
393 options.max_bytes_for_level_base = 4500 << 10;
394 options.target_file_size_base = 98 << 10;
395 options.max_write_buffer_number = 2;
396 options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
397 options.max_open_files = 11; // Make sure no proloading of table readers
398
399 // RocksDB sanitize max open files to at least 20. Modify it back.
400 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
401 "SanitizeOptions::AfterChangeMaxOpenFiles", [&](void* arg) {
402 int* max_open_files = static_cast<int*>(arg);
403 *max_open_files = 11;
404 });
405 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
406
407 BlockBasedTableOptions table_options;
408 table_options.no_block_cache = true;
409
410 CreateAndReopenWithCF({"pikachu"}, options);
411 int key_index = 0;
412 Random rnd(301);
413 for (int num = 0; num < 8; num++) {
414 ASSERT_OK(Put("foo", "bar"));
415 GenerateNewFile(&rnd, &key_index);
416 ASSERT_OK(dbfull()->TEST_WaitForCompact());
417 }
418 ASSERT_OK(dbfull()->TEST_WaitForCompact());
419
420 std::string prop;
421 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
422
423 // Get() after flushes, See latency histogram tracked.
424 for (int key = 0; key < key_index; key++) {
425 Get(Key(key));
426 }
427 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop));
428 ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
429 ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
430 ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
431
432 // Reopen and issue Get(). See thee latency tracked
433 ReopenWithColumnFamilies({"default", "pikachu"}, options);
434 ASSERT_OK(dbfull()->TEST_WaitForCompact());
435 for (int key = 0; key < key_index; key++) {
436 Get(Key(key));
437 }
438
439 // Test for getting immutable_db_options_.statistics
440 ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(),
441 "rocksdb.options-statistics", &prop));
442 ASSERT_NE(std::string::npos, prop.find("rocksdb.block.cache.miss"));
443 ASSERT_EQ(std::string::npos, prop.find("rocksdb.db.f.micros"));
444
445 ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(),
446 "rocksdb.cf-file-histogram", &prop));
447 ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
448 ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
449 ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
450
451 // Reopen and issue iterating. See thee latency tracked
452 ReopenWithColumnFamilies({"default", "pikachu"}, options);
453 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
454 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop));
455 ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
456 ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
457 ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
458 {
459 std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
460 for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) {
461 }
462 ASSERT_OK(iter->status());
463 }
464 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop));
465 ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
466 ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
467 ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
468
469 // CF 1 should show no histogram.
470 ASSERT_TRUE(
471 dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop));
472 ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
473 ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
474 ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
475 // put something and read it back , CF 1 should show histogram.
476 ASSERT_OK(Put(1, "foo", "bar"));
477 ASSERT_OK(Flush(1));
478 ASSERT_OK(dbfull()->TEST_WaitForCompact());
479 ASSERT_EQ("bar", Get(1, "foo"));
480
481 ASSERT_TRUE(
482 dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop));
483 ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
484 ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
485 ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
486
487 // options.max_open_files preloads table readers.
488 options.max_open_files = -1;
489 ReopenWithColumnFamilies({"default", "pikachu"}, options);
490 ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(),
491 "rocksdb.cf-file-histogram", &prop));
492 ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
493 ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
494 ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
495 for (int key = 0; key < key_index; key++) {
496 Get(Key(key));
497 }
498 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop));
499 ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
500 ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
501 ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
502
503 // Clear internal stats
504 ASSERT_OK(dbfull()->ResetStats());
505 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop));
506 ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
507 ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
508 ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
509 }
510
TEST_F(DBPropertiesTest,AggregatedTablePropertiesAtLevel)511 TEST_F(DBPropertiesTest, AggregatedTablePropertiesAtLevel) {
512 const int kTableCount = 100;
513 const int kDeletionsPerTable = 2;
514 const int kMergeOperandsPerTable = 2;
515 const int kRangeDeletionsPerTable = 2;
516 const int kPutsPerTable = 10;
517 const int kKeySize = 50;
518 const int kValueSize = 400;
519 const int kMaxLevel = 7;
520 const int kBloomBitsPerKey = 20;
521 Random rnd(301);
522 Options options = CurrentOptions();
523 options.level0_file_num_compaction_trigger = 8;
524 options.compression = kNoCompression;
525 options.create_if_missing = true;
526 options.level0_file_num_compaction_trigger = 2;
527 options.target_file_size_base = 8192;
528 options.max_bytes_for_level_base = 10000;
529 options.max_bytes_for_level_multiplier = 2;
530 // This ensures there no compaction happening when we call GetProperty().
531 options.disable_auto_compactions = true;
532 options.preserve_deletes = true;
533 options.merge_operator.reset(new TestPutOperator());
534
535 BlockBasedTableOptions table_options;
536 table_options.filter_policy.reset(
537 NewBloomFilterPolicy(kBloomBitsPerKey, false));
538 table_options.block_size = 1024;
539 options.table_factory.reset(NewBlockBasedTableFactory(table_options));
540
541 DestroyAndReopen(options);
542
543 // Hold open a snapshot to prevent range tombstones from being compacted away.
544 ManagedSnapshot snapshot(db_);
545
546 std::string level_tp_strings[kMaxLevel];
547 std::string tp_string;
548 TableProperties level_tps[kMaxLevel];
549 TableProperties tp, sum_tp, expected_tp;
550 for (int table = 1; table <= kTableCount; ++table) {
551 for (int i = 0; i < kPutsPerTable; ++i) {
552 ASSERT_OK(db_->Put(WriteOptions(), rnd.RandomString(kKeySize),
553 rnd.RandomString(kValueSize)));
554 }
555 for (int i = 0; i < kDeletionsPerTable; i++) {
556 ASSERT_OK(db_->Delete(WriteOptions(), rnd.RandomString(kKeySize)));
557 }
558 for (int i = 0; i < kMergeOperandsPerTable; i++) {
559 ASSERT_OK(db_->Merge(WriteOptions(), rnd.RandomString(kKeySize),
560 rnd.RandomString(kValueSize)));
561 }
562 for (int i = 0; i < kRangeDeletionsPerTable; i++) {
563 std::string start = rnd.RandomString(kKeySize);
564 std::string end = start;
565 end.resize(kValueSize);
566 ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
567 start, end));
568 }
569 ASSERT_OK(db_->Flush(FlushOptions()));
570 ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
571 ResetTableProperties(&sum_tp);
572 for (int level = 0; level < kMaxLevel; ++level) {
573 db_->GetProperty(
574 DB::Properties::kAggregatedTablePropertiesAtLevel + ToString(level),
575 &level_tp_strings[level]);
576 ParseTablePropertiesString(level_tp_strings[level], &level_tps[level]);
577 sum_tp.data_size += level_tps[level].data_size;
578 sum_tp.index_size += level_tps[level].index_size;
579 sum_tp.filter_size += level_tps[level].filter_size;
580 sum_tp.raw_key_size += level_tps[level].raw_key_size;
581 sum_tp.raw_value_size += level_tps[level].raw_value_size;
582 sum_tp.num_data_blocks += level_tps[level].num_data_blocks;
583 sum_tp.num_entries += level_tps[level].num_entries;
584 sum_tp.num_deletions += level_tps[level].num_deletions;
585 sum_tp.num_merge_operands += level_tps[level].num_merge_operands;
586 sum_tp.num_range_deletions += level_tps[level].num_range_deletions;
587 }
588 db_->GetProperty(DB::Properties::kAggregatedTableProperties, &tp_string);
589 ParseTablePropertiesString(tp_string, &tp);
590 bool index_key_is_user_key = tp.index_key_is_user_key > 0;
591 bool value_is_delta_encoded = tp.index_value_is_delta_encoded > 0;
592 ASSERT_EQ(sum_tp.data_size, tp.data_size);
593 ASSERT_EQ(sum_tp.index_size, tp.index_size);
594 ASSERT_EQ(sum_tp.filter_size, tp.filter_size);
595 ASSERT_EQ(sum_tp.raw_key_size, tp.raw_key_size);
596 ASSERT_EQ(sum_tp.raw_value_size, tp.raw_value_size);
597 ASSERT_EQ(sum_tp.num_data_blocks, tp.num_data_blocks);
598 ASSERT_EQ(sum_tp.num_entries, tp.num_entries);
599 ASSERT_EQ(sum_tp.num_deletions, tp.num_deletions);
600 ASSERT_EQ(sum_tp.num_merge_operands, tp.num_merge_operands);
601 ASSERT_EQ(sum_tp.num_range_deletions, tp.num_range_deletions);
602 if (table > 3) {
603 GetExpectedTableProperties(
604 &expected_tp, kKeySize, kValueSize, kPutsPerTable, kDeletionsPerTable,
605 kMergeOperandsPerTable, kRangeDeletionsPerTable, table,
606 kBloomBitsPerKey, table_options.block_size, index_key_is_user_key,
607 value_is_delta_encoded);
608 // Gives larger bias here as index block size, filter block size,
609 // and data block size become much harder to estimate in this test.
610 VerifyTableProperties(expected_tp, tp, 0.5, 0.4, 0.4, 0.25);
611 }
612 }
613 }
614
TEST_F(DBPropertiesTest,NumImmutableMemTable)615 TEST_F(DBPropertiesTest, NumImmutableMemTable) {
616 do {
617 Options options = CurrentOptions();
618 WriteOptions writeOpt = WriteOptions();
619 writeOpt.disableWAL = true;
620 options.max_write_buffer_number = 4;
621 options.min_write_buffer_number_to_merge = 3;
622 options.write_buffer_size = 1000000;
623 options.max_write_buffer_size_to_maintain =
624 5 * static_cast<int64_t>(options.write_buffer_size);
625 CreateAndReopenWithCF({"pikachu"}, options);
626
627 std::string big_value(1000000 * 2, 'x');
628 std::string num;
629 uint64_t value;
630 SetPerfLevel(kEnableTime);
631 ASSERT_TRUE(GetPerfLevel() == kEnableTime);
632
633 ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k1", big_value));
634 ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
635 "rocksdb.num-immutable-mem-table", &num));
636 ASSERT_EQ(num, "0");
637 ASSERT_TRUE(dbfull()->GetProperty(
638 handles_[1], DB::Properties::kNumImmutableMemTableFlushed, &num));
639 ASSERT_EQ(num, "0");
640 ASSERT_TRUE(dbfull()->GetProperty(
641 handles_[1], "rocksdb.num-entries-active-mem-table", &num));
642 ASSERT_EQ(num, "1");
643 get_perf_context()->Reset();
644 Get(1, "k1");
645 ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
646
647 ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
648 ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
649 "rocksdb.num-immutable-mem-table", &num));
650 ASSERT_EQ(num, "1");
651 ASSERT_TRUE(dbfull()->GetProperty(
652 handles_[1], "rocksdb.num-entries-active-mem-table", &num));
653 ASSERT_EQ(num, "1");
654 ASSERT_TRUE(dbfull()->GetProperty(
655 handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
656 ASSERT_EQ(num, "1");
657
658 get_perf_context()->Reset();
659 Get(1, "k1");
660 ASSERT_EQ(2, static_cast<int>(get_perf_context()->get_from_memtable_count));
661 get_perf_context()->Reset();
662 Get(1, "k2");
663 ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
664
665 ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", big_value));
666 ASSERT_TRUE(dbfull()->GetProperty(
667 handles_[1], "rocksdb.cur-size-active-mem-table", &num));
668 ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
669 "rocksdb.num-immutable-mem-table", &num));
670 ASSERT_EQ(num, "2");
671 ASSERT_TRUE(dbfull()->GetProperty(
672 handles_[1], "rocksdb.num-entries-active-mem-table", &num));
673 ASSERT_EQ(num, "1");
674 ASSERT_TRUE(dbfull()->GetProperty(
675 handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
676 ASSERT_EQ(num, "2");
677 get_perf_context()->Reset();
678 Get(1, "k2");
679 ASSERT_EQ(2, static_cast<int>(get_perf_context()->get_from_memtable_count));
680 get_perf_context()->Reset();
681 Get(1, "k3");
682 ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
683 get_perf_context()->Reset();
684 Get(1, "k1");
685 ASSERT_EQ(3, static_cast<int>(get_perf_context()->get_from_memtable_count));
686
687 ASSERT_OK(Flush(1));
688 ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
689 "rocksdb.num-immutable-mem-table", &num));
690 ASSERT_EQ(num, "0");
691 ASSERT_TRUE(dbfull()->GetProperty(
692 handles_[1], DB::Properties::kNumImmutableMemTableFlushed, &num));
693 ASSERT_EQ(num, "3");
694 ASSERT_TRUE(dbfull()->GetIntProperty(
695 handles_[1], "rocksdb.cur-size-active-mem-table", &value));
696 // "192" is the size of the metadata of two empty skiplists, this would
697 // break if we change the default skiplist implementation
698 ASSERT_GE(value, 192);
699
700 uint64_t int_num;
701 uint64_t base_total_size;
702 ASSERT_TRUE(dbfull()->GetIntProperty(
703 handles_[1], "rocksdb.estimate-num-keys", &base_total_size));
704
705 ASSERT_OK(dbfull()->Delete(writeOpt, handles_[1], "k2"));
706 ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", ""));
707 ASSERT_OK(dbfull()->Delete(writeOpt, handles_[1], "k3"));
708 ASSERT_TRUE(dbfull()->GetIntProperty(
709 handles_[1], "rocksdb.num-deletes-active-mem-table", &int_num));
710 ASSERT_EQ(int_num, 2U);
711 ASSERT_TRUE(dbfull()->GetIntProperty(
712 handles_[1], "rocksdb.num-entries-active-mem-table", &int_num));
713 ASSERT_EQ(int_num, 3U);
714
715 ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
716 ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
717 ASSERT_TRUE(dbfull()->GetIntProperty(
718 handles_[1], "rocksdb.num-entries-imm-mem-tables", &int_num));
719 ASSERT_EQ(int_num, 4U);
720 ASSERT_TRUE(dbfull()->GetIntProperty(
721 handles_[1], "rocksdb.num-deletes-imm-mem-tables", &int_num));
722 ASSERT_EQ(int_num, 2U);
723
724 ASSERT_TRUE(dbfull()->GetIntProperty(
725 handles_[1], "rocksdb.estimate-num-keys", &int_num));
726 ASSERT_EQ(int_num, base_total_size + 1);
727
728 SetPerfLevel(kDisable);
729 ASSERT_TRUE(GetPerfLevel() == kDisable);
730 } while (ChangeCompactOptions());
731 }
732
733 // TODO(techdept) : Disabled flaky test #12863555
TEST_F(DBPropertiesTest,DISABLED_GetProperty)734 TEST_F(DBPropertiesTest, DISABLED_GetProperty) {
735 // Set sizes to both background thread pool to be 1 and block them.
736 env_->SetBackgroundThreads(1, Env::HIGH);
737 env_->SetBackgroundThreads(1, Env::LOW);
738 test::SleepingBackgroundTask sleeping_task_low;
739 env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
740 Env::Priority::LOW);
741 test::SleepingBackgroundTask sleeping_task_high;
742 env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
743 &sleeping_task_high, Env::Priority::HIGH);
744
745 Options options = CurrentOptions();
746 WriteOptions writeOpt = WriteOptions();
747 writeOpt.disableWAL = true;
748 options.compaction_style = kCompactionStyleUniversal;
749 options.level0_file_num_compaction_trigger = 1;
750 options.compaction_options_universal.size_ratio = 50;
751 options.max_background_compactions = 1;
752 options.max_background_flushes = 1;
753 options.max_write_buffer_number = 10;
754 options.min_write_buffer_number_to_merge = 1;
755 options.max_write_buffer_size_to_maintain = 0;
756 options.write_buffer_size = 1000000;
757 Reopen(options);
758
759 std::string big_value(1000000 * 2, 'x');
760 std::string num;
761 uint64_t int_num;
762 SetPerfLevel(kEnableTime);
763
764 ASSERT_TRUE(
765 dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
766 ASSERT_EQ(int_num, 0U);
767 ASSERT_TRUE(
768 dbfull()->GetIntProperty("rocksdb.estimate-live-data-size", &int_num));
769 ASSERT_EQ(int_num, 0U);
770
771 ASSERT_OK(dbfull()->Put(writeOpt, "k1", big_value));
772 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
773 ASSERT_EQ(num, "0");
774 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
775 ASSERT_EQ(num, "0");
776 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
777 ASSERT_EQ(num, "0");
778 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
779 ASSERT_EQ(num, "1");
780 get_perf_context()->Reset();
781
782 ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value));
783 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
784 ASSERT_EQ(num, "1");
785 ASSERT_OK(dbfull()->Delete(writeOpt, "k-non-existing"));
786 ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value));
787 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
788 ASSERT_EQ(num, "2");
789 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
790 ASSERT_EQ(num, "1");
791 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
792 ASSERT_EQ(num, "0");
793 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
794 ASSERT_EQ(num, "2");
795 // Verify the same set of properties through GetIntProperty
796 ASSERT_TRUE(
797 dbfull()->GetIntProperty("rocksdb.num-immutable-mem-table", &int_num));
798 ASSERT_EQ(int_num, 2U);
799 ASSERT_TRUE(
800 dbfull()->GetIntProperty("rocksdb.mem-table-flush-pending", &int_num));
801 ASSERT_EQ(int_num, 1U);
802 ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.compaction-pending", &int_num));
803 ASSERT_EQ(int_num, 0U);
804 ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &int_num));
805 ASSERT_EQ(int_num, 2U);
806
807 ASSERT_TRUE(
808 dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
809 ASSERT_EQ(int_num, 0U);
810
811 sleeping_task_high.WakeUp();
812 sleeping_task_high.WaitUntilDone();
813 dbfull()->TEST_WaitForFlushMemTable();
814
815 ASSERT_OK(dbfull()->Put(writeOpt, "k4", big_value));
816 ASSERT_OK(dbfull()->Put(writeOpt, "k5", big_value));
817 dbfull()->TEST_WaitForFlushMemTable();
818 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
819 ASSERT_EQ(num, "0");
820 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
821 ASSERT_EQ(num, "1");
822 ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
823 ASSERT_EQ(num, "4");
824
825 ASSERT_TRUE(
826 dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
827 ASSERT_GT(int_num, 0U);
828
829 sleeping_task_low.WakeUp();
830 sleeping_task_low.WaitUntilDone();
831
832 // Wait for compaction to be done. This is important because otherwise RocksDB
833 // might schedule a compaction when reopening the database, failing assertion
834 // (A) as a result.
835 ASSERT_OK(dbfull()->TEST_WaitForCompact());
836 options.max_open_files = 10;
837 Reopen(options);
838 // After reopening, no table reader is loaded, so no memory for table readers
839 ASSERT_TRUE(
840 dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
841 ASSERT_EQ(int_num, 0U); // (A)
842 ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &int_num));
843 ASSERT_GT(int_num, 0U);
844
845 // After reading a key, at least one table reader is loaded.
846 Get("k5");
847 ASSERT_TRUE(
848 dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));
849 ASSERT_GT(int_num, 0U);
850
851 // Test rocksdb.num-live-versions
852 {
853 options.level0_file_num_compaction_trigger = 20;
854 Reopen(options);
855 ASSERT_TRUE(
856 dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num));
857 ASSERT_EQ(int_num, 1U);
858
859 // Use an iterator to hold current version
860 std::unique_ptr<Iterator> iter1(dbfull()->NewIterator(ReadOptions()));
861
862 ASSERT_OK(dbfull()->Put(writeOpt, "k6", big_value));
863 ASSERT_OK(Flush());
864 ASSERT_TRUE(
865 dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num));
866 ASSERT_EQ(int_num, 2U);
867
868 // Use an iterator to hold current version
869 std::unique_ptr<Iterator> iter2(dbfull()->NewIterator(ReadOptions()));
870
871 ASSERT_OK(dbfull()->Put(writeOpt, "k7", big_value));
872 ASSERT_OK(Flush());
873 ASSERT_TRUE(
874 dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num));
875 ASSERT_EQ(int_num, 3U);
876
877 iter2.reset();
878 ASSERT_TRUE(
879 dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num));
880 ASSERT_EQ(int_num, 2U);
881
882 iter1.reset();
883 ASSERT_TRUE(
884 dbfull()->GetIntProperty("rocksdb.num-live-versions", &int_num));
885 ASSERT_EQ(int_num, 1U);
886 }
887 }
888
TEST_F(DBPropertiesTest,ApproximateMemoryUsage)889 TEST_F(DBPropertiesTest, ApproximateMemoryUsage) {
890 const int kNumRounds = 10;
891 // TODO(noetzli) kFlushesPerRound does not really correlate with how many
892 // flushes happen.
893 const int kFlushesPerRound = 10;
894 const int kWritesPerFlush = 10;
895 const int kKeySize = 100;
896 const int kValueSize = 1000;
897 Options options;
898 options.write_buffer_size = 1000; // small write buffer
899 options.min_write_buffer_number_to_merge = 4;
900 options.compression = kNoCompression;
901 options.create_if_missing = true;
902 options = CurrentOptions(options);
903 DestroyAndReopen(options);
904
905 Random rnd(301);
906
907 std::vector<Iterator*> iters;
908
909 uint64_t active_mem;
910 uint64_t unflushed_mem;
911 uint64_t all_mem;
912 uint64_t prev_all_mem;
913
914 // Phase 0. The verify the initial value of all these properties are the same
915 // as we have no mem-tables.
916 dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem);
917 dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem);
918 dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
919 ASSERT_EQ(all_mem, active_mem);
920 ASSERT_EQ(all_mem, unflushed_mem);
921
922 // Phase 1. Simply issue Put() and expect "cur-size-all-mem-tables" equals to
923 // "size-all-mem-tables"
924 for (int r = 0; r < kNumRounds; ++r) {
925 for (int f = 0; f < kFlushesPerRound; ++f) {
926 for (int w = 0; w < kWritesPerFlush; ++w) {
927 ASSERT_OK(
928 Put(rnd.RandomString(kKeySize), rnd.RandomString(kValueSize)));
929 }
930 }
931 // Make sure that there is no flush between getting the two properties.
932 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
933 dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem);
934 dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
935 // in no iterator case, these two number should be the same.
936 ASSERT_EQ(unflushed_mem, all_mem);
937 }
938 prev_all_mem = all_mem;
939
940 // Phase 2. Keep issuing Put() but also create new iterators. This time we
941 // expect "size-all-mem-tables" > "cur-size-all-mem-tables".
942 for (int r = 0; r < kNumRounds; ++r) {
943 iters.push_back(db_->NewIterator(ReadOptions()));
944 for (int f = 0; f < kFlushesPerRound; ++f) {
945 for (int w = 0; w < kWritesPerFlush; ++w) {
946 ASSERT_OK(
947 Put(rnd.RandomString(kKeySize), rnd.RandomString(kValueSize)));
948 }
949 }
950 // Force flush to prevent flush from happening between getting the
951 // properties or after getting the properties and before the new round.
952 ASSERT_OK(Flush());
953
954 // In the second round, add iterators.
955 dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem);
956 dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem);
957 dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
958 ASSERT_GT(all_mem, active_mem);
959 ASSERT_GT(all_mem, unflushed_mem);
960 ASSERT_GT(all_mem, prev_all_mem);
961 prev_all_mem = all_mem;
962 }
963
964 // Phase 3. Delete iterators and expect "size-all-mem-tables" shrinks
965 // whenever we release an iterator.
966 for (auto* iter : iters) {
967 ASSERT_OK(iter->status());
968 delete iter;
969 dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
970 // Expect the size shrinking
971 ASSERT_LT(all_mem, prev_all_mem);
972 prev_all_mem = all_mem;
973 }
974
975 // Expect all these three counters to be the same.
976 dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem);
977 dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem);
978 dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
979 ASSERT_EQ(active_mem, unflushed_mem);
980 ASSERT_EQ(unflushed_mem, all_mem);
981
982 // Phase 5. Reopen, and expect all these three counters to be the same again.
983 Reopen(options);
984 dbfull()->GetIntProperty("rocksdb.cur-size-active-mem-table", &active_mem);
985 dbfull()->GetIntProperty("rocksdb.cur-size-all-mem-tables", &unflushed_mem);
986 dbfull()->GetIntProperty("rocksdb.size-all-mem-tables", &all_mem);
987 ASSERT_EQ(active_mem, unflushed_mem);
988 ASSERT_EQ(unflushed_mem, all_mem);
989 }
990
TEST_F(DBPropertiesTest,EstimatePendingCompBytes)991 TEST_F(DBPropertiesTest, EstimatePendingCompBytes) {
992 // Set sizes to both background thread pool to be 1 and block them.
993 env_->SetBackgroundThreads(1, Env::HIGH);
994 env_->SetBackgroundThreads(1, Env::LOW);
995 test::SleepingBackgroundTask sleeping_task_low;
996 env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
997 Env::Priority::LOW);
998
999 Options options = CurrentOptions();
1000 WriteOptions writeOpt = WriteOptions();
1001 writeOpt.disableWAL = true;
1002 options.compaction_style = kCompactionStyleLevel;
1003 options.level0_file_num_compaction_trigger = 2;
1004 options.max_background_compactions = 1;
1005 options.max_background_flushes = 1;
1006 options.max_write_buffer_number = 10;
1007 options.min_write_buffer_number_to_merge = 1;
1008 options.max_write_buffer_size_to_maintain = 0;
1009 options.write_buffer_size = 1000000;
1010 Reopen(options);
1011
1012 std::string big_value(1000000 * 2, 'x');
1013 std::string num;
1014 uint64_t int_num;
1015
1016 ASSERT_OK(dbfull()->Put(writeOpt, "k1", big_value));
1017 ASSERT_OK(Flush());
1018 ASSERT_TRUE(dbfull()->GetIntProperty(
1019 "rocksdb.estimate-pending-compaction-bytes", &int_num));
1020 ASSERT_EQ(int_num, 0U);
1021
1022 ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value));
1023 ASSERT_OK(Flush());
1024 ASSERT_TRUE(dbfull()->GetIntProperty(
1025 "rocksdb.estimate-pending-compaction-bytes", &int_num));
1026 ASSERT_GT(int_num, 0U);
1027
1028 ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value));
1029 ASSERT_OK(Flush());
1030 ASSERT_TRUE(dbfull()->GetIntProperty(
1031 "rocksdb.estimate-pending-compaction-bytes", &int_num));
1032 ASSERT_GT(int_num, 0U);
1033
1034 sleeping_task_low.WakeUp();
1035 sleeping_task_low.WaitUntilDone();
1036
1037 ASSERT_OK(dbfull()->TEST_WaitForCompact());
1038 ASSERT_TRUE(dbfull()->GetIntProperty(
1039 "rocksdb.estimate-pending-compaction-bytes", &int_num));
1040 ASSERT_EQ(int_num, 0U);
1041 }
1042
TEST_F(DBPropertiesTest,EstimateCompressionRatio)1043 TEST_F(DBPropertiesTest, EstimateCompressionRatio) {
1044 if (!Snappy_Supported()) {
1045 return;
1046 }
1047 const int kNumL0Files = 3;
1048 const int kNumEntriesPerFile = 1000;
1049
1050 Options options = CurrentOptions();
1051 options.compression_per_level = {kNoCompression, kSnappyCompression};
1052 options.disable_auto_compactions = true;
1053 options.num_levels = 2;
1054 Reopen(options);
1055
1056 // compression ratio is -1.0 when no open files at level
1057 ASSERT_EQ(CompressionRatioAtLevel(0), -1.0);
1058
1059 const std::string kVal(100, 'a');
1060 for (int i = 0; i < kNumL0Files; ++i) {
1061 for (int j = 0; j < kNumEntriesPerFile; ++j) {
1062 // Put common data ("key") at end to prevent delta encoding from
1063 // compressing the key effectively
1064 std::string key = ToString(i) + ToString(j) + "key";
1065 ASSERT_OK(dbfull()->Put(WriteOptions(), key, kVal));
1066 }
1067 ASSERT_OK(Flush());
1068 }
1069
1070 // no compression at L0, so ratio is less than one
1071 ASSERT_LT(CompressionRatioAtLevel(0), 1.0);
1072 ASSERT_GT(CompressionRatioAtLevel(0), 0.0);
1073 ASSERT_EQ(CompressionRatioAtLevel(1), -1.0);
1074
1075 ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
1076
1077 ASSERT_EQ(CompressionRatioAtLevel(0), -1.0);
1078 // Data at L1 should be highly compressed thanks to Snappy and redundant data
1079 // in values (ratio is 12.846 as of 4/19/2016).
1080 ASSERT_GT(CompressionRatioAtLevel(1), 10.0);
1081 }
1082
1083 #endif // ROCKSDB_LITE
1084
1085 class CountingUserTblPropCollector : public TablePropertiesCollector {
1086 public:
Name() const1087 const char* Name() const override { return "CountingUserTblPropCollector"; }
1088
Finish(UserCollectedProperties * properties)1089 Status Finish(UserCollectedProperties* properties) override {
1090 std::string encoded;
1091 PutVarint32(&encoded, count_);
1092 *properties = UserCollectedProperties{
1093 {"CountingUserTblPropCollector", message_}, {"Count", encoded},
1094 };
1095 return Status::OK();
1096 }
1097
AddUserKey(const Slice &,const Slice &,EntryType,SequenceNumber,uint64_t)1098 Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/,
1099 EntryType /*type*/, SequenceNumber /*seq*/,
1100 uint64_t /*file_size*/) override {
1101 ++count_;
1102 return Status::OK();
1103 }
1104
GetReadableProperties() const1105 UserCollectedProperties GetReadableProperties() const override {
1106 return UserCollectedProperties{};
1107 }
1108
1109 private:
1110 std::string message_ = "Rocksdb";
1111 uint32_t count_ = 0;
1112 };
1113
1114 class CountingUserTblPropCollectorFactory
1115 : public TablePropertiesCollectorFactory {
1116 public:
CountingUserTblPropCollectorFactory(uint32_t expected_column_family_id)1117 explicit CountingUserTblPropCollectorFactory(
1118 uint32_t expected_column_family_id)
1119 : expected_column_family_id_(expected_column_family_id),
1120 num_created_(0) {}
CreateTablePropertiesCollector(TablePropertiesCollectorFactory::Context context)1121 TablePropertiesCollector* CreateTablePropertiesCollector(
1122 TablePropertiesCollectorFactory::Context context) override {
1123 EXPECT_EQ(expected_column_family_id_, context.column_family_id);
1124 num_created_++;
1125 return new CountingUserTblPropCollector();
1126 }
Name() const1127 const char* Name() const override {
1128 return "CountingUserTblPropCollectorFactory";
1129 }
set_expected_column_family_id(uint32_t v)1130 void set_expected_column_family_id(uint32_t v) {
1131 expected_column_family_id_ = v;
1132 }
1133 uint32_t expected_column_family_id_;
1134 uint32_t num_created_;
1135 };
1136
1137 class CountingDeleteTabPropCollector : public TablePropertiesCollector {
1138 public:
Name() const1139 const char* Name() const override { return "CountingDeleteTabPropCollector"; }
1140
AddUserKey(const Slice &,const Slice &,EntryType type,SequenceNumber,uint64_t)1141 Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/,
1142 EntryType type, SequenceNumber /*seq*/,
1143 uint64_t /*file_size*/) override {
1144 if (type == kEntryDelete) {
1145 num_deletes_++;
1146 }
1147 return Status::OK();
1148 }
1149
NeedCompact() const1150 bool NeedCompact() const override { return num_deletes_ > 10; }
1151
GetReadableProperties() const1152 UserCollectedProperties GetReadableProperties() const override {
1153 return UserCollectedProperties{};
1154 }
1155
Finish(UserCollectedProperties * properties)1156 Status Finish(UserCollectedProperties* properties) override {
1157 *properties =
1158 UserCollectedProperties{{"num_delete", ToString(num_deletes_)}};
1159 return Status::OK();
1160 }
1161
1162 private:
1163 uint32_t num_deletes_ = 0;
1164 };
1165
1166 class CountingDeleteTabPropCollectorFactory
1167 : public TablePropertiesCollectorFactory {
1168 public:
CreateTablePropertiesCollector(TablePropertiesCollectorFactory::Context)1169 TablePropertiesCollector* CreateTablePropertiesCollector(
1170 TablePropertiesCollectorFactory::Context /*context*/) override {
1171 return new CountingDeleteTabPropCollector();
1172 }
Name() const1173 const char* Name() const override {
1174 return "CountingDeleteTabPropCollectorFactory";
1175 }
1176 };
1177
1178 class BlockCountingTablePropertiesCollector : public TablePropertiesCollector {
1179 public:
1180 static const std::string kNumSampledBlocksPropertyName;
1181
Name() const1182 const char* Name() const override {
1183 return "BlockCountingTablePropertiesCollector";
1184 }
1185
Finish(UserCollectedProperties * properties)1186 Status Finish(UserCollectedProperties* properties) override {
1187 (*properties)[kNumSampledBlocksPropertyName] =
1188 ToString(num_sampled_blocks_);
1189 return Status::OK();
1190 }
1191
AddUserKey(const Slice &,const Slice &,EntryType,SequenceNumber,uint64_t)1192 Status AddUserKey(const Slice& /*user_key*/, const Slice& /*value*/,
1193 EntryType /*type*/, SequenceNumber /*seq*/,
1194 uint64_t /*file_size*/) override {
1195 return Status::OK();
1196 }
1197
BlockAdd(uint64_t,uint64_t block_compressed_bytes_fast,uint64_t block_compressed_bytes_slow)1198 void BlockAdd(uint64_t /* block_raw_bytes */,
1199 uint64_t block_compressed_bytes_fast,
1200 uint64_t block_compressed_bytes_slow) override {
1201 if (block_compressed_bytes_fast > 0 || block_compressed_bytes_slow > 0) {
1202 num_sampled_blocks_++;
1203 }
1204 }
1205
GetReadableProperties() const1206 UserCollectedProperties GetReadableProperties() const override {
1207 return UserCollectedProperties{
1208 {kNumSampledBlocksPropertyName, ToString(num_sampled_blocks_)},
1209 };
1210 }
1211
1212 private:
1213 uint32_t num_sampled_blocks_ = 0;
1214 };
1215
1216 const std::string
1217 BlockCountingTablePropertiesCollector::kNumSampledBlocksPropertyName =
1218 "NumSampledBlocks";
1219
1220 class BlockCountingTablePropertiesCollectorFactory
1221 : public TablePropertiesCollectorFactory {
1222 public:
Name() const1223 const char* Name() const override {
1224 return "BlockCountingTablePropertiesCollectorFactory";
1225 }
1226
CreateTablePropertiesCollector(TablePropertiesCollectorFactory::Context)1227 TablePropertiesCollector* CreateTablePropertiesCollector(
1228 TablePropertiesCollectorFactory::Context /* context */) override {
1229 return new BlockCountingTablePropertiesCollector();
1230 }
1231 };
1232
1233 #ifndef ROCKSDB_LITE
TEST_F(DBPropertiesTest,GetUserDefinedTableProperties)1234 TEST_F(DBPropertiesTest, GetUserDefinedTableProperties) {
1235 Options options = CurrentOptions();
1236 options.level0_file_num_compaction_trigger = (1 << 30);
1237 options.table_properties_collector_factories.resize(1);
1238 std::shared_ptr<CountingUserTblPropCollectorFactory> collector_factory =
1239 std::make_shared<CountingUserTblPropCollectorFactory>(0);
1240 options.table_properties_collector_factories[0] = collector_factory;
1241 Reopen(options);
1242 // Create 4 tables
1243 for (int table = 0; table < 4; ++table) {
1244 for (int i = 0; i < 10 + table; ++i) {
1245 ASSERT_OK(db_->Put(WriteOptions(), ToString(table * 100 + i), "val"));
1246 }
1247 ASSERT_OK(db_->Flush(FlushOptions()));
1248 }
1249
1250 TablePropertiesCollection props;
1251 ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
1252 ASSERT_EQ(4U, props.size());
1253 uint32_t sum = 0;
1254 for (const auto& item : props) {
1255 auto& user_collected = item.second->user_collected_properties;
1256 ASSERT_TRUE(user_collected.find("CountingUserTblPropCollector") !=
1257 user_collected.end());
1258 ASSERT_EQ(user_collected.at("CountingUserTblPropCollector"), "Rocksdb");
1259 ASSERT_TRUE(user_collected.find("Count") != user_collected.end());
1260 Slice key(user_collected.at("Count"));
1261 uint32_t count;
1262 ASSERT_TRUE(GetVarint32(&key, &count));
1263 sum += count;
1264 }
1265 ASSERT_EQ(10u + 11u + 12u + 13u, sum);
1266
1267 ASSERT_GT(collector_factory->num_created_, 0U);
1268 collector_factory->num_created_ = 0;
1269 ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
1270 ASSERT_GT(collector_factory->num_created_, 0U);
1271 }
1272 #endif // ROCKSDB_LITE
1273
TEST_F(DBPropertiesTest,UserDefinedTablePropertiesContext)1274 TEST_F(DBPropertiesTest, UserDefinedTablePropertiesContext) {
1275 Options options = CurrentOptions();
1276 options.level0_file_num_compaction_trigger = 3;
1277 options.table_properties_collector_factories.resize(1);
1278 std::shared_ptr<CountingUserTblPropCollectorFactory> collector_factory =
1279 std::make_shared<CountingUserTblPropCollectorFactory>(1);
1280 options.table_properties_collector_factories[0] = collector_factory,
1281 CreateAndReopenWithCF({"pikachu"}, options);
1282 // Create 2 files
1283 for (int table = 0; table < 2; ++table) {
1284 for (int i = 0; i < 10 + table; ++i) {
1285 ASSERT_OK(Put(1, ToString(table * 100 + i), "val"));
1286 }
1287 ASSERT_OK(Flush(1));
1288 }
1289 ASSERT_GT(collector_factory->num_created_, 0U);
1290
1291 collector_factory->num_created_ = 0;
1292 // Trigger automatic compactions.
1293 for (int table = 0; table < 3; ++table) {
1294 for (int i = 0; i < 10 + table; ++i) {
1295 ASSERT_OK(Put(1, ToString(table * 100 + i), "val"));
1296 }
1297 ASSERT_OK(Flush(1));
1298 ASSERT_OK(dbfull()->TEST_WaitForCompact());
1299 }
1300 ASSERT_GT(collector_factory->num_created_, 0U);
1301
1302 collector_factory->num_created_ = 0;
1303 ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]));
1304 ASSERT_GT(collector_factory->num_created_, 0U);
1305
1306 // Come back to write to default column family
1307 collector_factory->num_created_ = 0;
1308 collector_factory->set_expected_column_family_id(0); // default CF
1309 // Create 4 tables in default column family
1310 for (int table = 0; table < 2; ++table) {
1311 for (int i = 0; i < 10 + table; ++i) {
1312 ASSERT_OK(Put(ToString(table * 100 + i), "val"));
1313 }
1314 ASSERT_OK(Flush());
1315 }
1316 ASSERT_GT(collector_factory->num_created_, 0U);
1317
1318 collector_factory->num_created_ = 0;
1319 // Trigger automatic compactions.
1320 for (int table = 0; table < 3; ++table) {
1321 for (int i = 0; i < 10 + table; ++i) {
1322 ASSERT_OK(Put(ToString(table * 100 + i), "val"));
1323 }
1324 ASSERT_OK(Flush());
1325 ASSERT_OK(dbfull()->TEST_WaitForCompact());
1326 }
1327 ASSERT_GT(collector_factory->num_created_, 0U);
1328
1329 collector_factory->num_created_ = 0;
1330 ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
1331 ASSERT_GT(collector_factory->num_created_, 0U);
1332 }
1333
1334 #ifndef ROCKSDB_LITE
TEST_F(DBPropertiesTest,TablePropertiesNeedCompactTest)1335 TEST_F(DBPropertiesTest, TablePropertiesNeedCompactTest) {
1336 Random rnd(301);
1337
1338 Options options;
1339 options.create_if_missing = true;
1340 options.write_buffer_size = 4096;
1341 options.max_write_buffer_number = 8;
1342 options.level0_file_num_compaction_trigger = 2;
1343 options.level0_slowdown_writes_trigger = 2;
1344 options.level0_stop_writes_trigger = 4;
1345 options.target_file_size_base = 2048;
1346 options.max_bytes_for_level_base = 10240;
1347 options.max_bytes_for_level_multiplier = 4;
1348 options.soft_pending_compaction_bytes_limit = 1024 * 1024;
1349 options.num_levels = 8;
1350 options.env = env_;
1351
1352 std::shared_ptr<TablePropertiesCollectorFactory> collector_factory =
1353 std::make_shared<CountingDeleteTabPropCollectorFactory>();
1354 options.table_properties_collector_factories.resize(1);
1355 options.table_properties_collector_factories[0] = collector_factory;
1356
1357 DestroyAndReopen(options);
1358
1359 const int kMaxKey = 1000;
1360 for (int i = 0; i < kMaxKey; i++) {
1361 ASSERT_OK(Put(Key(i), rnd.RandomString(102)));
1362 ASSERT_OK(Put(Key(kMaxKey + i), rnd.RandomString(102)));
1363 }
1364 ASSERT_OK(Flush());
1365 ASSERT_OK(dbfull()->TEST_WaitForCompact());
1366 if (NumTableFilesAtLevel(0) == 1) {
1367 // Clear Level 0 so that when later flush a file with deletions,
1368 // we don't trigger an organic compaction.
1369 ASSERT_OK(Put(Key(0), ""));
1370 ASSERT_OK(Put(Key(kMaxKey * 2), ""));
1371 ASSERT_OK(Flush());
1372 ASSERT_OK(dbfull()->TEST_WaitForCompact());
1373 }
1374 ASSERT_EQ(NumTableFilesAtLevel(0), 0);
1375
1376 {
1377 int c = 0;
1378 std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
1379 iter->Seek(Key(kMaxKey - 100));
1380 while (iter->Valid() && iter->key().compare(Key(kMaxKey + 100)) < 0) {
1381 iter->Next();
1382 ++c;
1383 }
1384 ASSERT_OK(iter->status());
1385 ASSERT_EQ(c, 200);
1386 }
1387
1388 ASSERT_OK(Delete(Key(0)));
1389 for (int i = kMaxKey - 100; i < kMaxKey + 100; i++) {
1390 ASSERT_OK(Delete(Key(i)));
1391 }
1392 ASSERT_OK(Delete(Key(kMaxKey * 2)));
1393
1394 ASSERT_OK(Flush());
1395 ASSERT_OK(dbfull()->TEST_WaitForCompact());
1396
1397 {
1398 SetPerfLevel(kEnableCount);
1399 get_perf_context()->Reset();
1400 int c = 0;
1401 std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
1402 iter->Seek(Key(kMaxKey - 100));
1403 while (iter->Valid() && iter->key().compare(Key(kMaxKey + 100)) < 0) {
1404 iter->Next();
1405 }
1406 ASSERT_OK(iter->status());
1407 ASSERT_EQ(c, 0);
1408 ASSERT_LT(get_perf_context()->internal_delete_skipped_count, 30u);
1409 ASSERT_LT(get_perf_context()->internal_key_skipped_count, 30u);
1410 SetPerfLevel(kDisable);
1411 }
1412 }
1413
TEST_F(DBPropertiesTest,NeedCompactHintPersistentTest)1414 TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
1415 Random rnd(301);
1416
1417 Options options;
1418 options.create_if_missing = true;
1419 options.max_write_buffer_number = 8;
1420 options.level0_file_num_compaction_trigger = 10;
1421 options.level0_slowdown_writes_trigger = 10;
1422 options.level0_stop_writes_trigger = 10;
1423 options.disable_auto_compactions = true;
1424 options.env = env_;
1425
1426 std::shared_ptr<TablePropertiesCollectorFactory> collector_factory =
1427 std::make_shared<CountingDeleteTabPropCollectorFactory>();
1428 options.table_properties_collector_factories.resize(1);
1429 options.table_properties_collector_factories[0] = collector_factory;
1430
1431 DestroyAndReopen(options);
1432
1433 const int kMaxKey = 100;
1434 for (int i = 0; i < kMaxKey; i++) {
1435 ASSERT_OK(Put(Key(i), ""));
1436 }
1437 ASSERT_OK(Flush());
1438 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
1439
1440 for (int i = 1; i < kMaxKey - 1; i++) {
1441 ASSERT_OK(Delete(Key(i)));
1442 }
1443 ASSERT_OK(Flush());
1444 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
1445 ASSERT_EQ(NumTableFilesAtLevel(0), 2);
1446
1447 // Restart the DB. Although number of files didn't reach
1448 // options.level0_file_num_compaction_trigger, compaction should
1449 // still be triggered because of the need-compaction hint.
1450 options.disable_auto_compactions = false;
1451 Reopen(options);
1452 ASSERT_OK(dbfull()->TEST_WaitForCompact());
1453 ASSERT_EQ(NumTableFilesAtLevel(0), 0);
1454 {
1455 SetPerfLevel(kEnableCount);
1456 get_perf_context()->Reset();
1457 int c = 0;
1458 std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
1459 for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) {
1460 c++;
1461 }
1462 ASSERT_OK(iter->status());
1463 ASSERT_EQ(c, 2);
1464 ASSERT_EQ(get_perf_context()->internal_delete_skipped_count, 0);
1465 // We iterate every key twice. Is it a bug?
1466 ASSERT_LE(get_perf_context()->internal_key_skipped_count, 2);
1467 SetPerfLevel(kDisable);
1468 }
1469 }
1470
1471 // Excluded from RocksDB lite tests due to `GetPropertiesOfAllTables()` usage.
TEST_F(DBPropertiesTest,BlockAddForCompressionSampling)1472 TEST_F(DBPropertiesTest, BlockAddForCompressionSampling) {
1473 // Sampled compression requires at least one of the following four types.
1474 if (!Snappy_Supported() && !Zlib_Supported() && !LZ4_Supported() &&
1475 !ZSTD_Supported()) {
1476 return;
1477 }
1478
1479 Options options = CurrentOptions();
1480 options.disable_auto_compactions = true;
1481 options.table_properties_collector_factories.emplace_back(
1482 std::make_shared<BlockCountingTablePropertiesCollectorFactory>());
1483
1484 for (bool sample_for_compression : {false, true}) {
1485 // For simplicity/determinism, sample 100% when enabled, or 0% when disabled
1486 options.sample_for_compression = sample_for_compression ? 1 : 0;
1487
1488 DestroyAndReopen(options);
1489
1490 // Setup the following LSM:
1491 //
1492 // L0_0 ["a", "b"]
1493 // L1_0 ["a", "b"]
1494 //
1495 // L0_0 was created by flush. L1_0 was created by compaction. Each file
1496 // contains one data block.
1497 for (int i = 0; i < 3; ++i) {
1498 ASSERT_OK(Put("a", "val"));
1499 ASSERT_OK(Put("b", "val"));
1500 ASSERT_OK(Flush());
1501 if (i == 1) {
1502 ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
1503 }
1504 }
1505
1506 // A `BlockAdd()` should have been seen for files generated by flush or
1507 // compaction when `sample_for_compression` is enabled.
1508 TablePropertiesCollection file_to_props;
1509 ASSERT_OK(db_->GetPropertiesOfAllTables(&file_to_props));
1510 ASSERT_EQ(2, file_to_props.size());
1511 for (const auto& file_and_props : file_to_props) {
1512 auto& user_props = file_and_props.second->user_collected_properties;
1513 ASSERT_TRUE(user_props.find(BlockCountingTablePropertiesCollector::
1514 kNumSampledBlocksPropertyName) !=
1515 user_props.end());
1516 ASSERT_EQ(user_props.at(BlockCountingTablePropertiesCollector::
1517 kNumSampledBlocksPropertyName),
1518 ToString(sample_for_compression ? 1 : 0));
1519 }
1520 }
1521 }
1522
1523 class CompressionSamplingDBPropertiesTest
1524 : public DBPropertiesTest,
1525 public ::testing::WithParamInterface<bool> {
1526 public:
CompressionSamplingDBPropertiesTest()1527 CompressionSamplingDBPropertiesTest() : fast_(GetParam()) {}
1528
1529 protected:
1530 const bool fast_;
1531 };
1532
1533 INSTANTIATE_TEST_CASE_P(CompressionSamplingDBPropertiesTest,
1534 CompressionSamplingDBPropertiesTest, ::testing::Bool());
1535
1536 // Excluded from RocksDB lite tests due to `GetPropertiesOfAllTables()` usage.
TEST_P(CompressionSamplingDBPropertiesTest,EstimateDataSizeWithCompressionSampling)1537 TEST_P(CompressionSamplingDBPropertiesTest,
1538 EstimateDataSizeWithCompressionSampling) {
1539 Options options = CurrentOptions();
1540 if (fast_) {
1541 // One of the following light compression libraries must be present.
1542 if (LZ4_Supported()) {
1543 options.compression = kLZ4Compression;
1544 } else if (Snappy_Supported()) {
1545 options.compression = kSnappyCompression;
1546 } else {
1547 return;
1548 }
1549 } else {
1550 // One of the following heavy compression libraries must be present.
1551 if (ZSTD_Supported()) {
1552 options.compression = kZSTD;
1553 } else if (Zlib_Supported()) {
1554 options.compression = kZlibCompression;
1555 } else {
1556 return;
1557 }
1558 }
1559 options.disable_auto_compactions = true;
1560 // For simplicity/determinism, sample 100%.
1561 options.sample_for_compression = 1;
1562 Reopen(options);
1563
1564 // Setup the following LSM:
1565 //
1566 // L0_0 ["a", "b"]
1567 // L1_0 ["a", "b"]
1568 //
1569 // L0_0 was created by flush. L1_0 was created by compaction. Each file
1570 // contains one data block. The value consists of compressible data so the
1571 // data block should be stored compressed.
1572 std::string val(1024, 'a');
1573 for (int i = 0; i < 3; ++i) {
1574 ASSERT_OK(Put("a", val));
1575 ASSERT_OK(Put("b", val));
1576 ASSERT_OK(Flush());
1577 if (i == 1) {
1578 ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
1579 }
1580 }
1581
1582 TablePropertiesCollection file_to_props;
1583 ASSERT_OK(db_->GetPropertiesOfAllTables(&file_to_props));
1584 ASSERT_EQ(2, file_to_props.size());
1585 for (const auto& file_and_props : file_to_props) {
1586 ASSERT_GT(file_and_props.second->data_size, 0);
1587 if (fast_) {
1588 ASSERT_EQ(file_and_props.second->data_size,
1589 file_and_props.second->fast_compression_estimated_data_size);
1590 } else {
1591 ASSERT_EQ(file_and_props.second->data_size,
1592 file_and_props.second->slow_compression_estimated_data_size);
1593 }
1594 }
1595 }
1596
TEST_F(DBPropertiesTest,EstimateNumKeysUnderflow)1597 TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) {
1598 Options options = CurrentOptions();
1599 Reopen(options);
1600 ASSERT_OK(Put("foo", "bar"));
1601 ASSERT_OK(Delete("foo"));
1602 ASSERT_OK(Delete("foo"));
1603 uint64_t num_keys = 0;
1604 ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &num_keys));
1605 ASSERT_EQ(0, num_keys);
1606 }
1607
TEST_F(DBPropertiesTest,EstimateOldestKeyTime)1608 TEST_F(DBPropertiesTest, EstimateOldestKeyTime) {
1609 uint64_t oldest_key_time = 0;
1610 Options options = CurrentOptions();
1611 SetTimeElapseOnlySleepOnReopen(&options);
1612
1613 // "rocksdb.estimate-oldest-key-time" only available to fifo compaction.
1614 for (auto compaction : {kCompactionStyleLevel, kCompactionStyleUniversal,
1615 kCompactionStyleNone}) {
1616 options.compaction_style = compaction;
1617 options.create_if_missing = true;
1618 DestroyAndReopen(options);
1619 ASSERT_OK(Put("foo", "bar"));
1620 ASSERT_FALSE(dbfull()->GetIntProperty(
1621 DB::Properties::kEstimateOldestKeyTime, &oldest_key_time));
1622 }
1623
1624 int64_t mock_start_time;
1625 ASSERT_OK(env_->GetCurrentTime(&mock_start_time));
1626
1627 options.compaction_style = kCompactionStyleFIFO;
1628 options.ttl = 300;
1629 options.max_open_files = -1;
1630 options.compaction_options_fifo.allow_compaction = false;
1631 DestroyAndReopen(options);
1632
1633 env_->MockSleepForSeconds(100);
1634 ASSERT_OK(Put("k1", "v1"));
1635 ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
1636 &oldest_key_time));
1637 ASSERT_EQ(100, oldest_key_time - mock_start_time);
1638 ASSERT_OK(Flush());
1639 ASSERT_EQ("1", FilesPerLevel());
1640 ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
1641 &oldest_key_time));
1642 ASSERT_EQ(100, oldest_key_time - mock_start_time);
1643
1644 env_->MockSleepForSeconds(100); // -> 200
1645 ASSERT_OK(Put("k2", "v2"));
1646 ASSERT_OK(Flush());
1647 ASSERT_EQ("2", FilesPerLevel());
1648 ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
1649 &oldest_key_time));
1650 ASSERT_EQ(100, oldest_key_time - mock_start_time);
1651
1652 env_->MockSleepForSeconds(100); // -> 300
1653 ASSERT_OK(Put("k3", "v3"));
1654 ASSERT_OK(Flush());
1655 ASSERT_EQ("3", FilesPerLevel());
1656 ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
1657 &oldest_key_time));
1658 ASSERT_EQ(100, oldest_key_time - mock_start_time);
1659
1660 env_->MockSleepForSeconds(150); // -> 450
1661 ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
1662 ASSERT_EQ("2", FilesPerLevel());
1663 ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
1664 &oldest_key_time));
1665 ASSERT_EQ(200, oldest_key_time - mock_start_time);
1666
1667 env_->MockSleepForSeconds(100); // -> 550
1668 ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
1669 ASSERT_EQ("1", FilesPerLevel());
1670 ASSERT_TRUE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
1671 &oldest_key_time));
1672 ASSERT_EQ(300, oldest_key_time - mock_start_time);
1673
1674 env_->MockSleepForSeconds(100); // -> 650
1675 ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
1676 ASSERT_EQ("", FilesPerLevel());
1677 ASSERT_FALSE(dbfull()->GetIntProperty(DB::Properties::kEstimateOldestKeyTime,
1678 &oldest_key_time));
1679 }
1680
TEST_F(DBPropertiesTest,SstFilesSize)1681 TEST_F(DBPropertiesTest, SstFilesSize) {
1682 struct TestListener : public EventListener {
1683 void OnCompactionCompleted(DB* db,
1684 const CompactionJobInfo& /*info*/) override {
1685 assert(callback_triggered == false);
1686 assert(size_before_compaction > 0);
1687 callback_triggered = true;
1688 uint64_t total_sst_size = 0;
1689 uint64_t live_sst_size = 0;
1690 bool ok = db->GetIntProperty(DB::Properties::kTotalSstFilesSize,
1691 &total_sst_size);
1692 ASSERT_TRUE(ok);
1693 // total_sst_size include files before and after compaction.
1694 ASSERT_GT(total_sst_size, size_before_compaction);
1695 ok =
1696 db->GetIntProperty(DB::Properties::kLiveSstFilesSize, &live_sst_size);
1697 ASSERT_TRUE(ok);
1698 // live_sst_size only include files after compaction.
1699 ASSERT_GT(live_sst_size, 0);
1700 ASSERT_LT(live_sst_size, size_before_compaction);
1701 }
1702
1703 uint64_t size_before_compaction = 0;
1704 bool callback_triggered = false;
1705 };
1706 std::shared_ptr<TestListener> listener = std::make_shared<TestListener>();
1707
1708 Options options;
1709 options.env = CurrentOptions().env;
1710 options.disable_auto_compactions = true;
1711 options.listeners.push_back(listener);
1712 Reopen(options);
1713
1714 for (int i = 0; i < 10; i++) {
1715 ASSERT_OK(Put("key" + ToString(i), std::string(1000, 'v')));
1716 }
1717 ASSERT_OK(Flush());
1718 for (int i = 0; i < 5; i++) {
1719 ASSERT_OK(Delete("key" + ToString(i)));
1720 }
1721 ASSERT_OK(Flush());
1722 uint64_t sst_size;
1723 bool ok = db_->GetIntProperty(DB::Properties::kTotalSstFilesSize, &sst_size);
1724 ASSERT_TRUE(ok);
1725 ASSERT_GT(sst_size, 0);
1726 listener->size_before_compaction = sst_size;
1727 // Compact to clean all keys and trigger listener.
1728 ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
1729 ASSERT_TRUE(listener->callback_triggered);
1730 }
1731
TEST_F(DBPropertiesTest,MinObsoleteSstNumberToKeep)1732 TEST_F(DBPropertiesTest, MinObsoleteSstNumberToKeep) {
1733 class TestListener : public EventListener {
1734 public:
1735 void OnTableFileCreated(const TableFileCreationInfo& info) override {
1736 if (info.reason == TableFileCreationReason::kCompaction) {
1737 // Verify the property indicates that SSTs created by a running
1738 // compaction cannot be deleted.
1739 uint64_t created_file_num;
1740 FileType created_file_type;
1741 std::string filename =
1742 info.file_path.substr(info.file_path.rfind('/') + 1);
1743 ASSERT_TRUE(
1744 ParseFileName(filename, &created_file_num, &created_file_type));
1745 ASSERT_EQ(kTableFile, created_file_type);
1746
1747 uint64_t keep_sst_lower_bound;
1748 ASSERT_TRUE(
1749 db_->GetIntProperty(DB::Properties::kMinObsoleteSstNumberToKeep,
1750 &keep_sst_lower_bound));
1751
1752 ASSERT_LE(keep_sst_lower_bound, created_file_num);
1753 validated_ = true;
1754 }
1755 }
1756
1757 void SetDB(DB* db) { db_ = db; }
1758
1759 int GetNumCompactions() { return num_compactions_; }
1760
1761 // True if we've verified the property for at least one output file
1762 bool Validated() { return validated_; }
1763
1764 private:
1765 int num_compactions_ = 0;
1766 bool validated_ = false;
1767 DB* db_ = nullptr;
1768 };
1769
1770 const int kNumL0Files = 4;
1771
1772 std::shared_ptr<TestListener> listener = std::make_shared<TestListener>();
1773
1774 Options options = CurrentOptions();
1775 options.listeners.push_back(listener);
1776 options.level0_file_num_compaction_trigger = kNumL0Files;
1777 DestroyAndReopen(options);
1778 listener->SetDB(db_);
1779
1780 for (int i = 0; i < kNumL0Files; ++i) {
1781 // Make sure they overlap in keyspace to prevent trivial move
1782 ASSERT_OK(Put("key1", "val"));
1783 ASSERT_OK(Put("key2", "val"));
1784 ASSERT_OK(Flush());
1785 }
1786 ASSERT_OK(dbfull()->TEST_WaitForCompact());
1787 ASSERT_TRUE(listener->Validated());
1788 }
1789
TEST_F(DBPropertiesTest,BlockCacheProperties)1790 TEST_F(DBPropertiesTest, BlockCacheProperties) {
1791 Options options;
1792 uint64_t value;
1793
1794 options.env = CurrentOptions().env;
1795
1796 // Block cache properties are not available for tables other than
1797 // block-based table.
1798 options.table_factory.reset(NewPlainTableFactory());
1799 Reopen(options);
1800 ASSERT_FALSE(
1801 db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
1802 ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
1803 ASSERT_FALSE(
1804 db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
1805
1806 options.table_factory.reset(NewCuckooTableFactory());
1807 Reopen(options);
1808 ASSERT_FALSE(
1809 db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
1810 ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
1811 ASSERT_FALSE(
1812 db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
1813
1814 // Block cache properties are not available if block cache is not used.
1815 BlockBasedTableOptions table_options;
1816 table_options.no_block_cache = true;
1817 options.table_factory.reset(NewBlockBasedTableFactory(table_options));
1818 Reopen(options);
1819 ASSERT_FALSE(
1820 db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
1821 ASSERT_FALSE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
1822 ASSERT_FALSE(
1823 db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
1824
1825 // Test with empty block cache.
1826 constexpr size_t kCapacity = 100;
1827 LRUCacheOptions co;
1828 co.capacity = kCapacity;
1829 co.num_shard_bits = 0;
1830 co.metadata_charge_policy = kDontChargeCacheMetadata;
1831 auto block_cache = NewLRUCache(co);
1832 table_options.block_cache = block_cache;
1833 table_options.no_block_cache = false;
1834 options.table_factory.reset(NewBlockBasedTableFactory(table_options));
1835 Reopen(options);
1836 ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
1837 ASSERT_EQ(kCapacity, value);
1838 ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
1839 ASSERT_EQ(0, value);
1840 ASSERT_TRUE(
1841 db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
1842 ASSERT_EQ(0, value);
1843
1844 // Insert unpinned item to the cache and check size.
1845 constexpr size_t kSize1 = 50;
1846 ASSERT_OK(block_cache->Insert("item1", nullptr /*value*/, kSize1,
1847 nullptr /*deleter*/));
1848 ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
1849 ASSERT_EQ(kCapacity, value);
1850 ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
1851 ASSERT_EQ(kSize1, value);
1852 ASSERT_TRUE(
1853 db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
1854 ASSERT_EQ(0, value);
1855
1856 // Insert pinned item to the cache and check size.
1857 constexpr size_t kSize2 = 30;
1858 Cache::Handle* item2 = nullptr;
1859 ASSERT_OK(block_cache->Insert("item2", nullptr /*value*/, kSize2,
1860 nullptr /*deleter*/, &item2));
1861 ASSERT_NE(nullptr, item2);
1862 ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
1863 ASSERT_EQ(kCapacity, value);
1864 ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
1865 ASSERT_EQ(kSize1 + kSize2, value);
1866 ASSERT_TRUE(
1867 db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
1868 ASSERT_EQ(kSize2, value);
1869
1870 // Insert another pinned item to make the cache over-sized.
1871 constexpr size_t kSize3 = 80;
1872 Cache::Handle* item3 = nullptr;
1873 ASSERT_OK(block_cache->Insert("item3", nullptr /*value*/, kSize3,
1874 nullptr /*deleter*/, &item3));
1875 ASSERT_NE(nullptr, item2);
1876 ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
1877 ASSERT_EQ(kCapacity, value);
1878 ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
1879 // Item 1 is evicted.
1880 ASSERT_EQ(kSize2 + kSize3, value);
1881 ASSERT_TRUE(
1882 db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
1883 ASSERT_EQ(kSize2 + kSize3, value);
1884
1885 // Check size after release.
1886 block_cache->Release(item2);
1887 block_cache->Release(item3);
1888 ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
1889 ASSERT_EQ(kCapacity, value);
1890 ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
1891 // item2 will be evicted, while item3 remain in cache after release.
1892 ASSERT_EQ(kSize3, value);
1893 ASSERT_TRUE(
1894 db_->GetIntProperty(DB::Properties::kBlockCachePinnedUsage, &value));
1895 ASSERT_EQ(0, value);
1896 }
1897
1898 #endif // ROCKSDB_LITE
1899 } // namespace ROCKSDB_NAMESPACE
1900
main(int argc,char ** argv)1901 int main(int argc, char** argv) {
1902 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
1903 ::testing::InitGoogleTest(&argc, argv);
1904 return RUN_ALL_TESTS();
1905 }
1906