1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #ifndef ROCKSDB_LITE
11
12 #include <algorithm>
13 #include <set>
14
15 #include "db/db_impl/db_impl.h"
16 #include "db/version_set.h"
17 #include "db/write_batch_internal.h"
18 #include "file/filename.h"
19 #include "logging/logging.h"
20 #include "rocksdb/cache.h"
21 #include "rocksdb/compaction_filter.h"
22 #include "rocksdb/db.h"
23 #include "rocksdb/env.h"
24 #include "rocksdb/filter_policy.h"
25 #include "rocksdb/slice_transform.h"
26 #include "rocksdb/table.h"
27 #include "table/meta_blocks.h"
28 #include "table/plain/plain_table_bloom.h"
29 #include "table/plain/plain_table_factory.h"
30 #include "table/plain/plain_table_key_coding.h"
31 #include "table/plain/plain_table_reader.h"
32 #include "table/table_builder.h"
33 #include "test_util/testharness.h"
34 #include "test_util/testutil.h"
35 #include "util/hash.h"
36 #include "util/mutexlock.h"
37 #include "util/string_util.h"
38 #include "utilities/merge_operators.h"
39
40 using std::unique_ptr;
41
42 namespace rocksdb {
43 class PlainTableKeyDecoderTest : public testing::Test {};
44
TEST_F(PlainTableKeyDecoderTest,ReadNonMmap)45 TEST_F(PlainTableKeyDecoderTest, ReadNonMmap) {
46 std::string tmp;
47 Random rnd(301);
48 const uint32_t kLength = 2222;
49 Slice contents = test::RandomString(&rnd, kLength, &tmp);
50 test::StringSource* string_source =
51 new test::StringSource(contents, 0, false);
52
53 std::unique_ptr<RandomAccessFileReader> file_reader(
54 test::GetRandomAccessFileReader(string_source));
55 std::unique_ptr<PlainTableReaderFileInfo> file_info(
56 new PlainTableReaderFileInfo(std::move(file_reader), EnvOptions(),
57 kLength));
58
59 {
60 PlainTableFileReader reader(file_info.get());
61
62 const uint32_t kReadSize = 77;
63 for (uint32_t pos = 0; pos < kLength; pos += kReadSize) {
64 uint32_t read_size = std::min(kLength - pos, kReadSize);
65 Slice out;
66 ASSERT_TRUE(reader.Read(pos, read_size, &out));
67 ASSERT_EQ(0, out.compare(tmp.substr(pos, read_size)));
68 }
69
70 ASSERT_LT(uint32_t(string_source->total_reads()), kLength / kReadSize / 2);
71 }
72
73 std::vector<std::vector<std::pair<uint32_t, uint32_t>>> reads = {
74 {{600, 30}, {590, 30}, {600, 20}, {600, 40}},
75 {{800, 20}, {100, 20}, {500, 20}, {1500, 20}, {100, 20}, {80, 20}},
76 {{1000, 20}, {500, 20}, {1000, 50}},
77 {{1000, 20}, {500, 20}, {500, 20}},
78 {{1000, 20}, {500, 20}, {200, 20}, {500, 20}},
79 {{1000, 20}, {500, 20}, {200, 20}, {1000, 50}},
80 {{600, 500}, {610, 20}, {100, 20}},
81 {{500, 100}, {490, 100}, {550, 50}},
82 };
83
84 std::vector<int> num_file_reads = {2, 6, 2, 2, 4, 3, 2, 2};
85
86 for (size_t i = 0; i < reads.size(); i++) {
87 string_source->set_total_reads(0);
88 PlainTableFileReader reader(file_info.get());
89 for (auto p : reads[i]) {
90 Slice out;
91 ASSERT_TRUE(reader.Read(p.first, p.second, &out));
92 ASSERT_EQ(0, out.compare(tmp.substr(p.first, p.second)));
93 }
94 ASSERT_EQ(num_file_reads[i], string_source->total_reads());
95 }
96 }
97
98 class PlainTableDBTest : public testing::Test,
99 public testing::WithParamInterface<bool> {
100 protected:
101 private:
102 std::string dbname_;
103 Env* env_;
104 DB* db_;
105
106 bool mmap_mode_;
107 Options last_options_;
108
109 public:
PlainTableDBTest()110 PlainTableDBTest() : env_(Env::Default()) {}
111
~PlainTableDBTest()112 ~PlainTableDBTest() override {
113 delete db_;
114 EXPECT_OK(DestroyDB(dbname_, Options()));
115 }
116
SetUp()117 void SetUp() override {
118 mmap_mode_ = GetParam();
119 dbname_ = test::PerThreadDBPath("plain_table_db_test");
120 EXPECT_OK(DestroyDB(dbname_, Options()));
121 db_ = nullptr;
122 Reopen();
123 }
124
125 // Return the current option configuration.
CurrentOptions()126 Options CurrentOptions() {
127 Options options;
128
129 PlainTableOptions plain_table_options;
130 plain_table_options.user_key_len = 0;
131 plain_table_options.bloom_bits_per_key = 2;
132 plain_table_options.hash_table_ratio = 0.8;
133 plain_table_options.index_sparseness = 3;
134 plain_table_options.huge_page_tlb_size = 0;
135 plain_table_options.encoding_type = kPrefix;
136 plain_table_options.full_scan_mode = false;
137 plain_table_options.store_index_in_file = false;
138
139 options.table_factory.reset(NewPlainTableFactory(plain_table_options));
140 options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true));
141
142 options.prefix_extractor.reset(NewFixedPrefixTransform(8));
143 options.allow_mmap_reads = mmap_mode_;
144 options.allow_concurrent_memtable_write = false;
145 options.unordered_write = false;
146 return options;
147 }
148
dbfull()149 DBImpl* dbfull() {
150 return reinterpret_cast<DBImpl*>(db_);
151 }
152
Reopen(Options * options=nullptr)153 void Reopen(Options* options = nullptr) {
154 ASSERT_OK(TryReopen(options));
155 }
156
Close()157 void Close() {
158 delete db_;
159 db_ = nullptr;
160 }
161
mmap_mode() const162 bool mmap_mode() const { return mmap_mode_; }
163
DestroyAndReopen(Options * options=nullptr)164 void DestroyAndReopen(Options* options = nullptr) {
165 //Destroy using last options
166 Destroy(&last_options_);
167 ASSERT_OK(TryReopen(options));
168 }
169
Destroy(Options * options)170 void Destroy(Options* options) {
171 delete db_;
172 db_ = nullptr;
173 ASSERT_OK(DestroyDB(dbname_, *options));
174 }
175
PureReopen(Options * options,DB ** db)176 Status PureReopen(Options* options, DB** db) {
177 return DB::Open(*options, dbname_, db);
178 }
179
ReopenForReadOnly(Options * options)180 Status ReopenForReadOnly(Options* options) {
181 delete db_;
182 db_ = nullptr;
183 return DB::OpenForReadOnly(*options, dbname_, &db_);
184 }
185
TryReopen(Options * options=nullptr)186 Status TryReopen(Options* options = nullptr) {
187 delete db_;
188 db_ = nullptr;
189 Options opts;
190 if (options != nullptr) {
191 opts = *options;
192 } else {
193 opts = CurrentOptions();
194 opts.create_if_missing = true;
195 }
196 last_options_ = opts;
197
198 return DB::Open(opts, dbname_, &db_);
199 }
200
Put(const Slice & k,const Slice & v)201 Status Put(const Slice& k, const Slice& v) {
202 return db_->Put(WriteOptions(), k, v);
203 }
204
Delete(const std::string & k)205 Status Delete(const std::string& k) {
206 return db_->Delete(WriteOptions(), k);
207 }
208
Get(const std::string & k,const Snapshot * snapshot=nullptr)209 std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
210 ReadOptions options;
211 options.snapshot = snapshot;
212 std::string result;
213 Status s = db_->Get(options, k, &result);
214 if (s.IsNotFound()) {
215 result = "NOT_FOUND";
216 } else if (!s.ok()) {
217 result = s.ToString();
218 }
219 return result;
220 }
221
222
NumTableFilesAtLevel(int level)223 int NumTableFilesAtLevel(int level) {
224 std::string property;
225 EXPECT_TRUE(db_->GetProperty(
226 "rocksdb.num-files-at-level" + NumberToString(level), &property));
227 return atoi(property.c_str());
228 }
229
230 // Return spread of files per level
FilesPerLevel()231 std::string FilesPerLevel() {
232 std::string result;
233 size_t last_non_zero_offset = 0;
234 for (int level = 0; level < db_->NumberLevels(); level++) {
235 int f = NumTableFilesAtLevel(level);
236 char buf[100];
237 snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
238 result += buf;
239 if (f > 0) {
240 last_non_zero_offset = result.size();
241 }
242 }
243 result.resize(last_non_zero_offset);
244 return result;
245 }
246
IterStatus(Iterator * iter)247 std::string IterStatus(Iterator* iter) {
248 std::string result;
249 if (iter->Valid()) {
250 result = iter->key().ToString() + "->" + iter->value().ToString();
251 } else {
252 result = "(invalid)";
253 }
254 return result;
255 }
256 };
257
TEST_P(PlainTableDBTest,Empty)258 TEST_P(PlainTableDBTest, Empty) {
259 ASSERT_TRUE(dbfull() != nullptr);
260 ASSERT_EQ("NOT_FOUND", Get("0000000000000foo"));
261 }
262
263 extern const uint64_t kPlainTableMagicNumber;
264
265 class TestPlainTableReader : public PlainTableReader {
266 public:
TestPlainTableReader(const EnvOptions & env_options,const InternalKeyComparator & icomparator,EncodingType encoding_type,uint64_t file_size,int bloom_bits_per_key,double hash_table_ratio,size_t index_sparseness,const TableProperties * table_properties,std::unique_ptr<RandomAccessFileReader> && file,const ImmutableCFOptions & ioptions,const SliceTransform * prefix_extractor,bool * expect_bloom_not_match,bool store_index_in_file,uint32_t column_family_id,const std::string & column_family_name)267 TestPlainTableReader(const EnvOptions& env_options,
268 const InternalKeyComparator& icomparator,
269 EncodingType encoding_type, uint64_t file_size,
270 int bloom_bits_per_key, double hash_table_ratio,
271 size_t index_sparseness,
272 const TableProperties* table_properties,
273 std::unique_ptr<RandomAccessFileReader>&& file,
274 const ImmutableCFOptions& ioptions,
275 const SliceTransform* prefix_extractor,
276 bool* expect_bloom_not_match, bool store_index_in_file,
277 uint32_t column_family_id,
278 const std::string& column_family_name)
279 : PlainTableReader(ioptions, std::move(file), env_options, icomparator,
280 encoding_type, file_size, table_properties,
281 prefix_extractor),
282 expect_bloom_not_match_(expect_bloom_not_match) {
283 Status s = MmapDataIfNeeded();
284 EXPECT_TRUE(s.ok());
285
286 s = PopulateIndex(const_cast<TableProperties*>(table_properties),
287 bloom_bits_per_key, hash_table_ratio, index_sparseness,
288 2 * 1024 * 1024);
289 EXPECT_TRUE(s.ok());
290
291 TableProperties* props = const_cast<TableProperties*>(table_properties);
292 EXPECT_EQ(column_family_id, static_cast<uint32_t>(props->column_family_id));
293 EXPECT_EQ(column_family_name, props->column_family_name);
294 if (store_index_in_file) {
295 auto bloom_version_ptr = props->user_collected_properties.find(
296 PlainTablePropertyNames::kBloomVersion);
297 EXPECT_TRUE(bloom_version_ptr != props->user_collected_properties.end());
298 EXPECT_EQ(bloom_version_ptr->second, std::string("1"));
299 if (ioptions.bloom_locality > 0) {
300 auto num_blocks_ptr = props->user_collected_properties.find(
301 PlainTablePropertyNames::kNumBloomBlocks);
302 EXPECT_TRUE(num_blocks_ptr != props->user_collected_properties.end());
303 }
304 }
305 table_properties_.reset(props);
306 }
307
~TestPlainTableReader()308 ~TestPlainTableReader() override {}
309
310 private:
MatchBloom(uint32_t hash) const311 bool MatchBloom(uint32_t hash) const override {
312 bool ret = PlainTableReader::MatchBloom(hash);
313 if (*expect_bloom_not_match_) {
314 EXPECT_TRUE(!ret);
315 } else {
316 EXPECT_TRUE(ret);
317 }
318 return ret;
319 }
320 bool* expect_bloom_not_match_;
321 };
322
323 extern const uint64_t kPlainTableMagicNumber;
324 class TestPlainTableFactory : public PlainTableFactory {
325 public:
TestPlainTableFactory(bool * expect_bloom_not_match,const PlainTableOptions & options,uint32_t column_family_id,std::string column_family_name)326 explicit TestPlainTableFactory(bool* expect_bloom_not_match,
327 const PlainTableOptions& options,
328 uint32_t column_family_id,
329 std::string column_family_name)
330 : PlainTableFactory(options),
331 bloom_bits_per_key_(options.bloom_bits_per_key),
332 hash_table_ratio_(options.hash_table_ratio),
333 index_sparseness_(options.index_sparseness),
334 store_index_in_file_(options.store_index_in_file),
335 expect_bloom_not_match_(expect_bloom_not_match),
336 column_family_id_(column_family_id),
337 column_family_name_(std::move(column_family_name)) {}
338
NewTableReader(const TableReaderOptions & table_reader_options,std::unique_ptr<RandomAccessFileReader> && file,uint64_t file_size,std::unique_ptr<TableReader> * table,bool) const339 Status NewTableReader(
340 const TableReaderOptions& table_reader_options,
341 std::unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
342 std::unique_ptr<TableReader>* table,
343 bool /*prefetch_index_and_filter_in_cache*/) const override {
344 TableProperties* props = nullptr;
345 auto s =
346 ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber,
347 table_reader_options.ioptions, &props,
348 true /* compression_type_missing */);
349 EXPECT_TRUE(s.ok());
350
351 if (store_index_in_file_) {
352 BlockHandle bloom_block_handle;
353 s = FindMetaBlock(file.get(), file_size, kPlainTableMagicNumber,
354 table_reader_options.ioptions,
355 BloomBlockBuilder::kBloomBlock, &bloom_block_handle,
356 /* compression_type_missing */ true);
357 EXPECT_TRUE(s.ok());
358
359 BlockHandle index_block_handle;
360 s = FindMetaBlock(file.get(), file_size, kPlainTableMagicNumber,
361 table_reader_options.ioptions,
362 PlainTableIndexBuilder::kPlainTableIndexBlock,
363 &index_block_handle, /* compression_type_missing */ true);
364 EXPECT_TRUE(s.ok());
365 }
366
367 auto& user_props = props->user_collected_properties;
368 auto encoding_type_prop =
369 user_props.find(PlainTablePropertyNames::kEncodingType);
370 assert(encoding_type_prop != user_props.end());
371 EncodingType encoding_type = static_cast<EncodingType>(
372 DecodeFixed32(encoding_type_prop->second.c_str()));
373
374 std::unique_ptr<PlainTableReader> new_reader(new TestPlainTableReader(
375 table_reader_options.env_options,
376 table_reader_options.internal_comparator, encoding_type, file_size,
377 bloom_bits_per_key_, hash_table_ratio_, index_sparseness_, props,
378 std::move(file), table_reader_options.ioptions,
379 table_reader_options.prefix_extractor, expect_bloom_not_match_,
380 store_index_in_file_, column_family_id_, column_family_name_));
381
382 *table = std::move(new_reader);
383 return s;
384 }
385
386 private:
387 int bloom_bits_per_key_;
388 double hash_table_ratio_;
389 size_t index_sparseness_;
390 bool store_index_in_file_;
391 bool* expect_bloom_not_match_;
392 const uint32_t column_family_id_;
393 const std::string column_family_name_;
394 };
395
TEST_P(PlainTableDBTest,BadOptions1)396 TEST_P(PlainTableDBTest, BadOptions1) {
397 // Build with a prefix extractor
398 ASSERT_OK(Put("1000000000000foo", "v1"));
399 dbfull()->TEST_FlushMemTable();
400
401 // Bad attempt to re-open without a prefix extractor
402 Options options = CurrentOptions();
403 options.prefix_extractor.reset();
404 Reopen(&options);
405 ASSERT_EQ(
406 "Invalid argument: Prefix extractor is missing when opening a PlainTable "
407 "built using a prefix extractor",
408 Get("1000000000000foo"));
409
410 // Bad attempt to re-open with different prefix extractor
411 options.prefix_extractor.reset(NewFixedPrefixTransform(6));
412 Reopen(&options);
413 ASSERT_EQ(
414 "Invalid argument: Prefix extractor given doesn't match the one used to "
415 "build PlainTable",
416 Get("1000000000000foo"));
417
418 // Correct prefix extractor
419 options.prefix_extractor.reset(NewFixedPrefixTransform(8));
420 Reopen(&options);
421 ASSERT_EQ("v1", Get("1000000000000foo"));
422 }
423
TEST_P(PlainTableDBTest,BadOptions2)424 TEST_P(PlainTableDBTest, BadOptions2) {
425 Options options = CurrentOptions();
426 options.prefix_extractor.reset();
427 options.create_if_missing = true;
428 DestroyAndReopen(&options);
429 // Build without a prefix extractor
430 // (apparently works even if hash_table_ratio > 0)
431 ASSERT_OK(Put("1000000000000foo", "v1"));
432 dbfull()->TEST_FlushMemTable();
433
434 // Bad attempt to re-open with hash_table_ratio > 0 and no prefix extractor
435 Status s = TryReopen(&options);
436 ASSERT_EQ(
437 "Not implemented: PlainTable requires a prefix extractor enable prefix "
438 "hash mode.",
439 s.ToString());
440
441 // OK to open with hash_table_ratio == 0 and no prefix extractor
442 PlainTableOptions plain_table_options;
443 plain_table_options.hash_table_ratio = 0;
444 options.table_factory.reset(NewPlainTableFactory(plain_table_options));
445 Reopen(&options);
446 ASSERT_EQ("v1", Get("1000000000000foo"));
447
448 // OK to open newly with a prefix_extractor and hash table; builds index
449 // in memory.
450 options = CurrentOptions();
451 Reopen(&options);
452 ASSERT_EQ("v1", Get("1000000000000foo"));
453 }
454
TEST_P(PlainTableDBTest,Flush)455 TEST_P(PlainTableDBTest, Flush) {
456 for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
457 huge_page_tlb_size += 2 * 1024 * 1024) {
458 for (EncodingType encoding_type : {kPlain, kPrefix}) {
459 for (int bloom = -1; bloom <= 117; bloom += 117) {
460 const int bloom_bits = std::max(bloom, 0);
461 const bool full_scan_mode = bloom < 0;
462 for (int total_order = 0; total_order <= 1; total_order++) {
463 for (int store_index_in_file = 0; store_index_in_file <= 1;
464 ++store_index_in_file) {
465 Options options = CurrentOptions();
466 options.create_if_missing = true;
467 // Set only one bucket to force bucket conflict.
468 // Test index interval for the same prefix to be 1, 2 and 4
469 if (total_order) {
470 options.prefix_extractor.reset();
471
472 PlainTableOptions plain_table_options;
473 plain_table_options.user_key_len = 0;
474 plain_table_options.bloom_bits_per_key = bloom_bits;
475 plain_table_options.hash_table_ratio = 0;
476 plain_table_options.index_sparseness = 2;
477 plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
478 plain_table_options.encoding_type = encoding_type;
479 plain_table_options.full_scan_mode = full_scan_mode;
480 plain_table_options.store_index_in_file = store_index_in_file;
481
482 options.table_factory.reset(
483 NewPlainTableFactory(plain_table_options));
484 } else {
485 PlainTableOptions plain_table_options;
486 plain_table_options.user_key_len = 0;
487 plain_table_options.bloom_bits_per_key = bloom_bits;
488 plain_table_options.hash_table_ratio = 0.75;
489 plain_table_options.index_sparseness = 16;
490 plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
491 plain_table_options.encoding_type = encoding_type;
492 plain_table_options.full_scan_mode = full_scan_mode;
493 plain_table_options.store_index_in_file = store_index_in_file;
494
495 options.table_factory.reset(
496 NewPlainTableFactory(plain_table_options));
497 }
498 DestroyAndReopen(&options);
499 uint64_t int_num;
500 ASSERT_TRUE(dbfull()->GetIntProperty(
501 "rocksdb.estimate-table-readers-mem", &int_num));
502 ASSERT_EQ(int_num, 0U);
503
504 ASSERT_OK(Put("1000000000000foo", "v1"));
505 ASSERT_OK(Put("0000000000000bar", "v2"));
506 ASSERT_OK(Put("1000000000000foo", "v3"));
507 dbfull()->TEST_FlushMemTable();
508
509 ASSERT_TRUE(dbfull()->GetIntProperty(
510 "rocksdb.estimate-table-readers-mem", &int_num));
511 ASSERT_GT(int_num, 0U);
512
513 TablePropertiesCollection ptc;
514 reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc);
515 ASSERT_EQ(1U, ptc.size());
516 auto row = ptc.begin();
517 auto tp = row->second;
518
519 if (full_scan_mode) {
520 // Does not support Get/Seek
521 std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ReadOptions()));
522 iter->SeekToFirst();
523 ASSERT_TRUE(iter->Valid());
524 ASSERT_EQ("0000000000000bar", iter->key().ToString());
525 ASSERT_EQ("v2", iter->value().ToString());
526 iter->Next();
527 ASSERT_TRUE(iter->Valid());
528 ASSERT_EQ("1000000000000foo", iter->key().ToString());
529 ASSERT_EQ("v3", iter->value().ToString());
530 iter->Next();
531 ASSERT_TRUE(!iter->Valid());
532 ASSERT_TRUE(iter->status().ok());
533 } else {
534 if (!store_index_in_file) {
535 ASSERT_EQ(total_order ? "4" : "12",
536 (tp->user_collected_properties)
537 .at("plain_table_hash_table_size"));
538 ASSERT_EQ("0", (tp->user_collected_properties)
539 .at("plain_table_sub_index_size"));
540 } else {
541 ASSERT_EQ("0", (tp->user_collected_properties)
542 .at("plain_table_hash_table_size"));
543 ASSERT_EQ("0", (tp->user_collected_properties)
544 .at("plain_table_sub_index_size"));
545 }
546 ASSERT_EQ("v3", Get("1000000000000foo"));
547 ASSERT_EQ("v2", Get("0000000000000bar"));
548 }
549 }
550 }
551 }
552 }
553 }
554 }
555
TEST_P(PlainTableDBTest,Flush2)556 TEST_P(PlainTableDBTest, Flush2) {
557 for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
558 huge_page_tlb_size += 2 * 1024 * 1024) {
559 for (EncodingType encoding_type : {kPlain, kPrefix}) {
560 for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
561 for (int total_order = 0; total_order <= 1; total_order++) {
562 for (int store_index_in_file = 0; store_index_in_file <= 1;
563 ++store_index_in_file) {
564 if (encoding_type == kPrefix && total_order) {
565 continue;
566 }
567 if (!bloom_bits && store_index_in_file) {
568 continue;
569 }
570 if (total_order && store_index_in_file) {
571 continue;
572 }
573 bool expect_bloom_not_match = false;
574 Options options = CurrentOptions();
575 options.create_if_missing = true;
576 // Set only one bucket to force bucket conflict.
577 // Test index interval for the same prefix to be 1, 2 and 4
578 PlainTableOptions plain_table_options;
579 if (total_order) {
580 options.prefix_extractor = nullptr;
581 plain_table_options.hash_table_ratio = 0;
582 plain_table_options.index_sparseness = 2;
583 } else {
584 plain_table_options.hash_table_ratio = 0.75;
585 plain_table_options.index_sparseness = 16;
586 }
587 plain_table_options.user_key_len = kPlainTableVariableLength;
588 plain_table_options.bloom_bits_per_key = bloom_bits;
589 plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
590 plain_table_options.encoding_type = encoding_type;
591 plain_table_options.store_index_in_file = store_index_in_file;
592 options.table_factory.reset(new TestPlainTableFactory(
593 &expect_bloom_not_match, plain_table_options,
594 0 /* column_family_id */, kDefaultColumnFamilyName));
595
596 DestroyAndReopen(&options);
597 ASSERT_OK(Put("0000000000000bar", "b"));
598 ASSERT_OK(Put("1000000000000foo", "v1"));
599 dbfull()->TEST_FlushMemTable();
600
601 ASSERT_OK(Put("1000000000000foo", "v2"));
602 dbfull()->TEST_FlushMemTable();
603 ASSERT_EQ("v2", Get("1000000000000foo"));
604
605 ASSERT_OK(Put("0000000000000eee", "v3"));
606 dbfull()->TEST_FlushMemTable();
607 ASSERT_EQ("v3", Get("0000000000000eee"));
608
609 ASSERT_OK(Delete("0000000000000bar"));
610 dbfull()->TEST_FlushMemTable();
611 ASSERT_EQ("NOT_FOUND", Get("0000000000000bar"));
612
613 ASSERT_OK(Put("0000000000000eee", "v5"));
614 ASSERT_OK(Put("9000000000000eee", "v5"));
615 dbfull()->TEST_FlushMemTable();
616 ASSERT_EQ("v5", Get("0000000000000eee"));
617
618 // Test Bloom Filter
619 if (bloom_bits > 0) {
620 // Neither key nor value should exist.
621 expect_bloom_not_match = true;
622 ASSERT_EQ("NOT_FOUND", Get("5_not00000000bar"));
623 // Key doesn't exist any more but prefix exists.
624 if (total_order) {
625 ASSERT_EQ("NOT_FOUND", Get("1000000000000not"));
626 ASSERT_EQ("NOT_FOUND", Get("0000000000000not"));
627 }
628 expect_bloom_not_match = false;
629 }
630 }
631 }
632 }
633 }
634 }
635 }
636
TEST_P(PlainTableDBTest,Immortal)637 TEST_P(PlainTableDBTest, Immortal) {
638 for (EncodingType encoding_type : {kPlain, kPrefix}) {
639 Options options = CurrentOptions();
640 options.create_if_missing = true;
641 options.max_open_files = -1;
642 // Set only one bucket to force bucket conflict.
643 // Test index interval for the same prefix to be 1, 2 and 4
644 PlainTableOptions plain_table_options;
645 plain_table_options.hash_table_ratio = 0.75;
646 plain_table_options.index_sparseness = 16;
647 plain_table_options.user_key_len = kPlainTableVariableLength;
648 plain_table_options.bloom_bits_per_key = 10;
649 plain_table_options.encoding_type = encoding_type;
650 options.table_factory.reset(NewPlainTableFactory(plain_table_options));
651
652 DestroyAndReopen(&options);
653 ASSERT_OK(Put("0000000000000bar", "b"));
654 ASSERT_OK(Put("1000000000000foo", "v1"));
655 dbfull()->TEST_FlushMemTable();
656
657 int copied = 0;
658 rocksdb::SyncPoint::GetInstance()->SetCallBack(
659 "GetContext::SaveValue::PinSelf", [&](void* /*arg*/) { copied++; });
660 rocksdb::SyncPoint::GetInstance()->EnableProcessing();
661 ASSERT_EQ("b", Get("0000000000000bar"));
662 ASSERT_EQ("v1", Get("1000000000000foo"));
663 ASSERT_EQ(2, copied);
664 copied = 0;
665
666 Close();
667 ASSERT_OK(ReopenForReadOnly(&options));
668
669 ASSERT_EQ("b", Get("0000000000000bar"));
670 ASSERT_EQ("v1", Get("1000000000000foo"));
671 ASSERT_EQ("NOT_FOUND", Get("1000000000000bar"));
672 if (mmap_mode()) {
673 ASSERT_EQ(0, copied);
674 } else {
675 ASSERT_EQ(2, copied);
676 }
677 rocksdb::SyncPoint::GetInstance()->DisableProcessing();
678 }
679 }
680
TEST_P(PlainTableDBTest,Iterator)681 TEST_P(PlainTableDBTest, Iterator) {
682 for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
683 huge_page_tlb_size += 2 * 1024 * 1024) {
684 for (EncodingType encoding_type : {kPlain, kPrefix}) {
685 for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
686 for (int total_order = 0; total_order <= 1; total_order++) {
687 if (encoding_type == kPrefix && total_order == 1) {
688 continue;
689 }
690 bool expect_bloom_not_match = false;
691 Options options = CurrentOptions();
692 options.create_if_missing = true;
693 // Set only one bucket to force bucket conflict.
694 // Test index interval for the same prefix to be 1, 2 and 4
695 if (total_order) {
696 options.prefix_extractor = nullptr;
697
698 PlainTableOptions plain_table_options;
699 plain_table_options.user_key_len = 16;
700 plain_table_options.bloom_bits_per_key = bloom_bits;
701 plain_table_options.hash_table_ratio = 0;
702 plain_table_options.index_sparseness = 2;
703 plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
704 plain_table_options.encoding_type = encoding_type;
705
706 options.table_factory.reset(new TestPlainTableFactory(
707 &expect_bloom_not_match, plain_table_options,
708 0 /* column_family_id */, kDefaultColumnFamilyName));
709 } else {
710 PlainTableOptions plain_table_options;
711 plain_table_options.user_key_len = 16;
712 plain_table_options.bloom_bits_per_key = bloom_bits;
713 plain_table_options.hash_table_ratio = 0.75;
714 plain_table_options.index_sparseness = 16;
715 plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
716 plain_table_options.encoding_type = encoding_type;
717
718 options.table_factory.reset(new TestPlainTableFactory(
719 &expect_bloom_not_match, plain_table_options,
720 0 /* column_family_id */, kDefaultColumnFamilyName));
721 }
722 DestroyAndReopen(&options);
723
724 ASSERT_OK(Put("1000000000foo002", "v_2"));
725 ASSERT_OK(Put("0000000000000bar", "random"));
726 ASSERT_OK(Put("1000000000foo001", "v1"));
727 ASSERT_OK(Put("3000000000000bar", "bar_v"));
728 ASSERT_OK(Put("1000000000foo003", "v__3"));
729 ASSERT_OK(Put("1000000000foo004", "v__4"));
730 ASSERT_OK(Put("1000000000foo005", "v__5"));
731 ASSERT_OK(Put("1000000000foo007", "v__7"));
732 ASSERT_OK(Put("1000000000foo008", "v__8"));
733 dbfull()->TEST_FlushMemTable();
734 ASSERT_EQ("v1", Get("1000000000foo001"));
735 ASSERT_EQ("v__3", Get("1000000000foo003"));
736 Iterator* iter = dbfull()->NewIterator(ReadOptions());
737 iter->Seek("1000000000foo000");
738 ASSERT_TRUE(iter->Valid());
739 ASSERT_EQ("1000000000foo001", iter->key().ToString());
740 ASSERT_EQ("v1", iter->value().ToString());
741
742 iter->Next();
743 ASSERT_TRUE(iter->Valid());
744 ASSERT_EQ("1000000000foo002", iter->key().ToString());
745 ASSERT_EQ("v_2", iter->value().ToString());
746
747 iter->Next();
748 ASSERT_TRUE(iter->Valid());
749 ASSERT_EQ("1000000000foo003", iter->key().ToString());
750 ASSERT_EQ("v__3", iter->value().ToString());
751
752 iter->Next();
753 ASSERT_TRUE(iter->Valid());
754 ASSERT_EQ("1000000000foo004", iter->key().ToString());
755 ASSERT_EQ("v__4", iter->value().ToString());
756
757 iter->Seek("3000000000000bar");
758 ASSERT_TRUE(iter->Valid());
759 ASSERT_EQ("3000000000000bar", iter->key().ToString());
760 ASSERT_EQ("bar_v", iter->value().ToString());
761
762 iter->Seek("1000000000foo000");
763 ASSERT_TRUE(iter->Valid());
764 ASSERT_EQ("1000000000foo001", iter->key().ToString());
765 ASSERT_EQ("v1", iter->value().ToString());
766
767 iter->Seek("1000000000foo005");
768 ASSERT_TRUE(iter->Valid());
769 ASSERT_EQ("1000000000foo005", iter->key().ToString());
770 ASSERT_EQ("v__5", iter->value().ToString());
771
772 iter->Seek("1000000000foo006");
773 ASSERT_TRUE(iter->Valid());
774 ASSERT_EQ("1000000000foo007", iter->key().ToString());
775 ASSERT_EQ("v__7", iter->value().ToString());
776
777 iter->Seek("1000000000foo008");
778 ASSERT_TRUE(iter->Valid());
779 ASSERT_EQ("1000000000foo008", iter->key().ToString());
780 ASSERT_EQ("v__8", iter->value().ToString());
781
782 if (total_order == 0) {
783 iter->Seek("1000000000foo009");
784 ASSERT_TRUE(iter->Valid());
785 ASSERT_EQ("3000000000000bar", iter->key().ToString());
786 }
787
788 // Test Bloom Filter
789 if (bloom_bits > 0) {
790 if (!total_order) {
791 // Neither key nor value should exist.
792 expect_bloom_not_match = true;
793 iter->Seek("2not000000000bar");
794 ASSERT_TRUE(!iter->Valid());
795 ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
796 expect_bloom_not_match = false;
797 } else {
798 expect_bloom_not_match = true;
799 ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
800 expect_bloom_not_match = false;
801 }
802 }
803
804 delete iter;
805 }
806 }
807 }
808 }
809 }
810
811 namespace {
NthKey(size_t n,char filler)812 std::string NthKey(size_t n, char filler) {
813 std::string rv(16, filler);
814 rv[0] = n % 10;
815 rv[1] = (n / 10) % 10;
816 rv[2] = (n / 100) % 10;
817 rv[3] = (n / 1000) % 10;
818 return rv;
819 }
820 } // anonymous namespace
821
TEST_P(PlainTableDBTest,BloomSchema)822 TEST_P(PlainTableDBTest, BloomSchema) {
823 Options options = CurrentOptions();
824 options.create_if_missing = true;
825 for (int bloom_locality = 0; bloom_locality <= 1; bloom_locality++) {
826 options.bloom_locality = bloom_locality;
827 PlainTableOptions plain_table_options;
828 plain_table_options.user_key_len = 16;
829 plain_table_options.bloom_bits_per_key = 3; // high FP rate for test
830 plain_table_options.hash_table_ratio = 0.75;
831 plain_table_options.index_sparseness = 16;
832 plain_table_options.huge_page_tlb_size = 0;
833 plain_table_options.encoding_type = kPlain;
834
835 bool expect_bloom_not_match = false;
836 options.table_factory.reset(new TestPlainTableFactory(
837 &expect_bloom_not_match, plain_table_options, 0 /* column_family_id */,
838 kDefaultColumnFamilyName));
839 DestroyAndReopen(&options);
840
841 for (unsigned i = 0; i < 2345; ++i) {
842 ASSERT_OK(Put(NthKey(i, 'y'), "added"));
843 }
844 dbfull()->TEST_FlushMemTable();
845 ASSERT_EQ("added", Get(NthKey(42, 'y')));
846
847 for (unsigned i = 0; i < 32; ++i) {
848 // Known pattern of Bloom filter false positives can detect schema change
849 // with high probability. Known FPs stuffed into bits:
850 uint32_t pattern;
851 if (!bloom_locality) {
852 pattern = 1785868347UL;
853 } else if (CACHE_LINE_SIZE == 64U) {
854 pattern = 2421694657UL;
855 } else if (CACHE_LINE_SIZE == 128U) {
856 pattern = 788710956UL;
857 } else {
858 ASSERT_EQ(CACHE_LINE_SIZE, 256U);
859 pattern = 163905UL;
860 }
861 bool expect_fp = pattern & (1UL << i);
862 // fprintf(stderr, "expect_fp@%u: %d\n", i, (int)expect_fp);
863 expect_bloom_not_match = !expect_fp;
864 ASSERT_EQ("NOT_FOUND", Get(NthKey(i, 'n')));
865 }
866 }
867 }
868
869 namespace {
MakeLongKey(size_t length,char c)870 std::string MakeLongKey(size_t length, char c) {
871 return std::string(length, c);
872 }
873 } // namespace
874
TEST_P(PlainTableDBTest,IteratorLargeKeys)875 TEST_P(PlainTableDBTest, IteratorLargeKeys) {
876 Options options = CurrentOptions();
877
878 PlainTableOptions plain_table_options;
879 plain_table_options.user_key_len = 0;
880 plain_table_options.bloom_bits_per_key = 0;
881 plain_table_options.hash_table_ratio = 0;
882
883 options.table_factory.reset(NewPlainTableFactory(plain_table_options));
884 options.create_if_missing = true;
885 options.prefix_extractor.reset();
886 DestroyAndReopen(&options);
887
888 std::string key_list[] = {
889 MakeLongKey(30, '0'),
890 MakeLongKey(16, '1'),
891 MakeLongKey(32, '2'),
892 MakeLongKey(60, '3'),
893 MakeLongKey(90, '4'),
894 MakeLongKey(50, '5'),
895 MakeLongKey(26, '6')
896 };
897
898 for (size_t i = 0; i < 7; i++) {
899 ASSERT_OK(Put(key_list[i], ToString(i)));
900 }
901
902 dbfull()->TEST_FlushMemTable();
903
904 Iterator* iter = dbfull()->NewIterator(ReadOptions());
905 iter->Seek(key_list[0]);
906
907 for (size_t i = 0; i < 7; i++) {
908 ASSERT_TRUE(iter->Valid());
909 ASSERT_EQ(key_list[i], iter->key().ToString());
910 ASSERT_EQ(ToString(i), iter->value().ToString());
911 iter->Next();
912 }
913
914 ASSERT_TRUE(!iter->Valid());
915
916 delete iter;
917 }
918
919 namespace {
MakeLongKeyWithPrefix(size_t length,char c)920 std::string MakeLongKeyWithPrefix(size_t length, char c) {
921 return "00000000" + std::string(length - 8, c);
922 }
923 } // namespace
924
TEST_P(PlainTableDBTest,IteratorLargeKeysWithPrefix)925 TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
926 Options options = CurrentOptions();
927
928 PlainTableOptions plain_table_options;
929 plain_table_options.user_key_len = 16;
930 plain_table_options.bloom_bits_per_key = 0;
931 plain_table_options.hash_table_ratio = 0.8;
932 plain_table_options.index_sparseness = 3;
933 plain_table_options.huge_page_tlb_size = 0;
934 plain_table_options.encoding_type = kPrefix;
935
936 options.table_factory.reset(NewPlainTableFactory(plain_table_options));
937 options.create_if_missing = true;
938 DestroyAndReopen(&options);
939
940 std::string key_list[] = {
941 MakeLongKeyWithPrefix(30, '0'), MakeLongKeyWithPrefix(16, '1'),
942 MakeLongKeyWithPrefix(32, '2'), MakeLongKeyWithPrefix(60, '3'),
943 MakeLongKeyWithPrefix(90, '4'), MakeLongKeyWithPrefix(50, '5'),
944 MakeLongKeyWithPrefix(26, '6')};
945
946 for (size_t i = 0; i < 7; i++) {
947 ASSERT_OK(Put(key_list[i], ToString(i)));
948 }
949
950 dbfull()->TEST_FlushMemTable();
951
952 Iterator* iter = dbfull()->NewIterator(ReadOptions());
953 iter->Seek(key_list[0]);
954
955 for (size_t i = 0; i < 7; i++) {
956 ASSERT_TRUE(iter->Valid());
957 ASSERT_EQ(key_list[i], iter->key().ToString());
958 ASSERT_EQ(ToString(i), iter->value().ToString());
959 iter->Next();
960 }
961
962 ASSERT_TRUE(!iter->Valid());
963
964 delete iter;
965 }
966
TEST_P(PlainTableDBTest,IteratorReverseSuffixComparator)967 TEST_P(PlainTableDBTest, IteratorReverseSuffixComparator) {
968 Options options = CurrentOptions();
969 options.create_if_missing = true;
970 // Set only one bucket to force bucket conflict.
971 // Test index interval for the same prefix to be 1, 2 and 4
972 test::SimpleSuffixReverseComparator comp;
973 options.comparator = ∁
974 DestroyAndReopen(&options);
975
976 ASSERT_OK(Put("1000000000foo002", "v_2"));
977 ASSERT_OK(Put("0000000000000bar", "random"));
978 ASSERT_OK(Put("1000000000foo001", "v1"));
979 ASSERT_OK(Put("3000000000000bar", "bar_v"));
980 ASSERT_OK(Put("1000000000foo003", "v__3"));
981 ASSERT_OK(Put("1000000000foo004", "v__4"));
982 ASSERT_OK(Put("1000000000foo005", "v__5"));
983 ASSERT_OK(Put("1000000000foo007", "v__7"));
984 ASSERT_OK(Put("1000000000foo008", "v__8"));
985 dbfull()->TEST_FlushMemTable();
986 ASSERT_EQ("v1", Get("1000000000foo001"));
987 ASSERT_EQ("v__3", Get("1000000000foo003"));
988 Iterator* iter = dbfull()->NewIterator(ReadOptions());
989 iter->Seek("1000000000foo009");
990 ASSERT_TRUE(iter->Valid());
991 ASSERT_EQ("1000000000foo008", iter->key().ToString());
992 ASSERT_EQ("v__8", iter->value().ToString());
993
994 iter->Next();
995 ASSERT_TRUE(iter->Valid());
996 ASSERT_EQ("1000000000foo007", iter->key().ToString());
997 ASSERT_EQ("v__7", iter->value().ToString());
998
999 iter->Next();
1000 ASSERT_TRUE(iter->Valid());
1001 ASSERT_EQ("1000000000foo005", iter->key().ToString());
1002 ASSERT_EQ("v__5", iter->value().ToString());
1003
1004 iter->Next();
1005 ASSERT_TRUE(iter->Valid());
1006 ASSERT_EQ("1000000000foo004", iter->key().ToString());
1007 ASSERT_EQ("v__4", iter->value().ToString());
1008
1009 iter->Seek("3000000000000bar");
1010 ASSERT_TRUE(iter->Valid());
1011 ASSERT_EQ("3000000000000bar", iter->key().ToString());
1012 ASSERT_EQ("bar_v", iter->value().ToString());
1013
1014 iter->Seek("1000000000foo005");
1015 ASSERT_TRUE(iter->Valid());
1016 ASSERT_EQ("1000000000foo005", iter->key().ToString());
1017 ASSERT_EQ("v__5", iter->value().ToString());
1018
1019 iter->Seek("1000000000foo006");
1020 ASSERT_TRUE(iter->Valid());
1021 ASSERT_EQ("1000000000foo005", iter->key().ToString());
1022 ASSERT_EQ("v__5", iter->value().ToString());
1023
1024 iter->Seek("1000000000foo008");
1025 ASSERT_TRUE(iter->Valid());
1026 ASSERT_EQ("1000000000foo008", iter->key().ToString());
1027 ASSERT_EQ("v__8", iter->value().ToString());
1028
1029 iter->Seek("1000000000foo000");
1030 ASSERT_TRUE(iter->Valid());
1031 ASSERT_EQ("3000000000000bar", iter->key().ToString());
1032
1033 delete iter;
1034 }
1035
TEST_P(PlainTableDBTest,HashBucketConflict)1036 TEST_P(PlainTableDBTest, HashBucketConflict) {
1037 for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
1038 huge_page_tlb_size += 2 * 1024 * 1024) {
1039 for (unsigned char i = 1; i <= 3; i++) {
1040 Options options = CurrentOptions();
1041 options.create_if_missing = true;
1042 // Set only one bucket to force bucket conflict.
1043 // Test index interval for the same prefix to be 1, 2 and 4
1044
1045 PlainTableOptions plain_table_options;
1046 plain_table_options.user_key_len = 16;
1047 plain_table_options.bloom_bits_per_key = 0;
1048 plain_table_options.hash_table_ratio = 0;
1049 plain_table_options.index_sparseness = 2 ^ i;
1050 plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
1051
1052 options.table_factory.reset(NewPlainTableFactory(plain_table_options));
1053
1054 DestroyAndReopen(&options);
1055 ASSERT_OK(Put("5000000000000fo0", "v1"));
1056 ASSERT_OK(Put("5000000000000fo1", "v2"));
1057 ASSERT_OK(Put("5000000000000fo2", "v"));
1058 ASSERT_OK(Put("2000000000000fo0", "v3"));
1059 ASSERT_OK(Put("2000000000000fo1", "v4"));
1060 ASSERT_OK(Put("2000000000000fo2", "v"));
1061 ASSERT_OK(Put("2000000000000fo3", "v"));
1062
1063 dbfull()->TEST_FlushMemTable();
1064
1065 ASSERT_EQ("v1", Get("5000000000000fo0"));
1066 ASSERT_EQ("v2", Get("5000000000000fo1"));
1067 ASSERT_EQ("v3", Get("2000000000000fo0"));
1068 ASSERT_EQ("v4", Get("2000000000000fo1"));
1069
1070 ASSERT_EQ("NOT_FOUND", Get("5000000000000bar"));
1071 ASSERT_EQ("NOT_FOUND", Get("2000000000000bar"));
1072 ASSERT_EQ("NOT_FOUND", Get("5000000000000fo8"));
1073 ASSERT_EQ("NOT_FOUND", Get("2000000000000fo8"));
1074
1075 ReadOptions ro;
1076 Iterator* iter = dbfull()->NewIterator(ro);
1077
1078 iter->Seek("5000000000000fo0");
1079 ASSERT_TRUE(iter->Valid());
1080 ASSERT_EQ("5000000000000fo0", iter->key().ToString());
1081 iter->Next();
1082 ASSERT_TRUE(iter->Valid());
1083 ASSERT_EQ("5000000000000fo1", iter->key().ToString());
1084
1085 iter->Seek("5000000000000fo1");
1086 ASSERT_TRUE(iter->Valid());
1087 ASSERT_EQ("5000000000000fo1", iter->key().ToString());
1088
1089 iter->Seek("2000000000000fo0");
1090 ASSERT_TRUE(iter->Valid());
1091 ASSERT_EQ("2000000000000fo0", iter->key().ToString());
1092 iter->Next();
1093 ASSERT_TRUE(iter->Valid());
1094 ASSERT_EQ("2000000000000fo1", iter->key().ToString());
1095
1096 iter->Seek("2000000000000fo1");
1097 ASSERT_TRUE(iter->Valid());
1098 ASSERT_EQ("2000000000000fo1", iter->key().ToString());
1099
1100 iter->Seek("2000000000000bar");
1101 ASSERT_TRUE(iter->Valid());
1102 ASSERT_EQ("2000000000000fo0", iter->key().ToString());
1103
1104 iter->Seek("5000000000000bar");
1105 ASSERT_TRUE(iter->Valid());
1106 ASSERT_EQ("5000000000000fo0", iter->key().ToString());
1107
1108 iter->Seek("2000000000000fo8");
1109 ASSERT_TRUE(!iter->Valid() ||
1110 options.comparator->Compare(iter->key(), "20000001") > 0);
1111
1112 iter->Seek("5000000000000fo8");
1113 ASSERT_TRUE(!iter->Valid());
1114
1115 iter->Seek("1000000000000fo2");
1116 ASSERT_TRUE(!iter->Valid());
1117
1118 iter->Seek("3000000000000fo2");
1119 ASSERT_TRUE(!iter->Valid());
1120
1121 iter->Seek("8000000000000fo2");
1122 ASSERT_TRUE(!iter->Valid());
1123
1124 delete iter;
1125 }
1126 }
1127 }
1128
TEST_P(PlainTableDBTest,HashBucketConflictReverseSuffixComparator)1129 TEST_P(PlainTableDBTest, HashBucketConflictReverseSuffixComparator) {
1130 for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
1131 huge_page_tlb_size += 2 * 1024 * 1024) {
1132 for (unsigned char i = 1; i <= 3; i++) {
1133 Options options = CurrentOptions();
1134 options.create_if_missing = true;
1135 test::SimpleSuffixReverseComparator comp;
1136 options.comparator = ∁
1137 // Set only one bucket to force bucket conflict.
1138 // Test index interval for the same prefix to be 1, 2 and 4
1139
1140 PlainTableOptions plain_table_options;
1141 plain_table_options.user_key_len = 16;
1142 plain_table_options.bloom_bits_per_key = 0;
1143 plain_table_options.hash_table_ratio = 0;
1144 plain_table_options.index_sparseness = 2 ^ i;
1145 plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
1146
1147 options.table_factory.reset(NewPlainTableFactory(plain_table_options));
1148 DestroyAndReopen(&options);
1149 ASSERT_OK(Put("5000000000000fo0", "v1"));
1150 ASSERT_OK(Put("5000000000000fo1", "v2"));
1151 ASSERT_OK(Put("5000000000000fo2", "v"));
1152 ASSERT_OK(Put("2000000000000fo0", "v3"));
1153 ASSERT_OK(Put("2000000000000fo1", "v4"));
1154 ASSERT_OK(Put("2000000000000fo2", "v"));
1155 ASSERT_OK(Put("2000000000000fo3", "v"));
1156
1157 dbfull()->TEST_FlushMemTable();
1158
1159 ASSERT_EQ("v1", Get("5000000000000fo0"));
1160 ASSERT_EQ("v2", Get("5000000000000fo1"));
1161 ASSERT_EQ("v3", Get("2000000000000fo0"));
1162 ASSERT_EQ("v4", Get("2000000000000fo1"));
1163
1164 ASSERT_EQ("NOT_FOUND", Get("5000000000000bar"));
1165 ASSERT_EQ("NOT_FOUND", Get("2000000000000bar"));
1166 ASSERT_EQ("NOT_FOUND", Get("5000000000000fo8"));
1167 ASSERT_EQ("NOT_FOUND", Get("2000000000000fo8"));
1168
1169 ReadOptions ro;
1170 Iterator* iter = dbfull()->NewIterator(ro);
1171
1172 iter->Seek("5000000000000fo1");
1173 ASSERT_TRUE(iter->Valid());
1174 ASSERT_EQ("5000000000000fo1", iter->key().ToString());
1175 iter->Next();
1176 ASSERT_TRUE(iter->Valid());
1177 ASSERT_EQ("5000000000000fo0", iter->key().ToString());
1178
1179 iter->Seek("5000000000000fo1");
1180 ASSERT_TRUE(iter->Valid());
1181 ASSERT_EQ("5000000000000fo1", iter->key().ToString());
1182
1183 iter->Seek("2000000000000fo1");
1184 ASSERT_TRUE(iter->Valid());
1185 ASSERT_EQ("2000000000000fo1", iter->key().ToString());
1186 iter->Next();
1187 ASSERT_TRUE(iter->Valid());
1188 ASSERT_EQ("2000000000000fo0", iter->key().ToString());
1189
1190 iter->Seek("2000000000000fo1");
1191 ASSERT_TRUE(iter->Valid());
1192 ASSERT_EQ("2000000000000fo1", iter->key().ToString());
1193
1194 iter->Seek("2000000000000var");
1195 ASSERT_TRUE(iter->Valid());
1196 ASSERT_EQ("2000000000000fo3", iter->key().ToString());
1197
1198 iter->Seek("5000000000000var");
1199 ASSERT_TRUE(iter->Valid());
1200 ASSERT_EQ("5000000000000fo2", iter->key().ToString());
1201
1202 std::string seek_key = "2000000000000bar";
1203 iter->Seek(seek_key);
1204 ASSERT_TRUE(!iter->Valid() ||
1205 options.prefix_extractor->Transform(iter->key()) !=
1206 options.prefix_extractor->Transform(seek_key));
1207
1208 iter->Seek("1000000000000fo2");
1209 ASSERT_TRUE(!iter->Valid());
1210
1211 iter->Seek("3000000000000fo2");
1212 ASSERT_TRUE(!iter->Valid());
1213
1214 iter->Seek("8000000000000fo2");
1215 ASSERT_TRUE(!iter->Valid());
1216
1217 delete iter;
1218 }
1219 }
1220 }
1221
TEST_P(PlainTableDBTest,NonExistingKeyToNonEmptyBucket)1222 TEST_P(PlainTableDBTest, NonExistingKeyToNonEmptyBucket) {
1223 Options options = CurrentOptions();
1224 options.create_if_missing = true;
1225 // Set only one bucket to force bucket conflict.
1226 // Test index interval for the same prefix to be 1, 2 and 4
1227 PlainTableOptions plain_table_options;
1228 plain_table_options.user_key_len = 16;
1229 plain_table_options.bloom_bits_per_key = 0;
1230 plain_table_options.hash_table_ratio = 0;
1231 plain_table_options.index_sparseness = 5;
1232
1233 options.table_factory.reset(NewPlainTableFactory(plain_table_options));
1234 DestroyAndReopen(&options);
1235 ASSERT_OK(Put("5000000000000fo0", "v1"));
1236 ASSERT_OK(Put("5000000000000fo1", "v2"));
1237 ASSERT_OK(Put("5000000000000fo2", "v3"));
1238
1239 dbfull()->TEST_FlushMemTable();
1240
1241 ASSERT_EQ("v1", Get("5000000000000fo0"));
1242 ASSERT_EQ("v2", Get("5000000000000fo1"));
1243 ASSERT_EQ("v3", Get("5000000000000fo2"));
1244
1245 ASSERT_EQ("NOT_FOUND", Get("8000000000000bar"));
1246 ASSERT_EQ("NOT_FOUND", Get("1000000000000bar"));
1247
1248 Iterator* iter = dbfull()->NewIterator(ReadOptions());
1249
1250 iter->Seek("5000000000000bar");
1251 ASSERT_TRUE(iter->Valid());
1252 ASSERT_EQ("5000000000000fo0", iter->key().ToString());
1253
1254 iter->Seek("5000000000000fo8");
1255 ASSERT_TRUE(!iter->Valid());
1256
1257 iter->Seek("1000000000000fo2");
1258 ASSERT_TRUE(!iter->Valid());
1259
1260 iter->Seek("8000000000000fo2");
1261 ASSERT_TRUE(!iter->Valid());
1262
1263 delete iter;
1264 }
1265
Key(int i)1266 static std::string Key(int i) {
1267 char buf[100];
1268 snprintf(buf, sizeof(buf), "key_______%06d", i);
1269 return std::string(buf);
1270 }
1271
RandomString(Random * rnd,int len)1272 static std::string RandomString(Random* rnd, int len) {
1273 std::string r;
1274 test::RandomString(rnd, len, &r);
1275 return r;
1276 }
1277
TEST_P(PlainTableDBTest,CompactionTrigger)1278 TEST_P(PlainTableDBTest, CompactionTrigger) {
1279 Options options = CurrentOptions();
1280 options.write_buffer_size = 120 << 10; // 100KB
1281 options.num_levels = 3;
1282 options.level0_file_num_compaction_trigger = 3;
1283 Reopen(&options);
1284
1285 Random rnd(301);
1286
1287 for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
1288 num++) {
1289 std::vector<std::string> values;
1290 // Write 120KB (10 values, each 12K)
1291 for (int i = 0; i < 10; i++) {
1292 values.push_back(RandomString(&rnd, 12000));
1293 ASSERT_OK(Put(Key(i), values[i]));
1294 }
1295 ASSERT_OK(Put(Key(999), ""));
1296 dbfull()->TEST_WaitForFlushMemTable();
1297 ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
1298 }
1299
1300 //generate one more file in level-0, and should trigger level-0 compaction
1301 std::vector<std::string> values;
1302 for (int i = 0; i < 12; i++) {
1303 values.push_back(RandomString(&rnd, 10000));
1304 ASSERT_OK(Put(Key(i), values[i]));
1305 }
1306 ASSERT_OK(Put(Key(999), ""));
1307 dbfull()->TEST_WaitForCompact();
1308
1309 ASSERT_EQ(NumTableFilesAtLevel(0), 0);
1310 ASSERT_EQ(NumTableFilesAtLevel(1), 1);
1311 }
1312
TEST_P(PlainTableDBTest,AdaptiveTable)1313 TEST_P(PlainTableDBTest, AdaptiveTable) {
1314 Options options = CurrentOptions();
1315 options.create_if_missing = true;
1316
1317 options.table_factory.reset(NewPlainTableFactory());
1318 DestroyAndReopen(&options);
1319
1320 ASSERT_OK(Put("1000000000000foo", "v1"));
1321 ASSERT_OK(Put("0000000000000bar", "v2"));
1322 ASSERT_OK(Put("1000000000000foo", "v3"));
1323 dbfull()->TEST_FlushMemTable();
1324
1325 options.create_if_missing = false;
1326 std::shared_ptr<TableFactory> dummy_factory;
1327 std::shared_ptr<TableFactory> block_based_factory(
1328 NewBlockBasedTableFactory());
1329 options.table_factory.reset(NewAdaptiveTableFactory(
1330 block_based_factory, dummy_factory, dummy_factory));
1331 Reopen(&options);
1332 ASSERT_EQ("v3", Get("1000000000000foo"));
1333 ASSERT_EQ("v2", Get("0000000000000bar"));
1334
1335 ASSERT_OK(Put("2000000000000foo", "v4"));
1336 ASSERT_OK(Put("3000000000000bar", "v5"));
1337 dbfull()->TEST_FlushMemTable();
1338 ASSERT_EQ("v4", Get("2000000000000foo"));
1339 ASSERT_EQ("v5", Get("3000000000000bar"));
1340
1341 Reopen(&options);
1342 ASSERT_EQ("v3", Get("1000000000000foo"));
1343 ASSERT_EQ("v2", Get("0000000000000bar"));
1344 ASSERT_EQ("v4", Get("2000000000000foo"));
1345 ASSERT_EQ("v5", Get("3000000000000bar"));
1346
1347 options.table_factory.reset(NewBlockBasedTableFactory());
1348 Reopen(&options);
1349 ASSERT_NE("v3", Get("1000000000000foo"));
1350
1351 options.table_factory.reset(NewPlainTableFactory());
1352 Reopen(&options);
1353 ASSERT_NE("v5", Get("3000000000000bar"));
1354 }
1355
1356 INSTANTIATE_TEST_CASE_P(PlainTableDBTest, PlainTableDBTest, ::testing::Bool());
1357
1358 } // namespace rocksdb
1359
main(int argc,char ** argv)1360 int main(int argc, char** argv) {
1361 ::testing::InitGoogleTest(&argc, argv);
1362 return RUN_ALL_TESTS();
1363 }
1364
1365 #else
1366 #include <stdio.h>
1367
main(int,char **)1368 int main(int /*argc*/, char** /*argv*/) {
1369 fprintf(stderr, "SKIPPED as plain table is not supported in ROCKSDB_LITE\n");
1370 return 0;
1371 }
1372
1373 #endif // !ROCKSDB_LITE
1374