1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5
6 #include "table/block_based/full_filter_block.h"
7
8 #include <set>
9
10 #include "rocksdb/filter_policy.h"
11 #include "rocksdb/status.h"
12 #include "table/block_based/block_based_table_reader.h"
13 #include "table/block_based/filter_policy_internal.h"
14 #include "table/block_based/mock_block_based_table.h"
15 #include "table/format.h"
16 #include "test_util/testharness.h"
17 #include "test_util/testutil.h"
18 #include "util/coding.h"
19 #include "util/hash.h"
20 #include "util/string_util.h"
21
22 namespace ROCKSDB_NAMESPACE {
23
24 class TestFilterBitsBuilder : public FilterBitsBuilder {
25 public:
TestFilterBitsBuilder()26 explicit TestFilterBitsBuilder() {}
27
28 // Add Key to filter
AddKey(const Slice & key)29 void AddKey(const Slice& key) override {
30 hash_entries_.push_back(Hash(key.data(), key.size(), 1));
31 }
32
33 // Generate the filter using the keys that are added
Finish(std::unique_ptr<const char[]> * buf)34 Slice Finish(std::unique_ptr<const char[]>* buf) override {
35 uint32_t len = static_cast<uint32_t>(hash_entries_.size()) * 4;
36 char* data = new char[len];
37 for (size_t i = 0; i < hash_entries_.size(); i++) {
38 EncodeFixed32(data + i * 4, hash_entries_[i]);
39 }
40 const char* const_data = data;
41 buf->reset(const_data);
42 return Slice(data, len);
43 }
44
45 private:
46 std::vector<uint32_t> hash_entries_;
47 };
48
49 class MockBlockBasedTable : public BlockBasedTable {
50 public:
MockBlockBasedTable(Rep * rep)51 explicit MockBlockBasedTable(Rep* rep)
52 : BlockBasedTable(rep, nullptr /* block_cache_tracer */) {}
53 };
54
55 class TestFilterBitsReader : public FilterBitsReader {
56 public:
TestFilterBitsReader(const Slice & contents)57 explicit TestFilterBitsReader(const Slice& contents)
58 : data_(contents.data()), len_(static_cast<uint32_t>(contents.size())) {}
59
60 // Silence compiler warning about overloaded virtual
61 using FilterBitsReader::MayMatch;
MayMatch(const Slice & entry)62 bool MayMatch(const Slice& entry) override {
63 uint32_t h = Hash(entry.data(), entry.size(), 1);
64 for (size_t i = 0; i + 4 <= len_; i += 4) {
65 if (h == DecodeFixed32(data_ + i)) {
66 return true;
67 }
68 }
69 return false;
70 }
71
72 private:
73 const char* data_;
74 uint32_t len_;
75 };
76
77
78 class TestHashFilter : public FilterPolicy {
79 public:
Name() const80 const char* Name() const override { return "TestHashFilter"; }
81
CreateFilter(const Slice * keys,int n,std::string * dst) const82 void CreateFilter(const Slice* keys, int n, std::string* dst) const override {
83 for (int i = 0; i < n; i++) {
84 uint32_t h = Hash(keys[i].data(), keys[i].size(), 1);
85 PutFixed32(dst, h);
86 }
87 }
88
KeyMayMatch(const Slice & key,const Slice & filter) const89 bool KeyMayMatch(const Slice& key, const Slice& filter) const override {
90 uint32_t h = Hash(key.data(), key.size(), 1);
91 for (unsigned int i = 0; i + 4 <= filter.size(); i += 4) {
92 if (h == DecodeFixed32(filter.data() + i)) {
93 return true;
94 }
95 }
96 return false;
97 }
98
GetFilterBitsBuilder() const99 FilterBitsBuilder* GetFilterBitsBuilder() const override {
100 return new TestFilterBitsBuilder();
101 }
102
GetFilterBitsReader(const Slice & contents) const103 FilterBitsReader* GetFilterBitsReader(const Slice& contents) const override {
104 return new TestFilterBitsReader(contents);
105 }
106 };
107
108 class PluginFullFilterBlockTest : public mock::MockBlockBasedTableTester,
109 public testing::Test {
110 public:
PluginFullFilterBlockTest()111 PluginFullFilterBlockTest()
112 : mock::MockBlockBasedTableTester(new TestHashFilter) {}
113 };
114
TEST_F(PluginFullFilterBlockTest,PluginEmptyBuilder)115 TEST_F(PluginFullFilterBlockTest, PluginEmptyBuilder) {
116 FullFilterBlockBuilder builder(nullptr, true, GetBuilder());
117 Slice slice = builder.Finish();
118 ASSERT_EQ("", EscapeString(slice));
119
120 CachableEntry<ParsedFullFilterBlock> block(
121 new ParsedFullFilterBlock(table_options_.filter_policy.get(),
122 BlockContents(slice)),
123 nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */);
124
125 FullFilterBlockReader reader(table_.get(), std::move(block));
126 // Remain same symantic with blockbased filter
127 ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr,
128 /*block_offset=*/kNotValid,
129 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
130 /*get_context=*/nullptr,
131 /*lookup_context=*/nullptr));
132 }
133
TEST_F(PluginFullFilterBlockTest,PluginSingleChunk)134 TEST_F(PluginFullFilterBlockTest, PluginSingleChunk) {
135 FullFilterBlockBuilder builder(nullptr, true, GetBuilder());
136 builder.Add("foo");
137 builder.Add("bar");
138 builder.Add("box");
139 builder.Add("box");
140 builder.Add("hello");
141 Slice slice = builder.Finish();
142
143 CachableEntry<ParsedFullFilterBlock> block(
144 new ParsedFullFilterBlock(table_options_.filter_policy.get(),
145 BlockContents(slice)),
146 nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */);
147
148 FullFilterBlockReader reader(table_.get(), std::move(block));
149 ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr,
150 /*block_offset=*/kNotValid,
151 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
152 /*get_context=*/nullptr,
153 /*lookup_context=*/nullptr));
154 ASSERT_TRUE(reader.KeyMayMatch("bar", /*prefix_extractor=*/nullptr,
155 /*block_offset=*/kNotValid,
156 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
157 /*get_context=*/nullptr,
158 /*lookup_context=*/nullptr));
159 ASSERT_TRUE(reader.KeyMayMatch("box", /*prefix_extractor=*/nullptr,
160 /*block_offset=*/kNotValid,
161 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
162 /*get_context=*/nullptr,
163 /*lookup_context=*/nullptr));
164 ASSERT_TRUE(reader.KeyMayMatch("hello", /*prefix_extractor=*/nullptr,
165 /*block_offset=*/kNotValid,
166 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
167 /*get_context=*/nullptr,
168 /*lookup_context=*/nullptr));
169 ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr,
170 /*block_offset=*/kNotValid,
171 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
172 /*get_context=*/nullptr,
173 /*lookup_context=*/nullptr));
174 ASSERT_TRUE(!reader.KeyMayMatch(
175 "missing", /*prefix_extractor=*/nullptr, /*block_offset=*/kNotValid,
176 /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
177 /*lookup_context=*/nullptr));
178 ASSERT_TRUE(!reader.KeyMayMatch(
179 "other", /*prefix_extractor=*/nullptr, /*block_offset=*/kNotValid,
180 /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
181 /*lookup_context=*/nullptr));
182 }
183
184 class FullFilterBlockTest : public mock::MockBlockBasedTableTester,
185 public testing::Test {
186 public:
FullFilterBlockTest()187 FullFilterBlockTest()
188 : mock::MockBlockBasedTableTester(NewBloomFilterPolicy(10, false)) {}
189 };
190
TEST_F(FullFilterBlockTest,EmptyBuilder)191 TEST_F(FullFilterBlockTest, EmptyBuilder) {
192 FullFilterBlockBuilder builder(nullptr, true, GetBuilder());
193 Slice slice = builder.Finish();
194 ASSERT_EQ("", EscapeString(slice));
195
196 CachableEntry<ParsedFullFilterBlock> block(
197 new ParsedFullFilterBlock(table_options_.filter_policy.get(),
198 BlockContents(slice)),
199 nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */);
200
201 FullFilterBlockReader reader(table_.get(), std::move(block));
202 // Remain same symantic with blockbased filter
203 ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr,
204 /*block_offset=*/kNotValid,
205 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
206 /*get_context=*/nullptr,
207 /*lookup_context=*/nullptr));
208 }
209
210 class CountUniqueFilterBitsBuilderWrapper : public FilterBitsBuilder {
211 std::unique_ptr<FilterBitsBuilder> b_;
212 std::set<std::string> uniq_;
213
214 public:
CountUniqueFilterBitsBuilderWrapper(FilterBitsBuilder * b)215 explicit CountUniqueFilterBitsBuilderWrapper(FilterBitsBuilder* b) : b_(b) {}
216
~CountUniqueFilterBitsBuilderWrapper()217 ~CountUniqueFilterBitsBuilderWrapper() override {}
218
AddKey(const Slice & key)219 void AddKey(const Slice& key) override {
220 b_->AddKey(key);
221 uniq_.insert(key.ToString());
222 }
223
Finish(std::unique_ptr<const char[]> * buf)224 Slice Finish(std::unique_ptr<const char[]>* buf) override {
225 Slice rv = b_->Finish(buf);
226 uniq_.clear();
227 return rv;
228 }
229
ApproximateNumEntries(size_t bytes)230 size_t ApproximateNumEntries(size_t bytes) override {
231 return b_->ApproximateNumEntries(bytes);
232 }
233
CountUnique()234 size_t CountUnique() { return uniq_.size(); }
235 };
236
TEST_F(FullFilterBlockTest,DuplicateEntries)237 TEST_F(FullFilterBlockTest, DuplicateEntries) {
238 { // empty prefixes
239 std::unique_ptr<const SliceTransform> prefix_extractor(
240 NewFixedPrefixTransform(0));
241 auto bits_builder = new CountUniqueFilterBitsBuilderWrapper(GetBuilder());
242 const bool WHOLE_KEY = true;
243 FullFilterBlockBuilder builder(prefix_extractor.get(), WHOLE_KEY,
244 bits_builder);
245 ASSERT_EQ(0, bits_builder->CountUnique());
246 // adds key and empty prefix; both abstractions count them
247 builder.Add("key1");
248 ASSERT_EQ(2, bits_builder->CountUnique());
249 // Add different key (unique) and also empty prefix (not unique).
250 // From here in this test, it's immaterial whether the block builder
251 // can count unique keys.
252 builder.Add("key2");
253 ASSERT_EQ(3, bits_builder->CountUnique());
254 // Empty key -> nothing unique
255 builder.Add("");
256 ASSERT_EQ(3, bits_builder->CountUnique());
257 }
258
259 // mix of empty and non-empty
260 std::unique_ptr<const SliceTransform> prefix_extractor(
261 NewFixedPrefixTransform(7));
262 auto bits_builder = new CountUniqueFilterBitsBuilderWrapper(GetBuilder());
263 const bool WHOLE_KEY = true;
264 FullFilterBlockBuilder builder(prefix_extractor.get(), WHOLE_KEY,
265 bits_builder);
266 builder.Add(""); // test with empty key too
267 builder.Add("prefix1key1");
268 builder.Add("prefix1key1");
269 builder.Add("prefix1key2");
270 builder.Add("prefix1key3");
271 builder.Add("prefix2key4");
272 // 1 empty, 2 non-empty prefixes, and 4 non-empty keys
273 ASSERT_EQ(1 + 2 + 4, bits_builder->CountUnique());
274 }
275
TEST_F(FullFilterBlockTest,SingleChunk)276 TEST_F(FullFilterBlockTest, SingleChunk) {
277 FullFilterBlockBuilder builder(nullptr, true, GetBuilder());
278 ASSERT_TRUE(builder.IsEmpty());
279 builder.Add("foo");
280 ASSERT_FALSE(builder.IsEmpty());
281 builder.Add("bar");
282 builder.Add("box");
283 builder.Add("box");
284 builder.Add("hello");
285 // "box" only counts once
286 ASSERT_EQ(4, builder.EstimateEntriesAdded());
287 ASSERT_FALSE(builder.IsEmpty());
288 Status s;
289 Slice slice = builder.Finish(BlockHandle(), &s);
290 ASSERT_OK(s);
291
292 CachableEntry<ParsedFullFilterBlock> block(
293 new ParsedFullFilterBlock(table_options_.filter_policy.get(),
294 BlockContents(slice)),
295 nullptr /* cache */, nullptr /* cache_handle */, true /* own_value */);
296
297 FullFilterBlockReader reader(table_.get(), std::move(block));
298 ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr,
299 /*block_offset=*/kNotValid,
300 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
301 /*get_context=*/nullptr,
302 /*lookup_context=*/nullptr));
303 ASSERT_TRUE(reader.KeyMayMatch("bar", /*prefix_extractor=*/nullptr,
304 /*block_offset=*/kNotValid,
305 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
306 /*get_context=*/nullptr,
307 /*lookup_context=*/nullptr));
308 ASSERT_TRUE(reader.KeyMayMatch("box", /*prefix_extractor=*/nullptr,
309 /*block_offset=*/kNotValid,
310 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
311 /*get_context=*/nullptr,
312 /*lookup_context=*/nullptr));
313 ASSERT_TRUE(reader.KeyMayMatch("hello", /*prefix_extractor=*/nullptr,
314 /*block_offset=*/kNotValid,
315 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
316 /*get_context=*/nullptr,
317 /*lookup_context=*/nullptr));
318 ASSERT_TRUE(reader.KeyMayMatch("foo", /*prefix_extractor=*/nullptr,
319 /*block_offset=*/kNotValid,
320 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
321 /*get_context=*/nullptr,
322 /*lookup_context=*/nullptr));
323 ASSERT_TRUE(!reader.KeyMayMatch(
324 "missing", /*prefix_extractor=*/nullptr, /*block_offset=*/kNotValid,
325 /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
326 /*lookup_context=*/nullptr));
327 ASSERT_TRUE(!reader.KeyMayMatch(
328 "other", /*prefix_extractor=*/nullptr, /*block_offset=*/kNotValid,
329 /*no_io=*/false, /*const_ikey_ptr=*/nullptr, /*get_context=*/nullptr,
330 /*lookup_context=*/nullptr));
331 }
332
333 } // namespace ROCKSDB_NAMESPACE
334
main(int argc,char ** argv)335 int main(int argc, char** argv) {
336 ::testing::InitGoogleTest(&argc, argv);
337 return RUN_ALL_TESTS();
338 }
339