1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2012 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #include <array>
11 #include <deque>
12
13 #include "rocksdb/filter_policy.h"
14
15 #include "rocksdb/slice.h"
16 #include "table/block_based/block_based_filter_block.h"
17 #include "table/block_based/full_filter_block.h"
18 #include "table/block_based/filter_policy_internal.h"
19 #include "third-party/folly/folly/ConstexprMath.h"
20 #include "util/bloom_impl.h"
21 #include "util/coding.h"
22 #include "util/hash.h"
23
24 namespace ROCKSDB_NAMESPACE {
25
26 namespace {
27
28 // See description in FastLocalBloomImpl
29 class FastLocalBloomBitsBuilder : public BuiltinFilterBitsBuilder {
30 public:
FastLocalBloomBitsBuilder(const int millibits_per_key)31 explicit FastLocalBloomBitsBuilder(const int millibits_per_key)
32 : millibits_per_key_(millibits_per_key),
33 num_probes_(FastLocalBloomImpl::ChooseNumProbes(millibits_per_key_)) {
34 assert(millibits_per_key >= 1000);
35 }
36
37 // No Copy allowed
38 FastLocalBloomBitsBuilder(const FastLocalBloomBitsBuilder&) = delete;
39 void operator=(const FastLocalBloomBitsBuilder&) = delete;
40
~FastLocalBloomBitsBuilder()41 ~FastLocalBloomBitsBuilder() override {}
42
AddKey(const Slice & key)43 virtual void AddKey(const Slice& key) override {
44 uint64_t hash = GetSliceHash64(key);
45 if (hash_entries_.empty() || hash != hash_entries_.back()) {
46 hash_entries_.push_back(hash);
47 }
48 }
49
Finish(std::unique_ptr<const char[]> * buf)50 virtual Slice Finish(std::unique_ptr<const char[]>* buf) override {
51 uint32_t len_with_metadata =
52 CalculateSpace(static_cast<uint32_t>(hash_entries_.size()));
53 char* data = new char[len_with_metadata];
54 memset(data, 0, len_with_metadata);
55
56 assert(data);
57 assert(len_with_metadata >= 5);
58
59 uint32_t len = len_with_metadata - 5;
60 if (len > 0) {
61 AddAllEntries(data, len);
62 }
63
64 // See BloomFilterPolicy::GetBloomBitsReader re: metadata
65 // -1 = Marker for newer Bloom implementations
66 data[len] = static_cast<char>(-1);
67 // 0 = Marker for this sub-implementation
68 data[len + 1] = static_cast<char>(0);
69 // num_probes (and 0 in upper bits for 64-byte block size)
70 data[len + 2] = static_cast<char>(num_probes_);
71 // rest of metadata stays zero
72
73 const char* const_data = data;
74 buf->reset(const_data);
75 assert(hash_entries_.empty());
76
77 return Slice(data, len_with_metadata);
78 }
79
CalculateNumEntry(const uint32_t bytes)80 int CalculateNumEntry(const uint32_t bytes) override {
81 uint32_t bytes_no_meta = bytes >= 5u ? bytes - 5u : 0;
82 return static_cast<int>(uint64_t{8000} * bytes_no_meta /
83 millibits_per_key_);
84 }
85
CalculateSpace(const int num_entry)86 uint32_t CalculateSpace(const int num_entry) override {
87 uint32_t num_cache_lines = 0;
88 if (millibits_per_key_ > 0 && num_entry > 0) {
89 num_cache_lines = static_cast<uint32_t>(
90 (int64_t{num_entry} * millibits_per_key_ + 511999) / 512000);
91 }
92 return num_cache_lines * 64 + /*metadata*/ 5;
93 }
94
EstimatedFpRate(size_t keys,size_t bytes)95 double EstimatedFpRate(size_t keys, size_t bytes) override {
96 return FastLocalBloomImpl::EstimatedFpRate(keys, bytes - /*metadata*/ 5,
97 num_probes_, /*hash bits*/ 64);
98 }
99
100 private:
AddAllEntries(char * data,uint32_t len)101 void AddAllEntries(char* data, uint32_t len) {
102 // Simple version without prefetching:
103 //
104 // for (auto h : hash_entries_) {
105 // FastLocalBloomImpl::AddHash(Lower32of64(h), Upper32of64(h), len,
106 // num_probes_, data);
107 // }
108
109 const size_t num_entries = hash_entries_.size();
110 constexpr size_t kBufferMask = 7;
111 static_assert(((kBufferMask + 1) & kBufferMask) == 0,
112 "Must be power of 2 minus 1");
113
114 std::array<uint32_t, kBufferMask + 1> hashes;
115 std::array<uint32_t, kBufferMask + 1> byte_offsets;
116
117 // Prime the buffer
118 size_t i = 0;
119 for (; i <= kBufferMask && i < num_entries; ++i) {
120 uint64_t h = hash_entries_.front();
121 hash_entries_.pop_front();
122 FastLocalBloomImpl::PrepareHash(Lower32of64(h), len, data,
123 /*out*/ &byte_offsets[i]);
124 hashes[i] = Upper32of64(h);
125 }
126
127 // Process and buffer
128 for (; i < num_entries; ++i) {
129 uint32_t& hash_ref = hashes[i & kBufferMask];
130 uint32_t& byte_offset_ref = byte_offsets[i & kBufferMask];
131 // Process (add)
132 FastLocalBloomImpl::AddHashPrepared(hash_ref, num_probes_,
133 data + byte_offset_ref);
134 // And buffer
135 uint64_t h = hash_entries_.front();
136 hash_entries_.pop_front();
137 FastLocalBloomImpl::PrepareHash(Lower32of64(h), len, data,
138 /*out*/ &byte_offset_ref);
139 hash_ref = Upper32of64(h);
140 }
141
142 // Finish processing
143 for (i = 0; i <= kBufferMask && i < num_entries; ++i) {
144 FastLocalBloomImpl::AddHashPrepared(hashes[i], num_probes_,
145 data + byte_offsets[i]);
146 }
147 }
148
149 int millibits_per_key_;
150 int num_probes_;
151 // A deque avoids unnecessary copying of already-saved values
152 // and has near-minimal peak memory use.
153 std::deque<uint64_t> hash_entries_;
154 };
155
156 // See description in FastLocalBloomImpl
157 class FastLocalBloomBitsReader : public FilterBitsReader {
158 public:
FastLocalBloomBitsReader(const char * data,int num_probes,uint32_t len_bytes)159 FastLocalBloomBitsReader(const char* data, int num_probes, uint32_t len_bytes)
160 : data_(data), num_probes_(num_probes), len_bytes_(len_bytes) {}
161
162 // No Copy allowed
163 FastLocalBloomBitsReader(const FastLocalBloomBitsReader&) = delete;
164 void operator=(const FastLocalBloomBitsReader&) = delete;
165
~FastLocalBloomBitsReader()166 ~FastLocalBloomBitsReader() override {}
167
MayMatch(const Slice & key)168 bool MayMatch(const Slice& key) override {
169 uint64_t h = GetSliceHash64(key);
170 uint32_t byte_offset;
171 FastLocalBloomImpl::PrepareHash(Lower32of64(h), len_bytes_, data_,
172 /*out*/ &byte_offset);
173 return FastLocalBloomImpl::HashMayMatchPrepared(Upper32of64(h), num_probes_,
174 data_ + byte_offset);
175 }
176
MayMatch(int num_keys,Slice ** keys,bool * may_match)177 virtual void MayMatch(int num_keys, Slice** keys, bool* may_match) override {
178 std::array<uint32_t, MultiGetContext::MAX_BATCH_SIZE> hashes;
179 std::array<uint32_t, MultiGetContext::MAX_BATCH_SIZE> byte_offsets;
180 for (int i = 0; i < num_keys; ++i) {
181 uint64_t h = GetSliceHash64(*keys[i]);
182 FastLocalBloomImpl::PrepareHash(Lower32of64(h), len_bytes_, data_,
183 /*out*/ &byte_offsets[i]);
184 hashes[i] = Upper32of64(h);
185 }
186 for (int i = 0; i < num_keys; ++i) {
187 may_match[i] = FastLocalBloomImpl::HashMayMatchPrepared(
188 hashes[i], num_probes_, data_ + byte_offsets[i]);
189 }
190 }
191
192 private:
193 const char* data_;
194 const int num_probes_;
195 const uint32_t len_bytes_;
196 };
197
198 using LegacyBloomImpl = LegacyLocalityBloomImpl</*ExtraRotates*/ false>;
199
200 class LegacyBloomBitsBuilder : public BuiltinFilterBitsBuilder {
201 public:
202 explicit LegacyBloomBitsBuilder(const int bits_per_key, Logger* info_log);
203
204 // No Copy allowed
205 LegacyBloomBitsBuilder(const LegacyBloomBitsBuilder&) = delete;
206 void operator=(const LegacyBloomBitsBuilder&) = delete;
207
208 ~LegacyBloomBitsBuilder() override;
209
210 void AddKey(const Slice& key) override;
211
212 Slice Finish(std::unique_ptr<const char[]>* buf) override;
213
214 int CalculateNumEntry(const uint32_t bytes) override;
215
CalculateSpace(const int num_entry)216 uint32_t CalculateSpace(const int num_entry) override {
217 uint32_t dont_care1;
218 uint32_t dont_care2;
219 return CalculateSpace(num_entry, &dont_care1, &dont_care2);
220 }
221
EstimatedFpRate(size_t keys,size_t bytes)222 double EstimatedFpRate(size_t keys, size_t bytes) override {
223 return LegacyBloomImpl::EstimatedFpRate(keys, bytes - /*metadata*/ 5,
224 num_probes_);
225 }
226
227 private:
228 int bits_per_key_;
229 int num_probes_;
230 std::vector<uint32_t> hash_entries_;
231 Logger* info_log_;
232
233 // Get totalbits that optimized for cpu cache line
234 uint32_t GetTotalBitsForLocality(uint32_t total_bits);
235
236 // Reserve space for new filter
237 char* ReserveSpace(const int num_entry, uint32_t* total_bits,
238 uint32_t* num_lines);
239
240 // Implementation-specific variant of public CalculateSpace
241 uint32_t CalculateSpace(const int num_entry, uint32_t* total_bits,
242 uint32_t* num_lines);
243
244 // Assuming single threaded access to this function.
245 void AddHash(uint32_t h, char* data, uint32_t num_lines, uint32_t total_bits);
246 };
247
LegacyBloomBitsBuilder(const int bits_per_key,Logger * info_log)248 LegacyBloomBitsBuilder::LegacyBloomBitsBuilder(const int bits_per_key,
249 Logger* info_log)
250 : bits_per_key_(bits_per_key),
251 num_probes_(LegacyNoLocalityBloomImpl::ChooseNumProbes(bits_per_key_)),
252 info_log_(info_log) {
253 assert(bits_per_key_);
254 }
255
~LegacyBloomBitsBuilder()256 LegacyBloomBitsBuilder::~LegacyBloomBitsBuilder() {}
257
AddKey(const Slice & key)258 void LegacyBloomBitsBuilder::AddKey(const Slice& key) {
259 uint32_t hash = BloomHash(key);
260 if (hash_entries_.size() == 0 || hash != hash_entries_.back()) {
261 hash_entries_.push_back(hash);
262 }
263 }
264
Finish(std::unique_ptr<const char[]> * buf)265 Slice LegacyBloomBitsBuilder::Finish(std::unique_ptr<const char[]>* buf) {
266 uint32_t total_bits, num_lines;
267 size_t num_entries = hash_entries_.size();
268 char* data =
269 ReserveSpace(static_cast<int>(num_entries), &total_bits, &num_lines);
270 assert(data);
271
272 if (total_bits != 0 && num_lines != 0) {
273 for (auto h : hash_entries_) {
274 AddHash(h, data, num_lines, total_bits);
275 }
276
277 // Check for excessive entries for 32-bit hash function
278 if (num_entries >= /* minimum of 3 million */ 3000000U) {
279 // More specifically, we can detect that the 32-bit hash function
280 // is causing significant increase in FP rate by comparing current
281 // estimated FP rate to what we would get with a normal number of
282 // keys at same memory ratio.
283 double est_fp_rate = LegacyBloomImpl::EstimatedFpRate(
284 num_entries, total_bits / 8, num_probes_);
285 double vs_fp_rate = LegacyBloomImpl::EstimatedFpRate(
286 1U << 16, (1U << 16) * bits_per_key_ / 8, num_probes_);
287
288 if (est_fp_rate >= 1.50 * vs_fp_rate) {
289 // For more details, see
290 // https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter
291 ROCKS_LOG_WARN(
292 info_log_,
293 "Using legacy SST/BBT Bloom filter with excessive key count "
294 "(%.1fM @ %dbpk), causing estimated %.1fx higher filter FP rate. "
295 "Consider using new Bloom with format_version>=5, smaller SST "
296 "file size, or partitioned filters.",
297 num_entries / 1000000.0, bits_per_key_, est_fp_rate / vs_fp_rate);
298 }
299 }
300 }
301 // See BloomFilterPolicy::GetFilterBitsReader for metadata
302 data[total_bits / 8] = static_cast<char>(num_probes_);
303 EncodeFixed32(data + total_bits / 8 + 1, static_cast<uint32_t>(num_lines));
304
305 const char* const_data = data;
306 buf->reset(const_data);
307 hash_entries_.clear();
308
309 return Slice(data, total_bits / 8 + 5);
310 }
311
GetTotalBitsForLocality(uint32_t total_bits)312 uint32_t LegacyBloomBitsBuilder::GetTotalBitsForLocality(uint32_t total_bits) {
313 uint32_t num_lines =
314 (total_bits + CACHE_LINE_SIZE * 8 - 1) / (CACHE_LINE_SIZE * 8);
315
316 // Make num_lines an odd number to make sure more bits are involved
317 // when determining which block.
318 if (num_lines % 2 == 0) {
319 num_lines++;
320 }
321 return num_lines * (CACHE_LINE_SIZE * 8);
322 }
323
CalculateSpace(const int num_entry,uint32_t * total_bits,uint32_t * num_lines)324 uint32_t LegacyBloomBitsBuilder::CalculateSpace(const int num_entry,
325 uint32_t* total_bits,
326 uint32_t* num_lines) {
327 assert(bits_per_key_);
328 if (num_entry != 0) {
329 uint32_t total_bits_tmp = static_cast<uint32_t>(num_entry * bits_per_key_);
330
331 *total_bits = GetTotalBitsForLocality(total_bits_tmp);
332 *num_lines = *total_bits / (CACHE_LINE_SIZE * 8);
333 assert(*total_bits > 0 && *total_bits % 8 == 0);
334 } else {
335 // filter is empty, just leave space for metadata
336 *total_bits = 0;
337 *num_lines = 0;
338 }
339
340 // Reserve space for Filter
341 uint32_t sz = *total_bits / 8;
342 sz += 5; // 4 bytes for num_lines, 1 byte for num_probes
343 return sz;
344 }
345
ReserveSpace(const int num_entry,uint32_t * total_bits,uint32_t * num_lines)346 char* LegacyBloomBitsBuilder::ReserveSpace(const int num_entry,
347 uint32_t* total_bits,
348 uint32_t* num_lines) {
349 uint32_t sz = CalculateSpace(num_entry, total_bits, num_lines);
350 char* data = new char[sz];
351 memset(data, 0, sz);
352 return data;
353 }
354
CalculateNumEntry(const uint32_t bytes)355 int LegacyBloomBitsBuilder::CalculateNumEntry(const uint32_t bytes) {
356 assert(bits_per_key_);
357 assert(bytes > 0);
358 int high = static_cast<int>(bytes * 8 / bits_per_key_ + 1);
359 int low = 1;
360 int n = high;
361 for (; n >= low; n--) {
362 if (CalculateSpace(n) <= bytes) {
363 break;
364 }
365 }
366 assert(n < high); // High should be an overestimation
367 return n;
368 }
369
AddHash(uint32_t h,char * data,uint32_t num_lines,uint32_t total_bits)370 inline void LegacyBloomBitsBuilder::AddHash(uint32_t h, char* data,
371 uint32_t num_lines,
372 uint32_t total_bits) {
373 #ifdef NDEBUG
374 static_cast<void>(total_bits);
375 #endif
376 assert(num_lines > 0 && total_bits > 0);
377
378 LegacyBloomImpl::AddHash(h, num_lines, num_probes_, data,
379 folly::constexpr_log2(CACHE_LINE_SIZE));
380 }
381
382 class LegacyBloomBitsReader : public FilterBitsReader {
383 public:
LegacyBloomBitsReader(const char * data,int num_probes,uint32_t num_lines,uint32_t log2_cache_line_size)384 LegacyBloomBitsReader(const char* data, int num_probes, uint32_t num_lines,
385 uint32_t log2_cache_line_size)
386 : data_(data),
387 num_probes_(num_probes),
388 num_lines_(num_lines),
389 log2_cache_line_size_(log2_cache_line_size) {}
390
391 // No Copy allowed
392 LegacyBloomBitsReader(const LegacyBloomBitsReader&) = delete;
393 void operator=(const LegacyBloomBitsReader&) = delete;
394
~LegacyBloomBitsReader()395 ~LegacyBloomBitsReader() override {}
396
397 // "contents" contains the data built by a preceding call to
398 // FilterBitsBuilder::Finish. MayMatch must return true if the key was
399 // passed to FilterBitsBuilder::AddKey. This method may return true or false
400 // if the key was not on the list, but it should aim to return false with a
401 // high probability.
MayMatch(const Slice & key)402 bool MayMatch(const Slice& key) override {
403 uint32_t hash = BloomHash(key);
404 uint32_t byte_offset;
405 LegacyBloomImpl::PrepareHashMayMatch(
406 hash, num_lines_, data_, /*out*/ &byte_offset, log2_cache_line_size_);
407 return LegacyBloomImpl::HashMayMatchPrepared(
408 hash, num_probes_, data_ + byte_offset, log2_cache_line_size_);
409 }
410
MayMatch(int num_keys,Slice ** keys,bool * may_match)411 virtual void MayMatch(int num_keys, Slice** keys, bool* may_match) override {
412 std::array<uint32_t, MultiGetContext::MAX_BATCH_SIZE> hashes;
413 std::array<uint32_t, MultiGetContext::MAX_BATCH_SIZE> byte_offsets;
414 for (int i = 0; i < num_keys; ++i) {
415 hashes[i] = BloomHash(*keys[i]);
416 LegacyBloomImpl::PrepareHashMayMatch(hashes[i], num_lines_, data_,
417 /*out*/ &byte_offsets[i],
418 log2_cache_line_size_);
419 }
420 for (int i = 0; i < num_keys; ++i) {
421 may_match[i] = LegacyBloomImpl::HashMayMatchPrepared(
422 hashes[i], num_probes_, data_ + byte_offsets[i],
423 log2_cache_line_size_);
424 }
425 }
426
427 private:
428 const char* data_;
429 const int num_probes_;
430 const uint32_t num_lines_;
431 const uint32_t log2_cache_line_size_;
432 };
433
434 class AlwaysTrueFilter : public FilterBitsReader {
435 public:
MayMatch(const Slice &)436 bool MayMatch(const Slice&) override { return true; }
437 using FilterBitsReader::MayMatch; // inherit overload
438 };
439
440 class AlwaysFalseFilter : public FilterBitsReader {
441 public:
MayMatch(const Slice &)442 bool MayMatch(const Slice&) override { return false; }
443 using FilterBitsReader::MayMatch; // inherit overload
444 };
445
446 } // namespace
447
448 const std::vector<BloomFilterPolicy::Mode> BloomFilterPolicy::kAllFixedImpls = {
449 kLegacyBloom,
450 kDeprecatedBlock,
451 kFastLocalBloom,
452 };
453
454 const std::vector<BloomFilterPolicy::Mode> BloomFilterPolicy::kAllUserModes = {
455 kDeprecatedBlock,
456 kAuto,
457 };
458
BloomFilterPolicy(double bits_per_key,Mode mode)459 BloomFilterPolicy::BloomFilterPolicy(double bits_per_key, Mode mode)
460 : mode_(mode), warned_(false) {
461 // Sanitize bits_per_key
462 if (bits_per_key < 1.0) {
463 bits_per_key = 1.0;
464 } else if (!(bits_per_key < 100.0)) { // including NaN
465 bits_per_key = 100.0;
466 }
467
468 // Includes a nudge toward rounding up, to ensure on all platforms
469 // that doubles specified with three decimal digits after the decimal
470 // point are interpreted accurately.
471 millibits_per_key_ = static_cast<int>(bits_per_key * 1000.0 + 0.500001);
472
473 // For better or worse, this is a rounding up of a nudged rounding up,
474 // e.g. 7.4999999999999 will round up to 8, but that provides more
475 // predictability against small arithmetic errors in floating point.
476 whole_bits_per_key_ = (millibits_per_key_ + 500) / 1000;
477 }
478
~BloomFilterPolicy()479 BloomFilterPolicy::~BloomFilterPolicy() {}
480
Name() const481 const char* BloomFilterPolicy::Name() const {
482 return "rocksdb.BuiltinBloomFilter";
483 }
484
CreateFilter(const Slice * keys,int n,std::string * dst) const485 void BloomFilterPolicy::CreateFilter(const Slice* keys, int n,
486 std::string* dst) const {
487 // We should ideally only be using this deprecated interface for
488 // appropriately constructed BloomFilterPolicy
489 assert(mode_ == kDeprecatedBlock);
490
491 // Compute bloom filter size (in both bits and bytes)
492 uint32_t bits = static_cast<uint32_t>(n * whole_bits_per_key_);
493
494 // For small n, we can see a very high false positive rate. Fix it
495 // by enforcing a minimum bloom filter length.
496 if (bits < 64) bits = 64;
497
498 uint32_t bytes = (bits + 7) / 8;
499 bits = bytes * 8;
500
501 int num_probes =
502 LegacyNoLocalityBloomImpl::ChooseNumProbes(whole_bits_per_key_);
503
504 const size_t init_size = dst->size();
505 dst->resize(init_size + bytes, 0);
506 dst->push_back(static_cast<char>(num_probes)); // Remember # of probes
507 char* array = &(*dst)[init_size];
508 for (int i = 0; i < n; i++) {
509 LegacyNoLocalityBloomImpl::AddHash(BloomHash(keys[i]), bits, num_probes,
510 array);
511 }
512 }
513
KeyMayMatch(const Slice & key,const Slice & bloom_filter) const514 bool BloomFilterPolicy::KeyMayMatch(const Slice& key,
515 const Slice& bloom_filter) const {
516 const size_t len = bloom_filter.size();
517 if (len < 2 || len > 0xffffffffU) {
518 return false;
519 }
520
521 const char* array = bloom_filter.data();
522 const uint32_t bits = static_cast<uint32_t>(len - 1) * 8;
523
524 // Use the encoded k so that we can read filters generated by
525 // bloom filters created using different parameters.
526 const int k = static_cast<uint8_t>(array[len - 1]);
527 if (k > 30) {
528 // Reserved for potentially new encodings for short bloom filters.
529 // Consider it a match.
530 return true;
531 }
532 // NB: using stored k not num_probes for whole_bits_per_key_
533 return LegacyNoLocalityBloomImpl::HashMayMatch(BloomHash(key), bits, k,
534 array);
535 }
536
GetFilterBitsBuilder() const537 FilterBitsBuilder* BloomFilterPolicy::GetFilterBitsBuilder() const {
538 // This code path should no longer be used, for the built-in
539 // BloomFilterPolicy. Internal to RocksDB and outside
540 // BloomFilterPolicy, only get a FilterBitsBuilder with
541 // BloomFilterPolicy::GetBuilderFromContext(), which will call
542 // BloomFilterPolicy::GetBuilderWithContext(). RocksDB users have
543 // been warned (HISTORY.md) that they can no longer call this on
544 // the built-in BloomFilterPolicy (unlikely).
545 assert(false);
546 return GetBuilderWithContext(FilterBuildingContext(BlockBasedTableOptions()));
547 }
548
GetBuilderWithContext(const FilterBuildingContext & context) const549 FilterBitsBuilder* BloomFilterPolicy::GetBuilderWithContext(
550 const FilterBuildingContext& context) const {
551 Mode cur = mode_;
552 // Unusual code construction so that we can have just
553 // one exhaustive switch without (risky) recursion
554 for (int i = 0; i < 2; ++i) {
555 switch (cur) {
556 case kAuto:
557 if (context.table_options.format_version < 5) {
558 cur = kLegacyBloom;
559 } else {
560 cur = kFastLocalBloom;
561 }
562 break;
563 case kDeprecatedBlock:
564 return nullptr;
565 case kFastLocalBloom:
566 return new FastLocalBloomBitsBuilder(millibits_per_key_);
567 case kLegacyBloom:
568 if (whole_bits_per_key_ >= 14 && context.info_log &&
569 !warned_.load(std::memory_order_relaxed)) {
570 warned_ = true;
571 const char* adjective;
572 if (whole_bits_per_key_ >= 20) {
573 adjective = "Dramatic";
574 } else {
575 adjective = "Significant";
576 }
577 // For more details, see
578 // https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter
579 ROCKS_LOG_WARN(
580 context.info_log,
581 "Using legacy Bloom filter with high (%d) bits/key. "
582 "%s filter space and/or accuracy improvement is available "
583 "with format_version>=5.",
584 whole_bits_per_key_, adjective);
585 }
586 return new LegacyBloomBitsBuilder(whole_bits_per_key_,
587 context.info_log);
588 }
589 }
590 assert(false);
591 return nullptr; // something legal
592 }
593
GetBuilderFromContext(const FilterBuildingContext & context)594 FilterBitsBuilder* BloomFilterPolicy::GetBuilderFromContext(
595 const FilterBuildingContext& context) {
596 if (context.table_options.filter_policy) {
597 return context.table_options.filter_policy->GetBuilderWithContext(context);
598 } else {
599 return nullptr;
600 }
601 }
602
603 // Read metadata to determine what kind of FilterBitsReader is needed
604 // and return a new one.
GetFilterBitsReader(const Slice & contents) const605 FilterBitsReader* BloomFilterPolicy::GetFilterBitsReader(
606 const Slice& contents) const {
607 uint32_t len_with_meta = static_cast<uint32_t>(contents.size());
608 if (len_with_meta <= 5) {
609 // filter is empty or broken. Treat like zero keys added.
610 return new AlwaysFalseFilter();
611 }
612
613 // Legacy Bloom filter data:
614 // 0 +-----------------------------------+
615 // | Raw Bloom filter data |
616 // | ... |
617 // len +-----------------------------------+
618 // | byte for num_probes or |
619 // | marker for new implementations |
620 // len+1 +-----------------------------------+
621 // | four bytes for number of cache |
622 // | lines |
623 // len_with_meta +-----------------------------------+
624
625 int8_t raw_num_probes =
626 static_cast<int8_t>(contents.data()[len_with_meta - 5]);
627 // NB: *num_probes > 30 and < 128 probably have not been used, because of
628 // BloomFilterPolicy::initialize, unless directly calling
629 // LegacyBloomBitsBuilder as an API, but we are leaving those cases in
630 // limbo with LegacyBloomBitsReader for now.
631
632 if (raw_num_probes < 1) {
633 // Note: < 0 (or unsigned > 127) indicate special new implementations
634 // (or reserved for future use)
635 if (raw_num_probes == -1) {
636 // Marker for newer Bloom implementations
637 return GetBloomBitsReader(contents);
638 }
639 // otherwise
640 // Treat as zero probes (always FP) for now.
641 return new AlwaysTrueFilter();
642 }
643 // else attempt decode for LegacyBloomBitsReader
644
645 int num_probes = raw_num_probes;
646 assert(num_probes >= 1);
647 assert(num_probes <= 127);
648
649 uint32_t len = len_with_meta - 5;
650 assert(len > 0);
651
652 uint32_t num_lines = DecodeFixed32(contents.data() + len_with_meta - 4);
653 uint32_t log2_cache_line_size;
654
655 if (num_lines * CACHE_LINE_SIZE == len) {
656 // Common case
657 log2_cache_line_size = folly::constexpr_log2(CACHE_LINE_SIZE);
658 } else if (num_lines == 0 || len % num_lines != 0) {
659 // Invalid (no solution to num_lines * x == len)
660 // Treat as zero probes (always FP) for now.
661 return new AlwaysTrueFilter();
662 } else {
663 // Determine the non-native cache line size (from another system)
664 log2_cache_line_size = 0;
665 while ((num_lines << log2_cache_line_size) < len) {
666 ++log2_cache_line_size;
667 }
668 if ((num_lines << log2_cache_line_size) != len) {
669 // Invalid (block size not a power of two)
670 // Treat as zero probes (always FP) for now.
671 return new AlwaysTrueFilter();
672 }
673 }
674 // if not early return
675 return new LegacyBloomBitsReader(contents.data(), num_probes, num_lines,
676 log2_cache_line_size);
677 }
678
679 // For newer Bloom filter implementations
GetBloomBitsReader(const Slice & contents) const680 FilterBitsReader* BloomFilterPolicy::GetBloomBitsReader(
681 const Slice& contents) const {
682 uint32_t len_with_meta = static_cast<uint32_t>(contents.size());
683 uint32_t len = len_with_meta - 5;
684
685 assert(len > 0); // precondition
686
687 // New Bloom filter data:
688 // 0 +-----------------------------------+
689 // | Raw Bloom filter data |
690 // | ... |
691 // len +-----------------------------------+
692 // | char{-1} byte -> new Bloom filter |
693 // len+1 +-----------------------------------+
694 // | byte for subimplementation |
695 // | 0: FastLocalBloom |
696 // | other: reserved |
697 // len+2 +-----------------------------------+
698 // | byte for block_and_probes |
699 // | 0 in top 3 bits -> 6 -> 64-byte |
700 // | reserved: |
701 // | 1 in top 3 bits -> 7 -> 128-byte|
702 // | 2 in top 3 bits -> 8 -> 256-byte|
703 // | ... |
704 // | num_probes in bottom 5 bits, |
705 // | except 0 and 31 reserved |
706 // len+3 +-----------------------------------+
707 // | two bytes reserved |
708 // | possibly for hash seed |
709 // len_with_meta +-----------------------------------+
710
711 // Read more metadata (see above)
712 char sub_impl_val = contents.data()[len_with_meta - 4];
713 char block_and_probes = contents.data()[len_with_meta - 3];
714 int log2_block_bytes = ((block_and_probes >> 5) & 7) + 6;
715
716 int num_probes = (block_and_probes & 31);
717 if (num_probes < 1 || num_probes > 30) {
718 // Reserved / future safe
719 return new AlwaysTrueFilter();
720 }
721
722 uint16_t rest = DecodeFixed16(contents.data() + len_with_meta - 2);
723 if (rest != 0) {
724 // Reserved, possibly for hash seed
725 // Future safe
726 return new AlwaysTrueFilter();
727 }
728
729 if (sub_impl_val == 0) { // FastLocalBloom
730 if (log2_block_bytes == 6) { // Only block size supported for now
731 return new FastLocalBloomBitsReader(contents.data(), num_probes, len);
732 }
733 }
734 // otherwise
735 // Reserved / future safe
736 return new AlwaysTrueFilter();
737 }
738
NewBloomFilterPolicy(double bits_per_key,bool use_block_based_builder)739 const FilterPolicy* NewBloomFilterPolicy(double bits_per_key,
740 bool use_block_based_builder) {
741 BloomFilterPolicy::Mode m;
742 if (use_block_based_builder) {
743 m = BloomFilterPolicy::kDeprecatedBlock;
744 } else {
745 m = BloomFilterPolicy::kAuto;
746 }
747 assert(std::find(BloomFilterPolicy::kAllUserModes.begin(),
748 BloomFilterPolicy::kAllUserModes.end(),
749 m) != BloomFilterPolicy::kAllUserModes.end());
750 return new BloomFilterPolicy(bits_per_key, m);
751 }
752
FilterBuildingContext(const BlockBasedTableOptions & _table_options)753 FilterBuildingContext::FilterBuildingContext(
754 const BlockBasedTableOptions& _table_options)
755 : table_options(_table_options) {}
756
~FilterPolicy()757 FilterPolicy::~FilterPolicy() { }
758
759 } // namespace ROCKSDB_NAMESPACE
760