1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5
6 #include "rocksdb/utilities/sim_cache.h"
7 #include <cstdlib>
8 #include "db/db_test_util.h"
9 #include "port/stack_trace.h"
10
11 namespace ROCKSDB_NAMESPACE {
12
13 class SimCacheTest : public DBTestBase {
14 private:
15 size_t miss_count_ = 0;
16 size_t hit_count_ = 0;
17 size_t insert_count_ = 0;
18 size_t failure_count_ = 0;
19
20 public:
21 const size_t kNumBlocks = 5;
22 const size_t kValueSize = 1000;
23
SimCacheTest()24 SimCacheTest() : DBTestBase("/sim_cache_test") {}
25
GetTableOptions()26 BlockBasedTableOptions GetTableOptions() {
27 BlockBasedTableOptions table_options;
28 // Set a small enough block size so that each key-value get its own block.
29 table_options.block_size = 1;
30 return table_options;
31 }
32
GetOptions(const BlockBasedTableOptions & table_options)33 Options GetOptions(const BlockBasedTableOptions& table_options) {
34 Options options = CurrentOptions();
35 options.create_if_missing = true;
36 // options.compression = kNoCompression;
37 options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
38 options.table_factory.reset(new BlockBasedTableFactory(table_options));
39 return options;
40 }
41
InitTable(const Options &)42 void InitTable(const Options& /*options*/) {
43 std::string value(kValueSize, 'a');
44 for (size_t i = 0; i < kNumBlocks * 2; i++) {
45 ASSERT_OK(Put(ToString(i), value.c_str()));
46 }
47 }
48
RecordCacheCounters(const Options & options)49 void RecordCacheCounters(const Options& options) {
50 miss_count_ = TestGetTickerCount(options, BLOCK_CACHE_MISS);
51 hit_count_ = TestGetTickerCount(options, BLOCK_CACHE_HIT);
52 insert_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD);
53 failure_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES);
54 }
55
CheckCacheCounters(const Options & options,size_t expected_misses,size_t expected_hits,size_t expected_inserts,size_t expected_failures)56 void CheckCacheCounters(const Options& options, size_t expected_misses,
57 size_t expected_hits, size_t expected_inserts,
58 size_t expected_failures) {
59 size_t new_miss_count = TestGetTickerCount(options, BLOCK_CACHE_MISS);
60 size_t new_hit_count = TestGetTickerCount(options, BLOCK_CACHE_HIT);
61 size_t new_insert_count = TestGetTickerCount(options, BLOCK_CACHE_ADD);
62 size_t new_failure_count =
63 TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES);
64 ASSERT_EQ(miss_count_ + expected_misses, new_miss_count);
65 ASSERT_EQ(hit_count_ + expected_hits, new_hit_count);
66 ASSERT_EQ(insert_count_ + expected_inserts, new_insert_count);
67 ASSERT_EQ(failure_count_ + expected_failures, new_failure_count);
68 miss_count_ = new_miss_count;
69 hit_count_ = new_hit_count;
70 insert_count_ = new_insert_count;
71 failure_count_ = new_failure_count;
72 }
73 };
74
TEST_F(SimCacheTest,SimCache)75 TEST_F(SimCacheTest, SimCache) {
76 ReadOptions read_options;
77 auto table_options = GetTableOptions();
78 auto options = GetOptions(table_options);
79 InitTable(options);
80 LRUCacheOptions co;
81 co.capacity = 0;
82 co.num_shard_bits = 0;
83 co.strict_capacity_limit = false;
84 co.metadata_charge_policy = kDontChargeCacheMetadata;
85 std::shared_ptr<SimCache> simCache = NewSimCache(NewLRUCache(co), 20000, 0);
86 table_options.block_cache = simCache;
87 options.table_factory.reset(new BlockBasedTableFactory(table_options));
88 Reopen(options);
89 RecordCacheCounters(options);
90
91 std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks);
92 Iterator* iter = nullptr;
93
94 // Load blocks into cache.
95 for (size_t i = 0; i < kNumBlocks; i++) {
96 iter = db_->NewIterator(read_options);
97 iter->Seek(ToString(i));
98 ASSERT_OK(iter->status());
99 CheckCacheCounters(options, 1, 0, 1, 0);
100 iterators[i].reset(iter);
101 }
102 ASSERT_EQ(kNumBlocks,
103 simCache->get_hit_counter() + simCache->get_miss_counter());
104 ASSERT_EQ(0, simCache->get_hit_counter());
105 size_t usage = simCache->GetUsage();
106 ASSERT_LT(0, usage);
107 ASSERT_EQ(usage, simCache->GetSimUsage());
108 simCache->SetCapacity(usage);
109 ASSERT_EQ(usage, simCache->GetPinnedUsage());
110
111 // Test with strict capacity limit.
112 simCache->SetStrictCapacityLimit(true);
113 iter = db_->NewIterator(read_options);
114 iter->Seek(ToString(kNumBlocks * 2 - 1));
115 ASSERT_TRUE(iter->status().IsIncomplete());
116 CheckCacheCounters(options, 1, 0, 0, 1);
117 delete iter;
118 iter = nullptr;
119
120 // Release iterators and access cache again.
121 for (size_t i = 0; i < kNumBlocks; i++) {
122 iterators[i].reset();
123 CheckCacheCounters(options, 0, 0, 0, 0);
124 }
125 // Add kNumBlocks again
126 for (size_t i = 0; i < kNumBlocks; i++) {
127 std::unique_ptr<Iterator> it(db_->NewIterator(read_options));
128 it->Seek(ToString(i));
129 ASSERT_OK(it->status());
130 CheckCacheCounters(options, 0, 1, 0, 0);
131 }
132 ASSERT_EQ(5, simCache->get_hit_counter());
133 for (size_t i = kNumBlocks; i < kNumBlocks * 2; i++) {
134 std::unique_ptr<Iterator> it(db_->NewIterator(read_options));
135 it->Seek(ToString(i));
136 ASSERT_OK(it->status());
137 CheckCacheCounters(options, 1, 0, 1, 0);
138 }
139 ASSERT_EQ(0, simCache->GetPinnedUsage());
140 ASSERT_EQ(3 * kNumBlocks + 1,
141 simCache->get_hit_counter() + simCache->get_miss_counter());
142 ASSERT_EQ(6, simCache->get_hit_counter());
143 }
144
TEST_F(SimCacheTest,SimCacheLogging)145 TEST_F(SimCacheTest, SimCacheLogging) {
146 auto table_options = GetTableOptions();
147 auto options = GetOptions(table_options);
148 options.disable_auto_compactions = true;
149 LRUCacheOptions co;
150 co.capacity = 1024 * 1024;
151 co.metadata_charge_policy = kDontChargeCacheMetadata;
152 std::shared_ptr<SimCache> sim_cache = NewSimCache(NewLRUCache(co), 20000, 0);
153 table_options.block_cache = sim_cache;
154 options.table_factory.reset(new BlockBasedTableFactory(table_options));
155 Reopen(options);
156
157 int num_block_entries = 20;
158 for (int i = 0; i < num_block_entries; i++) {
159 Put(Key(i), "val");
160 Flush();
161 }
162
163 std::string log_file = test::PerThreadDBPath(env_, "cache_log.txt");
164 ASSERT_OK(sim_cache->StartActivityLogging(log_file, env_));
165 for (int i = 0; i < num_block_entries; i++) {
166 ASSERT_EQ(Get(Key(i)), "val");
167 }
168 for (int i = 0; i < num_block_entries; i++) {
169 ASSERT_EQ(Get(Key(i)), "val");
170 }
171 sim_cache->StopActivityLogging();
172 ASSERT_OK(sim_cache->GetActivityLoggingStatus());
173
174 std::string file_contents = "";
175 ReadFileToString(env_, log_file, &file_contents);
176
177 int lookup_num = 0;
178 int add_num = 0;
179 std::string::size_type pos;
180
181 // count number of lookups
182 pos = 0;
183 while ((pos = file_contents.find("LOOKUP -", pos)) != std::string::npos) {
184 ++lookup_num;
185 pos += 1;
186 }
187
188 // count number of additions
189 pos = 0;
190 while ((pos = file_contents.find("ADD -", pos)) != std::string::npos) {
191 ++add_num;
192 pos += 1;
193 }
194
195 // We asked for every block twice
196 ASSERT_EQ(lookup_num, num_block_entries * 2);
197
198 // We added every block only once, since the cache can hold all blocks
199 ASSERT_EQ(add_num, num_block_entries);
200
201 // Log things again but stop logging automatically after reaching 512 bytes
202 // @lint-ignore TXT2 T25377293 Grandfathered in
203 int max_size = 512;
204 ASSERT_OK(sim_cache->StartActivityLogging(log_file, env_, max_size));
205 for (int it = 0; it < 10; it++) {
206 for (int i = 0; i < num_block_entries; i++) {
207 ASSERT_EQ(Get(Key(i)), "val");
208 }
209 }
210 ASSERT_OK(sim_cache->GetActivityLoggingStatus());
211
212 uint64_t fsize = 0;
213 ASSERT_OK(env_->GetFileSize(log_file, &fsize));
214 // error margin of 100 bytes
215 ASSERT_LT(fsize, max_size + 100);
216 ASSERT_GT(fsize, max_size - 100);
217 }
218
219 } // namespace ROCKSDB_NAMESPACE
220
main(int argc,char ** argv)221 int main(int argc, char** argv) {
222 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
223 ::testing::InitGoogleTest(&argc, argv);
224 return RUN_ALL_TESTS();
225 }
226