1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5
6 #ifndef GFLAGS
7 #include <cstdio>
main()8 int main() {
9 fprintf(stderr, "Please install gflags to run rocksdb tools\n");
10 return 1;
11 }
12 #else
13
14 #include "db/db_impl/db_impl.h"
15 #include "db/dbformat.h"
16 #include "env/composite_env_wrapper.h"
17 #include "file/random_access_file_reader.h"
18 #include "monitoring/histogram.h"
19 #include "rocksdb/db.h"
20 #include "rocksdb/slice_transform.h"
21 #include "rocksdb/table.h"
22 #include "table/block_based/block_based_table_factory.h"
23 #include "table/get_context.h"
24 #include "table/internal_iterator.h"
25 #include "table/plain/plain_table_factory.h"
26 #include "table/table_builder.h"
27 #include "test_util/testharness.h"
28 #include "test_util/testutil.h"
29 #include "util/gflags_compat.h"
30
31 using GFLAGS_NAMESPACE::ParseCommandLineFlags;
32 using GFLAGS_NAMESPACE::SetUsageMessage;
33
34 namespace ROCKSDB_NAMESPACE {
35
36 namespace {
37 // Make a key that i determines the first 4 characters and j determines the
38 // last 4 characters.
MakeKey(int i,int j,bool through_db)39 static std::string MakeKey(int i, int j, bool through_db) {
40 char buf[100];
41 snprintf(buf, sizeof(buf), "%04d__key___%04d", i, j);
42 if (through_db) {
43 return std::string(buf);
44 }
45 // If we directly query table, which operates on internal keys
46 // instead of user keys, we need to add 8 bytes of internal
47 // information (row type etc) to user key to make an internal
48 // key.
49 InternalKey key(std::string(buf), 0, ValueType::kTypeValue);
50 return key.Encode().ToString();
51 }
52
Now(Env * env,bool measured_by_nanosecond)53 uint64_t Now(Env* env, bool measured_by_nanosecond) {
54 return measured_by_nanosecond ? env->NowNanos() : env->NowMicros();
55 }
56 } // namespace
57
58 // A very simple benchmark that.
59 // Create a table with roughly numKey1 * numKey2 keys,
60 // where there are numKey1 prefixes of the key, each has numKey2 number of
61 // distinguished key, differing in the suffix part.
62 // If if_query_empty_keys = false, query the existing keys numKey1 * numKey2
63 // times randomly.
64 // If if_query_empty_keys = true, query numKey1 * numKey2 random empty keys.
65 // Print out the total time.
66 // If through_db=true, a full DB will be created and queries will be against
67 // it. Otherwise, operations will be directly through table level.
68 //
69 // If for_terator=true, instead of just query one key each time, it queries
70 // a range sharing the same prefix.
71 namespace {
TableReaderBenchmark(Options & opts,EnvOptions & env_options,ReadOptions & read_options,int num_keys1,int num_keys2,int num_iter,int,bool if_query_empty_keys,bool for_iterator,bool through_db,bool measured_by_nanosecond)72 void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
73 ReadOptions& read_options, int num_keys1,
74 int num_keys2, int num_iter, int /*prefix_len*/,
75 bool if_query_empty_keys, bool for_iterator,
76 bool through_db, bool measured_by_nanosecond) {
77 ROCKSDB_NAMESPACE::InternalKeyComparator ikc(opts.comparator);
78
79 std::string file_name =
80 test::PerThreadDBPath("rocksdb_table_reader_benchmark");
81 std::string dbname = test::PerThreadDBPath("rocksdb_table_reader_bench_db");
82 WriteOptions wo;
83 Env* env = Env::Default();
84 TableBuilder* tb = nullptr;
85 DB* db = nullptr;
86 Status s;
87 const ImmutableCFOptions ioptions(opts);
88 const ColumnFamilyOptions cfo(opts);
89 const MutableCFOptions moptions(cfo);
90 std::unique_ptr<WritableFileWriter> file_writer;
91 if (!through_db) {
92 std::unique_ptr<WritableFile> file;
93 env->NewWritableFile(file_name, &file, env_options);
94
95 std::vector<std::unique_ptr<IntTblPropCollectorFactory> >
96 int_tbl_prop_collector_factories;
97
98 file_writer.reset(new WritableFileWriter(
99 NewLegacyWritableFileWrapper(std::move(file)), file_name, env_options));
100 int unknown_level = -1;
101 tb = opts.table_factory->NewTableBuilder(
102 TableBuilderOptions(
103 ioptions, moptions, ikc, &int_tbl_prop_collector_factories,
104 CompressionType::kNoCompression, 0 /* sample_for_compression */,
105 CompressionOptions(), false /* skip_filters */,
106 kDefaultColumnFamilyName, unknown_level),
107 0 /* column_family_id */, file_writer.get());
108 } else {
109 s = DB::Open(opts, dbname, &db);
110 ASSERT_OK(s);
111 ASSERT_TRUE(db != nullptr);
112 }
113 // Populate slightly more than 1M keys
114 for (int i = 0; i < num_keys1; i++) {
115 for (int j = 0; j < num_keys2; j++) {
116 std::string key = MakeKey(i * 2, j, through_db);
117 if (!through_db) {
118 tb->Add(key, key);
119 } else {
120 db->Put(wo, key, key);
121 }
122 }
123 }
124 if (!through_db) {
125 tb->Finish();
126 file_writer->Close();
127 } else {
128 db->Flush(FlushOptions());
129 }
130
131 std::unique_ptr<TableReader> table_reader;
132 if (!through_db) {
133 std::unique_ptr<RandomAccessFile> raf;
134 s = env->NewRandomAccessFile(file_name, &raf, env_options);
135 if (!s.ok()) {
136 fprintf(stderr, "Create File Error: %s\n", s.ToString().c_str());
137 exit(1);
138 }
139 uint64_t file_size;
140 env->GetFileSize(file_name, &file_size);
141 std::unique_ptr<RandomAccessFileReader> file_reader(
142 new RandomAccessFileReader(NewLegacyRandomAccessFileWrapper(raf),
143 file_name));
144 s = opts.table_factory->NewTableReader(
145 TableReaderOptions(ioptions, moptions.prefix_extractor.get(),
146 env_options, ikc),
147 std::move(file_reader), file_size, &table_reader);
148 if (!s.ok()) {
149 fprintf(stderr, "Open Table Error: %s\n", s.ToString().c_str());
150 exit(1);
151 }
152 }
153
154 Random rnd(301);
155 std::string result;
156 HistogramImpl hist;
157
158 for (int it = 0; it < num_iter; it++) {
159 for (int i = 0; i < num_keys1; i++) {
160 for (int j = 0; j < num_keys2; j++) {
161 int r1 = rnd.Uniform(num_keys1) * 2;
162 int r2 = rnd.Uniform(num_keys2);
163 if (if_query_empty_keys) {
164 r1++;
165 r2 = num_keys2 * 2 - r2;
166 }
167
168 if (!for_iterator) {
169 // Query one existing key;
170 std::string key = MakeKey(r1, r2, through_db);
171 uint64_t start_time = Now(env, measured_by_nanosecond);
172 if (!through_db) {
173 PinnableSlice value;
174 MergeContext merge_context;
175 SequenceNumber max_covering_tombstone_seq = 0;
176 GetContext get_context(ioptions.user_comparator,
177 ioptions.merge_operator, ioptions.info_log,
178 ioptions.statistics, GetContext::kNotFound,
179 Slice(key), &value, nullptr, &merge_context,
180 true, &max_covering_tombstone_seq, env);
181 s = table_reader->Get(read_options, key, &get_context, nullptr);
182 } else {
183 s = db->Get(read_options, key, &result);
184 }
185 hist.Add(Now(env, measured_by_nanosecond) - start_time);
186 } else {
187 int r2_len;
188 if (if_query_empty_keys) {
189 r2_len = 0;
190 } else {
191 r2_len = rnd.Uniform(num_keys2) + 1;
192 if (r2_len + r2 > num_keys2) {
193 r2_len = num_keys2 - r2;
194 }
195 }
196 std::string start_key = MakeKey(r1, r2, through_db);
197 std::string end_key = MakeKey(r1, r2 + r2_len, through_db);
198 uint64_t total_time = 0;
199 uint64_t start_time = Now(env, measured_by_nanosecond);
200 Iterator* iter = nullptr;
201 InternalIterator* iiter = nullptr;
202 if (!through_db) {
203 iiter = table_reader->NewIterator(
204 read_options, /*prefix_extractor=*/nullptr, /*arena=*/nullptr,
205 /*skip_filters=*/false, TableReaderCaller::kUncategorized);
206 } else {
207 iter = db->NewIterator(read_options);
208 }
209 int count = 0;
210 for (through_db ? iter->Seek(start_key) : iiter->Seek(start_key);
211 through_db ? iter->Valid() : iiter->Valid();
212 through_db ? iter->Next() : iiter->Next()) {
213 if (if_query_empty_keys) {
214 break;
215 }
216 // verify key;
217 total_time += Now(env, measured_by_nanosecond) - start_time;
218 assert(Slice(MakeKey(r1, r2 + count, through_db)) ==
219 (through_db ? iter->key() : iiter->key()));
220 start_time = Now(env, measured_by_nanosecond);
221 if (++count >= r2_len) {
222 break;
223 }
224 }
225 if (count != r2_len) {
226 fprintf(
227 stderr, "Iterator cannot iterate expected number of entries. "
228 "Expected %d but got %d\n", r2_len, count);
229 assert(false);
230 }
231 delete iter;
232 total_time += Now(env, measured_by_nanosecond) - start_time;
233 hist.Add(total_time);
234 }
235 }
236 }
237 }
238
239 fprintf(
240 stderr,
241 "==================================================="
242 "====================================================\n"
243 "InMemoryTableSimpleBenchmark: %20s num_key1: %5d "
244 "num_key2: %5d %10s\n"
245 "==================================================="
246 "===================================================="
247 "\nHistogram (unit: %s): \n%s",
248 opts.table_factory->Name(), num_keys1, num_keys2,
249 for_iterator ? "iterator" : (if_query_empty_keys ? "empty" : "non_empty"),
250 measured_by_nanosecond ? "nanosecond" : "microsecond",
251 hist.ToString().c_str());
252 if (!through_db) {
253 env->DeleteFile(file_name);
254 } else {
255 delete db;
256 db = nullptr;
257 DestroyDB(dbname, opts);
258 }
259 }
260 } // namespace
261 } // namespace ROCKSDB_NAMESPACE
262
263 DEFINE_bool(query_empty, false, "query non-existing keys instead of existing "
264 "ones.");
265 DEFINE_int32(num_keys1, 4096, "number of distinguish prefix of keys");
266 DEFINE_int32(num_keys2, 512, "number of distinguish keys for each prefix");
267 DEFINE_int32(iter, 3, "query non-existing keys instead of existing ones");
268 DEFINE_int32(prefix_len, 16, "Prefix length used for iterators and indexes");
269 DEFINE_bool(iterator, false, "For test iterator");
270 DEFINE_bool(through_db, false, "If enable, a DB instance will be created and "
271 "the query will be against DB. Otherwise, will be directly against "
272 "a table reader.");
273 DEFINE_bool(mmap_read, true, "Whether use mmap read");
274 DEFINE_string(table_factory, "block_based",
275 "Table factory to use: `block_based` (default), `plain_table` or "
276 "`cuckoo_hash`.");
277 DEFINE_string(time_unit, "microsecond",
278 "The time unit used for measuring performance. User can specify "
279 "`microsecond` (default) or `nanosecond`");
280
main(int argc,char ** argv)281 int main(int argc, char** argv) {
282 SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
283 " [OPTIONS]...");
284 ParseCommandLineFlags(&argc, &argv, true);
285
286 std::shared_ptr<ROCKSDB_NAMESPACE::TableFactory> tf;
287 ROCKSDB_NAMESPACE::Options options;
288 if (FLAGS_prefix_len < 16) {
289 options.prefix_extractor.reset(
290 ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_len));
291 }
292 ROCKSDB_NAMESPACE::ReadOptions ro;
293 ROCKSDB_NAMESPACE::EnvOptions env_options;
294 options.create_if_missing = true;
295 options.compression = ROCKSDB_NAMESPACE::CompressionType::kNoCompression;
296
297 if (FLAGS_table_factory == "cuckoo_hash") {
298 #ifndef ROCKSDB_LITE
299 options.allow_mmap_reads = FLAGS_mmap_read;
300 env_options.use_mmap_reads = FLAGS_mmap_read;
301 ROCKSDB_NAMESPACE::CuckooTableOptions table_options;
302 table_options.hash_table_ratio = 0.75;
303 tf.reset(ROCKSDB_NAMESPACE::NewCuckooTableFactory(table_options));
304 #else
305 fprintf(stderr, "Plain table is not supported in lite mode\n");
306 exit(1);
307 #endif // ROCKSDB_LITE
308 } else if (FLAGS_table_factory == "plain_table") {
309 #ifndef ROCKSDB_LITE
310 options.allow_mmap_reads = FLAGS_mmap_read;
311 env_options.use_mmap_reads = FLAGS_mmap_read;
312
313 ROCKSDB_NAMESPACE::PlainTableOptions plain_table_options;
314 plain_table_options.user_key_len = 16;
315 plain_table_options.bloom_bits_per_key = (FLAGS_prefix_len == 16) ? 0 : 8;
316 plain_table_options.hash_table_ratio = 0.75;
317
318 tf.reset(new ROCKSDB_NAMESPACE::PlainTableFactory(plain_table_options));
319 options.prefix_extractor.reset(
320 ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_len));
321 #else
322 fprintf(stderr, "Cuckoo table is not supported in lite mode\n");
323 exit(1);
324 #endif // ROCKSDB_LITE
325 } else if (FLAGS_table_factory == "block_based") {
326 tf.reset(new ROCKSDB_NAMESPACE::BlockBasedTableFactory());
327 } else {
328 fprintf(stderr, "Invalid table type %s\n", FLAGS_table_factory.c_str());
329 }
330
331 if (tf) {
332 // if user provides invalid options, just fall back to microsecond.
333 bool measured_by_nanosecond = FLAGS_time_unit == "nanosecond";
334
335 options.table_factory = tf;
336 ROCKSDB_NAMESPACE::TableReaderBenchmark(
337 options, env_options, ro, FLAGS_num_keys1, FLAGS_num_keys2, FLAGS_iter,
338 FLAGS_prefix_len, FLAGS_query_empty, FLAGS_iterator, FLAGS_through_db,
339 measured_by_nanosecond);
340 } else {
341 return 1;
342 }
343
344 return 0;
345 }
346
347 #endif // GFLAGS
348