1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2012 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #ifndef ROCKSDB_LITE
11 #ifndef GFLAGS
12 #include <cstdio>
main()13 int main() {
14 fprintf(stderr, "Please install gflags to run trace_analyzer test\n");
15 return 0;
16 }
17 #else
18
19 #include <chrono>
20 #include <cstdio>
21 #include <cstdlib>
22 #include <sstream>
23 #include <thread>
24
25 #include "db/db_test_util.h"
26 #include "file/line_file_reader.h"
27 #include "rocksdb/db.h"
28 #include "rocksdb/env.h"
29 #include "rocksdb/status.h"
30 #include "rocksdb/trace_reader_writer.h"
31 #include "test_util/testharness.h"
32 #include "test_util/testutil.h"
33 #include "tools/trace_analyzer_tool.h"
34 #include "trace_replay/trace_replay.h"
35
36 namespace ROCKSDB_NAMESPACE {
37
38 namespace {
39 static const int kMaxArgCount = 100;
40 static const size_t kArgBufferSize = 100000;
41 } // namespace
42
43 // The helper functions for the test
44 class TraceAnalyzerTest : public testing::Test {
45 public:
TraceAnalyzerTest()46 TraceAnalyzerTest() : rnd_(0xFB) {
47 // test_path_ = test::TmpDir() + "trace_analyzer_test";
48 test_path_ = test::PerThreadDBPath("trace_analyzer_test");
49 env_ = ROCKSDB_NAMESPACE::Env::Default();
50 env_->CreateDir(test_path_).PermitUncheckedError();
51 dbname_ = test_path_ + "/db";
52 }
53
~TraceAnalyzerTest()54 ~TraceAnalyzerTest() override {}
55
GenerateTrace(std::string trace_path)56 void GenerateTrace(std::string trace_path) {
57 Options options;
58 options.create_if_missing = true;
59 options.merge_operator = MergeOperators::CreatePutOperator();
60 Slice upper_bound("a");
61 Slice lower_bound("abce");
62 ReadOptions ro;
63 ro.iterate_upper_bound = &upper_bound;
64 ro.iterate_lower_bound = &lower_bound;
65 WriteOptions wo;
66 TraceOptions trace_opt;
67 DB* db_ = nullptr;
68 std::string value;
69 std::unique_ptr<TraceWriter> trace_writer;
70 Iterator* single_iter = nullptr;
71
72 ASSERT_OK(
73 NewFileTraceWriter(env_, env_options_, trace_path, &trace_writer));
74 ASSERT_OK(DB::Open(options, dbname_, &db_));
75 ASSERT_OK(db_->StartTrace(trace_opt, std::move(trace_writer)));
76
77 WriteBatch batch;
78 ASSERT_OK(batch.Put("a", "aaaaaaaaa"));
79 ASSERT_OK(batch.Merge("b", "aaaaaaaaaaaaaaaaaaaa"));
80 ASSERT_OK(batch.Delete("c"));
81 ASSERT_OK(batch.SingleDelete("d"));
82 ASSERT_OK(batch.DeleteRange("e", "f"));
83 ASSERT_OK(db_->Write(wo, &batch));
84 std::vector<Slice> keys;
85 keys.push_back("a");
86 keys.push_back("b");
87 keys.push_back("df");
88 keys.push_back("gege");
89 keys.push_back("hjhjhj");
90 std::vector<std::string> values;
91 std::vector<Status> ss = db_->MultiGet(ro, keys, &values);
92 ASSERT_GE(ss.size(), 0);
93 ASSERT_OK(ss[0]);
94 ASSERT_NOK(ss[2]);
95 std::vector<ColumnFamilyHandle*> cfs(2, db_->DefaultColumnFamily());
96 std::vector<PinnableSlice> values2(keys.size());
97 db_->MultiGet(ro, 2, cfs.data(), keys.data(), values2.data(), ss.data(),
98 false);
99 ASSERT_OK(ss[0]);
100 db_->MultiGet(ro, db_->DefaultColumnFamily(), 2, keys.data() + 3,
101 values2.data(), ss.data(), false);
102 ASSERT_OK(db_->Get(ro, "a", &value));
103
104 single_iter = db_->NewIterator(ro);
105 single_iter->Seek("a");
106 ASSERT_OK(single_iter->status());
107 single_iter->SeekForPrev("b");
108 ASSERT_OK(single_iter->status());
109 delete single_iter;
110 std::this_thread::sleep_for (std::chrono::seconds(1));
111
112 db_->Get(ro, "g", &value).PermitUncheckedError();
113
114 ASSERT_OK(db_->EndTrace());
115
116 ASSERT_OK(env_->FileExists(trace_path));
117
118 std::unique_ptr<WritableFile> whole_f;
119 std::string whole_path = test_path_ + "/0.txt";
120 ASSERT_OK(env_->NewWritableFile(whole_path, &whole_f, env_options_));
121 std::string whole_str = "0x61\n0x62\n0x63\n0x64\n0x65\n0x66\n";
122 ASSERT_OK(whole_f->Append(whole_str));
123 delete db_;
124 ASSERT_OK(DestroyDB(dbname_, options));
125 }
126
RunTraceAnalyzer(const std::vector<std::string> & args)127 void RunTraceAnalyzer(const std::vector<std::string>& args) {
128 char arg_buffer[kArgBufferSize];
129 char* argv[kMaxArgCount];
130 int argc = 0;
131 int cursor = 0;
132
133 for (const auto& arg : args) {
134 ASSERT_LE(cursor + arg.size() + 1, kArgBufferSize);
135 ASSERT_LE(argc + 1, kMaxArgCount);
136 snprintf(arg_buffer + cursor, arg.size() + 1, "%s", arg.c_str());
137
138 argv[argc++] = arg_buffer + cursor;
139 cursor += static_cast<int>(arg.size()) + 1;
140 }
141
142 ASSERT_EQ(0, ROCKSDB_NAMESPACE::trace_analyzer_tool(argc, argv));
143 }
144
CheckFileContent(const std::vector<std::string> & cnt,std::string file_path,bool full_content)145 void CheckFileContent(const std::vector<std::string>& cnt,
146 std::string file_path, bool full_content) {
147 const auto& fs = env_->GetFileSystem();
148 FileOptions fopts(env_options_);
149
150 ASSERT_OK(fs->FileExists(file_path, fopts.io_options, nullptr));
151 std::unique_ptr<FSSequentialFile> file;
152 ASSERT_OK(fs->NewSequentialFile(file_path, fopts, &file, nullptr));
153
154 LineFileReader lf_reader(std::move(file), file_path,
155 4096 /* filereadahead_size */);
156
157 std::vector<std::string> result;
158 std::string line;
159 while (lf_reader.ReadLine(&line)) {
160 result.push_back(line);
161 }
162
163 ASSERT_OK(lf_reader.GetStatus());
164
165 ASSERT_EQ(cnt.size(), result.size());
166 for (int i = 0; i < static_cast<int>(result.size()); i++) {
167 if (full_content) {
168 ASSERT_EQ(result[i], cnt[i]);
169 } else {
170 ASSERT_EQ(result[i][0], cnt[i][0]);
171 }
172 }
173
174 return;
175 }
176
AnalyzeTrace(std::vector<std::string> & paras_diff,std::string output_path,std::string trace_path)177 void AnalyzeTrace(std::vector<std::string>& paras_diff,
178 std::string output_path, std::string trace_path) {
179 std::vector<std::string> paras = {"./trace_analyzer",
180 "-convert_to_human_readable_trace",
181 "-output_key_stats",
182 "-output_access_count_stats",
183 "-output_prefix=test",
184 "-output_prefix_cut=1",
185 "-output_time_series",
186 "-output_value_distribution",
187 "-output_qps_stats",
188 "-no_key",
189 "-no_print"};
190 for (auto& para : paras_diff) {
191 paras.push_back(para);
192 }
193 Status s = env_->FileExists(trace_path);
194 if (!s.ok()) {
195 GenerateTrace(trace_path);
196 }
197 ASSERT_OK(env_->CreateDir(output_path));
198 RunTraceAnalyzer(paras);
199 }
200
201 ROCKSDB_NAMESPACE::Env* env_;
202 EnvOptions env_options_;
203 std::string test_path_;
204 std::string dbname_;
205 Random rnd_;
206 };
207
TEST_F(TraceAnalyzerTest,Get)208 TEST_F(TraceAnalyzerTest, Get) {
209 std::string trace_path = test_path_ + "/trace";
210 std::string output_path = test_path_ + "/get";
211 std::string file_path;
212 std::vector<std::string> paras = {
213 "-analyze_get=true", "-analyze_put=false",
214 "-analyze_delete=false", "-analyze_single_delete=false",
215 "-analyze_range_delete=false", "-analyze_iterator=false",
216 "-analyze_multiget=false"};
217 paras.push_back("-output_dir=" + output_path);
218 paras.push_back("-trace_path=" + trace_path);
219 paras.push_back("-key_space_dir=" + test_path_);
220 AnalyzeTrace(paras, output_path, trace_path);
221
222 // check the key_stats file
223 std::vector<std::string> k_stats = {"0 10 0 1 1.000000", "0 10 1 1 1.000000"};
224 file_path = output_path + "/test-get-0-accessed_key_stats.txt";
225 CheckFileContent(k_stats, file_path, true);
226
227 // Check the access count distribution
228 std::vector<std::string> k_dist = {"access_count: 1 num: 2"};
229 file_path = output_path + "/test-get-0-accessed_key_count_distribution.txt";
230 CheckFileContent(k_dist, file_path, true);
231
232 // Check the trace sequence
233 std::vector<std::string> k_sequence = {"1", "5", "2", "3", "4", "8",
234 "8", "8", "8", "8", "8", "8",
235 "8", "8", "0", "6", "7", "0"};
236 file_path = output_path + "/test-human_readable_trace.txt";
237 CheckFileContent(k_sequence, file_path, false);
238
239 // Check the prefix
240 std::vector<std::string> k_prefix = {"0 0 0 0.000000 0.000000 0x30",
241 "1 1 1 1.000000 1.000000 0x61"};
242 file_path = output_path + "/test-get-0-accessed_key_prefix_cut.txt";
243 CheckFileContent(k_prefix, file_path, true);
244
245 // Check the time series
246 std::vector<std::string> k_series = {"0 1533000630 0", "0 1533000630 1"};
247 file_path = output_path + "/test-get-0-time_series.txt";
248 CheckFileContent(k_series, file_path, false);
249
250 // Check the accessed key in whole key space
251 std::vector<std::string> k_whole_access = {"0 1"};
252 file_path = output_path + "/test-get-0-whole_key_stats.txt";
253 CheckFileContent(k_whole_access, file_path, true);
254
255 // Check the whole key prefix cut
256 std::vector<std::string> k_whole_prefix = {"0 0x61", "1 0x62", "2 0x63",
257 "3 0x64", "4 0x65", "5 0x66"};
258 file_path = output_path + "/test-get-0-whole_key_prefix_cut.txt";
259 CheckFileContent(k_whole_prefix, file_path, true);
260
261 // Check the overall qps
262 std::vector<std::string> all_qps = {"1 0 0 0 0 0 0 0 0 1"};
263 file_path = output_path + "/test-qps_stats.txt";
264 CheckFileContent(all_qps, file_path, true);
265
266 // Check the qps of get
267 std::vector<std::string> get_qps = {"1"};
268 file_path = output_path + "/test-get-0-qps_stats.txt";
269 CheckFileContent(get_qps, file_path, true);
270
271 // Check the top k qps prefix cut
272 std::vector<std::string> top_qps = {"At time: 0 with QPS: 1",
273 "The prefix: 0x61 Access count: 1"};
274 file_path = output_path + "/test-get-0-accessed_top_k_qps_prefix_cut.txt";
275 CheckFileContent(top_qps, file_path, true);
276 }
277
278 // Test analyzing of Put
TEST_F(TraceAnalyzerTest,Put)279 TEST_F(TraceAnalyzerTest, Put) {
280 std::string trace_path = test_path_ + "/trace";
281 std::string output_path = test_path_ + "/put";
282 std::string file_path;
283 std::vector<std::string> paras = {
284 "-analyze_get=false", "-analyze_put=true",
285 "-analyze_delete=false", "-analyze_single_delete=false",
286 "-analyze_range_delete=false", "-analyze_iterator=false",
287 "-analyze_multiget=false"};
288 paras.push_back("-output_dir=" + output_path);
289 paras.push_back("-trace_path=" + trace_path);
290 paras.push_back("-key_space_dir=" + test_path_);
291 AnalyzeTrace(paras, output_path, trace_path);
292
293 // check the key_stats file
294 std::vector<std::string> k_stats = {"0 9 0 1 1.000000"};
295 file_path = output_path + "/test-put-0-accessed_key_stats.txt";
296 CheckFileContent(k_stats, file_path, true);
297
298 // Check the access count distribution
299 std::vector<std::string> k_dist = {"access_count: 1 num: 1"};
300 file_path = output_path + "/test-put-0-accessed_key_count_distribution.txt";
301 CheckFileContent(k_dist, file_path, true);
302
303 // Check the trace sequence
304 std::vector<std::string> k_sequence = {"1", "5", "2", "3", "4", "8",
305 "8", "8", "8", "8", "8", "8",
306 "8", "8", "0", "6", "7", "0"};
307 file_path = output_path + "/test-human_readable_trace.txt";
308 CheckFileContent(k_sequence, file_path, false);
309
310 // Check the prefix
311 std::vector<std::string> k_prefix = {"0 0 0 0.000000 0.000000 0x30"};
312 file_path = output_path + "/test-put-0-accessed_key_prefix_cut.txt";
313 CheckFileContent(k_prefix, file_path, true);
314
315 // Check the time series
316 std::vector<std::string> k_series = {"1 1533056278 0"};
317 file_path = output_path + "/test-put-0-time_series.txt";
318 CheckFileContent(k_series, file_path, false);
319
320 // Check the accessed key in whole key space
321 std::vector<std::string> k_whole_access = {"0 1"};
322 file_path = output_path + "/test-put-0-whole_key_stats.txt";
323 CheckFileContent(k_whole_access, file_path, true);
324
325 // Check the whole key prefix cut
326 std::vector<std::string> k_whole_prefix = {"0 0x61", "1 0x62", "2 0x63",
327 "3 0x64", "4 0x65", "5 0x66"};
328 file_path = output_path + "/test-put-0-whole_key_prefix_cut.txt";
329 CheckFileContent(k_whole_prefix, file_path, true);
330
331 // Check the overall qps
332 std::vector<std::string> all_qps = {"0 1 0 0 0 0 0 0 0 1"};
333 file_path = output_path + "/test-qps_stats.txt";
334 CheckFileContent(all_qps, file_path, true);
335
336 // Check the qps of Put
337 std::vector<std::string> get_qps = {"1"};
338 file_path = output_path + "/test-put-0-qps_stats.txt";
339 CheckFileContent(get_qps, file_path, true);
340
341 // Check the top k qps prefix cut
342 std::vector<std::string> top_qps = {"At time: 0 with QPS: 1",
343 "The prefix: 0x61 Access count: 1"};
344 file_path = output_path + "/test-put-0-accessed_top_k_qps_prefix_cut.txt";
345 CheckFileContent(top_qps, file_path, true);
346
347 // Check the value size distribution
348 std::vector<std::string> value_dist = {
349 "Number_of_value_size_between 0 and 16 is: 1"};
350 file_path = output_path + "/test-put-0-accessed_value_size_distribution.txt";
351 CheckFileContent(value_dist, file_path, true);
352 }
353
354 // Test analyzing of delete
TEST_F(TraceAnalyzerTest,Delete)355 TEST_F(TraceAnalyzerTest, Delete) {
356 std::string trace_path = test_path_ + "/trace";
357 std::string output_path = test_path_ + "/delete";
358 std::string file_path;
359 std::vector<std::string> paras = {
360 "-analyze_get=false", "-analyze_put=false",
361 "-analyze_delete=true", "-analyze_single_delete=false",
362 "-analyze_range_delete=false", "-analyze_iterator=false",
363 "-analyze_multiget=false"};
364 paras.push_back("-output_dir=" + output_path);
365 paras.push_back("-trace_path=" + trace_path);
366 paras.push_back("-key_space_dir=" + test_path_);
367 AnalyzeTrace(paras, output_path, trace_path);
368
369 // check the key_stats file
370 std::vector<std::string> k_stats = {"0 10 0 1 1.000000"};
371 file_path = output_path + "/test-delete-0-accessed_key_stats.txt";
372 CheckFileContent(k_stats, file_path, true);
373
374 // Check the access count distribution
375 std::vector<std::string> k_dist = {"access_count: 1 num: 1"};
376 file_path =
377 output_path + "/test-delete-0-accessed_key_count_distribution.txt";
378 CheckFileContent(k_dist, file_path, true);
379
380 // Check the trace sequence
381 std::vector<std::string> k_sequence = {"1", "5", "2", "3", "4", "8",
382 "8", "8", "8", "8", "8", "8",
383 "8", "8", "0", "6", "7", "0"};
384 file_path = output_path + "/test-human_readable_trace.txt";
385 CheckFileContent(k_sequence, file_path, false);
386
387 // Check the prefix
388 std::vector<std::string> k_prefix = {"0 0 0 0.000000 0.000000 0x30"};
389 file_path = output_path + "/test-delete-0-accessed_key_prefix_cut.txt";
390 CheckFileContent(k_prefix, file_path, true);
391
392 // Check the time series
393 std::vector<std::string> k_series = {"2 1533000630 0"};
394 file_path = output_path + "/test-delete-0-time_series.txt";
395 CheckFileContent(k_series, file_path, false);
396
397 // Check the accessed key in whole key space
398 std::vector<std::string> k_whole_access = {"2 1"};
399 file_path = output_path + "/test-delete-0-whole_key_stats.txt";
400 CheckFileContent(k_whole_access, file_path, true);
401
402 // Check the whole key prefix cut
403 std::vector<std::string> k_whole_prefix = {"0 0x61", "1 0x62", "2 0x63",
404 "3 0x64", "4 0x65", "5 0x66"};
405 file_path = output_path + "/test-delete-0-whole_key_prefix_cut.txt";
406 CheckFileContent(k_whole_prefix, file_path, true);
407
408 // Check the overall qps
409 std::vector<std::string> all_qps = {"0 0 1 0 0 0 0 0 0 1"};
410 file_path = output_path + "/test-qps_stats.txt";
411 CheckFileContent(all_qps, file_path, true);
412
413 // Check the qps of Delete
414 std::vector<std::string> get_qps = {"1"};
415 file_path = output_path + "/test-delete-0-qps_stats.txt";
416 CheckFileContent(get_qps, file_path, true);
417
418 // Check the top k qps prefix cut
419 std::vector<std::string> top_qps = {"At time: 0 with QPS: 1",
420 "The prefix: 0x63 Access count: 1"};
421 file_path = output_path + "/test-delete-0-accessed_top_k_qps_prefix_cut.txt";
422 CheckFileContent(top_qps, file_path, true);
423 }
424
425 // Test analyzing of Merge
TEST_F(TraceAnalyzerTest,Merge)426 TEST_F(TraceAnalyzerTest, Merge) {
427 std::string trace_path = test_path_ + "/trace";
428 std::string output_path = test_path_ + "/merge";
429 std::string file_path;
430 std::vector<std::string> paras = {
431 "-analyze_get=false", "-analyze_put=false",
432 "-analyze_delete=false", "-analyze_merge=true",
433 "-analyze_single_delete=false", "-analyze_range_delete=false",
434 "-analyze_iterator=false", "-analyze_multiget=false"};
435 paras.push_back("-output_dir=" + output_path);
436 paras.push_back("-trace_path=" + trace_path);
437 paras.push_back("-key_space_dir=" + test_path_);
438 AnalyzeTrace(paras, output_path, trace_path);
439
440 // check the key_stats file
441 std::vector<std::string> k_stats = {"0 20 0 1 1.000000"};
442 file_path = output_path + "/test-merge-0-accessed_key_stats.txt";
443 CheckFileContent(k_stats, file_path, true);
444
445 // Check the access count distribution
446 std::vector<std::string> k_dist = {"access_count: 1 num: 1"};
447 file_path = output_path + "/test-merge-0-accessed_key_count_distribution.txt";
448 CheckFileContent(k_dist, file_path, true);
449
450 // Check the trace sequence
451 std::vector<std::string> k_sequence = {"1", "5", "2", "3", "4", "8",
452 "8", "8", "8", "8", "8", "8",
453 "8", "8", "0", "6", "7", "0"};
454 file_path = output_path + "/test-human_readable_trace.txt";
455 CheckFileContent(k_sequence, file_path, false);
456
457 // Check the prefix
458 std::vector<std::string> k_prefix = {"0 0 0 0.000000 0.000000 0x30"};
459 file_path = output_path + "/test-merge-0-accessed_key_prefix_cut.txt";
460 CheckFileContent(k_prefix, file_path, true);
461
462 // Check the time series
463 std::vector<std::string> k_series = {"5 1533000630 0"};
464 file_path = output_path + "/test-merge-0-time_series.txt";
465 CheckFileContent(k_series, file_path, false);
466
467 // Check the accessed key in whole key space
468 std::vector<std::string> k_whole_access = {"1 1"};
469 file_path = output_path + "/test-merge-0-whole_key_stats.txt";
470 CheckFileContent(k_whole_access, file_path, true);
471
472 // Check the whole key prefix cut
473 std::vector<std::string> k_whole_prefix = {"0 0x61", "1 0x62", "2 0x63",
474 "3 0x64", "4 0x65", "5 0x66"};
475 file_path = output_path + "/test-merge-0-whole_key_prefix_cut.txt";
476 CheckFileContent(k_whole_prefix, file_path, true);
477
478 // Check the overall qps
479 std::vector<std::string> all_qps = {"0 0 0 0 0 1 0 0 0 1"};
480 file_path = output_path + "/test-qps_stats.txt";
481 CheckFileContent(all_qps, file_path, true);
482
483 // Check the qps of Merge
484 std::vector<std::string> get_qps = {"1"};
485 file_path = output_path + "/test-merge-0-qps_stats.txt";
486 CheckFileContent(get_qps, file_path, true);
487
488 // Check the top k qps prefix cut
489 std::vector<std::string> top_qps = {"At time: 0 with QPS: 1",
490 "The prefix: 0x62 Access count: 1"};
491 file_path = output_path + "/test-merge-0-accessed_top_k_qps_prefix_cut.txt";
492 CheckFileContent(top_qps, file_path, true);
493
494 // Check the value size distribution
495 std::vector<std::string> value_dist = {
496 "Number_of_value_size_between 0 and 24 is: 1"};
497 file_path =
498 output_path + "/test-merge-0-accessed_value_size_distribution.txt";
499 CheckFileContent(value_dist, file_path, true);
500 }
501
502 // Test analyzing of SingleDelete
TEST_F(TraceAnalyzerTest,SingleDelete)503 TEST_F(TraceAnalyzerTest, SingleDelete) {
504 std::string trace_path = test_path_ + "/trace";
505 std::string output_path = test_path_ + "/single_delete";
506 std::string file_path;
507 std::vector<std::string> paras = {
508 "-analyze_get=false", "-analyze_put=false",
509 "-analyze_delete=false", "-analyze_merge=false",
510 "-analyze_single_delete=true", "-analyze_range_delete=false",
511 "-analyze_iterator=false", "-analyze_multiget=false"};
512 paras.push_back("-output_dir=" + output_path);
513 paras.push_back("-trace_path=" + trace_path);
514 paras.push_back("-key_space_dir=" + test_path_);
515 AnalyzeTrace(paras, output_path, trace_path);
516
517 // check the key_stats file
518 std::vector<std::string> k_stats = {"0 10 0 1 1.000000"};
519 file_path = output_path + "/test-single_delete-0-accessed_key_stats.txt";
520 CheckFileContent(k_stats, file_path, true);
521
522 // Check the access count distribution
523 std::vector<std::string> k_dist = {"access_count: 1 num: 1"};
524 file_path =
525 output_path + "/test-single_delete-0-accessed_key_count_distribution.txt";
526 CheckFileContent(k_dist, file_path, true);
527
528 // Check the trace sequence
529 std::vector<std::string> k_sequence = {"1", "5", "2", "3", "4", "8",
530 "8", "8", "8", "8", "8", "8",
531 "8", "8", "0", "6", "7", "0"};
532 file_path = output_path + "/test-human_readable_trace.txt";
533 CheckFileContent(k_sequence, file_path, false);
534
535 // Check the prefix
536 std::vector<std::string> k_prefix = {"0 0 0 0.000000 0.000000 0x30"};
537 file_path = output_path + "/test-single_delete-0-accessed_key_prefix_cut.txt";
538 CheckFileContent(k_prefix, file_path, true);
539
540 // Check the time series
541 std::vector<std::string> k_series = {"3 1533000630 0"};
542 file_path = output_path + "/test-single_delete-0-time_series.txt";
543 CheckFileContent(k_series, file_path, false);
544
545 // Check the accessed key in whole key space
546 std::vector<std::string> k_whole_access = {"3 1"};
547 file_path = output_path + "/test-single_delete-0-whole_key_stats.txt";
548 CheckFileContent(k_whole_access, file_path, true);
549
550 // Check the whole key prefix cut
551 std::vector<std::string> k_whole_prefix = {"0 0x61", "1 0x62", "2 0x63",
552 "3 0x64", "4 0x65", "5 0x66"};
553 file_path = output_path + "/test-single_delete-0-whole_key_prefix_cut.txt";
554 CheckFileContent(k_whole_prefix, file_path, true);
555
556 // Check the overall qps
557 std::vector<std::string> all_qps = {"0 0 0 1 0 0 0 0 0 1"};
558 file_path = output_path + "/test-qps_stats.txt";
559 CheckFileContent(all_qps, file_path, true);
560
561 // Check the qps of SingleDelete
562 std::vector<std::string> get_qps = {"1"};
563 file_path = output_path + "/test-single_delete-0-qps_stats.txt";
564 CheckFileContent(get_qps, file_path, true);
565
566 // Check the top k qps prefix cut
567 std::vector<std::string> top_qps = {"At time: 0 with QPS: 1",
568 "The prefix: 0x64 Access count: 1"};
569 file_path =
570 output_path + "/test-single_delete-0-accessed_top_k_qps_prefix_cut.txt";
571 CheckFileContent(top_qps, file_path, true);
572 }
573
574 // Test analyzing of delete
TEST_F(TraceAnalyzerTest,DeleteRange)575 TEST_F(TraceAnalyzerTest, DeleteRange) {
576 std::string trace_path = test_path_ + "/trace";
577 std::string output_path = test_path_ + "/range_delete";
578 std::string file_path;
579 std::vector<std::string> paras = {
580 "-analyze_get=false", "-analyze_put=false",
581 "-analyze_delete=false", "-analyze_merge=false",
582 "-analyze_single_delete=false", "-analyze_range_delete=true",
583 "-analyze_iterator=false", "-analyze_multiget=false"};
584 paras.push_back("-output_dir=" + output_path);
585 paras.push_back("-trace_path=" + trace_path);
586 paras.push_back("-key_space_dir=" + test_path_);
587 AnalyzeTrace(paras, output_path, trace_path);
588
589 // check the key_stats file
590 std::vector<std::string> k_stats = {"0 10 0 1 1.000000", "0 10 1 1 1.000000"};
591 file_path = output_path + "/test-range_delete-0-accessed_key_stats.txt";
592 CheckFileContent(k_stats, file_path, true);
593
594 // Check the access count distribution
595 std::vector<std::string> k_dist = {"access_count: 1 num: 2"};
596 file_path =
597 output_path + "/test-range_delete-0-accessed_key_count_distribution.txt";
598 CheckFileContent(k_dist, file_path, true);
599
600 // Check the trace sequence
601 std::vector<std::string> k_sequence = {"1", "5", "2", "3", "4", "8",
602 "8", "8", "8", "8", "8", "8",
603 "8", "8", "0", "6", "7", "0"};
604 file_path = output_path + "/test-human_readable_trace.txt";
605 CheckFileContent(k_sequence, file_path, false);
606
607 // Check the prefix
608 std::vector<std::string> k_prefix = {"0 0 0 0.000000 0.000000 0x30",
609 "1 1 1 1.000000 1.000000 0x65"};
610 file_path = output_path + "/test-range_delete-0-accessed_key_prefix_cut.txt";
611 CheckFileContent(k_prefix, file_path, true);
612
613 // Check the time series
614 std::vector<std::string> k_series = {"4 1533000630 0", "4 1533060100 1"};
615 file_path = output_path + "/test-range_delete-0-time_series.txt";
616 CheckFileContent(k_series, file_path, false);
617
618 // Check the accessed key in whole key space
619 std::vector<std::string> k_whole_access = {"4 1", "5 1"};
620 file_path = output_path + "/test-range_delete-0-whole_key_stats.txt";
621 CheckFileContent(k_whole_access, file_path, true);
622
623 // Check the whole key prefix cut
624 std::vector<std::string> k_whole_prefix = {"0 0x61", "1 0x62", "2 0x63",
625 "3 0x64", "4 0x65", "5 0x66"};
626 file_path = output_path + "/test-range_delete-0-whole_key_prefix_cut.txt";
627 CheckFileContent(k_whole_prefix, file_path, true);
628
629 // Check the overall qps
630 std::vector<std::string> all_qps = {"0 0 0 0 2 0 0 0 0 2"};
631 file_path = output_path + "/test-qps_stats.txt";
632 CheckFileContent(all_qps, file_path, true);
633
634 // Check the qps of DeleteRange
635 std::vector<std::string> get_qps = {"2"};
636 file_path = output_path + "/test-range_delete-0-qps_stats.txt";
637 CheckFileContent(get_qps, file_path, true);
638
639 // Check the top k qps prefix cut
640 std::vector<std::string> top_qps = {"At time: 0 with QPS: 2",
641 "The prefix: 0x65 Access count: 1",
642 "The prefix: 0x66 Access count: 1"};
643 file_path =
644 output_path + "/test-range_delete-0-accessed_top_k_qps_prefix_cut.txt";
645 CheckFileContent(top_qps, file_path, true);
646 }
647
648 // Test analyzing of Iterator
TEST_F(TraceAnalyzerTest,Iterator)649 TEST_F(TraceAnalyzerTest, Iterator) {
650 std::string trace_path = test_path_ + "/trace";
651 std::string output_path = test_path_ + "/iterator";
652 std::string file_path;
653 std::vector<std::string> paras = {
654 "-analyze_get=false", "-analyze_put=false",
655 "-analyze_delete=false", "-analyze_merge=false",
656 "-analyze_single_delete=false", "-analyze_range_delete=false",
657 "-analyze_iterator=true", "-analyze_multiget=false"};
658 paras.push_back("-output_dir=" + output_path);
659 paras.push_back("-trace_path=" + trace_path);
660 paras.push_back("-key_space_dir=" + test_path_);
661 AnalyzeTrace(paras, output_path, trace_path);
662
663 // Check the output of Seek
664 // check the key_stats file
665 std::vector<std::string> k_stats = {"0 10 0 1 1.000000"};
666 file_path = output_path + "/test-iterator_Seek-0-accessed_key_stats.txt";
667 CheckFileContent(k_stats, file_path, true);
668
669 // Check the access count distribution
670 std::vector<std::string> k_dist = {"access_count: 1 num: 1"};
671 file_path =
672 output_path + "/test-iterator_Seek-0-accessed_key_count_distribution.txt";
673 CheckFileContent(k_dist, file_path, true);
674
675 // Check the trace sequence
676 std::vector<std::string> k_sequence = {"1", "5", "2", "3", "4", "8",
677 "8", "8", "8", "8", "8", "8",
678 "8", "8", "0", "6", "7", "0"};
679 file_path = output_path + "/test-human_readable_trace.txt";
680 CheckFileContent(k_sequence, file_path, false);
681
682 // Check the prefix
683 std::vector<std::string> k_prefix = {"0 0 0 0.000000 0.000000 0x30"};
684 file_path = output_path + "/test-iterator_Seek-0-accessed_key_prefix_cut.txt";
685 CheckFileContent(k_prefix, file_path, true);
686
687 // Check the time series
688 std::vector<std::string> k_series = {"6 1 0"};
689 file_path = output_path + "/test-iterator_Seek-0-time_series.txt";
690 CheckFileContent(k_series, file_path, false);
691
692 // Check the accessed key in whole key space
693 std::vector<std::string> k_whole_access = {"0 1"};
694 file_path = output_path + "/test-iterator_Seek-0-whole_key_stats.txt";
695 CheckFileContent(k_whole_access, file_path, true);
696
697 // Check the whole key prefix cut
698 std::vector<std::string> k_whole_prefix = {"0 0x61", "1 0x62", "2 0x63",
699 "3 0x64", "4 0x65", "5 0x66"};
700 file_path = output_path + "/test-iterator_Seek-0-whole_key_prefix_cut.txt";
701 CheckFileContent(k_whole_prefix, file_path, true);
702
703 // Check the overall qps
704 std::vector<std::string> all_qps = {"0 0 0 0 0 0 1 1 0 2"};
705 file_path = output_path + "/test-qps_stats.txt";
706 CheckFileContent(all_qps, file_path, true);
707
708 // Check the qps of Iterator_Seek
709 std::vector<std::string> get_qps = {"1"};
710 file_path = output_path + "/test-iterator_Seek-0-qps_stats.txt";
711 CheckFileContent(get_qps, file_path, true);
712
713 // Check the top k qps prefix cut
714 std::vector<std::string> top_qps = {"At time: 0 with QPS: 1",
715 "The prefix: 0x61 Access count: 1"};
716 file_path =
717 output_path + "/test-iterator_Seek-0-accessed_top_k_qps_prefix_cut.txt";
718 CheckFileContent(top_qps, file_path, true);
719
720 // Check the output of SeekForPrev
721 // check the key_stats file
722 k_stats = {"0 10 0 1 1.000000"};
723 file_path =
724 output_path + "/test-iterator_SeekForPrev-0-accessed_key_stats.txt";
725 CheckFileContent(k_stats, file_path, true);
726
727 // Check the access count distribution
728 k_dist = {"access_count: 1 num: 1"};
729 file_path =
730 output_path +
731 "/test-iterator_SeekForPrev-0-accessed_key_count_distribution.txt";
732 CheckFileContent(k_dist, file_path, true);
733
734 // Check the prefix
735 k_prefix = {"0 0 0 0.000000 0.000000 0x30"};
736 file_path =
737 output_path + "/test-iterator_SeekForPrev-0-accessed_key_prefix_cut.txt";
738 CheckFileContent(k_prefix, file_path, true);
739
740 // Check the time series
741 k_series = {"7 0 0"};
742 file_path = output_path + "/test-iterator_SeekForPrev-0-time_series.txt";
743 CheckFileContent(k_series, file_path, false);
744
745 // Check the accessed key in whole key space
746 k_whole_access = {"1 1"};
747 file_path = output_path + "/test-iterator_SeekForPrev-0-whole_key_stats.txt";
748 CheckFileContent(k_whole_access, file_path, true);
749
750 // Check the whole key prefix cut
751 k_whole_prefix = {"0 0x61", "1 0x62", "2 0x63", "3 0x64", "4 0x65", "5 0x66"};
752 file_path =
753 output_path + "/test-iterator_SeekForPrev-0-whole_key_prefix_cut.txt";
754 CheckFileContent(k_whole_prefix, file_path, true);
755
756 // Check the qps of Iterator_SeekForPrev
757 get_qps = {"1"};
758 file_path = output_path + "/test-iterator_SeekForPrev-0-qps_stats.txt";
759 CheckFileContent(get_qps, file_path, true);
760
761 // Check the top k qps prefix cut
762 top_qps = {"At time: 0 with QPS: 1", "The prefix: 0x62 Access count: 1"};
763 file_path = output_path +
764 "/test-iterator_SeekForPrev-0-accessed_top_k_qps_prefix_cut.txt";
765 CheckFileContent(top_qps, file_path, true);
766 }
767
768 // Test analyzing of multiget
TEST_F(TraceAnalyzerTest,MultiGet)769 TEST_F(TraceAnalyzerTest, MultiGet) {
770 std::string trace_path = test_path_ + "/trace";
771 std::string output_path = test_path_ + "/multiget";
772 std::string file_path;
773 std::vector<std::string> paras = {
774 "-analyze_get=false", "-analyze_put=false",
775 "-analyze_delete=false", "-analyze_merge=false",
776 "-analyze_single_delete=false", "-analyze_range_delete=true",
777 "-analyze_iterator=false", "-analyze_multiget=true"};
778 paras.push_back("-output_dir=" + output_path);
779 paras.push_back("-trace_path=" + trace_path);
780 paras.push_back("-key_space_dir=" + test_path_);
781 AnalyzeTrace(paras, output_path, trace_path);
782
783 // check the key_stats file
784 std::vector<std::string> k_stats = {"0 10 0 2 1.000000", "0 10 1 2 1.000000",
785 "0 10 2 1 1.000000", "0 10 3 2 1.000000",
786 "0 10 4 2 1.000000"};
787 file_path = output_path + "/test-multiget-0-accessed_key_stats.txt";
788 CheckFileContent(k_stats, file_path, true);
789
790 // Check the access count distribution
791 std::vector<std::string> k_dist = {"access_count: 1 num: 1",
792 "access_count: 2 num: 4"};
793 file_path =
794 output_path + "/test-multiget-0-accessed_key_count_distribution.txt";
795 CheckFileContent(k_dist, file_path, true);
796
797 // Check the trace sequence
798 std::vector<std::string> k_sequence = {"1", "5", "2", "3", "4", "8",
799 "8", "8", "8", "8", "8", "8",
800 "8", "8", "0", "6", "7", "0"};
801 file_path = output_path + "/test-human_readable_trace.txt";
802 CheckFileContent(k_sequence, file_path, false);
803
804 // Check the prefix
805 std::vector<std::string> k_prefix = {
806 "0 0 0 0.000000 0.000000 0x30", "1 2 1 2.000000 1.000000 0x61",
807 "2 2 1 2.000000 1.000000 0x62", "3 1 1 1.000000 1.000000 0x64",
808 "4 2 1 2.000000 1.000000 0x67"};
809 file_path = output_path + "/test-multiget-0-accessed_key_prefix_cut.txt";
810 CheckFileContent(k_prefix, file_path, true);
811
812 // Check the time series
813 std::vector<std::string> k_series = {"8 0 0", "8 0 1", "8 0 2",
814 "8 0 3", "8 0 4", "8 0 0",
815 "8 0 1", "8 0 3", "8 0 4"};
816 file_path = output_path + "/test-multiget-0-time_series.txt";
817 CheckFileContent(k_series, file_path, false);
818
819 // Check the accessed key in whole key space
820 std::vector<std::string> k_whole_access = {"0 2", "1 2"};
821 file_path = output_path + "/test-multiget-0-whole_key_stats.txt";
822 CheckFileContent(k_whole_access, file_path, true);
823
824 // Check the whole key prefix cut
825 std::vector<std::string> k_whole_prefix = {"0 0x61", "1 0x62", "2 0x63",
826 "3 0x64", "4 0x65", "5 0x66"};
827 file_path = output_path + "/test-multiget-0-whole_key_prefix_cut.txt";
828 CheckFileContent(k_whole_prefix, file_path, true);
829
830 // Check the overall qps. We have 3 MultiGet queries and it requested 9 keys
831 // in total
832 std::vector<std::string> all_qps = {"0 0 0 0 2 0 0 0 9 11"};
833 file_path = output_path + "/test-qps_stats.txt";
834 CheckFileContent(all_qps, file_path, true);
835
836 // Check the qps of DeleteRange
837 std::vector<std::string> get_qps = {"9"};
838 file_path = output_path + "/test-multiget-0-qps_stats.txt";
839 CheckFileContent(get_qps, file_path, true);
840
841 // Check the top k qps prefix cut
842 std::vector<std::string> top_qps = {
843 "At time: 0 with QPS: 9", "The prefix: 0x61 Access count: 2",
844 "The prefix: 0x62 Access count: 2", "The prefix: 0x64 Access count: 1",
845 "The prefix: 0x67 Access count: 2", "The prefix: 0x68 Access count: 2"};
846 file_path =
847 output_path + "/test-multiget-0-accessed_top_k_qps_prefix_cut.txt";
848 CheckFileContent(top_qps, file_path, true);
849 }
850
851 } // namespace ROCKSDB_NAMESPACE
852
main(int argc,char ** argv)853 int main(int argc, char** argv) {
854 ::testing::InitGoogleTest(&argc, argv);
855 return RUN_ALL_TESTS();
856 }
857 #endif // GFLAG
858 #else
859 #include <stdio.h>
860
main(int,char **)861 int main(int /*argc*/, char** /*argv*/) {
862 fprintf(stderr, "Trace_analyzer test is not supported in ROCKSDB_LITE\n");
863 return 0;
864 }
865
866 #endif // !ROCKSDB_LITE return RUN_ALL_TESTS();
867