1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2012 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #ifndef ROCKSDB_LITE
11
12 #include <stdint.h>
13 #include "rocksdb/sst_dump_tool.h"
14
15 #include "file/random_access_file_reader.h"
16 #include "port/stack_trace.h"
17 #include "rocksdb/filter_policy.h"
18 #include "table/block_based/block_based_table_factory.h"
19 #include "table/table_builder.h"
20 #include "test_util/testharness.h"
21 #include "test_util/testutil.h"
22
23 namespace ROCKSDB_NAMESPACE {
24
25 const uint32_t kOptLength = 1024;
26
27 namespace {
MakeKey(int i)28 static std::string MakeKey(int i) {
29 char buf[100];
30 snprintf(buf, sizeof(buf), "k_%04d", i);
31 InternalKey key(std::string(buf), 0, ValueType::kTypeValue);
32 return key.Encode().ToString();
33 }
34
MakeValue(int i)35 static std::string MakeValue(int i) {
36 char buf[100];
37 snprintf(buf, sizeof(buf), "v_%04d", i);
38 InternalKey key(std::string(buf), 0, ValueType::kTypeValue);
39 return key.Encode().ToString();
40 }
41
cleanup(const Options & opts,const std::string & file_name)42 void cleanup(const Options& opts, const std::string& file_name) {
43 Env* env = opts.env;
44 ASSERT_OK(env->DeleteFile(file_name));
45 std::string outfile_name = file_name.substr(0, file_name.length() - 4);
46 outfile_name.append("_dump.txt");
47 env->DeleteFile(outfile_name).PermitUncheckedError();
48 }
49 } // namespace
50
51 // Test for sst dump tool "raw" mode
52 class SSTDumpToolTest : public testing::Test {
53 std::string test_dir_;
54 Env* env_;
55 std::shared_ptr<Env> env_guard_;
56
57 public:
SSTDumpToolTest()58 SSTDumpToolTest() : env_(Env::Default()) {
59 const char* test_env_uri = getenv("TEST_ENV_URI");
60 if (test_env_uri) {
61 Env::LoadEnv(test_env_uri, &env_, &env_guard_);
62 }
63 test_dir_ = test::PerThreadDBPath(env_, "sst_dump_test_db");
64 Status s = env_->CreateDirIfMissing(test_dir_);
65 EXPECT_OK(s);
66 }
67
~SSTDumpToolTest()68 ~SSTDumpToolTest() override {
69 if (getenv("KEEP_DB")) {
70 fprintf(stdout, "Data is still at %s\n", test_dir_.c_str());
71 } else {
72 EXPECT_OK(env_->DeleteDir(test_dir_));
73 }
74 }
75
env()76 Env* env() { return env_; }
77
MakeFilePath(const std::string & file_name) const78 std::string MakeFilePath(const std::string& file_name) const {
79 std::string path(test_dir_);
80 path.append("/").append(file_name);
81 return path;
82 }
83
84 template <std::size_t N>
PopulateCommandArgs(const std::string & file_path,const char * command,char * (& usage)[N]) const85 void PopulateCommandArgs(const std::string& file_path, const char* command,
86 char* (&usage)[N]) const {
87 for (int i = 0; i < static_cast<int>(N); ++i) {
88 usage[i] = new char[kOptLength];
89 }
90 snprintf(usage[0], kOptLength, "./sst_dump");
91 snprintf(usage[1], kOptLength, "%s", command);
92 snprintf(usage[2], kOptLength, "--file=%s", file_path.c_str());
93 }
94
createSST(const Options & opts,const std::string & file_name)95 void createSST(const Options& opts, const std::string& file_name) {
96 Env* test_env = opts.env;
97 FileOptions file_options(opts);
98 ReadOptions read_options;
99 const ImmutableOptions imoptions(opts);
100 const MutableCFOptions moptions(opts);
101 ROCKSDB_NAMESPACE::InternalKeyComparator ikc(opts.comparator);
102 std::unique_ptr<TableBuilder> tb;
103
104 IntTblPropCollectorFactories int_tbl_prop_collector_factories;
105 std::unique_ptr<WritableFileWriter> file_writer;
106 ASSERT_OK(WritableFileWriter::Create(test_env->GetFileSystem(), file_name,
107 file_options, &file_writer, nullptr));
108
109 std::string column_family_name;
110 int unknown_level = -1;
111 tb.reset(opts.table_factory->NewTableBuilder(
112 TableBuilderOptions(
113 imoptions, moptions, ikc, &int_tbl_prop_collector_factories,
114 CompressionType::kNoCompression, CompressionOptions(),
115 TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
116 column_family_name, unknown_level),
117 file_writer.get()));
118
119 // Populate slightly more than 1K keys
120 uint32_t num_keys = kNumKey;
121 for (uint32_t i = 0; i < num_keys; i++) {
122 tb->Add(MakeKey(i), MakeValue(i));
123 }
124 ASSERT_OK(tb->Finish());
125 ASSERT_OK(file_writer->Close());
126 }
127
128 protected:
129 constexpr static int kNumKey = 1024;
130 };
131
132 constexpr int SSTDumpToolTest::kNumKey;
133
TEST_F(SSTDumpToolTest,HelpAndVersion)134 TEST_F(SSTDumpToolTest, HelpAndVersion) {
135 Options opts;
136 opts.env = env();
137
138 ROCKSDB_NAMESPACE::SSTDumpTool tool;
139
140 static const char* help[] = {"./sst_dump", "--help"};
141 ASSERT_TRUE(!tool.Run(2, help, opts));
142 static const char* version[] = {"./sst_dump", "--version"};
143 ASSERT_TRUE(!tool.Run(2, version, opts));
144 static const char* bad[] = {"./sst_dump", "--not_an_option"};
145 ASSERT_TRUE(tool.Run(2, bad, opts));
146 }
147
TEST_F(SSTDumpToolTest,EmptyFilter)148 TEST_F(SSTDumpToolTest, EmptyFilter) {
149 Options opts;
150 opts.env = env();
151 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
152 createSST(opts, file_path);
153
154 char* usage[3];
155 PopulateCommandArgs(file_path, "--command=raw", usage);
156
157 ROCKSDB_NAMESPACE::SSTDumpTool tool;
158 ASSERT_TRUE(!tool.Run(3, usage, opts));
159
160 cleanup(opts, file_path);
161 for (int i = 0; i < 3; i++) {
162 delete[] usage[i];
163 }
164 }
165
TEST_F(SSTDumpToolTest,FilterBlock)166 TEST_F(SSTDumpToolTest, FilterBlock) {
167 Options opts;
168 opts.env = env();
169 BlockBasedTableOptions table_opts;
170 table_opts.filter_policy.reset(
171 ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, true));
172 opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
173 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
174 createSST(opts, file_path);
175
176 char* usage[3];
177 PopulateCommandArgs(file_path, "--command=raw", usage);
178
179 ROCKSDB_NAMESPACE::SSTDumpTool tool;
180 ASSERT_TRUE(!tool.Run(3, usage, opts));
181
182 cleanup(opts, file_path);
183 for (int i = 0; i < 3; i++) {
184 delete[] usage[i];
185 }
186 }
187
TEST_F(SSTDumpToolTest,FullFilterBlock)188 TEST_F(SSTDumpToolTest, FullFilterBlock) {
189 Options opts;
190 opts.env = env();
191 BlockBasedTableOptions table_opts;
192 table_opts.filter_policy.reset(
193 ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
194 opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
195 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
196 createSST(opts, file_path);
197
198 char* usage[3];
199 PopulateCommandArgs(file_path, "--command=raw", usage);
200
201 ROCKSDB_NAMESPACE::SSTDumpTool tool;
202 ASSERT_TRUE(!tool.Run(3, usage, opts));
203
204 cleanup(opts, file_path);
205 for (int i = 0; i < 3; i++) {
206 delete[] usage[i];
207 }
208 }
209
TEST_F(SSTDumpToolTest,GetProperties)210 TEST_F(SSTDumpToolTest, GetProperties) {
211 Options opts;
212 opts.env = env();
213 BlockBasedTableOptions table_opts;
214 table_opts.filter_policy.reset(
215 ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
216 opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
217 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
218 createSST(opts, file_path);
219
220 char* usage[3];
221 PopulateCommandArgs(file_path, "--show_properties", usage);
222
223 ROCKSDB_NAMESPACE::SSTDumpTool tool;
224 ASSERT_TRUE(!tool.Run(3, usage, opts));
225
226 cleanup(opts, file_path);
227 for (int i = 0; i < 3; i++) {
228 delete[] usage[i];
229 }
230 }
231
TEST_F(SSTDumpToolTest,CompressedSizes)232 TEST_F(SSTDumpToolTest, CompressedSizes) {
233 Options opts;
234 opts.env = env();
235 BlockBasedTableOptions table_opts;
236 table_opts.filter_policy.reset(
237 ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
238 opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
239 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
240 createSST(opts, file_path);
241
242 char* usage[3];
243 PopulateCommandArgs(file_path, "--command=recompress", usage);
244
245 ROCKSDB_NAMESPACE::SSTDumpTool tool;
246 ASSERT_TRUE(!tool.Run(3, usage, opts));
247
248 cleanup(opts, file_path);
249 for (int i = 0; i < 3; i++) {
250 delete[] usage[i];
251 }
252 }
253
TEST_F(SSTDumpToolTest,MemEnv)254 TEST_F(SSTDumpToolTest, MemEnv) {
255 std::unique_ptr<Env> mem_env(NewMemEnv(env()));
256 Options opts;
257 opts.env = mem_env.get();
258 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
259 createSST(opts, file_path);
260
261 char* usage[3];
262 PopulateCommandArgs(file_path, "--command=verify_checksum", usage);
263
264 ROCKSDB_NAMESPACE::SSTDumpTool tool;
265 ASSERT_TRUE(!tool.Run(3, usage, opts));
266
267 cleanup(opts, file_path);
268 for (int i = 0; i < 3; i++) {
269 delete[] usage[i];
270 }
271 }
272
TEST_F(SSTDumpToolTest,ReadaheadSize)273 TEST_F(SSTDumpToolTest, ReadaheadSize) {
274 Options opts;
275 opts.env = env();
276 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
277 createSST(opts, file_path);
278
279 char* usage[4];
280 PopulateCommandArgs(file_path, "--command=verify", usage);
281 snprintf(usage[3], kOptLength, "--readahead_size=4000000");
282
283 int num_reads = 0;
284 SyncPoint::GetInstance()->SetCallBack("RandomAccessFileReader::Read",
285 [&](void*) { num_reads++; });
286 SyncPoint::GetInstance()->EnableProcessing();
287
288 SSTDumpTool tool;
289 ASSERT_TRUE(!tool.Run(4, usage, opts));
290
291 // The file is approximately 10MB. Readahead is 4MB.
292 // We usually need 3 reads + one metadata read.
293 // One extra read is needed before opening the file for metadata.
294 ASSERT_EQ(5, num_reads);
295
296 SyncPoint::GetInstance()->ClearAllCallBacks();
297 SyncPoint::GetInstance()->DisableProcessing();
298
299 cleanup(opts, file_path);
300 for (int i = 0; i < 4; i++) {
301 delete[] usage[i];
302 }
303 }
304
TEST_F(SSTDumpToolTest,NoSstFile)305 TEST_F(SSTDumpToolTest, NoSstFile) {
306 Options opts;
307 opts.env = env();
308 std::string file_path = MakeFilePath("no_such_file.sst");
309 char* usage[3];
310 PopulateCommandArgs(file_path, "", usage);
311 ROCKSDB_NAMESPACE::SSTDumpTool tool;
312 for (const auto& command :
313 {"--command=check", "--command=dump", "--command=raw",
314 "--command=verify", "--command=recompress", "--command=verify_checksum",
315 "--show_properties"}) {
316 snprintf(usage[1], kOptLength, "%s", command);
317 ASSERT_TRUE(tool.Run(3, usage, opts));
318 }
319 for (int i = 0; i < 3; i++) {
320 delete[] usage[i];
321 }
322 }
323
TEST_F(SSTDumpToolTest,ValidSSTPath)324 TEST_F(SSTDumpToolTest, ValidSSTPath) {
325 Options opts;
326 opts.env = env();
327 char* usage[3];
328 PopulateCommandArgs("", "", usage);
329 SSTDumpTool tool;
330 std::string file_not_exists = MakeFilePath("file_not_exists.sst");
331 std::string sst_file = MakeFilePath("rocksdb_sst_test.sst");
332 createSST(opts, sst_file);
333 std::string text_file = MakeFilePath("text_file");
334 ASSERT_OK(WriteStringToFile(opts.env, "Hello World!", text_file));
335 std::string fake_sst = MakeFilePath("fake_sst.sst");
336 ASSERT_OK(WriteStringToFile(opts.env, "Not an SST file!", fake_sst));
337
338 for (const auto& command_arg : {"--command=verify", "--command=identify"}) {
339 snprintf(usage[1], kOptLength, "%s", command_arg);
340
341 snprintf(usage[2], kOptLength, "--file=%s", file_not_exists.c_str());
342 ASSERT_TRUE(tool.Run(3, usage, opts));
343
344 snprintf(usage[2], kOptLength, "--file=%s", sst_file.c_str());
345 ASSERT_TRUE(!tool.Run(3, usage, opts));
346
347 snprintf(usage[2], kOptLength, "--file=%s", text_file.c_str());
348 ASSERT_TRUE(tool.Run(3, usage, opts));
349
350 snprintf(usage[2], kOptLength, "--file=%s", fake_sst.c_str());
351 ASSERT_TRUE(tool.Run(3, usage, opts));
352 }
353 ASSERT_OK(opts.env->DeleteFile(sst_file));
354 ASSERT_OK(opts.env->DeleteFile(text_file));
355 ASSERT_OK(opts.env->DeleteFile(fake_sst));
356
357 for (int i = 0; i < 3; i++) {
358 delete[] usage[i];
359 }
360 }
361
TEST_F(SSTDumpToolTest,RawOutput)362 TEST_F(SSTDumpToolTest, RawOutput) {
363 Options opts;
364 opts.env = env();
365 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
366 createSST(opts, file_path);
367
368 char* usage[3];
369 PopulateCommandArgs(file_path, "--command=raw", usage);
370
371 ROCKSDB_NAMESPACE::SSTDumpTool tool;
372 ASSERT_TRUE(!tool.Run(3, usage, opts));
373
374 const std::string raw_path = MakeFilePath("rocksdb_sst_test_dump.txt");
375 std::ifstream raw_file(raw_path);
376
377 std::string tp;
378 bool is_data_block = false;
379 int key_count = 0;
380 while (getline(raw_file, tp)) {
381 if (tp.find("Data Block #") != std::string::npos) {
382 is_data_block = true;
383 }
384
385 if (is_data_block && tp.find("HEX") != std::string::npos) {
386 key_count++;
387 }
388 }
389
390 ASSERT_EQ(kNumKey, key_count);
391
392 raw_file.close();
393
394 cleanup(opts, file_path);
395 for (int i = 0; i < 3; i++) {
396 delete[] usage[i];
397 }
398 }
399
400 } // namespace ROCKSDB_NAMESPACE
401
402 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
403 extern "C" {
404 void RegisterCustomObjects(int argc, char** argv);
405 }
406 #else
RegisterCustomObjects(int,char **)407 void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
408 #endif // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
409
main(int argc,char ** argv)410 int main(int argc, char** argv) {
411 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
412 ::testing::InitGoogleTest(&argc, argv);
413 RegisterCustomObjects(argc, argv);
414 return RUN_ALL_TESTS();
415 }
416
417 #else
418 #include <stdio.h>
419
main(int,char **)420 int main(int /*argc*/, char** /*argv*/) {
421 fprintf(stderr, "SKIPPED as SSTDumpTool is not supported in ROCKSDB_LITE\n");
422 return 0;
423 }
424
425 #endif // !ROCKSDB_LITE return RUN_ALL_TESTS();
426