1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2012 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #ifndef ROCKSDB_LITE
11
12 #include <stdint.h>
13 #include "rocksdb/sst_dump_tool.h"
14
15 #include "file/random_access_file_reader.h"
16 #include "port/stack_trace.h"
17 #include "rocksdb/filter_policy.h"
18 #include "table/block_based/block_based_table_factory.h"
19 #include "table/table_builder.h"
20 #include "test_util/testharness.h"
21 #include "test_util/testutil.h"
22
23 namespace ROCKSDB_NAMESPACE {
24
25 const uint32_t optLength = 100;
26
27 namespace {
MakeKey(int i)28 static std::string MakeKey(int i) {
29 char buf[100];
30 snprintf(buf, sizeof(buf), "k_%04d", i);
31 InternalKey key(std::string(buf), 0, ValueType::kTypeValue);
32 return key.Encode().ToString();
33 }
34
MakeValue(int i)35 static std::string MakeValue(int i) {
36 char buf[100];
37 snprintf(buf, sizeof(buf), "v_%04d", i);
38 InternalKey key(std::string(buf), 0, ValueType::kTypeValue);
39 return key.Encode().ToString();
40 }
41
createSST(const Options & opts,const std::string & file_name)42 void createSST(const Options& opts, const std::string& file_name) {
43 Env* env = opts.env;
44 EnvOptions env_options(opts);
45 ReadOptions read_options;
46 const ImmutableCFOptions imoptions(opts);
47 const MutableCFOptions moptions(opts);
48 ROCKSDB_NAMESPACE::InternalKeyComparator ikc(opts.comparator);
49 std::unique_ptr<TableBuilder> tb;
50
51 std::unique_ptr<WritableFile> file;
52 ASSERT_OK(env->NewWritableFile(file_name, &file, env_options));
53
54 std::vector<std::unique_ptr<IntTblPropCollectorFactory> >
55 int_tbl_prop_collector_factories;
56 std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
57 NewLegacyWritableFileWrapper(std::move(file)), file_name, EnvOptions()));
58 std::string column_family_name;
59 int unknown_level = -1;
60 tb.reset(opts.table_factory->NewTableBuilder(
61 TableBuilderOptions(
62 imoptions, moptions, ikc, &int_tbl_prop_collector_factories,
63 CompressionType::kNoCompression, 0 /* sample_for_compression */,
64 CompressionOptions(), false /* skip_filters */, column_family_name,
65 unknown_level),
66 TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
67 file_writer.get()));
68
69 // Populate slightly more than 1K keys
70 uint32_t num_keys = 1024;
71 for (uint32_t i = 0; i < num_keys; i++) {
72 tb->Add(MakeKey(i), MakeValue(i));
73 }
74 tb->Finish();
75 file_writer->Close();
76 }
77
cleanup(const Options & opts,const std::string & file_name)78 void cleanup(const Options& opts, const std::string& file_name) {
79 Env* env = opts.env;
80 env->DeleteFile(file_name);
81 std::string outfile_name = file_name.substr(0, file_name.length() - 4);
82 outfile_name.append("_dump.txt");
83 env->DeleteFile(outfile_name);
84 }
85 } // namespace
86
87 // Test for sst dump tool "raw" mode
88 class SSTDumpToolTest : public testing::Test {
89 std::string test_dir_;
90 Env* env_;
91 std::shared_ptr<Env> env_guard_;
92
93 public:
SSTDumpToolTest()94 SSTDumpToolTest() : env_(Env::Default()) {
95 const char* test_env_uri = getenv("TEST_ENV_URI");
96 if (test_env_uri) {
97 Env::LoadEnv(test_env_uri, &env_, &env_guard_);
98 }
99 test_dir_ = test::PerThreadDBPath(env_, "sst_dump_test_db");
100 Status s = env_->CreateDirIfMissing(test_dir_);
101 EXPECT_OK(s);
102 }
103
~SSTDumpToolTest()104 ~SSTDumpToolTest() override {
105 if (getenv("KEEP_DB")) {
106 fprintf(stdout, "Data is still at %s\n", test_dir_.c_str());
107 } else {
108 EXPECT_OK(env_->DeleteDir(test_dir_));
109 }
110 }
111
env()112 Env* env() { return env_; }
113
MakeFilePath(const std::string & file_name) const114 std::string MakeFilePath(const std::string& file_name) const {
115 std::string path(test_dir_);
116 path.append("/").append(file_name);
117 return path;
118 }
119
120 template <std::size_t N>
PopulateCommandArgs(const std::string & file_path,const char * command,char * (& usage)[N]) const121 void PopulateCommandArgs(const std::string& file_path, const char* command,
122 char* (&usage)[N]) const {
123 for (int i = 0; i < static_cast<int>(N); ++i) {
124 usage[i] = new char[optLength];
125 }
126 snprintf(usage[0], optLength, "./sst_dump");
127 snprintf(usage[1], optLength, "%s", command);
128 snprintf(usage[2], optLength, "--file=%s", file_path.c_str());
129 }
130 };
131
TEST_F(SSTDumpToolTest,EmptyFilter)132 TEST_F(SSTDumpToolTest, EmptyFilter) {
133 Options opts;
134 opts.env = env();
135 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
136 createSST(opts, file_path);
137
138 char* usage[3];
139 PopulateCommandArgs(file_path, "--command=raw", usage);
140
141 ROCKSDB_NAMESPACE::SSTDumpTool tool;
142 ASSERT_TRUE(!tool.Run(3, usage, opts));
143
144 cleanup(opts, file_path);
145 for (int i = 0; i < 3; i++) {
146 delete[] usage[i];
147 }
148 }
149
TEST_F(SSTDumpToolTest,FilterBlock)150 TEST_F(SSTDumpToolTest, FilterBlock) {
151 Options opts;
152 opts.env = env();
153 BlockBasedTableOptions table_opts;
154 table_opts.filter_policy.reset(
155 ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, true));
156 opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
157 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
158 createSST(opts, file_path);
159
160 char* usage[3];
161 PopulateCommandArgs(file_path, "--command=raw", usage);
162
163 ROCKSDB_NAMESPACE::SSTDumpTool tool;
164 ASSERT_TRUE(!tool.Run(3, usage, opts));
165
166 cleanup(opts, file_path);
167 for (int i = 0; i < 3; i++) {
168 delete[] usage[i];
169 }
170 }
171
TEST_F(SSTDumpToolTest,FullFilterBlock)172 TEST_F(SSTDumpToolTest, FullFilterBlock) {
173 Options opts;
174 opts.env = env();
175 BlockBasedTableOptions table_opts;
176 table_opts.filter_policy.reset(
177 ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
178 opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
179 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
180 createSST(opts, file_path);
181
182 char* usage[3];
183 PopulateCommandArgs(file_path, "--command=raw", usage);
184
185 ROCKSDB_NAMESPACE::SSTDumpTool tool;
186 ASSERT_TRUE(!tool.Run(3, usage, opts));
187
188 cleanup(opts, file_path);
189 for (int i = 0; i < 3; i++) {
190 delete[] usage[i];
191 }
192 }
193
TEST_F(SSTDumpToolTest,GetProperties)194 TEST_F(SSTDumpToolTest, GetProperties) {
195 Options opts;
196 opts.env = env();
197 BlockBasedTableOptions table_opts;
198 table_opts.filter_policy.reset(
199 ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
200 opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
201 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
202 createSST(opts, file_path);
203
204 char* usage[3];
205 PopulateCommandArgs(file_path, "--show_properties", usage);
206
207 ROCKSDB_NAMESPACE::SSTDumpTool tool;
208 ASSERT_TRUE(!tool.Run(3, usage, opts));
209
210 cleanup(opts, file_path);
211 for (int i = 0; i < 3; i++) {
212 delete[] usage[i];
213 }
214 }
215
TEST_F(SSTDumpToolTest,CompressedSizes)216 TEST_F(SSTDumpToolTest, CompressedSizes) {
217 Options opts;
218 opts.env = env();
219 BlockBasedTableOptions table_opts;
220 table_opts.filter_policy.reset(
221 ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
222 opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
223 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
224 createSST(opts, file_path);
225
226 char* usage[3];
227 PopulateCommandArgs(file_path, "--command=recompress", usage);
228
229 ROCKSDB_NAMESPACE::SSTDumpTool tool;
230 ASSERT_TRUE(!tool.Run(3, usage, opts));
231
232 cleanup(opts, file_path);
233 for (int i = 0; i < 3; i++) {
234 delete[] usage[i];
235 }
236 }
237
TEST_F(SSTDumpToolTest,MemEnv)238 TEST_F(SSTDumpToolTest, MemEnv) {
239 std::unique_ptr<Env> mem_env(NewMemEnv(env()));
240 Options opts;
241 opts.env = mem_env.get();
242 std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
243 createSST(opts, file_path);
244
245 char* usage[3];
246 PopulateCommandArgs(file_path, "--command=verify_checksum", usage);
247
248 ROCKSDB_NAMESPACE::SSTDumpTool tool;
249 ASSERT_TRUE(!tool.Run(3, usage, opts));
250
251 cleanup(opts, file_path);
252 for (int i = 0; i < 3; i++) {
253 delete[] usage[i];
254 }
255 }
256
257 } // namespace ROCKSDB_NAMESPACE
258
259 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
260 extern "C" {
261 void RegisterCustomObjects(int argc, char** argv);
262 }
263 #else
RegisterCustomObjects(int,char **)264 void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
265 #endif // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
266
main(int argc,char ** argv)267 int main(int argc, char** argv) {
268 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
269 ::testing::InitGoogleTest(&argc, argv);
270 RegisterCustomObjects(argc, argv);
271 return RUN_ALL_TESTS();
272 }
273
274 #else
275 #include <stdio.h>
276
main(int,char **)277 int main(int /*argc*/, char** /*argv*/) {
278 fprintf(stderr, "SKIPPED as SSTDumpTool is not supported in ROCKSDB_LITE\n");
279 return 0;
280 }
281
282 #endif // !ROCKSDB_LITE return RUN_ALL_TESTS();
283