1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #include <cctype>
11 #include <cinttypes>
12 #include <cstring>
13 #include <unordered_map>
14
15 #include "cache/lru_cache.h"
16 #include "cache/sharded_cache.h"
17 #include "options/options_helper.h"
18 #include "options/options_parser.h"
19 #include "options/options_sanity_check.h"
20 #include "port/port.h"
21 #include "rocksdb/cache.h"
22 #include "rocksdb/convenience.h"
23 #include "rocksdb/memtablerep.h"
24 #include "rocksdb/utilities/leveldb_options.h"
25 #include "rocksdb/utilities/object_registry.h"
26 #include "table/block_based/filter_policy_internal.h"
27 #include "test_util/testharness.h"
28 #include "test_util/testutil.h"
29 #include "util/random.h"
30 #include "util/stderr_logger.h"
31 #include "util/string_util.h"
32 #include "utilities/merge_operators/bytesxor.h"
33
34 #ifndef GFLAGS
35 bool FLAGS_enable_print = false;
36 #else
37 #include "util/gflags_compat.h"
38 using GFLAGS_NAMESPACE::ParseCommandLineFlags;
39 DEFINE_bool(enable_print, false, "Print options generated to console.");
40 #endif // GFLAGS
41
42 namespace ROCKSDB_NAMESPACE {
43
44 class OptionsTest : public testing::Test {};
45
46 #ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE
TEST_F(OptionsTest,GetOptionsFromMapTest)47 TEST_F(OptionsTest, GetOptionsFromMapTest) {
48 std::unordered_map<std::string, std::string> cf_options_map = {
49 {"write_buffer_size", "1"},
50 {"max_write_buffer_number", "2"},
51 {"min_write_buffer_number_to_merge", "3"},
52 {"max_write_buffer_number_to_maintain", "99"},
53 {"max_write_buffer_size_to_maintain", "-99999"},
54 {"compression", "kSnappyCompression"},
55 {"compression_per_level",
56 "kNoCompression:"
57 "kSnappyCompression:"
58 "kZlibCompression:"
59 "kBZip2Compression:"
60 "kLZ4Compression:"
61 "kLZ4HCCompression:"
62 "kXpressCompression:"
63 "kZSTD:"
64 "kZSTDNotFinalCompression"},
65 {"bottommost_compression", "kLZ4Compression"},
66 {"bottommost_compression_opts", "5:6:7:8:9:true"},
67 {"compression_opts", "4:5:6:7:8:true"},
68 {"num_levels", "8"},
69 {"level0_file_num_compaction_trigger", "8"},
70 {"level0_slowdown_writes_trigger", "9"},
71 {"level0_stop_writes_trigger", "10"},
72 {"target_file_size_base", "12"},
73 {"target_file_size_multiplier", "13"},
74 {"max_bytes_for_level_base", "14"},
75 {"level_compaction_dynamic_level_bytes", "true"},
76 {"max_bytes_for_level_multiplier", "15.0"},
77 {"max_bytes_for_level_multiplier_additional", "16:17:18"},
78 {"max_compaction_bytes", "21"},
79 {"soft_rate_limit", "1.1"},
80 {"hard_rate_limit", "2.1"},
81 {"hard_pending_compaction_bytes_limit", "211"},
82 {"arena_block_size", "22"},
83 {"disable_auto_compactions", "true"},
84 {"compaction_style", "kCompactionStyleLevel"},
85 {"compaction_pri", "kOldestSmallestSeqFirst"},
86 {"verify_checksums_in_compaction", "false"},
87 {"compaction_options_fifo", "23"},
88 {"max_sequential_skip_in_iterations", "24"},
89 {"inplace_update_support", "true"},
90 {"report_bg_io_stats", "true"},
91 {"compaction_measure_io_stats", "false"},
92 {"inplace_update_num_locks", "25"},
93 {"memtable_prefix_bloom_size_ratio", "0.26"},
94 {"memtable_whole_key_filtering", "true"},
95 {"memtable_huge_page_size", "28"},
96 {"bloom_locality", "29"},
97 {"max_successive_merges", "30"},
98 {"min_partial_merge_operands", "31"},
99 {"prefix_extractor", "fixed:31"},
100 {"optimize_filters_for_hits", "true"},
101 };
102
103 std::unordered_map<std::string, std::string> db_options_map = {
104 {"create_if_missing", "false"},
105 {"create_missing_column_families", "true"},
106 {"error_if_exists", "false"},
107 {"paranoid_checks", "true"},
108 {"max_open_files", "32"},
109 {"max_total_wal_size", "33"},
110 {"use_fsync", "true"},
111 {"db_log_dir", "/db_log_dir"},
112 {"wal_dir", "/wal_dir"},
113 {"delete_obsolete_files_period_micros", "34"},
114 {"max_background_compactions", "35"},
115 {"max_background_flushes", "36"},
116 {"max_log_file_size", "37"},
117 {"log_file_time_to_roll", "38"},
118 {"keep_log_file_num", "39"},
119 {"recycle_log_file_num", "5"},
120 {"max_manifest_file_size", "40"},
121 {"table_cache_numshardbits", "41"},
122 {"WAL_ttl_seconds", "43"},
123 {"WAL_size_limit_MB", "44"},
124 {"manifest_preallocation_size", "45"},
125 {"allow_mmap_reads", "true"},
126 {"allow_mmap_writes", "false"},
127 {"use_direct_reads", "false"},
128 {"use_direct_io_for_flush_and_compaction", "false"},
129 {"is_fd_close_on_exec", "true"},
130 {"skip_log_error_on_recovery", "false"},
131 {"stats_dump_period_sec", "46"},
132 {"stats_persist_period_sec", "57"},
133 {"persist_stats_to_disk", "false"},
134 {"stats_history_buffer_size", "69"},
135 {"advise_random_on_open", "true"},
136 {"use_adaptive_mutex", "false"},
137 {"new_table_reader_for_compaction_inputs", "true"},
138 {"compaction_readahead_size", "100"},
139 {"random_access_max_buffer_size", "3145728"},
140 {"writable_file_max_buffer_size", "314159"},
141 {"bytes_per_sync", "47"},
142 {"wal_bytes_per_sync", "48"},
143 {"strict_bytes_per_sync", "true"},
144 };
145
146 ColumnFamilyOptions base_cf_opt;
147 ColumnFamilyOptions new_cf_opt;
148 ASSERT_OK(GetColumnFamilyOptionsFromMap(
149 base_cf_opt, cf_options_map, &new_cf_opt));
150 ASSERT_EQ(new_cf_opt.write_buffer_size, 1U);
151 ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2);
152 ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3);
153 ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99);
154 ASSERT_EQ(new_cf_opt.max_write_buffer_size_to_maintain, -99999);
155 ASSERT_EQ(new_cf_opt.compression, kSnappyCompression);
156 ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U);
157 ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression);
158 ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression);
159 ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression);
160 ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression);
161 ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression);
162 ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression);
163 ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression);
164 ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD);
165 ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression);
166 ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
167 ASSERT_EQ(new_cf_opt.compression_opts.level, 5);
168 ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6);
169 ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u);
170 ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u);
171 ASSERT_EQ(new_cf_opt.compression_opts.enabled, true);
172 ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression);
173 ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5);
174 ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6);
175 ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7);
176 ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u);
177 ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u);
178 ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true);
179 ASSERT_EQ(new_cf_opt.num_levels, 8);
180 ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8);
181 ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9);
182 ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10);
183 ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast<uint64_t>(12));
184 ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13);
185 ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U);
186 ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true);
187 ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0);
188 ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U);
189 ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16);
190 ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17);
191 ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18);
192 ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21);
193 ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211);
194 ASSERT_EQ(new_cf_opt.arena_block_size, 22U);
195 ASSERT_EQ(new_cf_opt.disable_auto_compactions, true);
196 ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
197 ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst);
198 ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
199 static_cast<uint64_t>(23));
200 ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
201 static_cast<uint64_t>(24));
202 ASSERT_EQ(new_cf_opt.inplace_update_support, true);
203 ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U);
204 ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26);
205 ASSERT_EQ(new_cf_opt.memtable_whole_key_filtering, true);
206 ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U);
207 ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
208 ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
209 ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
210 ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
211 ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()),
212 "rocksdb.FixedPrefix.31");
213
214 cf_options_map["write_buffer_size"] = "hello";
215 ASSERT_NOK(GetColumnFamilyOptionsFromMap(
216 base_cf_opt, cf_options_map, &new_cf_opt));
217 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
218
219 cf_options_map["write_buffer_size"] = "1";
220 ASSERT_OK(GetColumnFamilyOptionsFromMap(
221 base_cf_opt, cf_options_map, &new_cf_opt));
222
223 cf_options_map["unknown_option"] = "1";
224 ASSERT_NOK(GetColumnFamilyOptionsFromMap(
225 base_cf_opt, cf_options_map, &new_cf_opt));
226 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
227
228 ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map,
229 &new_cf_opt,
230 false, /* input_strings_escaped */
231 true /* ignore_unknown_options */));
232 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
233 base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */
234 kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility*/));
235 ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
236 base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */
237 kSanityLevelExactMatch /* default for VerifyCFOptions */));
238
239 DBOptions base_db_opt;
240 DBOptions new_db_opt;
241 ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
242 ASSERT_EQ(new_db_opt.create_if_missing, false);
243 ASSERT_EQ(new_db_opt.create_missing_column_families, true);
244 ASSERT_EQ(new_db_opt.error_if_exists, false);
245 ASSERT_EQ(new_db_opt.paranoid_checks, true);
246 ASSERT_EQ(new_db_opt.max_open_files, 32);
247 ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33));
248 ASSERT_EQ(new_db_opt.use_fsync, true);
249 ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir");
250 ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir");
251 ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros,
252 static_cast<uint64_t>(34));
253 ASSERT_EQ(new_db_opt.max_background_compactions, 35);
254 ASSERT_EQ(new_db_opt.max_background_flushes, 36);
255 ASSERT_EQ(new_db_opt.max_log_file_size, 37U);
256 ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U);
257 ASSERT_EQ(new_db_opt.keep_log_file_num, 39U);
258 ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U);
259 ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast<uint64_t>(40));
260 ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41);
261 ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast<uint64_t>(43));
262 ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast<uint64_t>(44));
263 ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U);
264 ASSERT_EQ(new_db_opt.allow_mmap_reads, true);
265 ASSERT_EQ(new_db_opt.allow_mmap_writes, false);
266 ASSERT_EQ(new_db_opt.use_direct_reads, false);
267 ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false);
268 ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true);
269 ASSERT_EQ(new_db_opt.skip_log_error_on_recovery, false);
270 ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U);
271 ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U);
272 ASSERT_EQ(new_db_opt.persist_stats_to_disk, false);
273 ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U);
274 ASSERT_EQ(new_db_opt.advise_random_on_open, true);
275 ASSERT_EQ(new_db_opt.use_adaptive_mutex, false);
276 ASSERT_EQ(new_db_opt.new_table_reader_for_compaction_inputs, true);
277 ASSERT_EQ(new_db_opt.compaction_readahead_size, 100);
278 ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728);
279 ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159);
280 ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47));
281 ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48));
282 ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true);
283
284 db_options_map["max_open_files"] = "hello";
285 ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
286 ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt));
287 ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
288 base_db_opt, new_db_opt, nullptr, /* new_opt_map */
289 kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */));
290
291 // unknow options should fail parsing without ignore_unknown_options = true
292 db_options_map["unknown_db_option"] = "1";
293 ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
294 ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt));
295
296 ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt,
297 false, /* input_strings_escaped */
298 true /* ignore_unknown_options */));
299 ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
300 base_db_opt, new_db_opt, nullptr, /* new_opt_map */
301 kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */));
302 ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(
303 base_db_opt, new_db_opt, nullptr, /* new_opt_mat */
304 kSanityLevelExactMatch /* default for VerifyDBOptions */));
305 }
306 #endif // !ROCKSDB_LITE
307
308 #ifndef ROCKSDB_LITE // GetColumnFamilyOptionsFromString is not supported in
309 // ROCKSDB_LITE
TEST_F(OptionsTest,GetColumnFamilyOptionsFromStringTest)310 TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
311 ColumnFamilyOptions base_cf_opt;
312 ColumnFamilyOptions new_cf_opt;
313 base_cf_opt.table_factory.reset();
314 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, "", &new_cf_opt));
315 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
316 "write_buffer_size=5", &new_cf_opt));
317 ASSERT_EQ(new_cf_opt.write_buffer_size, 5U);
318 ASSERT_TRUE(new_cf_opt.table_factory == nullptr);
319 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
320 "write_buffer_size=6;", &new_cf_opt));
321 ASSERT_EQ(new_cf_opt.write_buffer_size, 6U);
322 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
323 " write_buffer_size = 7 ", &new_cf_opt));
324 ASSERT_EQ(new_cf_opt.write_buffer_size, 7U);
325 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
326 " write_buffer_size = 8 ; ", &new_cf_opt));
327 ASSERT_EQ(new_cf_opt.write_buffer_size, 8U);
328 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
329 "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt));
330 ASSERT_EQ(new_cf_opt.write_buffer_size, 9U);
331 ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10);
332 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
333 "write_buffer_size=11; max_write_buffer_number = 12 ;",
334 &new_cf_opt));
335 ASSERT_EQ(new_cf_opt.write_buffer_size, 11U);
336 ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12);
337 // Wrong name "max_write_buffer_number_"
338 ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
339 "write_buffer_size=13;max_write_buffer_number_=14;",
340 &new_cf_opt));
341 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
342
343 // Comparator from object registry
344 std::string kCompName = "reverse_comp";
345 ObjectLibrary::Default()->Register<const Comparator>(
346 kCompName,
347 [](const std::string& /*name*/,
348 std::unique_ptr<const Comparator>* /*guard*/,
349 std::string* /* errmsg */) { return ReverseBytewiseComparator(); });
350
351 ASSERT_OK(GetColumnFamilyOptionsFromString(
352 base_cf_opt, "comparator=" + kCompName + ";", &new_cf_opt));
353 ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator());
354
355 // MergeOperator from object registry
356 std::unique_ptr<BytesXOROperator> bxo(new BytesXOROperator());
357 std::string kMoName = bxo->Name();
358 ObjectLibrary::Default()->Register<MergeOperator>(
359 kMoName,
360 [](const std::string& /*name*/, std::unique_ptr<MergeOperator>* guard,
361 std::string* /* errmsg */) {
362 guard->reset(new BytesXOROperator());
363 return guard->get();
364 });
365
366 ASSERT_OK(GetColumnFamilyOptionsFromString(
367 base_cf_opt, "merge_operator=" + kMoName + ";", &new_cf_opt));
368 ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name()));
369
370 // Wrong key/value pair
371 ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
372 "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt));
373 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
374
375 // Error Paring value
376 ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
377 "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt));
378 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
379
380 // Missing option name
381 ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
382 "write_buffer_size=13; =100;", &new_cf_opt));
383 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
384
385 const uint64_t kilo = 1024UL;
386 const uint64_t mega = 1024 * kilo;
387 const uint64_t giga = 1024 * mega;
388 const uint64_t tera = 1024 * giga;
389
390 // Units (k)
391 ASSERT_OK(GetColumnFamilyOptionsFromString(
392 base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt));
393 ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo);
394 // Units (m)
395 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
396 "max_write_buffer_number=16m;inplace_update_num_locks=17M",
397 &new_cf_opt));
398 ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega);
399 ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17u * mega);
400 // Units (g)
401 ASSERT_OK(GetColumnFamilyOptionsFromString(
402 base_cf_opt,
403 "write_buffer_size=18g;prefix_extractor=capped:8;"
404 "arena_block_size=19G",
405 &new_cf_opt));
406
407 ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
408 ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
409 ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
410 std::string prefix_name(new_cf_opt.prefix_extractor->Name());
411 ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8");
412
413 // Units (t)
414 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
415 "write_buffer_size=20t;arena_block_size=21T", &new_cf_opt));
416 ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera);
417 ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera);
418
419 // Nested block based table options
420 // Empty
421 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
422 "write_buffer_size=10;max_write_buffer_number=16;"
423 "block_based_table_factory={};arena_block_size=1024",
424 &new_cf_opt));
425 ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
426 // Non-empty
427 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
428 "write_buffer_size=10;max_write_buffer_number=16;"
429 "block_based_table_factory={block_cache=1M;block_size=4;};"
430 "arena_block_size=1024",
431 &new_cf_opt));
432 ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
433 // Last one
434 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
435 "write_buffer_size=10;max_write_buffer_number=16;"
436 "block_based_table_factory={block_cache=1M;block_size=4;}",
437 &new_cf_opt));
438 ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
439 // Mismatch curly braces
440 ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
441 "write_buffer_size=10;max_write_buffer_number=16;"
442 "block_based_table_factory={{{block_size=4;};"
443 "arena_block_size=1024",
444 &new_cf_opt));
445 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
446
447 // Unexpected chars after closing curly brace
448 ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
449 "write_buffer_size=10;max_write_buffer_number=16;"
450 "block_based_table_factory={block_size=4;}};"
451 "arena_block_size=1024",
452 &new_cf_opt));
453 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
454
455 ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
456 "write_buffer_size=10;max_write_buffer_number=16;"
457 "block_based_table_factory={block_size=4;}xdfa;"
458 "arena_block_size=1024",
459 &new_cf_opt));
460 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
461
462 ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
463 "write_buffer_size=10;max_write_buffer_number=16;"
464 "block_based_table_factory={block_size=4;}xdfa",
465 &new_cf_opt));
466 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
467
468 // Invalid block based table option
469 ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
470 "write_buffer_size=10;max_write_buffer_number=16;"
471 "block_based_table_factory={xx_block_size=4;}",
472 &new_cf_opt));
473 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
474
475 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
476 "optimize_filters_for_hits=true",
477 &new_cf_opt));
478 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
479 "optimize_filters_for_hits=false",
480 &new_cf_opt));
481
482 ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
483 "optimize_filters_for_hits=junk",
484 &new_cf_opt));
485 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
486
487 // Nested plain table options
488 // Empty
489 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
490 "write_buffer_size=10;max_write_buffer_number=16;"
491 "plain_table_factory={};arena_block_size=1024",
492 &new_cf_opt));
493 ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
494 ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
495 // Non-empty
496 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
497 "write_buffer_size=10;max_write_buffer_number=16;"
498 "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};"
499 "arena_block_size=1024",
500 &new_cf_opt));
501 ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
502 ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
503
504 // memtable factory
505 ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
506 "write_buffer_size=10;max_write_buffer_number=16;"
507 "memtable=skip_list:10;arena_block_size=1024",
508 &new_cf_opt));
509 ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
510 ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory");
511 }
512 #endif // !ROCKSDB_LITE
513
514 #ifndef ROCKSDB_LITE // GetBlockBasedTableOptionsFromString is not supported
TEST_F(OptionsTest,GetBlockBasedTableOptionsFromString)515 TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
516 BlockBasedTableOptions table_opt;
517 BlockBasedTableOptions new_opt;
518 // make sure default values are overwritten by something else
519 ASSERT_OK(GetBlockBasedTableOptionsFromString(
520 table_opt,
521 "cache_index_and_filter_blocks=1;index_type=kHashSearch;"
522 "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
523 "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
524 "block_size_deviation=8;block_restart_interval=4;"
525 "format_version=5;whole_key_filtering=1;"
526 "filter_policy=bloomfilter:4.567:false;",
527 &new_opt));
528 ASSERT_TRUE(new_opt.cache_index_and_filter_blocks);
529 ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch);
530 ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash);
531 ASSERT_TRUE(new_opt.hash_index_allow_collision);
532 ASSERT_TRUE(new_opt.no_block_cache);
533 ASSERT_TRUE(new_opt.block_cache != nullptr);
534 ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
535 ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
536 ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL);
537 ASSERT_EQ(new_opt.block_size, 1024UL);
538 ASSERT_EQ(new_opt.block_size_deviation, 8);
539 ASSERT_EQ(new_opt.block_restart_interval, 4);
540 ASSERT_EQ(new_opt.format_version, 5U);
541 ASSERT_EQ(new_opt.whole_key_filtering, true);
542 ASSERT_TRUE(new_opt.filter_policy != nullptr);
543 const BloomFilterPolicy& bfp =
544 dynamic_cast<const BloomFilterPolicy&>(*new_opt.filter_policy);
545 EXPECT_EQ(bfp.GetMillibitsPerKey(), 4567);
546 EXPECT_EQ(bfp.GetWholeBitsPerKey(), 5);
547
548 // unknown option
549 ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
550 "cache_index_and_filter_blocks=1;index_type=kBinarySearch;"
551 "bad_option=1",
552 &new_opt));
553 ASSERT_EQ(static_cast<bool>(table_opt.cache_index_and_filter_blocks),
554 new_opt.cache_index_and_filter_blocks);
555 ASSERT_EQ(table_opt.index_type, new_opt.index_type);
556
557 // unrecognized index type
558 ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
559 "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX",
560 &new_opt));
561 ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
562 new_opt.cache_index_and_filter_blocks);
563 ASSERT_EQ(table_opt.index_type, new_opt.index_type);
564
565 // unrecognized checksum type
566 ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
567 "cache_index_and_filter_blocks=1;checksum=kxxHashXX",
568 &new_opt));
569 ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
570 new_opt.cache_index_and_filter_blocks);
571 ASSERT_EQ(table_opt.index_type, new_opt.index_type);
572
573 // unrecognized filter policy name
574 ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
575 "cache_index_and_filter_blocks=1;"
576 "filter_policy=bloomfilterxx:4:true",
577 &new_opt));
578 ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
579 new_opt.cache_index_and_filter_blocks);
580 ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
581
582 // unrecognized filter policy config
583 ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
584 "cache_index_and_filter_blocks=1;"
585 "filter_policy=bloomfilter:4",
586 &new_opt));
587 ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
588 new_opt.cache_index_and_filter_blocks);
589 ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
590
591 // Check block cache options are overwritten when specified
592 // in new format as a struct.
593 ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
594 "block_cache={capacity=1M;num_shard_bits=4;"
595 "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};"
596 "block_cache_compressed={capacity=1M;num_shard_bits=4;"
597 "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}",
598 &new_opt));
599 ASSERT_TRUE(new_opt.block_cache != nullptr);
600 ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
601 ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
602 new_opt.block_cache)->GetNumShardBits(), 4);
603 ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
604 ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
605 new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
606 ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
607 ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
608 ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
609 new_opt.block_cache_compressed)->GetNumShardBits(), 4);
610 ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
611 ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
612 new_opt.block_cache_compressed)->GetHighPriPoolRatio(),
613 0.5);
614
615 // Set only block cache capacity. Check other values are
616 // reset to default values.
617 ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
618 "block_cache={capacity=2M};"
619 "block_cache_compressed={capacity=2M}",
620 &new_opt));
621 ASSERT_TRUE(new_opt.block_cache != nullptr);
622 ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL);
623 // Default values
624 ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
625 new_opt.block_cache)->GetNumShardBits(),
626 GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity()));
627 ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
628 ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache)
629 ->GetHighPriPoolRatio(),
630 0.5);
631 ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
632 ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 2*1024UL*1024UL);
633 // Default values
634 ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
635 new_opt.block_cache_compressed)->GetNumShardBits(),
636 GetDefaultCacheShardBits(
637 new_opt.block_cache_compressed->GetCapacity()));
638 ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
639 ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
640 ->GetHighPriPoolRatio(),
641 0.5);
642
643 // Set couple of block cache options.
644 ASSERT_OK(GetBlockBasedTableOptionsFromString(
645 table_opt,
646 "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};"
647 "block_cache_compressed={num_shard_bits=5;"
648 "high_pri_pool_ratio=0.0;}",
649 &new_opt));
650 ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0);
651 ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
652 new_opt.block_cache)->GetNumShardBits(), 5);
653 ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
654 ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
655 new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
656 ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
657 ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 0);
658 ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
659 new_opt.block_cache_compressed)->GetNumShardBits(), 5);
660 ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
661 ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
662 ->GetHighPriPoolRatio(),
663 0.0);
664
665 // Set couple of block cache options.
666 ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
667 "block_cache={capacity=1M;num_shard_bits=4;"
668 "strict_capacity_limit=true;};"
669 "block_cache_compressed={capacity=1M;num_shard_bits=4;"
670 "strict_capacity_limit=true;}",
671 &new_opt));
672 ASSERT_TRUE(new_opt.block_cache != nullptr);
673 ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
674 ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
675 new_opt.block_cache)->GetNumShardBits(), 4);
676 ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
677 ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache)
678 ->GetHighPriPoolRatio(),
679 0.5);
680 ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
681 ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
682 ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
683 new_opt.block_cache_compressed)->GetNumShardBits(), 4);
684 ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
685 ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
686 ->GetHighPriPoolRatio(),
687 0.5);
688 }
689 #endif // !ROCKSDB_LITE
690
691
692 #ifndef ROCKSDB_LITE // GetPlainTableOptionsFromString is not supported
TEST_F(OptionsTest,GetPlainTableOptionsFromString)693 TEST_F(OptionsTest, GetPlainTableOptionsFromString) {
694 PlainTableOptions table_opt;
695 PlainTableOptions new_opt;
696 // make sure default values are overwritten by something else
697 ASSERT_OK(GetPlainTableOptionsFromString(table_opt,
698 "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
699 "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;"
700 "full_scan_mode=true;store_index_in_file=true",
701 &new_opt));
702 ASSERT_EQ(new_opt.user_key_len, 66u);
703 ASSERT_EQ(new_opt.bloom_bits_per_key, 20);
704 ASSERT_EQ(new_opt.hash_table_ratio, 0.5);
705 ASSERT_EQ(new_opt.index_sparseness, 8);
706 ASSERT_EQ(new_opt.huge_page_tlb_size, 4);
707 ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix);
708 ASSERT_TRUE(new_opt.full_scan_mode);
709 ASSERT_TRUE(new_opt.store_index_in_file);
710
711 // unknown option
712 ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
713 "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
714 "bad_option=1",
715 &new_opt));
716
717 // unrecognized EncodingType
718 ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
719 "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
720 "encoding_type=kPrefixXX",
721 &new_opt));
722 }
723 #endif // !ROCKSDB_LITE
724
725 #ifndef ROCKSDB_LITE // GetMemTableRepFactoryFromString is not supported
TEST_F(OptionsTest,GetMemTableRepFactoryFromString)726 TEST_F(OptionsTest, GetMemTableRepFactoryFromString) {
727 std::unique_ptr<MemTableRepFactory> new_mem_factory = nullptr;
728
729 ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory));
730 ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory));
731 ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory");
732 ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt",
733 &new_mem_factory));
734
735 ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory));
736 ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000",
737 &new_mem_factory));
738 ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory");
739 ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt",
740 &new_mem_factory));
741
742 ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist",
743 &new_mem_factory));
744 ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist:1000",
745 &new_mem_factory));
746 ASSERT_EQ(std::string(new_mem_factory->Name()), "HashLinkListRepFactory");
747 ASSERT_NOK(GetMemTableRepFactoryFromString("hash_linkedlist:1000:invalid_opt",
748 &new_mem_factory));
749
750 ASSERT_OK(GetMemTableRepFactoryFromString("vector", &new_mem_factory));
751 ASSERT_OK(GetMemTableRepFactoryFromString("vector:1024", &new_mem_factory));
752 ASSERT_EQ(std::string(new_mem_factory->Name()), "VectorRepFactory");
753 ASSERT_NOK(GetMemTableRepFactoryFromString("vector:1024:invalid_opt",
754 &new_mem_factory));
755
756 ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo", &new_mem_factory));
757 // CuckooHash memtable is already removed.
758 ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo:1024", &new_mem_factory));
759
760 ASSERT_NOK(GetMemTableRepFactoryFromString("bad_factory", &new_mem_factory));
761 }
762 #endif // !ROCKSDB_LITE
763
764 #ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite
TEST_F(OptionsTest,GetOptionsFromStringTest)765 TEST_F(OptionsTest, GetOptionsFromStringTest) {
766 Options base_options, new_options;
767 base_options.write_buffer_size = 20;
768 base_options.min_write_buffer_number_to_merge = 15;
769 BlockBasedTableOptions block_based_table_options;
770 block_based_table_options.cache_index_and_filter_blocks = true;
771 base_options.table_factory.reset(
772 NewBlockBasedTableFactory(block_based_table_options));
773
774 // Register an Env with object registry.
775 const static char* kCustomEnvName = "CustomEnv";
776 class CustomEnv : public EnvWrapper {
777 public:
778 explicit CustomEnv(Env* _target) : EnvWrapper(_target) {}
779 };
780
781 ObjectLibrary::Default()->Register<Env>(
782 kCustomEnvName,
783 [](const std::string& /*name*/, std::unique_ptr<Env>* /*env_guard*/,
784 std::string* /* errmsg */) {
785 static CustomEnv env(Env::Default());
786 return &env;
787 });
788
789 ASSERT_OK(GetOptionsFromString(
790 base_options,
791 "write_buffer_size=10;max_write_buffer_number=16;"
792 "block_based_table_factory={block_cache=1M;block_size=4;};"
793 "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;"
794 "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files="
795 "1;"
796 "rate_limiter_bytes_per_sec=1024;env=CustomEnv",
797 &new_options));
798
799 ASSERT_EQ(new_options.compression_opts.window_bits, 4);
800 ASSERT_EQ(new_options.compression_opts.level, 5);
801 ASSERT_EQ(new_options.compression_opts.strategy, 6);
802 ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0u);
803 ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0u);
804 ASSERT_EQ(new_options.compression_opts.enabled, false);
805 ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption);
806 ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5);
807 ASSERT_EQ(new_options.bottommost_compression_opts.level, 6);
808 ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7);
809 ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0u);
810 ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0u);
811 ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false);
812 ASSERT_EQ(new_options.write_buffer_size, 10U);
813 ASSERT_EQ(new_options.max_write_buffer_number, 16);
814 BlockBasedTableOptions new_block_based_table_options =
815 dynamic_cast<BlockBasedTableFactory*>(new_options.table_factory.get())
816 ->table_options();
817 ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20);
818 ASSERT_EQ(new_block_based_table_options.block_size, 4U);
819 // don't overwrite block based table options
820 ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks);
821
822 ASSERT_EQ(new_options.create_if_missing, true);
823 ASSERT_EQ(new_options.max_open_files, 1);
824 ASSERT_TRUE(new_options.rate_limiter.get() != nullptr);
825 Env* newEnv = new_options.env;
826 ASSERT_OK(Env::LoadEnv(kCustomEnvName, &newEnv));
827 ASSERT_EQ(newEnv, new_options.env);
828 }
829
TEST_F(OptionsTest,DBOptionsSerialization)830 TEST_F(OptionsTest, DBOptionsSerialization) {
831 Options base_options, new_options;
832 Random rnd(301);
833
834 // Phase 1: Make big change in base_options
835 test::RandomInitDBOptions(&base_options, &rnd);
836
837 // Phase 2: obtain a string from base_option
838 std::string base_options_file_content;
839 ASSERT_OK(GetStringFromDBOptions(&base_options_file_content, base_options));
840
841 // Phase 3: Set new_options from the derived string and expect
842 // new_options == base_options
843 ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_options_file_content,
844 &new_options));
845 ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_options, new_options));
846 }
847
TEST_F(OptionsTest,OptionsComposeDecompose)848 TEST_F(OptionsTest, OptionsComposeDecompose) {
849 // build an Options from DBOptions + CFOptions, then decompose it to verify
850 // we get same constituent options.
851 DBOptions base_db_opts;
852 ColumnFamilyOptions base_cf_opts;
853
854 Random rnd(301);
855 test::RandomInitDBOptions(&base_db_opts, &rnd);
856 test::RandomInitCFOptions(&base_cf_opts, base_db_opts, &rnd);
857
858 Options base_opts(base_db_opts, base_cf_opts);
859 DBOptions new_db_opts(base_opts);
860 ColumnFamilyOptions new_cf_opts(base_opts);
861
862 ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opts, new_db_opts));
863 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opts, new_cf_opts));
864 delete new_cf_opts.compaction_filter;
865 }
866
TEST_F(OptionsTest,ColumnFamilyOptionsSerialization)867 TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
868 Options options;
869 ColumnFamilyOptions base_opt, new_opt;
870 Random rnd(302);
871 // Phase 1: randomly assign base_opt
872 // custom type options
873 test::RandomInitCFOptions(&base_opt, options, &rnd);
874
875 // Phase 2: obtain a string from base_opt
876 std::string base_options_file_content;
877 ASSERT_OK(
878 GetStringFromColumnFamilyOptions(&base_options_file_content, base_opt));
879
880 // Phase 3: Set new_opt from the derived string and expect
881 // new_opt == base_opt
882 ASSERT_OK(GetColumnFamilyOptionsFromString(
883 ColumnFamilyOptions(), base_options_file_content, &new_opt));
884 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_opt, new_opt));
885 if (base_opt.compaction_filter) {
886 delete base_opt.compaction_filter;
887 }
888 }
889
890 #endif // !ROCKSDB_LITE
891
892 Status StringToMap(
893 const std::string& opts_str,
894 std::unordered_map<std::string, std::string>* opts_map);
895
896 #ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE
TEST_F(OptionsTest,StringToMapTest)897 TEST_F(OptionsTest, StringToMapTest) {
898 std::unordered_map<std::string, std::string> opts_map;
899 // Regular options
900 ASSERT_OK(StringToMap("k1=v1;k2=v2;k3=v3", &opts_map));
901 ASSERT_EQ(opts_map["k1"], "v1");
902 ASSERT_EQ(opts_map["k2"], "v2");
903 ASSERT_EQ(opts_map["k3"], "v3");
904 // Value with '='
905 opts_map.clear();
906 ASSERT_OK(StringToMap("k1==v1;k2=v2=;", &opts_map));
907 ASSERT_EQ(opts_map["k1"], "=v1");
908 ASSERT_EQ(opts_map["k2"], "v2=");
909 // Overwrriten option
910 opts_map.clear();
911 ASSERT_OK(StringToMap("k1=v1;k1=v2;k3=v3", &opts_map));
912 ASSERT_EQ(opts_map["k1"], "v2");
913 ASSERT_EQ(opts_map["k3"], "v3");
914 // Empty value
915 opts_map.clear();
916 ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=", &opts_map));
917 ASSERT_EQ(opts_map["k1"], "v1");
918 ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
919 ASSERT_EQ(opts_map["k2"], "");
920 ASSERT_EQ(opts_map["k3"], "v3");
921 ASSERT_TRUE(opts_map.find("k4") != opts_map.end());
922 ASSERT_EQ(opts_map["k4"], "");
923 opts_map.clear();
924 ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4= ", &opts_map));
925 ASSERT_EQ(opts_map["k1"], "v1");
926 ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
927 ASSERT_EQ(opts_map["k2"], "");
928 ASSERT_EQ(opts_map["k3"], "v3");
929 ASSERT_TRUE(opts_map.find("k4") != opts_map.end());
930 ASSERT_EQ(opts_map["k4"], "");
931 opts_map.clear();
932 ASSERT_OK(StringToMap("k1=v1;k2=;k3=", &opts_map));
933 ASSERT_EQ(opts_map["k1"], "v1");
934 ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
935 ASSERT_EQ(opts_map["k2"], "");
936 ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
937 ASSERT_EQ(opts_map["k3"], "");
938 opts_map.clear();
939 ASSERT_OK(StringToMap("k1=v1;k2=;k3=;", &opts_map));
940 ASSERT_EQ(opts_map["k1"], "v1");
941 ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
942 ASSERT_EQ(opts_map["k2"], "");
943 ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
944 ASSERT_EQ(opts_map["k3"], "");
945 // Regular nested options
946 opts_map.clear();
947 ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map));
948 ASSERT_EQ(opts_map["k1"], "v1");
949 ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2");
950 ASSERT_EQ(opts_map["k3"], "v3");
951 // Multi-level nested options
952 opts_map.clear();
953 ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2={nnk1=nnk2}};"
954 "k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4",
955 &opts_map));
956 ASSERT_EQ(opts_map["k1"], "v1");
957 ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}");
958 ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}");
959 ASSERT_EQ(opts_map["k4"], "v4");
960 // Garbage inside curly braces
961 opts_map.clear();
962 ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4",
963 &opts_map));
964 ASSERT_EQ(opts_map["k1"], "v1");
965 ASSERT_EQ(opts_map["k2"], "dfad=");
966 ASSERT_EQ(opts_map["k3"], "=");
967 ASSERT_EQ(opts_map["k4"], "v4");
968 // Empty nested options
969 opts_map.clear();
970 ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map));
971 ASSERT_EQ(opts_map["k1"], "v1");
972 ASSERT_EQ(opts_map["k2"], "");
973 opts_map.clear();
974 ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map));
975 ASSERT_EQ(opts_map["k1"], "v1");
976 ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}");
977 // With random spaces
978 opts_map.clear();
979 ASSERT_OK(StringToMap(" k1 = v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}} ; "
980 "k3={ { } }; k4= v4 ",
981 &opts_map));
982 ASSERT_EQ(opts_map["k1"], "v1");
983 ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}");
984 ASSERT_EQ(opts_map["k3"], "{ }");
985 ASSERT_EQ(opts_map["k4"], "v4");
986
987 // Empty key
988 ASSERT_NOK(StringToMap("k1=v1;k2=v2;=", &opts_map));
989 ASSERT_NOK(StringToMap("=v1;k2=v2", &opts_map));
990 ASSERT_NOK(StringToMap("k1=v1;k2v2;", &opts_map));
991 ASSERT_NOK(StringToMap("k1=v1;k2=v2;fadfa", &opts_map));
992 ASSERT_NOK(StringToMap("k1=v1;k2=v2;;", &opts_map));
993 // Mismatch curly braces
994 ASSERT_NOK(StringToMap("k1=v1;k2={;k3=v3", &opts_map));
995 ASSERT_NOK(StringToMap("k1=v1;k2={{};k3=v3", &opts_map));
996 ASSERT_NOK(StringToMap("k1=v1;k2={}};k3=v3", &opts_map));
997 ASSERT_NOK(StringToMap("k1=v1;k2={{}{}}};k3=v3", &opts_map));
998 // However this is valid!
999 opts_map.clear();
1000 ASSERT_OK(StringToMap("k1=v1;k2=};k3=v3", &opts_map));
1001 ASSERT_EQ(opts_map["k1"], "v1");
1002 ASSERT_EQ(opts_map["k2"], "}");
1003 ASSERT_EQ(opts_map["k3"], "v3");
1004
1005 // Invalid chars after closing curly brace
1006 ASSERT_NOK(StringToMap("k1=v1;k2={{}}{};k3=v3", &opts_map));
1007 ASSERT_NOK(StringToMap("k1=v1;k2={{}}cfda;k3=v3", &opts_map));
1008 ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda;k3=v3", &opts_map));
1009 ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda", &opts_map));
1010 ASSERT_NOK(StringToMap("k1=v1;k2={{}}{}", &opts_map));
1011 ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map));
1012 }
1013 #endif // ROCKSDB_LITE
1014
1015 #ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE
TEST_F(OptionsTest,StringToMapRandomTest)1016 TEST_F(OptionsTest, StringToMapRandomTest) {
1017 std::unordered_map<std::string, std::string> opts_map;
1018 // Make sure segfault is not hit by semi-random strings
1019
1020 std::vector<std::string> bases = {
1021 "a={aa={};tt={xxx={}}};c=defff",
1022 "a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}",
1023 "abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"};
1024
1025 for (std::string base : bases) {
1026 for (int rand_seed = 301; rand_seed < 401; rand_seed++) {
1027 Random rnd(rand_seed);
1028 for (int attempt = 0; attempt < 10; attempt++) {
1029 std::string str = base;
1030 // Replace random position to space
1031 size_t pos = static_cast<size_t>(
1032 rnd.Uniform(static_cast<int>(base.size())));
1033 str[pos] = ' ';
1034 Status s = StringToMap(str, &opts_map);
1035 ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
1036 opts_map.clear();
1037 }
1038 }
1039 }
1040
1041 // Random Construct a string
1042 std::vector<char> chars = {'{', '}', ' ', '=', ';', 'c'};
1043 for (int rand_seed = 301; rand_seed < 1301; rand_seed++) {
1044 Random rnd(rand_seed);
1045 int len = rnd.Uniform(30);
1046 std::string str = "";
1047 for (int attempt = 0; attempt < len; attempt++) {
1048 // Add a random character
1049 size_t pos = static_cast<size_t>(
1050 rnd.Uniform(static_cast<int>(chars.size())));
1051 str.append(1, chars[pos]);
1052 }
1053 Status s = StringToMap(str, &opts_map);
1054 ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
1055 s = StringToMap("name=" + str, &opts_map);
1056 ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
1057 opts_map.clear();
1058 }
1059 }
1060
TEST_F(OptionsTest,GetStringFromCompressionType)1061 TEST_F(OptionsTest, GetStringFromCompressionType) {
1062 std::string res;
1063
1064 ASSERT_OK(GetStringFromCompressionType(&res, kNoCompression));
1065 ASSERT_EQ(res, "kNoCompression");
1066
1067 ASSERT_OK(GetStringFromCompressionType(&res, kSnappyCompression));
1068 ASSERT_EQ(res, "kSnappyCompression");
1069
1070 ASSERT_OK(GetStringFromCompressionType(&res, kDisableCompressionOption));
1071 ASSERT_EQ(res, "kDisableCompressionOption");
1072
1073 ASSERT_OK(GetStringFromCompressionType(&res, kLZ4Compression));
1074 ASSERT_EQ(res, "kLZ4Compression");
1075
1076 ASSERT_OK(GetStringFromCompressionType(&res, kZlibCompression));
1077 ASSERT_EQ(res, "kZlibCompression");
1078
1079 ASSERT_NOK(
1080 GetStringFromCompressionType(&res, static_cast<CompressionType>(-10)));
1081 }
1082 #endif // !ROCKSDB_LITE
1083
TEST_F(OptionsTest,ConvertOptionsTest)1084 TEST_F(OptionsTest, ConvertOptionsTest) {
1085 LevelDBOptions leveldb_opt;
1086 Options converted_opt = ConvertOptions(leveldb_opt);
1087
1088 ASSERT_EQ(converted_opt.create_if_missing, leveldb_opt.create_if_missing);
1089 ASSERT_EQ(converted_opt.error_if_exists, leveldb_opt.error_if_exists);
1090 ASSERT_EQ(converted_opt.paranoid_checks, leveldb_opt.paranoid_checks);
1091 ASSERT_EQ(converted_opt.env, leveldb_opt.env);
1092 ASSERT_EQ(converted_opt.info_log.get(), leveldb_opt.info_log);
1093 ASSERT_EQ(converted_opt.write_buffer_size, leveldb_opt.write_buffer_size);
1094 ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files);
1095 ASSERT_EQ(converted_opt.compression, leveldb_opt.compression);
1096
1097 std::shared_ptr<TableFactory> tb_guard = converted_opt.table_factory;
1098 BlockBasedTableFactory* table_factory =
1099 dynamic_cast<BlockBasedTableFactory*>(converted_opt.table_factory.get());
1100
1101 ASSERT_TRUE(table_factory != nullptr);
1102
1103 const BlockBasedTableOptions table_opt = table_factory->table_options();
1104
1105 ASSERT_EQ(table_opt.block_cache->GetCapacity(), 8UL << 20);
1106 ASSERT_EQ(table_opt.block_size, leveldb_opt.block_size);
1107 ASSERT_EQ(table_opt.block_restart_interval,
1108 leveldb_opt.block_restart_interval);
1109 ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy);
1110 }
1111
1112 #ifndef ROCKSDB_LITE
1113 class OptionsParserTest : public testing::Test {
1114 public:
OptionsParserTest()1115 OptionsParserTest() {
1116 env_.reset(new test::StringEnv(Env::Default()));
1117 fs_.reset(new LegacyFileSystemWrapper(env_.get()));
1118 }
1119
1120 protected:
1121 std::unique_ptr<test::StringEnv> env_;
1122 std::unique_ptr<LegacyFileSystemWrapper> fs_;
1123 };
1124
TEST_F(OptionsParserTest,Comment)1125 TEST_F(OptionsParserTest, Comment) {
1126 DBOptions db_opt;
1127 db_opt.max_open_files = 12345;
1128 db_opt.max_background_flushes = 301;
1129 db_opt.max_total_wal_size = 1024;
1130 ColumnFamilyOptions cf_opt;
1131
1132 std::string options_file_content =
1133 "# This is a testing option string.\n"
1134 "# Currently we only support \"#\" styled comment.\n"
1135 "\n"
1136 "[Version]\n"
1137 " rocksdb_version=3.14.0\n"
1138 " options_file_version=1\n"
1139 "[ DBOptions ]\n"
1140 " # note that we don't support space around \"=\"\n"
1141 " max_open_files=12345;\n"
1142 " max_background_flushes=301 # comment after a statement is fine\n"
1143 " # max_background_flushes=1000 # this line would be ignored\n"
1144 " # max_background_compactions=2000 # so does this one\n"
1145 " max_total_wal_size=1024 # keep_log_file_num=1000\n"
1146 "[CFOptions \"default\"] # column family must be specified\n"
1147 " # in the correct order\n"
1148 " # if a section is blank, we will use the default\n";
1149
1150 const std::string kTestFileName = "test-rocksdb-options.ini";
1151 env_->WriteToNewFile(kTestFileName, options_file_content);
1152 RocksDBOptionsParser parser;
1153 ASSERT_OK(
1154 parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1155
1156 ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), db_opt));
1157 ASSERT_EQ(parser.NumColumnFamilies(), 1U);
1158 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
1159 *parser.GetCFOptions("default"), cf_opt));
1160 }
1161
TEST_F(OptionsParserTest,ExtraSpace)1162 TEST_F(OptionsParserTest, ExtraSpace) {
1163 std::string options_file_content =
1164 "# This is a testing option string.\n"
1165 "# Currently we only support \"#\" styled comment.\n"
1166 "\n"
1167 "[ Version ]\n"
1168 " rocksdb_version = 3.14.0 \n"
1169 " options_file_version=1 # some comment\n"
1170 "[DBOptions ] # some comment\n"
1171 "max_open_files=12345 \n"
1172 " max_background_flushes = 301 \n"
1173 " max_total_wal_size = 1024 # keep_log_file_num=1000\n"
1174 " [CFOptions \"default\" ]\n"
1175 " # if a section is blank, we will use the default\n";
1176
1177 const std::string kTestFileName = "test-rocksdb-options.ini";
1178 env_->WriteToNewFile(kTestFileName, options_file_content);
1179 RocksDBOptionsParser parser;
1180 ASSERT_OK(
1181 parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1182 }
1183
TEST_F(OptionsParserTest,MissingDBOptions)1184 TEST_F(OptionsParserTest, MissingDBOptions) {
1185 std::string options_file_content =
1186 "# This is a testing option string.\n"
1187 "# Currently we only support \"#\" styled comment.\n"
1188 "\n"
1189 "[Version]\n"
1190 " rocksdb_version=3.14.0\n"
1191 " options_file_version=1\n"
1192 "[CFOptions \"default\"]\n"
1193 " # if a section is blank, we will use the default\n";
1194
1195 const std::string kTestFileName = "test-rocksdb-options.ini";
1196 env_->WriteToNewFile(kTestFileName, options_file_content);
1197 RocksDBOptionsParser parser;
1198 ASSERT_NOK(
1199 parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1200 }
1201
TEST_F(OptionsParserTest,DoubleDBOptions)1202 TEST_F(OptionsParserTest, DoubleDBOptions) {
1203 DBOptions db_opt;
1204 db_opt.max_open_files = 12345;
1205 db_opt.max_background_flushes = 301;
1206 db_opt.max_total_wal_size = 1024;
1207 ColumnFamilyOptions cf_opt;
1208
1209 std::string options_file_content =
1210 "# This is a testing option string.\n"
1211 "# Currently we only support \"#\" styled comment.\n"
1212 "\n"
1213 "[Version]\n"
1214 " rocksdb_version=3.14.0\n"
1215 " options_file_version=1\n"
1216 "[DBOptions]\n"
1217 " max_open_files=12345\n"
1218 " max_background_flushes=301\n"
1219 " max_total_wal_size=1024 # keep_log_file_num=1000\n"
1220 "[DBOptions]\n"
1221 "[CFOptions \"default\"]\n"
1222 " # if a section is blank, we will use the default\n";
1223
1224 const std::string kTestFileName = "test-rocksdb-options.ini";
1225 env_->WriteToNewFile(kTestFileName, options_file_content);
1226 RocksDBOptionsParser parser;
1227 ASSERT_NOK(
1228 parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1229 }
1230
TEST_F(OptionsParserTest,NoDefaultCFOptions)1231 TEST_F(OptionsParserTest, NoDefaultCFOptions) {
1232 DBOptions db_opt;
1233 db_opt.max_open_files = 12345;
1234 db_opt.max_background_flushes = 301;
1235 db_opt.max_total_wal_size = 1024;
1236 ColumnFamilyOptions cf_opt;
1237
1238 std::string options_file_content =
1239 "# This is a testing option string.\n"
1240 "# Currently we only support \"#\" styled comment.\n"
1241 "\n"
1242 "[Version]\n"
1243 " rocksdb_version=3.14.0\n"
1244 " options_file_version=1\n"
1245 "[DBOptions]\n"
1246 " max_open_files=12345\n"
1247 " max_background_flushes=301\n"
1248 " max_total_wal_size=1024 # keep_log_file_num=1000\n"
1249 "[CFOptions \"something_else\"]\n"
1250 " # if a section is blank, we will use the default\n";
1251
1252 const std::string kTestFileName = "test-rocksdb-options.ini";
1253 env_->WriteToNewFile(kTestFileName, options_file_content);
1254 RocksDBOptionsParser parser;
1255 ASSERT_NOK(
1256 parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1257 }
1258
TEST_F(OptionsParserTest,DefaultCFOptionsMustBeTheFirst)1259 TEST_F(OptionsParserTest, DefaultCFOptionsMustBeTheFirst) {
1260 DBOptions db_opt;
1261 db_opt.max_open_files = 12345;
1262 db_opt.max_background_flushes = 301;
1263 db_opt.max_total_wal_size = 1024;
1264 ColumnFamilyOptions cf_opt;
1265
1266 std::string options_file_content =
1267 "# This is a testing option string.\n"
1268 "# Currently we only support \"#\" styled comment.\n"
1269 "\n"
1270 "[Version]\n"
1271 " rocksdb_version=3.14.0\n"
1272 " options_file_version=1\n"
1273 "[DBOptions]\n"
1274 " max_open_files=12345\n"
1275 " max_background_flushes=301\n"
1276 " max_total_wal_size=1024 # keep_log_file_num=1000\n"
1277 "[CFOptions \"something_else\"]\n"
1278 " # if a section is blank, we will use the default\n"
1279 "[CFOptions \"default\"]\n"
1280 " # if a section is blank, we will use the default\n";
1281
1282 const std::string kTestFileName = "test-rocksdb-options.ini";
1283 env_->WriteToNewFile(kTestFileName, options_file_content);
1284 RocksDBOptionsParser parser;
1285 ASSERT_NOK(
1286 parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1287 }
1288
TEST_F(OptionsParserTest,DuplicateCFOptions)1289 TEST_F(OptionsParserTest, DuplicateCFOptions) {
1290 DBOptions db_opt;
1291 db_opt.max_open_files = 12345;
1292 db_opt.max_background_flushes = 301;
1293 db_opt.max_total_wal_size = 1024;
1294 ColumnFamilyOptions cf_opt;
1295
1296 std::string options_file_content =
1297 "# This is a testing option string.\n"
1298 "# Currently we only support \"#\" styled comment.\n"
1299 "\n"
1300 "[Version]\n"
1301 " rocksdb_version=3.14.0\n"
1302 " options_file_version=1\n"
1303 "[DBOptions]\n"
1304 " max_open_files=12345\n"
1305 " max_background_flushes=301\n"
1306 " max_total_wal_size=1024 # keep_log_file_num=1000\n"
1307 "[CFOptions \"default\"]\n"
1308 "[CFOptions \"something_else\"]\n"
1309 "[CFOptions \"something_else\"]\n";
1310
1311 const std::string kTestFileName = "test-rocksdb-options.ini";
1312 env_->WriteToNewFile(kTestFileName, options_file_content);
1313 RocksDBOptionsParser parser;
1314 ASSERT_NOK(
1315 parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1316 }
1317
TEST_F(OptionsParserTest,IgnoreUnknownOptions)1318 TEST_F(OptionsParserTest, IgnoreUnknownOptions) {
1319 for (int case_id = 0; case_id < 5; case_id++) {
1320 DBOptions db_opt;
1321 db_opt.max_open_files = 12345;
1322 db_opt.max_background_flushes = 301;
1323 db_opt.max_total_wal_size = 1024;
1324 ColumnFamilyOptions cf_opt;
1325
1326 std::string version_string;
1327 bool should_ignore = true;
1328 if (case_id == 0) {
1329 // same version
1330 should_ignore = false;
1331 version_string =
1332 ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR) + ".0";
1333 } else if (case_id == 1) {
1334 // higher minor version
1335 should_ignore = true;
1336 version_string =
1337 ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR + 1) + ".0";
1338 } else if (case_id == 2) {
1339 // higher major version.
1340 should_ignore = true;
1341 version_string = ToString(ROCKSDB_MAJOR + 1) + ".0.0";
1342 } else if (case_id == 3) {
1343 // lower minor version
1344 #if ROCKSDB_MINOR == 0
1345 continue;
1346 #else
1347 version_string =
1348 ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR - 1) + ".0";
1349 should_ignore = false;
1350 #endif
1351 } else {
1352 // lower major version
1353 should_ignore = false;
1354 version_string =
1355 ToString(ROCKSDB_MAJOR - 1) + "." + ToString(ROCKSDB_MINOR) + ".0";
1356 }
1357
1358 std::string options_file_content =
1359 "# This is a testing option string.\n"
1360 "# Currently we only support \"#\" styled comment.\n"
1361 "\n"
1362 "[Version]\n"
1363 " rocksdb_version=" +
1364 version_string +
1365 "\n"
1366 " options_file_version=1\n"
1367 "[DBOptions]\n"
1368 " max_open_files=12345\n"
1369 " max_background_flushes=301\n"
1370 " max_total_wal_size=1024 # keep_log_file_num=1000\n"
1371 " unknown_db_option1=321\n"
1372 " unknown_db_option2=false\n"
1373 "[CFOptions \"default\"]\n"
1374 " unknown_cf_option1=hello\n"
1375 "[CFOptions \"something_else\"]\n"
1376 " unknown_cf_option2=world\n"
1377 " # if a section is blank, we will use the default\n";
1378
1379 const std::string kTestFileName = "test-rocksdb-options.ini";
1380 env_->DeleteFile(kTestFileName);
1381 env_->WriteToNewFile(kTestFileName, options_file_content);
1382 RocksDBOptionsParser parser;
1383 ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(), false,
1384 4096 /* readahead_size */));
1385 if (should_ignore) {
1386 ASSERT_OK(parser.Parse(kTestFileName, fs_.get(),
1387 true /* ignore_unknown_options */,
1388 4096 /* readahead_size */));
1389 } else {
1390 ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(),
1391 true /* ignore_unknown_options */,
1392 4096 /* readahead_size */));
1393 }
1394 }
1395 }
1396
TEST_F(OptionsParserTest,ParseVersion)1397 TEST_F(OptionsParserTest, ParseVersion) {
1398 DBOptions db_opt;
1399 db_opt.max_open_files = 12345;
1400 db_opt.max_background_flushes = 301;
1401 db_opt.max_total_wal_size = 1024;
1402 ColumnFamilyOptions cf_opt;
1403
1404 std::string file_template =
1405 "# This is a testing option string.\n"
1406 "# Currently we only support \"#\" styled comment.\n"
1407 "\n"
1408 "[Version]\n"
1409 " rocksdb_version=3.13.1\n"
1410 " options_file_version=%s\n"
1411 "[DBOptions]\n"
1412 "[CFOptions \"default\"]\n";
1413 const int kLength = 1000;
1414 char buffer[kLength];
1415 RocksDBOptionsParser parser;
1416
1417 const std::vector<std::string> invalid_versions = {
1418 "a.b.c", "3.2.2b", "3.-12", "3. 1", // only digits and dots are allowed
1419 "1.2.3.4",
1420 "1.2.3" // can only contains at most one dot.
1421 "0", // options_file_version must be at least one
1422 "3..2",
1423 ".", ".1.2", // must have at least one digit before each dot
1424 "1.2.", "1.", "2.34."}; // must have at least one digit after each dot
1425 for (auto iv : invalid_versions) {
1426 snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str());
1427
1428 parser.Reset();
1429 env_->WriteToNewFile(iv, buffer);
1430 ASSERT_NOK(parser.Parse(iv, fs_.get(), false, 0 /* readahead_size */));
1431 }
1432
1433 const std::vector<std::string> valid_versions = {
1434 "1.232", "100", "3.12", "1", "12.3 ", " 1.25 "};
1435 for (auto vv : valid_versions) {
1436 snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str());
1437 parser.Reset();
1438 env_->WriteToNewFile(vv, buffer);
1439 ASSERT_OK(parser.Parse(vv, fs_.get(), false, 0 /* readahead_size */));
1440 }
1441 }
1442
VerifyCFPointerTypedOptions(ColumnFamilyOptions * base_cf_opt,const ColumnFamilyOptions * new_cf_opt,const std::unordered_map<std::string,std::string> * new_cf_opt_map)1443 void VerifyCFPointerTypedOptions(
1444 ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt,
1445 const std::unordered_map<std::string, std::string>* new_cf_opt_map) {
1446 std::string name_buffer;
1447 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1448 new_cf_opt_map));
1449
1450 // change the name of merge operator back-and-forth
1451 {
1452 auto* merge_operator = dynamic_cast<test::ChanglingMergeOperator*>(
1453 base_cf_opt->merge_operator.get());
1454 if (merge_operator != nullptr) {
1455 name_buffer = merge_operator->Name();
1456 // change the name and expect non-ok status
1457 merge_operator->SetName("some-other-name");
1458 ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
1459 *base_cf_opt, *new_cf_opt, new_cf_opt_map));
1460 // change the name back and expect ok status
1461 merge_operator->SetName(name_buffer);
1462 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1463 new_cf_opt_map));
1464 }
1465 }
1466
1467 // change the name of the compaction filter factory back-and-forth
1468 {
1469 auto* compaction_filter_factory =
1470 dynamic_cast<test::ChanglingCompactionFilterFactory*>(
1471 base_cf_opt->compaction_filter_factory.get());
1472 if (compaction_filter_factory != nullptr) {
1473 name_buffer = compaction_filter_factory->Name();
1474 // change the name and expect non-ok status
1475 compaction_filter_factory->SetName("some-other-name");
1476 ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
1477 *base_cf_opt, *new_cf_opt, new_cf_opt_map));
1478 // change the name back and expect ok status
1479 compaction_filter_factory->SetName(name_buffer);
1480 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1481 new_cf_opt_map));
1482 }
1483 }
1484
1485 // test by setting compaction_filter to nullptr
1486 {
1487 auto* tmp_compaction_filter = base_cf_opt->compaction_filter;
1488 if (tmp_compaction_filter != nullptr) {
1489 base_cf_opt->compaction_filter = nullptr;
1490 // set compaction_filter to nullptr and expect non-ok status
1491 ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
1492 *base_cf_opt, *new_cf_opt, new_cf_opt_map));
1493 // set the value back and expect ok status
1494 base_cf_opt->compaction_filter = tmp_compaction_filter;
1495 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1496 new_cf_opt_map));
1497 }
1498 }
1499
1500 // test by setting table_factory to nullptr
1501 {
1502 auto tmp_table_factory = base_cf_opt->table_factory;
1503 if (tmp_table_factory != nullptr) {
1504 base_cf_opt->table_factory.reset();
1505 // set table_factory to nullptr and expect non-ok status
1506 ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
1507 *base_cf_opt, *new_cf_opt, new_cf_opt_map));
1508 // set the value back and expect ok status
1509 base_cf_opt->table_factory = tmp_table_factory;
1510 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1511 new_cf_opt_map));
1512 }
1513 }
1514
1515 // test by setting memtable_factory to nullptr
1516 {
1517 auto tmp_memtable_factory = base_cf_opt->memtable_factory;
1518 if (tmp_memtable_factory != nullptr) {
1519 base_cf_opt->memtable_factory.reset();
1520 // set memtable_factory to nullptr and expect non-ok status
1521 ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
1522 *base_cf_opt, *new_cf_opt, new_cf_opt_map));
1523 // set the value back and expect ok status
1524 base_cf_opt->memtable_factory = tmp_memtable_factory;
1525 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1526 new_cf_opt_map));
1527 }
1528 }
1529 }
1530
TEST_F(OptionsParserTest,Readahead)1531 TEST_F(OptionsParserTest, Readahead) {
1532 DBOptions base_db_opt;
1533 std::vector<ColumnFamilyOptions> base_cf_opts;
1534 base_cf_opts.emplace_back();
1535 base_cf_opts.emplace_back();
1536
1537 std::string one_mb_string = std::string(1024 * 1024, 'x');
1538 std::vector<std::string> cf_names = {"default", one_mb_string};
1539 const std::string kOptionsFileName = "test-persisted-options.ini";
1540
1541 ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts,
1542 kOptionsFileName, fs_.get()));
1543
1544 uint64_t file_size = 0;
1545 ASSERT_OK(env_->GetFileSize(kOptionsFileName, &file_size));
1546 assert(file_size > 0);
1547
1548 RocksDBOptionsParser parser;
1549
1550 env_->num_seq_file_read_ = 0;
1551 size_t readahead_size = 128 * 1024;
1552
1553 ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size));
1554 ASSERT_EQ(env_->num_seq_file_read_.load(),
1555 (file_size - 1) / readahead_size + 1);
1556
1557 env_->num_seq_file_read_.store(0);
1558 readahead_size = 1024 * 1024;
1559 ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size));
1560 ASSERT_EQ(env_->num_seq_file_read_.load(),
1561 (file_size - 1) / readahead_size + 1);
1562
1563 // Tiny readahead. 8 KB is read each time.
1564 env_->num_seq_file_read_.store(0);
1565 ASSERT_OK(
1566 parser.Parse(kOptionsFileName, fs_.get(), false, 1 /* readahead_size */));
1567 ASSERT_GE(env_->num_seq_file_read_.load(), file_size / (8 * 1024));
1568 ASSERT_LT(env_->num_seq_file_read_.load(), file_size / (8 * 1024) * 2);
1569
1570 // Disable readahead means 512KB readahead.
1571 env_->num_seq_file_read_.store(0);
1572 ASSERT_OK(
1573 parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
1574 ASSERT_GE(env_->num_seq_file_read_.load(),
1575 (file_size - 1) / (512 * 1024) + 1);
1576 }
1577
TEST_F(OptionsParserTest,DumpAndParse)1578 TEST_F(OptionsParserTest, DumpAndParse) {
1579 DBOptions base_db_opt;
1580 std::vector<ColumnFamilyOptions> base_cf_opts;
1581 std::vector<std::string> cf_names = {"default", "cf1", "cf2", "cf3",
1582 "c:f:4:4:4"
1583 "p\\i\\k\\a\\chu\\\\\\",
1584 "###rocksdb#1-testcf#2###"};
1585 const int num_cf = static_cast<int>(cf_names.size());
1586 Random rnd(302);
1587 test::RandomInitDBOptions(&base_db_opt, &rnd);
1588 base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG";
1589
1590 BlockBasedTableOptions special_bbto;
1591 special_bbto.cache_index_and_filter_blocks = true;
1592 special_bbto.block_size = 999999;
1593
1594 for (int c = 0; c < num_cf; ++c) {
1595 ColumnFamilyOptions cf_opt;
1596 Random cf_rnd(0xFB + c);
1597 test::RandomInitCFOptions(&cf_opt, base_db_opt, &cf_rnd);
1598 if (c < 4) {
1599 cf_opt.prefix_extractor.reset(test::RandomSliceTransform(&rnd, c));
1600 }
1601 if (c < 3) {
1602 cf_opt.table_factory.reset(test::RandomTableFactory(&rnd, c));
1603 } else if (c == 4) {
1604 cf_opt.table_factory.reset(NewBlockBasedTableFactory(special_bbto));
1605 }
1606 base_cf_opts.emplace_back(cf_opt);
1607 }
1608
1609 const std::string kOptionsFileName = "test-persisted-options.ini";
1610 ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts,
1611 kOptionsFileName, fs_.get()));
1612
1613 RocksDBOptionsParser parser;
1614 ASSERT_OK(
1615 parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
1616
1617 // Make sure block-based table factory options was deserialized correctly
1618 std::shared_ptr<TableFactory> ttf = (*parser.cf_opts())[4].table_factory;
1619 ASSERT_EQ(BlockBasedTableFactory::kName, std::string(ttf->Name()));
1620 const BlockBasedTableOptions& parsed_bbto =
1621 static_cast<BlockBasedTableFactory*>(ttf.get())->table_options();
1622 ASSERT_EQ(special_bbto.block_size, parsed_bbto.block_size);
1623 ASSERT_EQ(special_bbto.cache_index_and_filter_blocks,
1624 parsed_bbto.cache_index_and_filter_blocks);
1625
1626 ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
1627 base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get()));
1628
1629 ASSERT_OK(
1630 RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), base_db_opt));
1631 for (int c = 0; c < num_cf; ++c) {
1632 const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
1633 ASSERT_NE(cf_opt, nullptr);
1634 ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
1635 base_cf_opts[c], *cf_opt, &(parser.cf_opt_maps()->at(c))));
1636 }
1637
1638 // Further verify pointer-typed options
1639 for (int c = 0; c < num_cf; ++c) {
1640 const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
1641 ASSERT_NE(cf_opt, nullptr);
1642 VerifyCFPointerTypedOptions(&base_cf_opts[c], cf_opt,
1643 &(parser.cf_opt_maps()->at(c)));
1644 }
1645
1646 ASSERT_EQ(parser.GetCFOptions("does not exist"), nullptr);
1647
1648 base_db_opt.max_open_files++;
1649 ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
1650 base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get()));
1651
1652 for (int c = 0; c < num_cf; ++c) {
1653 if (base_cf_opts[c].compaction_filter) {
1654 delete base_cf_opts[c].compaction_filter;
1655 }
1656 }
1657 }
1658
TEST_F(OptionsParserTest,DifferentDefault)1659 TEST_F(OptionsParserTest, DifferentDefault) {
1660 const std::string kOptionsFileName = "test-persisted-options.ini";
1661
1662 ColumnFamilyOptions cf_level_opts;
1663 ASSERT_EQ(CompactionPri::kMinOverlappingRatio, cf_level_opts.compaction_pri);
1664 cf_level_opts.OptimizeLevelStyleCompaction();
1665
1666 ColumnFamilyOptions cf_univ_opts;
1667 cf_univ_opts.OptimizeUniversalStyleCompaction();
1668
1669 ASSERT_OK(PersistRocksDBOptions(DBOptions(), {"default", "universal"},
1670 {cf_level_opts, cf_univ_opts},
1671 kOptionsFileName, fs_.get()));
1672
1673 RocksDBOptionsParser parser;
1674 ASSERT_OK(
1675 parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
1676
1677 {
1678 Options old_default_opts;
1679 old_default_opts.OldDefaults();
1680 ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base);
1681 ASSERT_EQ(5000, old_default_opts.max_open_files);
1682 ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
1683 ASSERT_EQ(WALRecoveryMode::kTolerateCorruptedTailRecords,
1684 old_default_opts.wal_recovery_mode);
1685 }
1686 {
1687 Options old_default_opts;
1688 old_default_opts.OldDefaults(4, 6);
1689 ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base);
1690 ASSERT_EQ(5000, old_default_opts.max_open_files);
1691 }
1692 {
1693 Options old_default_opts;
1694 old_default_opts.OldDefaults(4, 7);
1695 ASSERT_NE(10 * 1048576, old_default_opts.max_bytes_for_level_base);
1696 ASSERT_NE(4, old_default_opts.table_cache_numshardbits);
1697 ASSERT_EQ(5000, old_default_opts.max_open_files);
1698 ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
1699 }
1700 {
1701 ColumnFamilyOptions old_default_cf_opts;
1702 old_default_cf_opts.OldDefaults();
1703 ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
1704 ASSERT_EQ(4 << 20, old_default_cf_opts.write_buffer_size);
1705 ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
1706 ASSERT_EQ(0, old_default_cf_opts.soft_pending_compaction_bytes_limit);
1707 ASSERT_EQ(0, old_default_cf_opts.hard_pending_compaction_bytes_limit);
1708 ASSERT_EQ(CompactionPri::kByCompensatedSize,
1709 old_default_cf_opts.compaction_pri);
1710 }
1711 {
1712 ColumnFamilyOptions old_default_cf_opts;
1713 old_default_cf_opts.OldDefaults(4, 6);
1714 ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
1715 ASSERT_EQ(CompactionPri::kByCompensatedSize,
1716 old_default_cf_opts.compaction_pri);
1717 }
1718 {
1719 ColumnFamilyOptions old_default_cf_opts;
1720 old_default_cf_opts.OldDefaults(4, 7);
1721 ASSERT_NE(2 * 1048576, old_default_cf_opts.target_file_size_base);
1722 ASSERT_EQ(CompactionPri::kByCompensatedSize,
1723 old_default_cf_opts.compaction_pri);
1724 }
1725 {
1726 Options old_default_opts;
1727 old_default_opts.OldDefaults(5, 1);
1728 ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
1729 }
1730 {
1731 Options old_default_opts;
1732 old_default_opts.OldDefaults(5, 2);
1733 ASSERT_EQ(16 * 1024U * 1024U, old_default_opts.delayed_write_rate);
1734 ASSERT_TRUE(old_default_opts.compaction_pri ==
1735 CompactionPri::kByCompensatedSize);
1736 }
1737 {
1738 Options old_default_opts;
1739 old_default_opts.OldDefaults(5, 18);
1740 ASSERT_TRUE(old_default_opts.compaction_pri ==
1741 CompactionPri::kByCompensatedSize);
1742 }
1743
1744 Options small_opts;
1745 small_opts.OptimizeForSmallDb();
1746 ASSERT_EQ(2 << 20, small_opts.write_buffer_size);
1747 ASSERT_EQ(5000, small_opts.max_open_files);
1748 }
1749
1750 class OptionsSanityCheckTest : public OptionsParserTest {
1751 public:
OptionsSanityCheckTest()1752 OptionsSanityCheckTest() {}
1753
1754 protected:
SanityCheckCFOptions(const ColumnFamilyOptions & cf_opts,OptionsSanityCheckLevel level)1755 Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts,
1756 OptionsSanityCheckLevel level) {
1757 return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
1758 DBOptions(), {"default"}, {cf_opts}, kOptionsFileName, fs_.get(),
1759 level);
1760 }
1761
PersistCFOptions(const ColumnFamilyOptions & cf_opts)1762 Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) {
1763 Status s = env_->DeleteFile(kOptionsFileName);
1764 if (!s.ok()) {
1765 return s;
1766 }
1767 return PersistRocksDBOptions(DBOptions(), {"default"}, {cf_opts},
1768 kOptionsFileName, fs_.get());
1769 }
1770
1771 const std::string kOptionsFileName = "OPTIONS";
1772 };
1773
TEST_F(OptionsSanityCheckTest,SanityCheck)1774 TEST_F(OptionsSanityCheckTest, SanityCheck) {
1775 ColumnFamilyOptions opts;
1776 Random rnd(301);
1777
1778 // default ColumnFamilyOptions
1779 {
1780 ASSERT_OK(PersistCFOptions(opts));
1781 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1782 }
1783
1784 // prefix_extractor
1785 {
1786 // Okay to change prefix_extractor form nullptr to non-nullptr
1787 ASSERT_EQ(opts.prefix_extractor.get(), nullptr);
1788 opts.prefix_extractor.reset(NewCappedPrefixTransform(10));
1789 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1790 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1791
1792 // persist the change
1793 ASSERT_OK(PersistCFOptions(opts));
1794 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1795
1796 // use same prefix extractor but with different parameter
1797 opts.prefix_extractor.reset(NewCappedPrefixTransform(15));
1798 // expect pass only in kSanityLevelLooselyCompatible
1799 ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1800 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1801 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1802
1803 // repeat the test with FixedPrefixTransform
1804 opts.prefix_extractor.reset(NewFixedPrefixTransform(10));
1805 ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1806 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1807 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1808
1809 // persist the change of prefix_extractor
1810 ASSERT_OK(PersistCFOptions(opts));
1811 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1812
1813 // use same prefix extractor but with different parameter
1814 opts.prefix_extractor.reset(NewFixedPrefixTransform(15));
1815 // expect pass only in kSanityLevelLooselyCompatible
1816 ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1817 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1818 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1819
1820 // Change prefix extractor from non-nullptr to nullptr
1821 opts.prefix_extractor.reset();
1822 // expect pass as it's safe to change prefix_extractor
1823 // from non-null to null
1824 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1825 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1826 }
1827 // persist the change
1828 ASSERT_OK(PersistCFOptions(opts));
1829 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1830
1831 // table_factory
1832 {
1833 for (int tb = 0; tb <= 2; ++tb) {
1834 // change the table factory
1835 opts.table_factory.reset(test::RandomTableFactory(&rnd, tb));
1836 ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1837 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1838
1839 // persist the change
1840 ASSERT_OK(PersistCFOptions(opts));
1841 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1842 }
1843 }
1844
1845 // merge_operator
1846 {
1847 // Test when going from nullptr -> merge operator
1848 opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
1849 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1850 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1851
1852 // persist the change
1853 ASSERT_OK(PersistCFOptions(opts));
1854 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1855
1856 for (int test = 0; test < 5; ++test) {
1857 // change the merge operator
1858 opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
1859 ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1860 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1861
1862 // persist the change
1863 ASSERT_OK(PersistCFOptions(opts));
1864 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1865 }
1866
1867 // Test when going from merge operator -> nullptr
1868 opts.merge_operator = nullptr;
1869 ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1870 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1871
1872 // persist the change
1873 ASSERT_OK(PersistCFOptions(opts));
1874 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1875 }
1876
1877 // compaction_filter
1878 {
1879 for (int test = 0; test < 5; ++test) {
1880 // change the compaction filter
1881 opts.compaction_filter = test::RandomCompactionFilter(&rnd);
1882 ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1883 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1884
1885 // persist the change
1886 ASSERT_OK(PersistCFOptions(opts));
1887 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1888 delete opts.compaction_filter;
1889 opts.compaction_filter = nullptr;
1890 }
1891 }
1892
1893 // compaction_filter_factory
1894 {
1895 for (int test = 0; test < 5; ++test) {
1896 // change the compaction filter factory
1897 opts.compaction_filter_factory.reset(
1898 test::RandomCompactionFilterFactory(&rnd));
1899 ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1900 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1901
1902 // persist the change
1903 ASSERT_OK(PersistCFOptions(opts));
1904 ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1905 }
1906 }
1907 }
1908
1909 namespace {
IsEscapedString(const std::string & str)1910 bool IsEscapedString(const std::string& str) {
1911 for (size_t i = 0; i < str.size(); ++i) {
1912 if (str[i] == '\\') {
1913 // since we already handle those two consecutive '\'s in
1914 // the next if-then branch, any '\' appear at the end
1915 // of an escaped string in such case is not valid.
1916 if (i == str.size() - 1) {
1917 return false;
1918 }
1919 if (str[i + 1] == '\\') {
1920 // if there're two consecutive '\'s, skip the second one.
1921 i++;
1922 continue;
1923 }
1924 switch (str[i + 1]) {
1925 case ':':
1926 case '\\':
1927 case '#':
1928 continue;
1929 default:
1930 // if true, '\' together with str[i + 1] is not a valid escape.
1931 if (UnescapeChar(str[i + 1]) == str[i + 1]) {
1932 return false;
1933 }
1934 }
1935 } else if (isSpecialChar(str[i]) && (i == 0 || str[i - 1] != '\\')) {
1936 return false;
1937 }
1938 }
1939 return true;
1940 }
1941 } // namespace
1942
TEST_F(OptionsParserTest,IntegerParsing)1943 TEST_F(OptionsParserTest, IntegerParsing) {
1944 ASSERT_EQ(ParseUint64("18446744073709551615"), 18446744073709551615U);
1945 ASSERT_EQ(ParseUint32("4294967295"), 4294967295U);
1946 ASSERT_EQ(ParseSizeT("18446744073709551615"), 18446744073709551615U);
1947 ASSERT_EQ(ParseInt64("9223372036854775807"), 9223372036854775807);
1948 ASSERT_EQ(ParseInt64("-9223372036854775808"), port::kMinInt64);
1949 ASSERT_EQ(ParseInt32("2147483647"), 2147483647);
1950 ASSERT_EQ(ParseInt32("-2147483648"), port::kMinInt32);
1951 ASSERT_EQ(ParseInt("-32767"), -32767);
1952 ASSERT_EQ(ParseDouble("-1.234567"), -1.234567);
1953 }
1954
TEST_F(OptionsParserTest,EscapeOptionString)1955 TEST_F(OptionsParserTest, EscapeOptionString) {
1956 ASSERT_EQ(UnescapeOptionString(
1957 "This is a test string with \\# \\: and \\\\ escape chars."),
1958 "This is a test string with # : and \\ escape chars.");
1959
1960 ASSERT_EQ(
1961 EscapeOptionString("This is a test string with # : and \\ escape chars."),
1962 "This is a test string with \\# \\: and \\\\ escape chars.");
1963
1964 std::string readible_chars =
1965 "A String like this \"1234567890-=_)(*&^%$#@!ertyuiop[]{POIU"
1966 "YTREWQasdfghjkl;':LKJHGFDSAzxcvbnm,.?>"
1967 "<MNBVCXZ\\\" should be okay to \\#\\\\\\:\\#\\#\\#\\ "
1968 "be serialized and deserialized";
1969
1970 std::string escaped_string = EscapeOptionString(readible_chars);
1971 ASSERT_TRUE(IsEscapedString(escaped_string));
1972 // This two transformations should be canceled and should output
1973 // the original input.
1974 ASSERT_EQ(UnescapeOptionString(escaped_string), readible_chars);
1975
1976 std::string all_chars;
1977 for (unsigned char c = 0;; ++c) {
1978 all_chars += c;
1979 if (c == 255) {
1980 break;
1981 }
1982 }
1983 escaped_string = EscapeOptionString(all_chars);
1984 ASSERT_TRUE(IsEscapedString(escaped_string));
1985 ASSERT_EQ(UnescapeOptionString(escaped_string), all_chars);
1986
1987 ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment(
1988 " A simple statement with a comment. # like this :)"),
1989 "A simple statement with a comment.");
1990
1991 ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment(
1992 "Escape \\# and # comment together ."),
1993 "Escape \\# and");
1994 }
1995 #endif // !ROCKSDB_LITE
1996 } // namespace ROCKSDB_NAMESPACE
1997
main(int argc,char ** argv)1998 int main(int argc, char** argv) {
1999 ::testing::InitGoogleTest(&argc, argv);
2000 #ifdef GFLAGS
2001 ParseCommandLineFlags(&argc, &argv, true);
2002 #endif // GFLAGS
2003 return RUN_ALL_TESTS();
2004 }
2005