1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2 //  This source code is licensed under both the GPLv2 (found in the
3 //  COPYING file in the root directory) and Apache 2.0 License
4 //  (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 
10 #include <cctype>
11 #include <cinttypes>
12 #include <cstring>
13 #include <unordered_map>
14 
15 #include "cache/lru_cache.h"
16 #include "cache/sharded_cache.h"
17 #include "options/options_helper.h"
18 #include "options/options_parser.h"
19 #include "options/options_sanity_check.h"
20 #include "port/port.h"
21 #include "rocksdb/cache.h"
22 #include "rocksdb/convenience.h"
23 #include "rocksdb/memtablerep.h"
24 #include "rocksdb/utilities/leveldb_options.h"
25 #include "rocksdb/utilities/object_registry.h"
26 #include "table/block_based/filter_policy_internal.h"
27 #include "test_util/testharness.h"
28 #include "test_util/testutil.h"
29 #include "util/random.h"
30 #include "util/stderr_logger.h"
31 #include "util/string_util.h"
32 #include "utilities/merge_operators/bytesxor.h"
33 
34 #ifndef GFLAGS
35 bool FLAGS_enable_print = false;
36 #else
37 #include "util/gflags_compat.h"
38 using GFLAGS_NAMESPACE::ParseCommandLineFlags;
39 DEFINE_bool(enable_print, false, "Print options generated to console.");
40 #endif  // GFLAGS
41 
42 namespace ROCKSDB_NAMESPACE {
43 
44 class OptionsTest : public testing::Test {};
45 
46 #ifndef ROCKSDB_LITE  // GetOptionsFromMap is not supported in ROCKSDB_LITE
TEST_F(OptionsTest,GetOptionsFromMapTest)47 TEST_F(OptionsTest, GetOptionsFromMapTest) {
48   std::unordered_map<std::string, std::string> cf_options_map = {
49       {"write_buffer_size", "1"},
50       {"max_write_buffer_number", "2"},
51       {"min_write_buffer_number_to_merge", "3"},
52       {"max_write_buffer_number_to_maintain", "99"},
53       {"max_write_buffer_size_to_maintain", "-99999"},
54       {"compression", "kSnappyCompression"},
55       {"compression_per_level",
56        "kNoCompression:"
57        "kSnappyCompression:"
58        "kZlibCompression:"
59        "kBZip2Compression:"
60        "kLZ4Compression:"
61        "kLZ4HCCompression:"
62        "kXpressCompression:"
63        "kZSTD:"
64        "kZSTDNotFinalCompression"},
65       {"bottommost_compression", "kLZ4Compression"},
66       {"bottommost_compression_opts", "5:6:7:8:9:true"},
67       {"compression_opts", "4:5:6:7:8:true"},
68       {"num_levels", "8"},
69       {"level0_file_num_compaction_trigger", "8"},
70       {"level0_slowdown_writes_trigger", "9"},
71       {"level0_stop_writes_trigger", "10"},
72       {"target_file_size_base", "12"},
73       {"target_file_size_multiplier", "13"},
74       {"max_bytes_for_level_base", "14"},
75       {"level_compaction_dynamic_level_bytes", "true"},
76       {"max_bytes_for_level_multiplier", "15.0"},
77       {"max_bytes_for_level_multiplier_additional", "16:17:18"},
78       {"max_compaction_bytes", "21"},
79       {"soft_rate_limit", "1.1"},
80       {"hard_rate_limit", "2.1"},
81       {"hard_pending_compaction_bytes_limit", "211"},
82       {"arena_block_size", "22"},
83       {"disable_auto_compactions", "true"},
84       {"compaction_style", "kCompactionStyleLevel"},
85       {"compaction_pri", "kOldestSmallestSeqFirst"},
86       {"verify_checksums_in_compaction", "false"},
87       {"compaction_options_fifo", "23"},
88       {"max_sequential_skip_in_iterations", "24"},
89       {"inplace_update_support", "true"},
90       {"report_bg_io_stats", "true"},
91       {"compaction_measure_io_stats", "false"},
92       {"inplace_update_num_locks", "25"},
93       {"memtable_prefix_bloom_size_ratio", "0.26"},
94       {"memtable_whole_key_filtering", "true"},
95       {"memtable_huge_page_size", "28"},
96       {"bloom_locality", "29"},
97       {"max_successive_merges", "30"},
98       {"min_partial_merge_operands", "31"},
99       {"prefix_extractor", "fixed:31"},
100       {"optimize_filters_for_hits", "true"},
101   };
102 
103   std::unordered_map<std::string, std::string> db_options_map = {
104       {"create_if_missing", "false"},
105       {"create_missing_column_families", "true"},
106       {"error_if_exists", "false"},
107       {"paranoid_checks", "true"},
108       {"max_open_files", "32"},
109       {"max_total_wal_size", "33"},
110       {"use_fsync", "true"},
111       {"db_log_dir", "/db_log_dir"},
112       {"wal_dir", "/wal_dir"},
113       {"delete_obsolete_files_period_micros", "34"},
114       {"max_background_compactions", "35"},
115       {"max_background_flushes", "36"},
116       {"max_log_file_size", "37"},
117       {"log_file_time_to_roll", "38"},
118       {"keep_log_file_num", "39"},
119       {"recycle_log_file_num", "5"},
120       {"max_manifest_file_size", "40"},
121       {"table_cache_numshardbits", "41"},
122       {"WAL_ttl_seconds", "43"},
123       {"WAL_size_limit_MB", "44"},
124       {"manifest_preallocation_size", "45"},
125       {"allow_mmap_reads", "true"},
126       {"allow_mmap_writes", "false"},
127       {"use_direct_reads", "false"},
128       {"use_direct_io_for_flush_and_compaction", "false"},
129       {"is_fd_close_on_exec", "true"},
130       {"skip_log_error_on_recovery", "false"},
131       {"stats_dump_period_sec", "46"},
132       {"stats_persist_period_sec", "57"},
133       {"persist_stats_to_disk", "false"},
134       {"stats_history_buffer_size", "69"},
135       {"advise_random_on_open", "true"},
136       {"use_adaptive_mutex", "false"},
137       {"new_table_reader_for_compaction_inputs", "true"},
138       {"compaction_readahead_size", "100"},
139       {"random_access_max_buffer_size", "3145728"},
140       {"writable_file_max_buffer_size", "314159"},
141       {"bytes_per_sync", "47"},
142       {"wal_bytes_per_sync", "48"},
143       {"strict_bytes_per_sync", "true"},
144   };
145 
146   ColumnFamilyOptions base_cf_opt;
147   ColumnFamilyOptions new_cf_opt;
148   ASSERT_OK(GetColumnFamilyOptionsFromMap(
149             base_cf_opt, cf_options_map, &new_cf_opt));
150   ASSERT_EQ(new_cf_opt.write_buffer_size, 1U);
151   ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2);
152   ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3);
153   ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99);
154   ASSERT_EQ(new_cf_opt.max_write_buffer_size_to_maintain, -99999);
155   ASSERT_EQ(new_cf_opt.compression, kSnappyCompression);
156   ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U);
157   ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression);
158   ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression);
159   ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression);
160   ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression);
161   ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression);
162   ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression);
163   ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression);
164   ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD);
165   ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression);
166   ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
167   ASSERT_EQ(new_cf_opt.compression_opts.level, 5);
168   ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6);
169   ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u);
170   ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u);
171   ASSERT_EQ(new_cf_opt.compression_opts.enabled, true);
172   ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression);
173   ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5);
174   ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6);
175   ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7);
176   ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u);
177   ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u);
178   ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true);
179   ASSERT_EQ(new_cf_opt.num_levels, 8);
180   ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8);
181   ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9);
182   ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10);
183   ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast<uint64_t>(12));
184   ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13);
185   ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U);
186   ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true);
187   ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0);
188   ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U);
189   ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16);
190   ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17);
191   ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18);
192   ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21);
193   ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211);
194   ASSERT_EQ(new_cf_opt.arena_block_size, 22U);
195   ASSERT_EQ(new_cf_opt.disable_auto_compactions, true);
196   ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
197   ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst);
198   ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
199             static_cast<uint64_t>(23));
200   ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
201             static_cast<uint64_t>(24));
202   ASSERT_EQ(new_cf_opt.inplace_update_support, true);
203   ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U);
204   ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26);
205   ASSERT_EQ(new_cf_opt.memtable_whole_key_filtering, true);
206   ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U);
207   ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
208   ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
209   ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
210   ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
211   ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()),
212             "rocksdb.FixedPrefix.31");
213 
214   cf_options_map["write_buffer_size"] = "hello";
215   ASSERT_NOK(GetColumnFamilyOptionsFromMap(
216              base_cf_opt, cf_options_map, &new_cf_opt));
217   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
218 
219   cf_options_map["write_buffer_size"] = "1";
220   ASSERT_OK(GetColumnFamilyOptionsFromMap(
221             base_cf_opt, cf_options_map, &new_cf_opt));
222 
223   cf_options_map["unknown_option"] = "1";
224   ASSERT_NOK(GetColumnFamilyOptionsFromMap(
225              base_cf_opt, cf_options_map, &new_cf_opt));
226   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
227 
228   ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map,
229                                           &new_cf_opt,
230                                           false, /* input_strings_escaped  */
231                                           true /* ignore_unknown_options */));
232   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
233       base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */
234       kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility*/));
235   ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
236       base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */
237       kSanityLevelExactMatch /* default for VerifyCFOptions */));
238 
239   DBOptions base_db_opt;
240   DBOptions new_db_opt;
241   ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
242   ASSERT_EQ(new_db_opt.create_if_missing, false);
243   ASSERT_EQ(new_db_opt.create_missing_column_families, true);
244   ASSERT_EQ(new_db_opt.error_if_exists, false);
245   ASSERT_EQ(new_db_opt.paranoid_checks, true);
246   ASSERT_EQ(new_db_opt.max_open_files, 32);
247   ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33));
248   ASSERT_EQ(new_db_opt.use_fsync, true);
249   ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir");
250   ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir");
251   ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros,
252             static_cast<uint64_t>(34));
253   ASSERT_EQ(new_db_opt.max_background_compactions, 35);
254   ASSERT_EQ(new_db_opt.max_background_flushes, 36);
255   ASSERT_EQ(new_db_opt.max_log_file_size, 37U);
256   ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U);
257   ASSERT_EQ(new_db_opt.keep_log_file_num, 39U);
258   ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U);
259   ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast<uint64_t>(40));
260   ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41);
261   ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast<uint64_t>(43));
262   ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast<uint64_t>(44));
263   ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U);
264   ASSERT_EQ(new_db_opt.allow_mmap_reads, true);
265   ASSERT_EQ(new_db_opt.allow_mmap_writes, false);
266   ASSERT_EQ(new_db_opt.use_direct_reads, false);
267   ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false);
268   ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true);
269   ASSERT_EQ(new_db_opt.skip_log_error_on_recovery, false);
270   ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U);
271   ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U);
272   ASSERT_EQ(new_db_opt.persist_stats_to_disk, false);
273   ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U);
274   ASSERT_EQ(new_db_opt.advise_random_on_open, true);
275   ASSERT_EQ(new_db_opt.use_adaptive_mutex, false);
276   ASSERT_EQ(new_db_opt.new_table_reader_for_compaction_inputs, true);
277   ASSERT_EQ(new_db_opt.compaction_readahead_size, 100);
278   ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728);
279   ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159);
280   ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47));
281   ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48));
282   ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true);
283 
284   db_options_map["max_open_files"] = "hello";
285   ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
286   ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt));
287   ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
288       base_db_opt, new_db_opt, nullptr, /* new_opt_map */
289       kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */));
290 
291   // unknow options should fail parsing without ignore_unknown_options = true
292   db_options_map["unknown_db_option"] = "1";
293   ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
294   ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt));
295 
296   ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt,
297                                 false, /* input_strings_escaped  */
298                                 true /* ignore_unknown_options */));
299   ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
300       base_db_opt, new_db_opt, nullptr, /* new_opt_map */
301       kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */));
302   ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(
303       base_db_opt, new_db_opt, nullptr, /* new_opt_mat */
304       kSanityLevelExactMatch /* default for VerifyDBOptions */));
305 }
306 #endif  // !ROCKSDB_LITE
307 
308 #ifndef ROCKSDB_LITE  // GetColumnFamilyOptionsFromString is not supported in
309                       // ROCKSDB_LITE
TEST_F(OptionsTest,GetColumnFamilyOptionsFromStringTest)310 TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
311   ColumnFamilyOptions base_cf_opt;
312   ColumnFamilyOptions new_cf_opt;
313   base_cf_opt.table_factory.reset();
314   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, "", &new_cf_opt));
315   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
316             "write_buffer_size=5", &new_cf_opt));
317   ASSERT_EQ(new_cf_opt.write_buffer_size, 5U);
318   ASSERT_TRUE(new_cf_opt.table_factory == nullptr);
319   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
320             "write_buffer_size=6;", &new_cf_opt));
321   ASSERT_EQ(new_cf_opt.write_buffer_size, 6U);
322   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
323             "  write_buffer_size =  7  ", &new_cf_opt));
324   ASSERT_EQ(new_cf_opt.write_buffer_size, 7U);
325   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
326             "  write_buffer_size =  8 ; ", &new_cf_opt));
327   ASSERT_EQ(new_cf_opt.write_buffer_size, 8U);
328   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
329             "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt));
330   ASSERT_EQ(new_cf_opt.write_buffer_size, 9U);
331   ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10);
332   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
333             "write_buffer_size=11; max_write_buffer_number  =  12 ;",
334             &new_cf_opt));
335   ASSERT_EQ(new_cf_opt.write_buffer_size, 11U);
336   ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12);
337   // Wrong name "max_write_buffer_number_"
338   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
339              "write_buffer_size=13;max_write_buffer_number_=14;",
340               &new_cf_opt));
341   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
342 
343   // Comparator from object registry
344   std::string kCompName = "reverse_comp";
345   ObjectLibrary::Default()->Register<const Comparator>(
346       kCompName,
347       [](const std::string& /*name*/,
348          std::unique_ptr<const Comparator>* /*guard*/,
349          std::string* /* errmsg */) { return ReverseBytewiseComparator(); });
350 
351   ASSERT_OK(GetColumnFamilyOptionsFromString(
352       base_cf_opt, "comparator=" + kCompName + ";", &new_cf_opt));
353   ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator());
354 
355   // MergeOperator from object registry
356   std::unique_ptr<BytesXOROperator> bxo(new BytesXOROperator());
357   std::string kMoName = bxo->Name();
358   ObjectLibrary::Default()->Register<MergeOperator>(
359       kMoName,
360       [](const std::string& /*name*/, std::unique_ptr<MergeOperator>* guard,
361          std::string* /* errmsg */) {
362         guard->reset(new BytesXOROperator());
363         return guard->get();
364       });
365 
366   ASSERT_OK(GetColumnFamilyOptionsFromString(
367       base_cf_opt, "merge_operator=" + kMoName + ";", &new_cf_opt));
368   ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name()));
369 
370   // Wrong key/value pair
371   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
372              "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt));
373   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
374 
375   // Error Paring value
376   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
377              "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt));
378   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
379 
380   // Missing option name
381   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
382              "write_buffer_size=13; =100;", &new_cf_opt));
383   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
384 
385   const uint64_t kilo = 1024UL;
386   const uint64_t mega = 1024 * kilo;
387   const uint64_t giga = 1024 * mega;
388   const uint64_t tera = 1024 * giga;
389 
390   // Units (k)
391   ASSERT_OK(GetColumnFamilyOptionsFromString(
392       base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt));
393   ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo);
394   // Units (m)
395   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
396             "max_write_buffer_number=16m;inplace_update_num_locks=17M",
397             &new_cf_opt));
398   ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega);
399   ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17u * mega);
400   // Units (g)
401   ASSERT_OK(GetColumnFamilyOptionsFromString(
402       base_cf_opt,
403       "write_buffer_size=18g;prefix_extractor=capped:8;"
404       "arena_block_size=19G",
405       &new_cf_opt));
406 
407   ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
408   ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
409   ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
410   std::string prefix_name(new_cf_opt.prefix_extractor->Name());
411   ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8");
412 
413   // Units (t)
414   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
415             "write_buffer_size=20t;arena_block_size=21T", &new_cf_opt));
416   ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera);
417   ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera);
418 
419   // Nested block based table options
420   // Empty
421   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
422             "write_buffer_size=10;max_write_buffer_number=16;"
423             "block_based_table_factory={};arena_block_size=1024",
424             &new_cf_opt));
425   ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
426   // Non-empty
427   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
428             "write_buffer_size=10;max_write_buffer_number=16;"
429             "block_based_table_factory={block_cache=1M;block_size=4;};"
430             "arena_block_size=1024",
431             &new_cf_opt));
432   ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
433   // Last one
434   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
435             "write_buffer_size=10;max_write_buffer_number=16;"
436             "block_based_table_factory={block_cache=1M;block_size=4;}",
437             &new_cf_opt));
438   ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
439   // Mismatch curly braces
440   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
441              "write_buffer_size=10;max_write_buffer_number=16;"
442              "block_based_table_factory={{{block_size=4;};"
443              "arena_block_size=1024",
444              &new_cf_opt));
445   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
446 
447   // Unexpected chars after closing curly brace
448   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
449              "write_buffer_size=10;max_write_buffer_number=16;"
450              "block_based_table_factory={block_size=4;}};"
451              "arena_block_size=1024",
452              &new_cf_opt));
453   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
454 
455   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
456              "write_buffer_size=10;max_write_buffer_number=16;"
457              "block_based_table_factory={block_size=4;}xdfa;"
458              "arena_block_size=1024",
459              &new_cf_opt));
460   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
461 
462   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
463              "write_buffer_size=10;max_write_buffer_number=16;"
464              "block_based_table_factory={block_size=4;}xdfa",
465              &new_cf_opt));
466   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
467 
468   // Invalid block based table option
469   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
470              "write_buffer_size=10;max_write_buffer_number=16;"
471              "block_based_table_factory={xx_block_size=4;}",
472              &new_cf_opt));
473   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
474 
475   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
476            "optimize_filters_for_hits=true",
477            &new_cf_opt));
478   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
479             "optimize_filters_for_hits=false",
480             &new_cf_opt));
481 
482   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
483               "optimize_filters_for_hits=junk",
484               &new_cf_opt));
485   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
486 
487   // Nested plain table options
488   // Empty
489   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
490             "write_buffer_size=10;max_write_buffer_number=16;"
491             "plain_table_factory={};arena_block_size=1024",
492             &new_cf_opt));
493   ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
494   ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
495   // Non-empty
496   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
497             "write_buffer_size=10;max_write_buffer_number=16;"
498             "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};"
499             "arena_block_size=1024",
500             &new_cf_opt));
501   ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
502   ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
503 
504   // memtable factory
505   ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
506             "write_buffer_size=10;max_write_buffer_number=16;"
507             "memtable=skip_list:10;arena_block_size=1024",
508             &new_cf_opt));
509   ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
510   ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory");
511 }
512 #endif  // !ROCKSDB_LITE
513 
514 #ifndef ROCKSDB_LITE  // GetBlockBasedTableOptionsFromString is not supported
TEST_F(OptionsTest,GetBlockBasedTableOptionsFromString)515 TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
516   BlockBasedTableOptions table_opt;
517   BlockBasedTableOptions new_opt;
518   // make sure default values are overwritten by something else
519   ASSERT_OK(GetBlockBasedTableOptionsFromString(
520       table_opt,
521       "cache_index_and_filter_blocks=1;index_type=kHashSearch;"
522       "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
523       "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
524       "block_size_deviation=8;block_restart_interval=4;"
525       "format_version=5;whole_key_filtering=1;"
526       "filter_policy=bloomfilter:4.567:false;",
527       &new_opt));
528   ASSERT_TRUE(new_opt.cache_index_and_filter_blocks);
529   ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch);
530   ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash);
531   ASSERT_TRUE(new_opt.hash_index_allow_collision);
532   ASSERT_TRUE(new_opt.no_block_cache);
533   ASSERT_TRUE(new_opt.block_cache != nullptr);
534   ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
535   ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
536   ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL);
537   ASSERT_EQ(new_opt.block_size, 1024UL);
538   ASSERT_EQ(new_opt.block_size_deviation, 8);
539   ASSERT_EQ(new_opt.block_restart_interval, 4);
540   ASSERT_EQ(new_opt.format_version, 5U);
541   ASSERT_EQ(new_opt.whole_key_filtering, true);
542   ASSERT_TRUE(new_opt.filter_policy != nullptr);
543   const BloomFilterPolicy& bfp =
544       dynamic_cast<const BloomFilterPolicy&>(*new_opt.filter_policy);
545   EXPECT_EQ(bfp.GetMillibitsPerKey(), 4567);
546   EXPECT_EQ(bfp.GetWholeBitsPerKey(), 5);
547 
548   // unknown option
549   ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
550              "cache_index_and_filter_blocks=1;index_type=kBinarySearch;"
551              "bad_option=1",
552              &new_opt));
553   ASSERT_EQ(static_cast<bool>(table_opt.cache_index_and_filter_blocks),
554             new_opt.cache_index_and_filter_blocks);
555   ASSERT_EQ(table_opt.index_type, new_opt.index_type);
556 
557   // unrecognized index type
558   ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
559              "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX",
560              &new_opt));
561   ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
562             new_opt.cache_index_and_filter_blocks);
563   ASSERT_EQ(table_opt.index_type, new_opt.index_type);
564 
565   // unrecognized checksum type
566   ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
567              "cache_index_and_filter_blocks=1;checksum=kxxHashXX",
568              &new_opt));
569   ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
570             new_opt.cache_index_and_filter_blocks);
571   ASSERT_EQ(table_opt.index_type, new_opt.index_type);
572 
573   // unrecognized filter policy name
574   ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
575              "cache_index_and_filter_blocks=1;"
576              "filter_policy=bloomfilterxx:4:true",
577              &new_opt));
578   ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
579             new_opt.cache_index_and_filter_blocks);
580   ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
581 
582   // unrecognized filter policy config
583   ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
584              "cache_index_and_filter_blocks=1;"
585              "filter_policy=bloomfilter:4",
586              &new_opt));
587   ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
588             new_opt.cache_index_and_filter_blocks);
589   ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
590 
591   // Check block cache options are overwritten when specified
592   // in new format as a struct.
593   ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
594              "block_cache={capacity=1M;num_shard_bits=4;"
595              "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};"
596              "block_cache_compressed={capacity=1M;num_shard_bits=4;"
597              "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}",
598              &new_opt));
599   ASSERT_TRUE(new_opt.block_cache != nullptr);
600   ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
601   ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
602                 new_opt.block_cache)->GetNumShardBits(), 4);
603   ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
604   ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
605                 new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
606   ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
607   ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
608   ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
609                 new_opt.block_cache_compressed)->GetNumShardBits(), 4);
610   ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
611   ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
612                 new_opt.block_cache_compressed)->GetHighPriPoolRatio(),
613                 0.5);
614 
615   // Set only block cache capacity. Check other values are
616   // reset to default values.
617   ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
618              "block_cache={capacity=2M};"
619              "block_cache_compressed={capacity=2M}",
620              &new_opt));
621   ASSERT_TRUE(new_opt.block_cache != nullptr);
622   ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL);
623   // Default values
624   ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
625                 new_opt.block_cache)->GetNumShardBits(),
626                 GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity()));
627   ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
628   ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache)
629                 ->GetHighPriPoolRatio(),
630             0.5);
631   ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
632   ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 2*1024UL*1024UL);
633   // Default values
634   ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
635                 new_opt.block_cache_compressed)->GetNumShardBits(),
636                 GetDefaultCacheShardBits(
637                     new_opt.block_cache_compressed->GetCapacity()));
638   ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
639   ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
640                 ->GetHighPriPoolRatio(),
641             0.5);
642 
643   // Set couple of block cache options.
644   ASSERT_OK(GetBlockBasedTableOptionsFromString(
645       table_opt,
646       "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};"
647       "block_cache_compressed={num_shard_bits=5;"
648       "high_pri_pool_ratio=0.0;}",
649       &new_opt));
650   ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0);
651   ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
652                 new_opt.block_cache)->GetNumShardBits(), 5);
653   ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
654   ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
655                 new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
656   ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
657   ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 0);
658   ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
659                 new_opt.block_cache_compressed)->GetNumShardBits(), 5);
660   ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
661   ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
662                 ->GetHighPriPoolRatio(),
663             0.0);
664 
665   // Set couple of block cache options.
666   ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
667              "block_cache={capacity=1M;num_shard_bits=4;"
668              "strict_capacity_limit=true;};"
669              "block_cache_compressed={capacity=1M;num_shard_bits=4;"
670              "strict_capacity_limit=true;}",
671              &new_opt));
672   ASSERT_TRUE(new_opt.block_cache != nullptr);
673   ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
674   ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
675                 new_opt.block_cache)->GetNumShardBits(), 4);
676   ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
677   ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache)
678                 ->GetHighPriPoolRatio(),
679             0.5);
680   ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
681   ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
682   ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
683                 new_opt.block_cache_compressed)->GetNumShardBits(), 4);
684   ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
685   ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(new_opt.block_cache_compressed)
686                 ->GetHighPriPoolRatio(),
687             0.5);
688 }
689 #endif  // !ROCKSDB_LITE
690 
691 
692 #ifndef ROCKSDB_LITE  // GetPlainTableOptionsFromString is not supported
TEST_F(OptionsTest,GetPlainTableOptionsFromString)693 TEST_F(OptionsTest, GetPlainTableOptionsFromString) {
694   PlainTableOptions table_opt;
695   PlainTableOptions new_opt;
696   // make sure default values are overwritten by something else
697   ASSERT_OK(GetPlainTableOptionsFromString(table_opt,
698             "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
699             "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;"
700             "full_scan_mode=true;store_index_in_file=true",
701             &new_opt));
702   ASSERT_EQ(new_opt.user_key_len, 66u);
703   ASSERT_EQ(new_opt.bloom_bits_per_key, 20);
704   ASSERT_EQ(new_opt.hash_table_ratio, 0.5);
705   ASSERT_EQ(new_opt.index_sparseness, 8);
706   ASSERT_EQ(new_opt.huge_page_tlb_size, 4);
707   ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix);
708   ASSERT_TRUE(new_opt.full_scan_mode);
709   ASSERT_TRUE(new_opt.store_index_in_file);
710 
711   // unknown option
712   ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
713              "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
714              "bad_option=1",
715              &new_opt));
716 
717   // unrecognized EncodingType
718   ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
719              "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
720              "encoding_type=kPrefixXX",
721              &new_opt));
722 }
723 #endif  // !ROCKSDB_LITE
724 
725 #ifndef ROCKSDB_LITE  // GetMemTableRepFactoryFromString is not supported
TEST_F(OptionsTest,GetMemTableRepFactoryFromString)726 TEST_F(OptionsTest, GetMemTableRepFactoryFromString) {
727   std::unique_ptr<MemTableRepFactory> new_mem_factory = nullptr;
728 
729   ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory));
730   ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory));
731   ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory");
732   ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt",
733                                              &new_mem_factory));
734 
735   ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory));
736   ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000",
737                                             &new_mem_factory));
738   ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory");
739   ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt",
740                                              &new_mem_factory));
741 
742   ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist",
743                                             &new_mem_factory));
744   ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist:1000",
745                                             &new_mem_factory));
746   ASSERT_EQ(std::string(new_mem_factory->Name()), "HashLinkListRepFactory");
747   ASSERT_NOK(GetMemTableRepFactoryFromString("hash_linkedlist:1000:invalid_opt",
748                                              &new_mem_factory));
749 
750   ASSERT_OK(GetMemTableRepFactoryFromString("vector", &new_mem_factory));
751   ASSERT_OK(GetMemTableRepFactoryFromString("vector:1024", &new_mem_factory));
752   ASSERT_EQ(std::string(new_mem_factory->Name()), "VectorRepFactory");
753   ASSERT_NOK(GetMemTableRepFactoryFromString("vector:1024:invalid_opt",
754                                              &new_mem_factory));
755 
756   ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo", &new_mem_factory));
757   // CuckooHash memtable is already removed.
758   ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo:1024", &new_mem_factory));
759 
760   ASSERT_NOK(GetMemTableRepFactoryFromString("bad_factory", &new_mem_factory));
761 }
762 #endif  // !ROCKSDB_LITE
763 
764 #ifndef ROCKSDB_LITE  // GetOptionsFromString is not supported in RocksDB Lite
TEST_F(OptionsTest,GetOptionsFromStringTest)765 TEST_F(OptionsTest, GetOptionsFromStringTest) {
766   Options base_options, new_options;
767   base_options.write_buffer_size = 20;
768   base_options.min_write_buffer_number_to_merge = 15;
769   BlockBasedTableOptions block_based_table_options;
770   block_based_table_options.cache_index_and_filter_blocks = true;
771   base_options.table_factory.reset(
772       NewBlockBasedTableFactory(block_based_table_options));
773 
774   // Register an Env with object registry.
775   const static char* kCustomEnvName = "CustomEnv";
776   class CustomEnv : public EnvWrapper {
777    public:
778     explicit CustomEnv(Env* _target) : EnvWrapper(_target) {}
779   };
780 
781   ObjectLibrary::Default()->Register<Env>(
782       kCustomEnvName,
783       [](const std::string& /*name*/, std::unique_ptr<Env>* /*env_guard*/,
784          std::string* /* errmsg */) {
785         static CustomEnv env(Env::Default());
786         return &env;
787       });
788 
789   ASSERT_OK(GetOptionsFromString(
790       base_options,
791       "write_buffer_size=10;max_write_buffer_number=16;"
792       "block_based_table_factory={block_cache=1M;block_size=4;};"
793       "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;"
794       "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files="
795       "1;"
796       "rate_limiter_bytes_per_sec=1024;env=CustomEnv",
797       &new_options));
798 
799   ASSERT_EQ(new_options.compression_opts.window_bits, 4);
800   ASSERT_EQ(new_options.compression_opts.level, 5);
801   ASSERT_EQ(new_options.compression_opts.strategy, 6);
802   ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0u);
803   ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0u);
804   ASSERT_EQ(new_options.compression_opts.enabled, false);
805   ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption);
806   ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5);
807   ASSERT_EQ(new_options.bottommost_compression_opts.level, 6);
808   ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7);
809   ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0u);
810   ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0u);
811   ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false);
812   ASSERT_EQ(new_options.write_buffer_size, 10U);
813   ASSERT_EQ(new_options.max_write_buffer_number, 16);
814   BlockBasedTableOptions new_block_based_table_options =
815       dynamic_cast<BlockBasedTableFactory*>(new_options.table_factory.get())
816           ->table_options();
817   ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20);
818   ASSERT_EQ(new_block_based_table_options.block_size, 4U);
819   // don't overwrite block based table options
820   ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks);
821 
822   ASSERT_EQ(new_options.create_if_missing, true);
823   ASSERT_EQ(new_options.max_open_files, 1);
824   ASSERT_TRUE(new_options.rate_limiter.get() != nullptr);
825   Env* newEnv = new_options.env;
826   ASSERT_OK(Env::LoadEnv(kCustomEnvName, &newEnv));
827   ASSERT_EQ(newEnv, new_options.env);
828 }
829 
TEST_F(OptionsTest,DBOptionsSerialization)830 TEST_F(OptionsTest, DBOptionsSerialization) {
831   Options base_options, new_options;
832   Random rnd(301);
833 
834   // Phase 1: Make big change in base_options
835   test::RandomInitDBOptions(&base_options, &rnd);
836 
837   // Phase 2: obtain a string from base_option
838   std::string base_options_file_content;
839   ASSERT_OK(GetStringFromDBOptions(&base_options_file_content, base_options));
840 
841   // Phase 3: Set new_options from the derived string and expect
842   //          new_options == base_options
843   ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_options_file_content,
844                                    &new_options));
845   ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_options, new_options));
846 }
847 
TEST_F(OptionsTest,OptionsComposeDecompose)848 TEST_F(OptionsTest, OptionsComposeDecompose) {
849   // build an Options from DBOptions + CFOptions, then decompose it to verify
850   // we get same constituent options.
851   DBOptions base_db_opts;
852   ColumnFamilyOptions base_cf_opts;
853 
854   Random rnd(301);
855   test::RandomInitDBOptions(&base_db_opts, &rnd);
856   test::RandomInitCFOptions(&base_cf_opts, base_db_opts, &rnd);
857 
858   Options base_opts(base_db_opts, base_cf_opts);
859   DBOptions new_db_opts(base_opts);
860   ColumnFamilyOptions new_cf_opts(base_opts);
861 
862   ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opts, new_db_opts));
863   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opts, new_cf_opts));
864   delete new_cf_opts.compaction_filter;
865 }
866 
TEST_F(OptionsTest,ColumnFamilyOptionsSerialization)867 TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
868   Options options;
869   ColumnFamilyOptions base_opt, new_opt;
870   Random rnd(302);
871   // Phase 1: randomly assign base_opt
872   // custom type options
873   test::RandomInitCFOptions(&base_opt, options, &rnd);
874 
875   // Phase 2: obtain a string from base_opt
876   std::string base_options_file_content;
877   ASSERT_OK(
878       GetStringFromColumnFamilyOptions(&base_options_file_content, base_opt));
879 
880   // Phase 3: Set new_opt from the derived string and expect
881   //          new_opt == base_opt
882   ASSERT_OK(GetColumnFamilyOptionsFromString(
883       ColumnFamilyOptions(), base_options_file_content, &new_opt));
884   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_opt, new_opt));
885   if (base_opt.compaction_filter) {
886     delete base_opt.compaction_filter;
887   }
888 }
889 
890 #endif  // !ROCKSDB_LITE
891 
892 Status StringToMap(
893     const std::string& opts_str,
894     std::unordered_map<std::string, std::string>* opts_map);
895 
896 #ifndef ROCKSDB_LITE  // StringToMap is not supported in ROCKSDB_LITE
TEST_F(OptionsTest,StringToMapTest)897 TEST_F(OptionsTest, StringToMapTest) {
898   std::unordered_map<std::string, std::string> opts_map;
899   // Regular options
900   ASSERT_OK(StringToMap("k1=v1;k2=v2;k3=v3", &opts_map));
901   ASSERT_EQ(opts_map["k1"], "v1");
902   ASSERT_EQ(opts_map["k2"], "v2");
903   ASSERT_EQ(opts_map["k3"], "v3");
904   // Value with '='
905   opts_map.clear();
906   ASSERT_OK(StringToMap("k1==v1;k2=v2=;", &opts_map));
907   ASSERT_EQ(opts_map["k1"], "=v1");
908   ASSERT_EQ(opts_map["k2"], "v2=");
909   // Overwrriten option
910   opts_map.clear();
911   ASSERT_OK(StringToMap("k1=v1;k1=v2;k3=v3", &opts_map));
912   ASSERT_EQ(opts_map["k1"], "v2");
913   ASSERT_EQ(opts_map["k3"], "v3");
914   // Empty value
915   opts_map.clear();
916   ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=", &opts_map));
917   ASSERT_EQ(opts_map["k1"], "v1");
918   ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
919   ASSERT_EQ(opts_map["k2"], "");
920   ASSERT_EQ(opts_map["k3"], "v3");
921   ASSERT_TRUE(opts_map.find("k4") != opts_map.end());
922   ASSERT_EQ(opts_map["k4"], "");
923   opts_map.clear();
924   ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=   ", &opts_map));
925   ASSERT_EQ(opts_map["k1"], "v1");
926   ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
927   ASSERT_EQ(opts_map["k2"], "");
928   ASSERT_EQ(opts_map["k3"], "v3");
929   ASSERT_TRUE(opts_map.find("k4") != opts_map.end());
930   ASSERT_EQ(opts_map["k4"], "");
931   opts_map.clear();
932   ASSERT_OK(StringToMap("k1=v1;k2=;k3=", &opts_map));
933   ASSERT_EQ(opts_map["k1"], "v1");
934   ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
935   ASSERT_EQ(opts_map["k2"], "");
936   ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
937   ASSERT_EQ(opts_map["k3"], "");
938   opts_map.clear();
939   ASSERT_OK(StringToMap("k1=v1;k2=;k3=;", &opts_map));
940   ASSERT_EQ(opts_map["k1"], "v1");
941   ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
942   ASSERT_EQ(opts_map["k2"], "");
943   ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
944   ASSERT_EQ(opts_map["k3"], "");
945   // Regular nested options
946   opts_map.clear();
947   ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map));
948   ASSERT_EQ(opts_map["k1"], "v1");
949   ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2");
950   ASSERT_EQ(opts_map["k3"], "v3");
951   // Multi-level nested options
952   opts_map.clear();
953   ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2={nnk1=nnk2}};"
954                         "k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4",
955                         &opts_map));
956   ASSERT_EQ(opts_map["k1"], "v1");
957   ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}");
958   ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}");
959   ASSERT_EQ(opts_map["k4"], "v4");
960   // Garbage inside curly braces
961   opts_map.clear();
962   ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4",
963                         &opts_map));
964   ASSERT_EQ(opts_map["k1"], "v1");
965   ASSERT_EQ(opts_map["k2"], "dfad=");
966   ASSERT_EQ(opts_map["k3"], "=");
967   ASSERT_EQ(opts_map["k4"], "v4");
968   // Empty nested options
969   opts_map.clear();
970   ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map));
971   ASSERT_EQ(opts_map["k1"], "v1");
972   ASSERT_EQ(opts_map["k2"], "");
973   opts_map.clear();
974   ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map));
975   ASSERT_EQ(opts_map["k1"], "v1");
976   ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}");
977   // With random spaces
978   opts_map.clear();
979   ASSERT_OK(StringToMap("  k1 =  v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}}  ; "
980                         "k3={  {   } }; k4= v4  ",
981                         &opts_map));
982   ASSERT_EQ(opts_map["k1"], "v1");
983   ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}");
984   ASSERT_EQ(opts_map["k3"], "{   }");
985   ASSERT_EQ(opts_map["k4"], "v4");
986 
987   // Empty key
988   ASSERT_NOK(StringToMap("k1=v1;k2=v2;=", &opts_map));
989   ASSERT_NOK(StringToMap("=v1;k2=v2", &opts_map));
990   ASSERT_NOK(StringToMap("k1=v1;k2v2;", &opts_map));
991   ASSERT_NOK(StringToMap("k1=v1;k2=v2;fadfa", &opts_map));
992   ASSERT_NOK(StringToMap("k1=v1;k2=v2;;", &opts_map));
993   // Mismatch curly braces
994   ASSERT_NOK(StringToMap("k1=v1;k2={;k3=v3", &opts_map));
995   ASSERT_NOK(StringToMap("k1=v1;k2={{};k3=v3", &opts_map));
996   ASSERT_NOK(StringToMap("k1=v1;k2={}};k3=v3", &opts_map));
997   ASSERT_NOK(StringToMap("k1=v1;k2={{}{}}};k3=v3", &opts_map));
998   // However this is valid!
999   opts_map.clear();
1000   ASSERT_OK(StringToMap("k1=v1;k2=};k3=v3", &opts_map));
1001   ASSERT_EQ(opts_map["k1"], "v1");
1002   ASSERT_EQ(opts_map["k2"], "}");
1003   ASSERT_EQ(opts_map["k3"], "v3");
1004 
1005   // Invalid chars after closing curly brace
1006   ASSERT_NOK(StringToMap("k1=v1;k2={{}}{};k3=v3", &opts_map));
1007   ASSERT_NOK(StringToMap("k1=v1;k2={{}}cfda;k3=v3", &opts_map));
1008   ASSERT_NOK(StringToMap("k1=v1;k2={{}}  cfda;k3=v3", &opts_map));
1009   ASSERT_NOK(StringToMap("k1=v1;k2={{}}  cfda", &opts_map));
1010   ASSERT_NOK(StringToMap("k1=v1;k2={{}}{}", &opts_map));
1011   ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map));
1012 }
1013 #endif  // ROCKSDB_LITE
1014 
1015 #ifndef ROCKSDB_LITE  // StringToMap is not supported in ROCKSDB_LITE
TEST_F(OptionsTest,StringToMapRandomTest)1016 TEST_F(OptionsTest, StringToMapRandomTest) {
1017   std::unordered_map<std::string, std::string> opts_map;
1018   // Make sure segfault is not hit by semi-random strings
1019 
1020   std::vector<std::string> bases = {
1021       "a={aa={};tt={xxx={}}};c=defff",
1022       "a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}",
1023       "abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"};
1024 
1025   for (std::string base : bases) {
1026     for (int rand_seed = 301; rand_seed < 401; rand_seed++) {
1027       Random rnd(rand_seed);
1028       for (int attempt = 0; attempt < 10; attempt++) {
1029         std::string str = base;
1030         // Replace random position to space
1031         size_t pos = static_cast<size_t>(
1032             rnd.Uniform(static_cast<int>(base.size())));
1033         str[pos] = ' ';
1034         Status s = StringToMap(str, &opts_map);
1035         ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
1036         opts_map.clear();
1037       }
1038     }
1039   }
1040 
1041   // Random Construct a string
1042   std::vector<char> chars = {'{', '}', ' ', '=', ';', 'c'};
1043   for (int rand_seed = 301; rand_seed < 1301; rand_seed++) {
1044     Random rnd(rand_seed);
1045     int len = rnd.Uniform(30);
1046     std::string str = "";
1047     for (int attempt = 0; attempt < len; attempt++) {
1048       // Add a random character
1049       size_t pos = static_cast<size_t>(
1050           rnd.Uniform(static_cast<int>(chars.size())));
1051       str.append(1, chars[pos]);
1052     }
1053     Status s = StringToMap(str, &opts_map);
1054     ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
1055     s = StringToMap("name=" + str, &opts_map);
1056     ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
1057     opts_map.clear();
1058   }
1059 }
1060 
TEST_F(OptionsTest,GetStringFromCompressionType)1061 TEST_F(OptionsTest, GetStringFromCompressionType) {
1062   std::string res;
1063 
1064   ASSERT_OK(GetStringFromCompressionType(&res, kNoCompression));
1065   ASSERT_EQ(res, "kNoCompression");
1066 
1067   ASSERT_OK(GetStringFromCompressionType(&res, kSnappyCompression));
1068   ASSERT_EQ(res, "kSnappyCompression");
1069 
1070   ASSERT_OK(GetStringFromCompressionType(&res, kDisableCompressionOption));
1071   ASSERT_EQ(res, "kDisableCompressionOption");
1072 
1073   ASSERT_OK(GetStringFromCompressionType(&res, kLZ4Compression));
1074   ASSERT_EQ(res, "kLZ4Compression");
1075 
1076   ASSERT_OK(GetStringFromCompressionType(&res, kZlibCompression));
1077   ASSERT_EQ(res, "kZlibCompression");
1078 
1079   ASSERT_NOK(
1080       GetStringFromCompressionType(&res, static_cast<CompressionType>(-10)));
1081 }
1082 #endif  // !ROCKSDB_LITE
1083 
TEST_F(OptionsTest,ConvertOptionsTest)1084 TEST_F(OptionsTest, ConvertOptionsTest) {
1085   LevelDBOptions leveldb_opt;
1086   Options converted_opt = ConvertOptions(leveldb_opt);
1087 
1088   ASSERT_EQ(converted_opt.create_if_missing, leveldb_opt.create_if_missing);
1089   ASSERT_EQ(converted_opt.error_if_exists, leveldb_opt.error_if_exists);
1090   ASSERT_EQ(converted_opt.paranoid_checks, leveldb_opt.paranoid_checks);
1091   ASSERT_EQ(converted_opt.env, leveldb_opt.env);
1092   ASSERT_EQ(converted_opt.info_log.get(), leveldb_opt.info_log);
1093   ASSERT_EQ(converted_opt.write_buffer_size, leveldb_opt.write_buffer_size);
1094   ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files);
1095   ASSERT_EQ(converted_opt.compression, leveldb_opt.compression);
1096 
1097   std::shared_ptr<TableFactory> tb_guard = converted_opt.table_factory;
1098   BlockBasedTableFactory* table_factory =
1099       dynamic_cast<BlockBasedTableFactory*>(converted_opt.table_factory.get());
1100 
1101   ASSERT_TRUE(table_factory != nullptr);
1102 
1103   const BlockBasedTableOptions table_opt = table_factory->table_options();
1104 
1105   ASSERT_EQ(table_opt.block_cache->GetCapacity(), 8UL << 20);
1106   ASSERT_EQ(table_opt.block_size, leveldb_opt.block_size);
1107   ASSERT_EQ(table_opt.block_restart_interval,
1108             leveldb_opt.block_restart_interval);
1109   ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy);
1110 }
1111 
1112 #ifndef ROCKSDB_LITE
1113 class OptionsParserTest : public testing::Test {
1114  public:
OptionsParserTest()1115   OptionsParserTest() {
1116     env_.reset(new test::StringEnv(Env::Default()));
1117     fs_.reset(new LegacyFileSystemWrapper(env_.get()));
1118   }
1119 
1120  protected:
1121   std::unique_ptr<test::StringEnv> env_;
1122   std::unique_ptr<LegacyFileSystemWrapper> fs_;
1123 };
1124 
TEST_F(OptionsParserTest,Comment)1125 TEST_F(OptionsParserTest, Comment) {
1126   DBOptions db_opt;
1127   db_opt.max_open_files = 12345;
1128   db_opt.max_background_flushes = 301;
1129   db_opt.max_total_wal_size = 1024;
1130   ColumnFamilyOptions cf_opt;
1131 
1132   std::string options_file_content =
1133       "# This is a testing option string.\n"
1134       "# Currently we only support \"#\" styled comment.\n"
1135       "\n"
1136       "[Version]\n"
1137       "  rocksdb_version=3.14.0\n"
1138       "  options_file_version=1\n"
1139       "[ DBOptions ]\n"
1140       "  # note that we don't support space around \"=\"\n"
1141       "  max_open_files=12345;\n"
1142       "  max_background_flushes=301  # comment after a statement is fine\n"
1143       "  # max_background_flushes=1000  # this line would be ignored\n"
1144       "  # max_background_compactions=2000 # so does this one\n"
1145       "  max_total_wal_size=1024  # keep_log_file_num=1000\n"
1146       "[CFOptions   \"default\"]  # column family must be specified\n"
1147       "                     # in the correct order\n"
1148       "  # if a section is blank, we will use the default\n";
1149 
1150   const std::string kTestFileName = "test-rocksdb-options.ini";
1151   env_->WriteToNewFile(kTestFileName, options_file_content);
1152   RocksDBOptionsParser parser;
1153   ASSERT_OK(
1154       parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1155 
1156   ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), db_opt));
1157   ASSERT_EQ(parser.NumColumnFamilies(), 1U);
1158   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
1159       *parser.GetCFOptions("default"), cf_opt));
1160 }
1161 
TEST_F(OptionsParserTest,ExtraSpace)1162 TEST_F(OptionsParserTest, ExtraSpace) {
1163   std::string options_file_content =
1164       "# This is a testing option string.\n"
1165       "# Currently we only support \"#\" styled comment.\n"
1166       "\n"
1167       "[      Version   ]\n"
1168       "  rocksdb_version     = 3.14.0      \n"
1169       "  options_file_version=1   # some comment\n"
1170       "[DBOptions  ]  # some comment\n"
1171       "max_open_files=12345   \n"
1172       "    max_background_flushes   =    301   \n"
1173       " max_total_wal_size     =   1024  # keep_log_file_num=1000\n"
1174       "        [CFOptions      \"default\"     ]\n"
1175       "  # if a section is blank, we will use the default\n";
1176 
1177   const std::string kTestFileName = "test-rocksdb-options.ini";
1178   env_->WriteToNewFile(kTestFileName, options_file_content);
1179   RocksDBOptionsParser parser;
1180   ASSERT_OK(
1181       parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1182 }
1183 
TEST_F(OptionsParserTest,MissingDBOptions)1184 TEST_F(OptionsParserTest, MissingDBOptions) {
1185   std::string options_file_content =
1186       "# This is a testing option string.\n"
1187       "# Currently we only support \"#\" styled comment.\n"
1188       "\n"
1189       "[Version]\n"
1190       "  rocksdb_version=3.14.0\n"
1191       "  options_file_version=1\n"
1192       "[CFOptions \"default\"]\n"
1193       "  # if a section is blank, we will use the default\n";
1194 
1195   const std::string kTestFileName = "test-rocksdb-options.ini";
1196   env_->WriteToNewFile(kTestFileName, options_file_content);
1197   RocksDBOptionsParser parser;
1198   ASSERT_NOK(
1199       parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1200 }
1201 
TEST_F(OptionsParserTest,DoubleDBOptions)1202 TEST_F(OptionsParserTest, DoubleDBOptions) {
1203   DBOptions db_opt;
1204   db_opt.max_open_files = 12345;
1205   db_opt.max_background_flushes = 301;
1206   db_opt.max_total_wal_size = 1024;
1207   ColumnFamilyOptions cf_opt;
1208 
1209   std::string options_file_content =
1210       "# This is a testing option string.\n"
1211       "# Currently we only support \"#\" styled comment.\n"
1212       "\n"
1213       "[Version]\n"
1214       "  rocksdb_version=3.14.0\n"
1215       "  options_file_version=1\n"
1216       "[DBOptions]\n"
1217       "  max_open_files=12345\n"
1218       "  max_background_flushes=301\n"
1219       "  max_total_wal_size=1024  # keep_log_file_num=1000\n"
1220       "[DBOptions]\n"
1221       "[CFOptions \"default\"]\n"
1222       "  # if a section is blank, we will use the default\n";
1223 
1224   const std::string kTestFileName = "test-rocksdb-options.ini";
1225   env_->WriteToNewFile(kTestFileName, options_file_content);
1226   RocksDBOptionsParser parser;
1227   ASSERT_NOK(
1228       parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1229 }
1230 
TEST_F(OptionsParserTest,NoDefaultCFOptions)1231 TEST_F(OptionsParserTest, NoDefaultCFOptions) {
1232   DBOptions db_opt;
1233   db_opt.max_open_files = 12345;
1234   db_opt.max_background_flushes = 301;
1235   db_opt.max_total_wal_size = 1024;
1236   ColumnFamilyOptions cf_opt;
1237 
1238   std::string options_file_content =
1239       "# This is a testing option string.\n"
1240       "# Currently we only support \"#\" styled comment.\n"
1241       "\n"
1242       "[Version]\n"
1243       "  rocksdb_version=3.14.0\n"
1244       "  options_file_version=1\n"
1245       "[DBOptions]\n"
1246       "  max_open_files=12345\n"
1247       "  max_background_flushes=301\n"
1248       "  max_total_wal_size=1024  # keep_log_file_num=1000\n"
1249       "[CFOptions \"something_else\"]\n"
1250       "  # if a section is blank, we will use the default\n";
1251 
1252   const std::string kTestFileName = "test-rocksdb-options.ini";
1253   env_->WriteToNewFile(kTestFileName, options_file_content);
1254   RocksDBOptionsParser parser;
1255   ASSERT_NOK(
1256       parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1257 }
1258 
TEST_F(OptionsParserTest,DefaultCFOptionsMustBeTheFirst)1259 TEST_F(OptionsParserTest, DefaultCFOptionsMustBeTheFirst) {
1260   DBOptions db_opt;
1261   db_opt.max_open_files = 12345;
1262   db_opt.max_background_flushes = 301;
1263   db_opt.max_total_wal_size = 1024;
1264   ColumnFamilyOptions cf_opt;
1265 
1266   std::string options_file_content =
1267       "# This is a testing option string.\n"
1268       "# Currently we only support \"#\" styled comment.\n"
1269       "\n"
1270       "[Version]\n"
1271       "  rocksdb_version=3.14.0\n"
1272       "  options_file_version=1\n"
1273       "[DBOptions]\n"
1274       "  max_open_files=12345\n"
1275       "  max_background_flushes=301\n"
1276       "  max_total_wal_size=1024  # keep_log_file_num=1000\n"
1277       "[CFOptions \"something_else\"]\n"
1278       "  # if a section is blank, we will use the default\n"
1279       "[CFOptions \"default\"]\n"
1280       "  # if a section is blank, we will use the default\n";
1281 
1282   const std::string kTestFileName = "test-rocksdb-options.ini";
1283   env_->WriteToNewFile(kTestFileName, options_file_content);
1284   RocksDBOptionsParser parser;
1285   ASSERT_NOK(
1286       parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1287 }
1288 
TEST_F(OptionsParserTest,DuplicateCFOptions)1289 TEST_F(OptionsParserTest, DuplicateCFOptions) {
1290   DBOptions db_opt;
1291   db_opt.max_open_files = 12345;
1292   db_opt.max_background_flushes = 301;
1293   db_opt.max_total_wal_size = 1024;
1294   ColumnFamilyOptions cf_opt;
1295 
1296   std::string options_file_content =
1297       "# This is a testing option string.\n"
1298       "# Currently we only support \"#\" styled comment.\n"
1299       "\n"
1300       "[Version]\n"
1301       "  rocksdb_version=3.14.0\n"
1302       "  options_file_version=1\n"
1303       "[DBOptions]\n"
1304       "  max_open_files=12345\n"
1305       "  max_background_flushes=301\n"
1306       "  max_total_wal_size=1024  # keep_log_file_num=1000\n"
1307       "[CFOptions \"default\"]\n"
1308       "[CFOptions \"something_else\"]\n"
1309       "[CFOptions \"something_else\"]\n";
1310 
1311   const std::string kTestFileName = "test-rocksdb-options.ini";
1312   env_->WriteToNewFile(kTestFileName, options_file_content);
1313   RocksDBOptionsParser parser;
1314   ASSERT_NOK(
1315       parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
1316 }
1317 
TEST_F(OptionsParserTest,IgnoreUnknownOptions)1318 TEST_F(OptionsParserTest, IgnoreUnknownOptions) {
1319   for (int case_id = 0; case_id < 5; case_id++) {
1320     DBOptions db_opt;
1321     db_opt.max_open_files = 12345;
1322     db_opt.max_background_flushes = 301;
1323     db_opt.max_total_wal_size = 1024;
1324     ColumnFamilyOptions cf_opt;
1325 
1326     std::string version_string;
1327     bool should_ignore = true;
1328     if (case_id == 0) {
1329       // same version
1330       should_ignore = false;
1331       version_string =
1332           ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR) + ".0";
1333     } else if (case_id == 1) {
1334       // higher minor version
1335       should_ignore = true;
1336       version_string =
1337           ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR + 1) + ".0";
1338     } else if (case_id == 2) {
1339       // higher major version.
1340       should_ignore = true;
1341       version_string = ToString(ROCKSDB_MAJOR + 1) + ".0.0";
1342     } else if (case_id == 3) {
1343       // lower minor version
1344 #if ROCKSDB_MINOR == 0
1345       continue;
1346 #else
1347       version_string =
1348           ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR - 1) + ".0";
1349       should_ignore = false;
1350 #endif
1351     } else {
1352       // lower major version
1353       should_ignore = false;
1354       version_string =
1355           ToString(ROCKSDB_MAJOR - 1) + "." + ToString(ROCKSDB_MINOR) + ".0";
1356     }
1357 
1358     std::string options_file_content =
1359         "# This is a testing option string.\n"
1360         "# Currently we only support \"#\" styled comment.\n"
1361         "\n"
1362         "[Version]\n"
1363         "  rocksdb_version=" +
1364         version_string +
1365         "\n"
1366         "  options_file_version=1\n"
1367         "[DBOptions]\n"
1368         "  max_open_files=12345\n"
1369         "  max_background_flushes=301\n"
1370         "  max_total_wal_size=1024  # keep_log_file_num=1000\n"
1371         "  unknown_db_option1=321\n"
1372         "  unknown_db_option2=false\n"
1373         "[CFOptions \"default\"]\n"
1374         "  unknown_cf_option1=hello\n"
1375         "[CFOptions \"something_else\"]\n"
1376         "  unknown_cf_option2=world\n"
1377         "  # if a section is blank, we will use the default\n";
1378 
1379     const std::string kTestFileName = "test-rocksdb-options.ini";
1380     env_->DeleteFile(kTestFileName);
1381     env_->WriteToNewFile(kTestFileName, options_file_content);
1382     RocksDBOptionsParser parser;
1383     ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(), false,
1384                             4096 /* readahead_size */));
1385     if (should_ignore) {
1386       ASSERT_OK(parser.Parse(kTestFileName, fs_.get(),
1387                              true /* ignore_unknown_options */,
1388                              4096 /* readahead_size */));
1389     } else {
1390       ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(),
1391                               true /* ignore_unknown_options */,
1392                               4096 /* readahead_size */));
1393     }
1394   }
1395 }
1396 
TEST_F(OptionsParserTest,ParseVersion)1397 TEST_F(OptionsParserTest, ParseVersion) {
1398   DBOptions db_opt;
1399   db_opt.max_open_files = 12345;
1400   db_opt.max_background_flushes = 301;
1401   db_opt.max_total_wal_size = 1024;
1402   ColumnFamilyOptions cf_opt;
1403 
1404   std::string file_template =
1405       "# This is a testing option string.\n"
1406       "# Currently we only support \"#\" styled comment.\n"
1407       "\n"
1408       "[Version]\n"
1409       "  rocksdb_version=3.13.1\n"
1410       "  options_file_version=%s\n"
1411       "[DBOptions]\n"
1412       "[CFOptions \"default\"]\n";
1413   const int kLength = 1000;
1414   char buffer[kLength];
1415   RocksDBOptionsParser parser;
1416 
1417   const std::vector<std::string> invalid_versions = {
1418       "a.b.c", "3.2.2b", "3.-12", "3. 1",  // only digits and dots are allowed
1419       "1.2.3.4",
1420       "1.2.3"  // can only contains at most one dot.
1421       "0",     // options_file_version must be at least one
1422       "3..2",
1423       ".", ".1.2",             // must have at least one digit before each dot
1424       "1.2.", "1.", "2.34."};  // must have at least one digit after each dot
1425   for (auto iv : invalid_versions) {
1426     snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str());
1427 
1428     parser.Reset();
1429     env_->WriteToNewFile(iv, buffer);
1430     ASSERT_NOK(parser.Parse(iv, fs_.get(), false, 0 /* readahead_size */));
1431   }
1432 
1433   const std::vector<std::string> valid_versions = {
1434       "1.232", "100", "3.12", "1", "12.3  ", "  1.25  "};
1435   for (auto vv : valid_versions) {
1436     snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str());
1437     parser.Reset();
1438     env_->WriteToNewFile(vv, buffer);
1439     ASSERT_OK(parser.Parse(vv, fs_.get(), false, 0 /* readahead_size */));
1440   }
1441 }
1442 
VerifyCFPointerTypedOptions(ColumnFamilyOptions * base_cf_opt,const ColumnFamilyOptions * new_cf_opt,const std::unordered_map<std::string,std::string> * new_cf_opt_map)1443 void VerifyCFPointerTypedOptions(
1444     ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt,
1445     const std::unordered_map<std::string, std::string>* new_cf_opt_map) {
1446   std::string name_buffer;
1447   ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1448                                                   new_cf_opt_map));
1449 
1450   // change the name of merge operator back-and-forth
1451   {
1452     auto* merge_operator = dynamic_cast<test::ChanglingMergeOperator*>(
1453         base_cf_opt->merge_operator.get());
1454     if (merge_operator != nullptr) {
1455       name_buffer = merge_operator->Name();
1456       // change the name  and expect non-ok status
1457       merge_operator->SetName("some-other-name");
1458       ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
1459           *base_cf_opt, *new_cf_opt, new_cf_opt_map));
1460       // change the name back and expect ok status
1461       merge_operator->SetName(name_buffer);
1462       ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1463                                                       new_cf_opt_map));
1464     }
1465   }
1466 
1467   // change the name of the compaction filter factory back-and-forth
1468   {
1469     auto* compaction_filter_factory =
1470         dynamic_cast<test::ChanglingCompactionFilterFactory*>(
1471             base_cf_opt->compaction_filter_factory.get());
1472     if (compaction_filter_factory != nullptr) {
1473       name_buffer = compaction_filter_factory->Name();
1474       // change the name and expect non-ok status
1475       compaction_filter_factory->SetName("some-other-name");
1476       ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
1477           *base_cf_opt, *new_cf_opt, new_cf_opt_map));
1478       // change the name back and expect ok status
1479       compaction_filter_factory->SetName(name_buffer);
1480       ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1481                                                       new_cf_opt_map));
1482     }
1483   }
1484 
1485   // test by setting compaction_filter to nullptr
1486   {
1487     auto* tmp_compaction_filter = base_cf_opt->compaction_filter;
1488     if (tmp_compaction_filter != nullptr) {
1489       base_cf_opt->compaction_filter = nullptr;
1490       // set compaction_filter to nullptr and expect non-ok status
1491       ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
1492           *base_cf_opt, *new_cf_opt, new_cf_opt_map));
1493       // set the value back and expect ok status
1494       base_cf_opt->compaction_filter = tmp_compaction_filter;
1495       ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1496                                                       new_cf_opt_map));
1497     }
1498   }
1499 
1500   // test by setting table_factory to nullptr
1501   {
1502     auto tmp_table_factory = base_cf_opt->table_factory;
1503     if (tmp_table_factory != nullptr) {
1504       base_cf_opt->table_factory.reset();
1505       // set table_factory to nullptr and expect non-ok status
1506       ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
1507           *base_cf_opt, *new_cf_opt, new_cf_opt_map));
1508       // set the value back and expect ok status
1509       base_cf_opt->table_factory = tmp_table_factory;
1510       ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1511                                                       new_cf_opt_map));
1512     }
1513   }
1514 
1515   // test by setting memtable_factory to nullptr
1516   {
1517     auto tmp_memtable_factory = base_cf_opt->memtable_factory;
1518     if (tmp_memtable_factory != nullptr) {
1519       base_cf_opt->memtable_factory.reset();
1520       // set memtable_factory to nullptr and expect non-ok status
1521       ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
1522           *base_cf_opt, *new_cf_opt, new_cf_opt_map));
1523       // set the value back and expect ok status
1524       base_cf_opt->memtable_factory = tmp_memtable_factory;
1525       ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
1526                                                       new_cf_opt_map));
1527     }
1528   }
1529 }
1530 
TEST_F(OptionsParserTest,Readahead)1531 TEST_F(OptionsParserTest, Readahead) {
1532   DBOptions base_db_opt;
1533   std::vector<ColumnFamilyOptions> base_cf_opts;
1534   base_cf_opts.emplace_back();
1535   base_cf_opts.emplace_back();
1536 
1537   std::string one_mb_string = std::string(1024 * 1024, 'x');
1538   std::vector<std::string> cf_names = {"default", one_mb_string};
1539   const std::string kOptionsFileName = "test-persisted-options.ini";
1540 
1541   ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts,
1542                                   kOptionsFileName, fs_.get()));
1543 
1544   uint64_t file_size = 0;
1545   ASSERT_OK(env_->GetFileSize(kOptionsFileName, &file_size));
1546   assert(file_size > 0);
1547 
1548   RocksDBOptionsParser parser;
1549 
1550   env_->num_seq_file_read_ = 0;
1551   size_t readahead_size = 128 * 1024;
1552 
1553   ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size));
1554   ASSERT_EQ(env_->num_seq_file_read_.load(),
1555             (file_size - 1) / readahead_size + 1);
1556 
1557   env_->num_seq_file_read_.store(0);
1558   readahead_size = 1024 * 1024;
1559   ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size));
1560   ASSERT_EQ(env_->num_seq_file_read_.load(),
1561             (file_size - 1) / readahead_size + 1);
1562 
1563   // Tiny readahead. 8 KB is read each time.
1564   env_->num_seq_file_read_.store(0);
1565   ASSERT_OK(
1566       parser.Parse(kOptionsFileName, fs_.get(), false, 1 /* readahead_size */));
1567   ASSERT_GE(env_->num_seq_file_read_.load(), file_size / (8 * 1024));
1568   ASSERT_LT(env_->num_seq_file_read_.load(), file_size / (8 * 1024) * 2);
1569 
1570   // Disable readahead means 512KB readahead.
1571   env_->num_seq_file_read_.store(0);
1572   ASSERT_OK(
1573       parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
1574   ASSERT_GE(env_->num_seq_file_read_.load(),
1575             (file_size - 1) / (512 * 1024) + 1);
1576 }
1577 
TEST_F(OptionsParserTest,DumpAndParse)1578 TEST_F(OptionsParserTest, DumpAndParse) {
1579   DBOptions base_db_opt;
1580   std::vector<ColumnFamilyOptions> base_cf_opts;
1581   std::vector<std::string> cf_names = {"default", "cf1", "cf2", "cf3",
1582                                        "c:f:4:4:4"
1583                                        "p\\i\\k\\a\\chu\\\\\\",
1584                                        "###rocksdb#1-testcf#2###"};
1585   const int num_cf = static_cast<int>(cf_names.size());
1586   Random rnd(302);
1587   test::RandomInitDBOptions(&base_db_opt, &rnd);
1588   base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG";
1589 
1590   BlockBasedTableOptions special_bbto;
1591   special_bbto.cache_index_and_filter_blocks = true;
1592   special_bbto.block_size = 999999;
1593 
1594   for (int c = 0; c < num_cf; ++c) {
1595     ColumnFamilyOptions cf_opt;
1596     Random cf_rnd(0xFB + c);
1597     test::RandomInitCFOptions(&cf_opt, base_db_opt, &cf_rnd);
1598     if (c < 4) {
1599       cf_opt.prefix_extractor.reset(test::RandomSliceTransform(&rnd, c));
1600     }
1601     if (c < 3) {
1602       cf_opt.table_factory.reset(test::RandomTableFactory(&rnd, c));
1603     } else if (c == 4) {
1604       cf_opt.table_factory.reset(NewBlockBasedTableFactory(special_bbto));
1605     }
1606     base_cf_opts.emplace_back(cf_opt);
1607   }
1608 
1609   const std::string kOptionsFileName = "test-persisted-options.ini";
1610   ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts,
1611                                   kOptionsFileName, fs_.get()));
1612 
1613   RocksDBOptionsParser parser;
1614   ASSERT_OK(
1615       parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
1616 
1617   // Make sure block-based table factory options was deserialized correctly
1618   std::shared_ptr<TableFactory> ttf = (*parser.cf_opts())[4].table_factory;
1619   ASSERT_EQ(BlockBasedTableFactory::kName, std::string(ttf->Name()));
1620   const BlockBasedTableOptions& parsed_bbto =
1621       static_cast<BlockBasedTableFactory*>(ttf.get())->table_options();
1622   ASSERT_EQ(special_bbto.block_size, parsed_bbto.block_size);
1623   ASSERT_EQ(special_bbto.cache_index_and_filter_blocks,
1624             parsed_bbto.cache_index_and_filter_blocks);
1625 
1626   ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
1627       base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get()));
1628 
1629   ASSERT_OK(
1630       RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), base_db_opt));
1631   for (int c = 0; c < num_cf; ++c) {
1632     const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
1633     ASSERT_NE(cf_opt, nullptr);
1634     ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
1635         base_cf_opts[c], *cf_opt, &(parser.cf_opt_maps()->at(c))));
1636   }
1637 
1638   // Further verify pointer-typed options
1639   for (int c = 0; c < num_cf; ++c) {
1640     const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
1641     ASSERT_NE(cf_opt, nullptr);
1642     VerifyCFPointerTypedOptions(&base_cf_opts[c], cf_opt,
1643                                 &(parser.cf_opt_maps()->at(c)));
1644   }
1645 
1646   ASSERT_EQ(parser.GetCFOptions("does not exist"), nullptr);
1647 
1648   base_db_opt.max_open_files++;
1649   ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
1650       base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get()));
1651 
1652   for (int c = 0; c < num_cf; ++c) {
1653     if (base_cf_opts[c].compaction_filter) {
1654       delete base_cf_opts[c].compaction_filter;
1655     }
1656   }
1657 }
1658 
TEST_F(OptionsParserTest,DifferentDefault)1659 TEST_F(OptionsParserTest, DifferentDefault) {
1660   const std::string kOptionsFileName = "test-persisted-options.ini";
1661 
1662   ColumnFamilyOptions cf_level_opts;
1663   ASSERT_EQ(CompactionPri::kMinOverlappingRatio, cf_level_opts.compaction_pri);
1664   cf_level_opts.OptimizeLevelStyleCompaction();
1665 
1666   ColumnFamilyOptions cf_univ_opts;
1667   cf_univ_opts.OptimizeUniversalStyleCompaction();
1668 
1669   ASSERT_OK(PersistRocksDBOptions(DBOptions(), {"default", "universal"},
1670                                   {cf_level_opts, cf_univ_opts},
1671                                   kOptionsFileName, fs_.get()));
1672 
1673   RocksDBOptionsParser parser;
1674   ASSERT_OK(
1675       parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
1676 
1677   {
1678     Options old_default_opts;
1679     old_default_opts.OldDefaults();
1680     ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base);
1681     ASSERT_EQ(5000, old_default_opts.max_open_files);
1682     ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
1683     ASSERT_EQ(WALRecoveryMode::kTolerateCorruptedTailRecords,
1684               old_default_opts.wal_recovery_mode);
1685   }
1686   {
1687     Options old_default_opts;
1688     old_default_opts.OldDefaults(4, 6);
1689     ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base);
1690     ASSERT_EQ(5000, old_default_opts.max_open_files);
1691   }
1692   {
1693     Options old_default_opts;
1694     old_default_opts.OldDefaults(4, 7);
1695     ASSERT_NE(10 * 1048576, old_default_opts.max_bytes_for_level_base);
1696     ASSERT_NE(4, old_default_opts.table_cache_numshardbits);
1697     ASSERT_EQ(5000, old_default_opts.max_open_files);
1698     ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
1699   }
1700   {
1701     ColumnFamilyOptions old_default_cf_opts;
1702     old_default_cf_opts.OldDefaults();
1703     ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
1704     ASSERT_EQ(4 << 20, old_default_cf_opts.write_buffer_size);
1705     ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
1706     ASSERT_EQ(0, old_default_cf_opts.soft_pending_compaction_bytes_limit);
1707     ASSERT_EQ(0, old_default_cf_opts.hard_pending_compaction_bytes_limit);
1708     ASSERT_EQ(CompactionPri::kByCompensatedSize,
1709               old_default_cf_opts.compaction_pri);
1710   }
1711   {
1712     ColumnFamilyOptions old_default_cf_opts;
1713     old_default_cf_opts.OldDefaults(4, 6);
1714     ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
1715     ASSERT_EQ(CompactionPri::kByCompensatedSize,
1716               old_default_cf_opts.compaction_pri);
1717   }
1718   {
1719     ColumnFamilyOptions old_default_cf_opts;
1720     old_default_cf_opts.OldDefaults(4, 7);
1721     ASSERT_NE(2 * 1048576, old_default_cf_opts.target_file_size_base);
1722     ASSERT_EQ(CompactionPri::kByCompensatedSize,
1723               old_default_cf_opts.compaction_pri);
1724   }
1725   {
1726     Options old_default_opts;
1727     old_default_opts.OldDefaults(5, 1);
1728     ASSERT_EQ(2 * 1024U * 1024U, old_default_opts.delayed_write_rate);
1729   }
1730   {
1731     Options old_default_opts;
1732     old_default_opts.OldDefaults(5, 2);
1733     ASSERT_EQ(16 * 1024U * 1024U, old_default_opts.delayed_write_rate);
1734     ASSERT_TRUE(old_default_opts.compaction_pri ==
1735                 CompactionPri::kByCompensatedSize);
1736   }
1737   {
1738     Options old_default_opts;
1739     old_default_opts.OldDefaults(5, 18);
1740     ASSERT_TRUE(old_default_opts.compaction_pri ==
1741                 CompactionPri::kByCompensatedSize);
1742   }
1743 
1744   Options small_opts;
1745   small_opts.OptimizeForSmallDb();
1746   ASSERT_EQ(2 << 20, small_opts.write_buffer_size);
1747   ASSERT_EQ(5000, small_opts.max_open_files);
1748 }
1749 
1750 class OptionsSanityCheckTest : public OptionsParserTest {
1751  public:
OptionsSanityCheckTest()1752   OptionsSanityCheckTest() {}
1753 
1754  protected:
SanityCheckCFOptions(const ColumnFamilyOptions & cf_opts,OptionsSanityCheckLevel level)1755   Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts,
1756                               OptionsSanityCheckLevel level) {
1757     return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
1758         DBOptions(), {"default"}, {cf_opts}, kOptionsFileName, fs_.get(),
1759         level);
1760   }
1761 
PersistCFOptions(const ColumnFamilyOptions & cf_opts)1762   Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) {
1763     Status s = env_->DeleteFile(kOptionsFileName);
1764     if (!s.ok()) {
1765       return s;
1766     }
1767     return PersistRocksDBOptions(DBOptions(), {"default"}, {cf_opts},
1768                                  kOptionsFileName, fs_.get());
1769   }
1770 
1771   const std::string kOptionsFileName = "OPTIONS";
1772 };
1773 
TEST_F(OptionsSanityCheckTest,SanityCheck)1774 TEST_F(OptionsSanityCheckTest, SanityCheck) {
1775   ColumnFamilyOptions opts;
1776   Random rnd(301);
1777 
1778   // default ColumnFamilyOptions
1779   {
1780     ASSERT_OK(PersistCFOptions(opts));
1781     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1782   }
1783 
1784   // prefix_extractor
1785   {
1786     // Okay to change prefix_extractor form nullptr to non-nullptr
1787     ASSERT_EQ(opts.prefix_extractor.get(), nullptr);
1788     opts.prefix_extractor.reset(NewCappedPrefixTransform(10));
1789     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1790     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1791 
1792     // persist the change
1793     ASSERT_OK(PersistCFOptions(opts));
1794     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1795 
1796     // use same prefix extractor but with different parameter
1797     opts.prefix_extractor.reset(NewCappedPrefixTransform(15));
1798     // expect pass only in kSanityLevelLooselyCompatible
1799     ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1800     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1801     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1802 
1803     // repeat the test with FixedPrefixTransform
1804     opts.prefix_extractor.reset(NewFixedPrefixTransform(10));
1805     ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1806     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1807     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1808 
1809     // persist the change of prefix_extractor
1810     ASSERT_OK(PersistCFOptions(opts));
1811     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1812 
1813     // use same prefix extractor but with different parameter
1814     opts.prefix_extractor.reset(NewFixedPrefixTransform(15));
1815     // expect pass only in kSanityLevelLooselyCompatible
1816     ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1817     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1818     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1819 
1820     // Change prefix extractor from non-nullptr to nullptr
1821     opts.prefix_extractor.reset();
1822     // expect pass as it's safe to change prefix_extractor
1823     // from non-null to null
1824     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1825     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1826   }
1827   // persist the change
1828   ASSERT_OK(PersistCFOptions(opts));
1829   ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1830 
1831   // table_factory
1832   {
1833     for (int tb = 0; tb <= 2; ++tb) {
1834       // change the table factory
1835       opts.table_factory.reset(test::RandomTableFactory(&rnd, tb));
1836       ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1837       ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1838 
1839       // persist the change
1840       ASSERT_OK(PersistCFOptions(opts));
1841       ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1842     }
1843   }
1844 
1845   // merge_operator
1846   {
1847     // Test when going from nullptr -> merge operator
1848     opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
1849     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1850     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1851 
1852     // persist the change
1853     ASSERT_OK(PersistCFOptions(opts));
1854     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1855 
1856     for (int test = 0; test < 5; ++test) {
1857       // change the merge operator
1858       opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
1859       ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1860       ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1861 
1862       // persist the change
1863       ASSERT_OK(PersistCFOptions(opts));
1864       ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1865     }
1866 
1867     // Test when going from merge operator -> nullptr
1868     opts.merge_operator = nullptr;
1869     ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1870     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
1871 
1872     // persist the change
1873     ASSERT_OK(PersistCFOptions(opts));
1874     ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1875   }
1876 
1877   // compaction_filter
1878   {
1879     for (int test = 0; test < 5; ++test) {
1880       // change the compaction filter
1881       opts.compaction_filter = test::RandomCompactionFilter(&rnd);
1882       ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1883       ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1884 
1885       // persist the change
1886       ASSERT_OK(PersistCFOptions(opts));
1887       ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1888       delete opts.compaction_filter;
1889       opts.compaction_filter = nullptr;
1890     }
1891   }
1892 
1893   // compaction_filter_factory
1894   {
1895     for (int test = 0; test < 5; ++test) {
1896       // change the compaction filter factory
1897       opts.compaction_filter_factory.reset(
1898           test::RandomCompactionFilterFactory(&rnd));
1899       ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1900       ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
1901 
1902       // persist the change
1903       ASSERT_OK(PersistCFOptions(opts));
1904       ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
1905     }
1906   }
1907 }
1908 
1909 namespace {
IsEscapedString(const std::string & str)1910 bool IsEscapedString(const std::string& str) {
1911   for (size_t i = 0; i < str.size(); ++i) {
1912     if (str[i] == '\\') {
1913       // since we already handle those two consecutive '\'s in
1914       // the next if-then branch, any '\' appear at the end
1915       // of an escaped string in such case is not valid.
1916       if (i == str.size() - 1) {
1917         return false;
1918       }
1919       if (str[i + 1] == '\\') {
1920         // if there're two consecutive '\'s, skip the second one.
1921         i++;
1922         continue;
1923       }
1924       switch (str[i + 1]) {
1925         case ':':
1926         case '\\':
1927         case '#':
1928           continue;
1929         default:
1930           // if true, '\' together with str[i + 1] is not a valid escape.
1931           if (UnescapeChar(str[i + 1]) == str[i + 1]) {
1932             return false;
1933           }
1934       }
1935     } else if (isSpecialChar(str[i]) && (i == 0 || str[i - 1] != '\\')) {
1936       return false;
1937     }
1938   }
1939   return true;
1940 }
1941 }  // namespace
1942 
TEST_F(OptionsParserTest,IntegerParsing)1943 TEST_F(OptionsParserTest, IntegerParsing) {
1944   ASSERT_EQ(ParseUint64("18446744073709551615"), 18446744073709551615U);
1945   ASSERT_EQ(ParseUint32("4294967295"), 4294967295U);
1946   ASSERT_EQ(ParseSizeT("18446744073709551615"), 18446744073709551615U);
1947   ASSERT_EQ(ParseInt64("9223372036854775807"), 9223372036854775807);
1948   ASSERT_EQ(ParseInt64("-9223372036854775808"), port::kMinInt64);
1949   ASSERT_EQ(ParseInt32("2147483647"), 2147483647);
1950   ASSERT_EQ(ParseInt32("-2147483648"), port::kMinInt32);
1951   ASSERT_EQ(ParseInt("-32767"), -32767);
1952   ASSERT_EQ(ParseDouble("-1.234567"), -1.234567);
1953 }
1954 
TEST_F(OptionsParserTest,EscapeOptionString)1955 TEST_F(OptionsParserTest, EscapeOptionString) {
1956   ASSERT_EQ(UnescapeOptionString(
1957                 "This is a test string with \\# \\: and \\\\ escape chars."),
1958             "This is a test string with # : and \\ escape chars.");
1959 
1960   ASSERT_EQ(
1961       EscapeOptionString("This is a test string with # : and \\ escape chars."),
1962       "This is a test string with \\# \\: and \\\\ escape chars.");
1963 
1964   std::string readible_chars =
1965       "A String like this \"1234567890-=_)(*&^%$#@!ertyuiop[]{POIU"
1966       "YTREWQasdfghjkl;':LKJHGFDSAzxcvbnm,.?>"
1967       "<MNBVCXZ\\\" should be okay to \\#\\\\\\:\\#\\#\\#\\ "
1968       "be serialized and deserialized";
1969 
1970   std::string escaped_string = EscapeOptionString(readible_chars);
1971   ASSERT_TRUE(IsEscapedString(escaped_string));
1972   // This two transformations should be canceled and should output
1973   // the original input.
1974   ASSERT_EQ(UnescapeOptionString(escaped_string), readible_chars);
1975 
1976   std::string all_chars;
1977   for (unsigned char c = 0;; ++c) {
1978     all_chars += c;
1979     if (c == 255) {
1980       break;
1981     }
1982   }
1983   escaped_string = EscapeOptionString(all_chars);
1984   ASSERT_TRUE(IsEscapedString(escaped_string));
1985   ASSERT_EQ(UnescapeOptionString(escaped_string), all_chars);
1986 
1987   ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment(
1988                 "     A simple statement with a comment.  # like this :)"),
1989             "A simple statement with a comment.");
1990 
1991   ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment(
1992                 "Escape \\# and # comment together   ."),
1993             "Escape \\# and");
1994 }
1995 #endif  // !ROCKSDB_LITE
1996 }  // namespace ROCKSDB_NAMESPACE
1997 
main(int argc,char ** argv)1998 int main(int argc, char** argv) {
1999   ::testing::InitGoogleTest(&argc, argv);
2000 #ifdef GFLAGS
2001   ParseCommandLineFlags(&argc, &argv, true);
2002 #endif  // GFLAGS
2003   return RUN_ALL_TESTS();
2004 }
2005