1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 // This source code is licensed under both the GPLv2 (found in the 3 // COPYING file in the root directory) and Apache 2.0 License 4 // (found in the LICENSE.Apache file in the root directory). 5 6 #pragma once 7 8 #include <string> 9 #include <unordered_map> 10 #include <vector> 11 12 #include "rocksdb/compression_type.h" 13 #include "rocksdb/db.h" 14 #include "rocksdb/status.h" 15 #include "rocksdb/table.h" 16 17 namespace ROCKSDB_NAMESPACE { 18 class Env; 19 class Logger; 20 class ObjectRegistry; 21 22 struct ColumnFamilyOptions; 23 struct DBOptions; 24 struct Options; 25 26 // ConfigOptions containing the parameters/controls for 27 // comparing objects and converting to/from strings. 28 // These settings control how the methods 29 // treat errors (e.g. ignore_unknown_objects), the format 30 // of the serialization (e.g. delimiter), and how to compare 31 // options (sanity_level). 32 struct ConfigOptions { 33 // Constructs a new ConfigOptions with a new object registry. 34 // This method should only be used when a DBOptions is not available, 35 // else registry settings may be lost 36 ConfigOptions(); 37 38 // Constructs a new ConfigOptions using the settings from 39 // the input DBOptions. Currently constructs a new object registry. 40 explicit ConfigOptions(const DBOptions&); 41 42 // This enum defines the RocksDB options sanity level. 43 enum SanityLevel : unsigned char { 44 kSanityLevelNone = 0x01, // Performs no sanity check at all. 45 // Performs minimum check to ensure the RocksDB instance can be 46 // opened without corrupting / mis-interpreting the data. 47 kSanityLevelLooselyCompatible = 0x02, 48 // Perform exact match sanity check. 49 kSanityLevelExactMatch = 0xFF, 50 }; 51 52 enum Depth { 53 kDepthDefault, // Traverse nested options that are not flagged as "shallow" 54 kDepthShallow, // Do not traverse into any nested options 55 kDepthDetailed, // Traverse nested options, overriding the options shallow 56 // setting 57 }; 58 59 // When true, any unused options will be ignored and OK will be returned 60 bool ignore_unknown_options = false; 61 62 // When true, any unsupported options will be ignored and OK will be returned 63 bool ignore_unsupported_options = true; 64 65 // If the strings are escaped (old-style?) 66 bool input_strings_escaped = true; 67 68 // Whether or not to invoke PrepareOptions after configure is called. 69 bool invoke_prepare_options = true; 70 71 // Options can be marked as Mutable (OptionTypeInfo::IsMutable()) or not. 72 // When "mutable_options_only=false", all options are evaluated. 73 // When "mutable_options_only="true", any option not marked as Mutable is 74 // either ignored (in the case of string/equals methods) or results in an 75 // error (in the case of Configure). 76 bool mutable_options_only = false; 77 78 // The separator between options when converting to a string 79 std::string delimiter = ";"; 80 81 // Controls how to traverse options during print/match stages 82 Depth depth = Depth::kDepthDefault; 83 84 // Controls how options are serialized 85 // Controls how pedantic the comparison must be for equivalency 86 SanityLevel sanity_level = SanityLevel::kSanityLevelExactMatch; 87 // `file_readahead_size` is used for readahead for the option file. 88 size_t file_readahead_size = 512 * 1024; 89 90 // The environment to use for this option 91 Env* env = Env::Default(); 92 93 #ifndef ROCKSDB_LITE 94 // The object registry to use for this options 95 std::shared_ptr<ObjectRegistry> registry; 96 #endif 97 IsShallowConfigOptions98 bool IsShallow() const { return depth == Depth::kDepthShallow; } IsDetailedConfigOptions99 bool IsDetailed() const { return depth == Depth::kDepthDetailed; } 100 IsCheckDisabledConfigOptions101 bool IsCheckDisabled() const { 102 return sanity_level == SanityLevel::kSanityLevelNone; 103 } 104 IsCheckEnabledConfigOptions105 bool IsCheckEnabled(SanityLevel level) const { 106 return (level > SanityLevel::kSanityLevelNone && level <= sanity_level); 107 } 108 }; 109 110 #ifndef ROCKSDB_LITE 111 112 // The following set of functions provide a way to construct RocksDB Options 113 // from a string or a string-to-string map. Here is the general rule of 114 // setting option values from strings by type. Some RocksDB types are also 115 // supported in these APIs. Please refer to the comment of the function itself 116 // to find more information about how to config those RocksDB types. 117 // 118 // * Strings: 119 // Strings will be used as values directly without any truncating or 120 // trimming. 121 // 122 // * Booleans: 123 // - "true" or "1" => true 124 // - "false" or "0" => false. 125 // [Example]: 126 // - {"optimize_filters_for_hits", "1"} in GetColumnFamilyOptionsFromMap, or 127 // - "optimize_filters_for_hits=true" in GetColumnFamilyOptionsFromString. 128 // 129 // * Integers: 130 // Integers are converted directly from string, in addition to the following 131 // units that we support: 132 // - 'k' or 'K' => 2^10 133 // - 'm' or 'M' => 2^20 134 // - 'g' or 'G' => 2^30 135 // - 't' or 'T' => 2^40 // only for unsigned int with sufficient bits. 136 // [Example]: 137 // - {"arena_block_size", "19G"} in GetColumnFamilyOptionsFromMap, or 138 // - "arena_block_size=19G" in GetColumnFamilyOptionsFromString. 139 // 140 // * Doubles / Floating Points: 141 // Doubles / Floating Points are converted directly from string. Note that 142 // currently we do not support units. 143 // [Example]: 144 // - {"hard_rate_limit", "2.1"} in GetColumnFamilyOptionsFromMap, or 145 // - "hard_rate_limit=2.1" in GetColumnFamilyOptionsFromString. 146 // * Array / Vectors: 147 // An array is specified by a list of values, where ':' is used as 148 // the delimiter to separate each value. 149 // [Example]: 150 // - {"compression_per_level", "kNoCompression:kSnappyCompression"} 151 // in GetColumnFamilyOptionsFromMap, or 152 // - "compression_per_level=kNoCompression:kSnappyCompression" in 153 // GetColumnFamilyOptionsFromMapString 154 // * Enums: 155 // The valid values of each enum are identical to the names of its constants. 156 // [Example]: 157 // - CompressionType: valid values are "kNoCompression", 158 // "kSnappyCompression", "kZlibCompression", "kBZip2Compression", ... 159 // - CompactionStyle: valid values are "kCompactionStyleLevel", 160 // "kCompactionStyleUniversal", "kCompactionStyleFIFO", and 161 // "kCompactionStyleNone". 162 // 163 164 // Take a default ColumnFamilyOptions "base_options" in addition to a 165 // map "opts_map" of option name to option value to construct the new 166 // ColumnFamilyOptions "new_options". 167 // 168 // Below are the instructions of how to config some non-primitive-typed 169 // options in ColumnFamilyOptions: 170 // 171 // * table_factory: 172 // table_factory can be configured using our custom nested-option syntax. 173 // 174 // {option_a=value_a; option_b=value_b; option_c=value_c; ... } 175 // 176 // A nested option is enclosed by two curly braces, within which there are 177 // multiple option assignments. Each assignment is of the form 178 // "variable_name=value;". 179 // 180 // Currently we support the following types of TableFactory: 181 // - BlockBasedTableFactory: 182 // Use name "block_based_table_factory" to initialize table_factory with 183 // BlockBasedTableFactory. Its BlockBasedTableFactoryOptions can be 184 // configured using the nested-option syntax. 185 // [Example]: 186 // * {"block_based_table_factory", "{block_cache=1M;block_size=4k;}"} 187 // is equivalent to assigning table_factory with a BlockBasedTableFactory 188 // that has 1M LRU block-cache with block size equals to 4k: 189 // ColumnFamilyOptions cf_opt; 190 // BlockBasedTableOptions blk_opt; 191 // blk_opt.block_cache = NewLRUCache(1 * 1024 * 1024); 192 // blk_opt.block_size = 4 * 1024; 193 // cf_opt.table_factory.reset(NewBlockBasedTableFactory(blk_opt)); 194 // - PlainTableFactory: 195 // Use name "plain_table_factory" to initialize table_factory with 196 // PlainTableFactory. Its PlainTableFactoryOptions can be configured using 197 // the nested-option syntax. 198 // [Example]: 199 // * {"plain_table_factory", "{user_key_len=66;bloom_bits_per_key=20;}"} 200 // 201 // * memtable_factory: 202 // Use "memtable" to config memtable_factory. Here are the supported 203 // memtable factories: 204 // - SkipList: 205 // Pass "skip_list:<lookahead>" to config memtable to use SkipList, 206 // or simply "skip_list" to use the default SkipList. 207 // [Example]: 208 // * {"memtable", "skip_list:5"} is equivalent to setting 209 // memtable to SkipListFactory(5). 210 // - PrefixHash: 211 // Pass "prefix_hash:<hash_bucket_count>" to config memtable 212 // to use PrefixHash, or simply "prefix_hash" to use the default 213 // PrefixHash. 214 // [Example]: 215 // * {"memtable", "prefix_hash:1000"} is equivalent to setting 216 // memtable to NewHashSkipListRepFactory(hash_bucket_count). 217 // - HashLinkedList: 218 // Pass "hash_linkedlist:<hash_bucket_count>" to config memtable 219 // to use HashLinkedList, or simply "hash_linkedlist" to use the default 220 // HashLinkedList. 221 // [Example]: 222 // * {"memtable", "hash_linkedlist:1000"} is equivalent to 223 // setting memtable to NewHashLinkListRepFactory(1000). 224 // - VectorRepFactory: 225 // Pass "vector:<count>" to config memtable to use VectorRepFactory, 226 // or simply "vector" to use the default Vector memtable. 227 // [Example]: 228 // * {"memtable", "vector:1024"} is equivalent to setting memtable 229 // to VectorRepFactory(1024). 230 // 231 // * compression_opts: 232 // Use "compression_opts" to config compression_opts. The value format 233 // is of the form "<window_bits>:<level>:<strategy>:<max_dict_bytes>". 234 // [Example]: 235 // * {"compression_opts", "4:5:6:7"} is equivalent to setting: 236 // ColumnFamilyOptions cf_opt; 237 // cf_opt.compression_opts.window_bits = 4; 238 // cf_opt.compression_opts.level = 5; 239 // cf_opt.compression_opts.strategy = 6; 240 // cf_opt.compression_opts.max_dict_bytes = 7; 241 // 242 // The GetColumnFamilyOptionsFromMap(ConfigOptions, ...) should be used; the 243 // alternative signature may be deprecated in a future release. The equivalent 244 // functionality can be achieved by setting the corresponding options in 245 // the ConfigOptions parameter. 246 // 247 // @param config_options controls how the map is processed. 248 // @param base_options the default options of the output "new_options". 249 // @param opts_map an option name to value map for specifying how "new_options" 250 // should be set. 251 // @param new_options the resulting options based on "base_options" with the 252 // change specified in "opts_map". 253 // @param input_strings_escaped when set to true, each escaped characters 254 // prefixed by '\' in the values of the opts_map will be further converted 255 // back to the raw string before assigning to the associated options. 256 // @param ignore_unknown_options when set to true, unknown options are ignored 257 // instead of resulting in an unknown-option error. 258 // @return Status::OK() on success. Otherwise, a non-ok status indicating 259 // error will be returned, and "new_options" will be set to "base_options". 260 // @return Status::NotFound means the one (or more) of the option name in 261 // the opts_map is not valid for this option 262 // @return Status::NotSupported means we do not know how to parse one of the 263 // value for this option 264 // @return Status::InvalidArgument means the one of the option values is not 265 // valid for this option. 266 Status GetColumnFamilyOptionsFromMap( 267 const ConfigOptions& config_options, 268 const ColumnFamilyOptions& base_options, 269 const std::unordered_map<std::string, std::string>& opts_map, 270 ColumnFamilyOptions* new_options); 271 Status GetColumnFamilyOptionsFromMap( 272 const ColumnFamilyOptions& base_options, 273 const std::unordered_map<std::string, std::string>& opts_map, 274 ColumnFamilyOptions* new_options, bool input_strings_escaped = false, 275 bool ignore_unknown_options = false); 276 277 // Take a default DBOptions "base_options" in addition to a 278 // map "opts_map" of option name to option value to construct the new 279 // DBOptions "new_options". 280 // 281 // Below are the instructions of how to config some non-primitive-typed 282 // options in DBOptions: 283 // 284 // * rate_limiter_bytes_per_sec: 285 // RateLimiter can be configured directly by specifying its bytes_per_sec. 286 // [Example]: 287 // - Passing {"rate_limiter_bytes_per_sec", "1024"} is equivalent to 288 // passing NewGenericRateLimiter(1024) to rate_limiter_bytes_per_sec. 289 // 290 // The GetDBOptionsFromMap(ConfigOptions, ...) should be used; the 291 // alternative signature may be deprecated in a future release. The equivalent 292 // functionality can be achieved by setting the corresponding options in 293 // the ConfigOptions parameter. 294 // 295 // @param config_options controls how the map is processed. 296 // @param base_options the default options of the output "new_options". 297 // @param opts_map an option name to value map for specifying how "new_options" 298 // should be set. 299 // @param new_options the resulting options based on "base_options" with the 300 // change specified in "opts_map". 301 // @param input_strings_escaped when set to true, each escaped characters 302 // prefixed by '\' in the values of the opts_map will be further converted 303 // back to the raw string before assigning to the associated options. 304 // @param ignore_unknown_options when set to true, unknown options are ignored 305 // instead of resulting in an unknown-option error. 306 // @return Status::OK() on success. Otherwise, a non-ok status indicating 307 // error will be returned, and "new_options" will be set to "base_options". 308 // @return Status::NotFound means the one (or more) of the option name in 309 // the opts_map is not valid for this option 310 // @return Status::NotSupported means we do not know how to parse one of the 311 // value for this option 312 // @return Status::InvalidArgument means the one of the option values is not 313 // valid for this option. 314 Status GetDBOptionsFromMap( 315 const ConfigOptions& cfg_options, const DBOptions& base_options, 316 const std::unordered_map<std::string, std::string>& opts_map, 317 DBOptions* new_options); 318 Status GetDBOptionsFromMap( 319 const DBOptions& base_options, 320 const std::unordered_map<std::string, std::string>& opts_map, 321 DBOptions* new_options, bool input_strings_escaped = false, 322 bool ignore_unknown_options = false); 323 324 // Take a default BlockBasedTableOptions "table_options" in addition to a 325 // map "opts_map" of option name to option value to construct the new 326 // BlockBasedTableOptions "new_table_options". 327 // 328 // Below are the instructions of how to config some non-primitive-typed 329 // options in BlockBasedTableOptions: 330 // 331 // * filter_policy: 332 // We currently only support the following FilterPolicy in the convenience 333 // functions: 334 // - BloomFilter: use "bloomfilter:[bits_per_key]:[use_block_based_builder]" 335 // to specify BloomFilter. The above string is equivalent to calling 336 // NewBloomFilterPolicy(bits_per_key, use_block_based_builder). 337 // [Example]: 338 // - Pass {"filter_policy", "bloomfilter:4:true"} in 339 // GetBlockBasedTableOptionsFromMap to use a BloomFilter with 4-bits 340 // per key and use_block_based_builder enabled. 341 // 342 // * block_cache / block_cache_compressed: 343 // We currently only support LRU cache in the GetOptions API. The LRU 344 // cache can be set by directly specifying its size. 345 // [Example]: 346 // - Passing {"block_cache", "1M"} in GetBlockBasedTableOptionsFromMap is 347 // equivalent to setting block_cache using NewLRUCache(1024 * 1024). 348 // 349 // The GetBlockBasedTableOptionsFromMap(ConfigOptions, ...) should be used; 350 // the alternative signature may be deprecated in a future release. The 351 // equivalent functionality can be achieved by setting the corresponding 352 // options in the ConfigOptions parameter. 353 // 354 // @param config_options controls how the map is processed. 355 // @param table_options the default options of the output "new_table_options". 356 // @param opts_map an option name to value map for specifying how 357 // "new_table_options" should be set. 358 // @param new_table_options the resulting options based on "table_options" 359 // with the change specified in "opts_map". 360 // @param input_strings_escaped when set to true, each escaped characters 361 // prefixed by '\' in the values of the opts_map will be further converted 362 // back to the raw string before assigning to the associated options. 363 // @param ignore_unknown_options when set to true, unknown options are ignored 364 // instead of resulting in an unknown-option error. 365 // @return Status::OK() on success. Otherwise, a non-ok status indicating 366 // error will be returned, and "new_table_options" will be set to 367 // "table_options". 368 Status GetBlockBasedTableOptionsFromMap( 369 const ConfigOptions& config_options, 370 const BlockBasedTableOptions& table_options, 371 const std::unordered_map<std::string, std::string>& opts_map, 372 BlockBasedTableOptions* new_table_options); 373 Status GetBlockBasedTableOptionsFromMap( 374 const BlockBasedTableOptions& table_options, 375 const std::unordered_map<std::string, std::string>& opts_map, 376 BlockBasedTableOptions* new_table_options, 377 bool input_strings_escaped = false, bool ignore_unknown_options = false); 378 379 // Take a default PlainTableOptions "table_options" in addition to a 380 // map "opts_map" of option name to option value to construct the new 381 // PlainTableOptions "new_table_options". 382 // 383 // The GetPlainTableOptionsFromMap(ConfigOptions, ...) should be used; the 384 // alternative signature may be deprecated in a future release. The equivalent 385 // functionality can be achieved by setting the corresponding options in 386 // the ConfigOptions parameter. 387 // 388 // @param config_options controls how the map is processed. 389 // @param table_options the default options of the output "new_table_options". 390 // @param opts_map an option name to value map for specifying how 391 // "new_table_options" should be set. 392 // @param new_table_options the resulting options based on "table_options" 393 // with the change specified in "opts_map". 394 // @param input_strings_escaped when set to true, each escaped characters 395 // prefixed by '\' in the values of the opts_map will be further converted 396 // back to the raw string before assigning to the associated options. 397 // @param ignore_unknown_options when set to true, unknown options are ignored 398 // instead of resulting in an unknown-option error. 399 // @return Status::OK() on success. Otherwise, a non-ok status indicating 400 // error will be returned, and "new_table_options" will be set to 401 // "table_options". 402 Status GetPlainTableOptionsFromMap( 403 const ConfigOptions& config_options, const PlainTableOptions& table_options, 404 const std::unordered_map<std::string, std::string>& opts_map, 405 PlainTableOptions* new_table_options); 406 Status GetPlainTableOptionsFromMap( 407 const PlainTableOptions& table_options, 408 const std::unordered_map<std::string, std::string>& opts_map, 409 PlainTableOptions* new_table_options, bool input_strings_escaped = false, 410 bool ignore_unknown_options = false); 411 412 // Take a string representation of option names and values, apply them into the 413 // base_options, and return the new options as a result. The string has the 414 // following format: 415 // "write_buffer_size=1024;max_write_buffer_number=2" 416 // Nested options config is also possible. For example, you can define 417 // BlockBasedTableOptions as part of the string for block-based table factory: 418 // "write_buffer_size=1024;block_based_table_factory={block_size=4k};" 419 // "max_write_buffer_num=2" 420 // 421 // 422 // The GetColumnFamilyOptionsFromString(ConfigOptions, ...) should be used; the 423 // alternative signature may be deprecated in a future release. The equivalent 424 // functionality can be achieved by setting the corresponding options in 425 // the ConfigOptions parameter. 426 Status GetColumnFamilyOptionsFromString(const ConfigOptions& config_options, 427 const ColumnFamilyOptions& base_options, 428 const std::string& opts_str, 429 ColumnFamilyOptions* new_options); 430 Status GetColumnFamilyOptionsFromString(const ColumnFamilyOptions& base_options, 431 const std::string& opts_str, 432 ColumnFamilyOptions* new_options); 433 434 Status GetDBOptionsFromString(const ConfigOptions& config_options, 435 const DBOptions& base_options, 436 const std::string& opts_str, 437 DBOptions* new_options); 438 439 Status GetDBOptionsFromString(const DBOptions& base_options, 440 const std::string& opts_str, 441 DBOptions* new_options); 442 443 Status GetStringFromDBOptions(const ConfigOptions& config_options, 444 const DBOptions& db_options, 445 std::string* opts_str); 446 447 Status GetStringFromDBOptions(std::string* opts_str, 448 const DBOptions& db_options, 449 const std::string& delimiter = "; "); 450 451 Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options, 452 const ColumnFamilyOptions& cf_options, 453 std::string* opts_str); 454 Status GetStringFromColumnFamilyOptions(std::string* opts_str, 455 const ColumnFamilyOptions& cf_options, 456 const std::string& delimiter = "; "); 457 Status GetStringFromCompressionType(std::string* compression_str, 458 CompressionType compression_type); 459 460 std::vector<CompressionType> GetSupportedCompressions(); 461 462 Status GetBlockBasedTableOptionsFromString( 463 const BlockBasedTableOptions& table_options, const std::string& opts_str, 464 BlockBasedTableOptions* new_table_options); 465 Status GetBlockBasedTableOptionsFromString( 466 const ConfigOptions& config_options, 467 const BlockBasedTableOptions& table_options, const std::string& opts_str, 468 BlockBasedTableOptions* new_table_options); 469 470 Status GetPlainTableOptionsFromString(const PlainTableOptions& table_options, 471 const std::string& opts_str, 472 PlainTableOptions* new_table_options); 473 Status GetPlainTableOptionsFromString(const ConfigOptions& config_options, 474 const PlainTableOptions& table_options, 475 const std::string& opts_str, 476 PlainTableOptions* new_table_options); 477 478 Status GetMemTableRepFactoryFromString( 479 const std::string& opts_str, 480 std::unique_ptr<MemTableRepFactory>* new_mem_factory); 481 482 Status GetOptionsFromString(const Options& base_options, 483 const std::string& opts_str, Options* new_options); 484 Status GetOptionsFromString(const ConfigOptions& config_options, 485 const Options& base_options, 486 const std::string& opts_str, Options* new_options); 487 488 Status StringToMap(const std::string& opts_str, 489 std::unordered_map<std::string, std::string>* opts_map); 490 491 // Request stopping background work, if wait is true wait until it's done 492 void CancelAllBackgroundWork(DB* db, bool wait = false); 493 494 // Delete files which are entirely in the given range 495 // Could leave some keys in the range which are in files which are not 496 // entirely in the range. Also leaves L0 files regardless of whether they're 497 // in the range. 498 // Snapshots before the delete might not see the data in the given range. 499 Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family, 500 const Slice* begin, const Slice* end, 501 bool include_end = true); 502 503 // Delete files in multiple ranges at once 504 // Delete files in a lot of ranges one at a time can be slow, use this API for 505 // better performance in that case. 506 Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family, 507 const RangePtr* ranges, size_t n, 508 bool include_end = true); 509 510 // Verify the checksum of file 511 Status VerifySstFileChecksum(const Options& options, 512 const EnvOptions& env_options, 513 const std::string& file_path); 514 515 // Verify the checksum of file 516 Status VerifySstFileChecksum(const Options& options, 517 const EnvOptions& env_options, 518 const ReadOptions& read_options, 519 const std::string& file_path); 520 #endif // ROCKSDB_LITE 521 522 } // namespace ROCKSDB_NAMESPACE 523