1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 // This source code is licensed under both the GPLv2 (found in the 3 // COPYING file in the root directory) and Apache 2.0 License 4 // (found in the LICENSE.Apache file in the root directory). 5 6 #pragma once 7 8 #include <string> 9 #include <unordered_map> 10 #include <vector> 11 12 #include "rocksdb/db.h" 13 #include "rocksdb/options.h" 14 #include "rocksdb/table.h" 15 16 namespace ROCKSDB_NAMESPACE { 17 18 #ifndef ROCKSDB_LITE 19 // The following set of functions provide a way to construct RocksDB Options 20 // from a string or a string-to-string map. Here're the general rule of 21 // setting option values from strings by type. Some RocksDB types are also 22 // supported in these APIs. Please refer to the comment of the function itself 23 // to find more information about how to config those RocksDB types. 24 // 25 // * Strings: 26 // Strings will be used as values directly without any truncating or 27 // trimming. 28 // 29 // * Booleans: 30 // - "true" or "1" => true 31 // - "false" or "0" => false. 32 // [Example]: 33 // - {"optimize_filters_for_hits", "1"} in GetColumnFamilyOptionsFromMap, or 34 // - "optimize_filters_for_hits=true" in GetColumnFamilyOptionsFromString. 35 // 36 // * Integers: 37 // Integers are converted directly from string, in addition to the following 38 // units that we support: 39 // - 'k' or 'K' => 2^10 40 // - 'm' or 'M' => 2^20 41 // - 'g' or 'G' => 2^30 42 // - 't' or 'T' => 2^40 // only for unsigned int with sufficient bits. 43 // [Example]: 44 // - {"arena_block_size", "19G"} in GetColumnFamilyOptionsFromMap, or 45 // - "arena_block_size=19G" in GetColumnFamilyOptionsFromString. 46 // 47 // * Doubles / Floating Points: 48 // Doubles / Floating Points are converted directly from string. Note that 49 // currently we do not support units. 50 // [Example]: 51 // - {"hard_rate_limit", "2.1"} in GetColumnFamilyOptionsFromMap, or 52 // - "hard_rate_limit=2.1" in GetColumnFamilyOptionsFromString. 53 // * Array / Vectors: 54 // An array is specified by a list of values, where ':' is used as 55 // the delimiter to separate each value. 56 // [Example]: 57 // - {"compression_per_level", "kNoCompression:kSnappyCompression"} 58 // in GetColumnFamilyOptionsFromMap, or 59 // - "compression_per_level=kNoCompression:kSnappyCompression" in 60 // GetColumnFamilyOptionsFromMapString 61 // * Enums: 62 // The valid values of each enum are identical to the names of its constants. 63 // [Example]: 64 // - CompressionType: valid values are "kNoCompression", 65 // "kSnappyCompression", "kZlibCompression", "kBZip2Compression", ... 66 // - CompactionStyle: valid values are "kCompactionStyleLevel", 67 // "kCompactionStyleUniversal", "kCompactionStyleFIFO", and 68 // "kCompactionStyleNone". 69 // 70 71 // Take a default ColumnFamilyOptions "base_options" in addition to a 72 // map "opts_map" of option name to option value to construct the new 73 // ColumnFamilyOptions "new_options". 74 // 75 // Below are the instructions of how to config some non-primitive-typed 76 // options in ColumnFOptions: 77 // 78 // * table_factory: 79 // table_factory can be configured using our custom nested-option syntax. 80 // 81 // {option_a=value_a; option_b=value_b; option_c=value_c; ... } 82 // 83 // A nested option is enclosed by two curly braces, within which there are 84 // multiple option assignments. Each assignment is of the form 85 // "variable_name=value;". 86 // 87 // Currently we support the following types of TableFactory: 88 // - BlockBasedTableFactory: 89 // Use name "block_based_table_factory" to initialize table_factory with 90 // BlockBasedTableFactory. Its BlockBasedTableFactoryOptions can be 91 // configured using the nested-option syntax. 92 // [Example]: 93 // * {"block_based_table_factory", "{block_cache=1M;block_size=4k;}"} 94 // is equivalent to assigning table_factory with a BlockBasedTableFactory 95 // that has 1M LRU block-cache with block size equals to 4k: 96 // ColumnFamilyOptions cf_opt; 97 // BlockBasedTableOptions blk_opt; 98 // blk_opt.block_cache = NewLRUCache(1 * 1024 * 1024); 99 // blk_opt.block_size = 4 * 1024; 100 // cf_opt.table_factory.reset(NewBlockBasedTableFactory(blk_opt)); 101 // - PlainTableFactory: 102 // Use name "plain_table_factory" to initialize table_factory with 103 // PlainTableFactory. Its PlainTableFactoryOptions can be configured using 104 // the nested-option syntax. 105 // [Example]: 106 // * {"plain_table_factory", "{user_key_len=66;bloom_bits_per_key=20;}"} 107 // 108 // * memtable_factory: 109 // Use "memtable" to config memtable_factory. Here are the supported 110 // memtable factories: 111 // - SkipList: 112 // Pass "skip_list:<lookahead>" to config memtable to use SkipList, 113 // or simply "skip_list" to use the default SkipList. 114 // [Example]: 115 // * {"memtable", "skip_list:5"} is equivalent to setting 116 // memtable to SkipListFactory(5). 117 // - PrefixHash: 118 // Pass "prfix_hash:<hash_bucket_count>" to config memtable 119 // to use PrefixHash, or simply "prefix_hash" to use the default 120 // PrefixHash. 121 // [Example]: 122 // * {"memtable", "prefix_hash:1000"} is equivalent to setting 123 // memtable to NewHashSkipListRepFactory(hash_bucket_count). 124 // - HashLinkedList: 125 // Pass "hash_linkedlist:<hash_bucket_count>" to config memtable 126 // to use HashLinkedList, or simply "hash_linkedlist" to use the default 127 // HashLinkedList. 128 // [Example]: 129 // * {"memtable", "hash_linkedlist:1000"} is equivalent to 130 // setting memtable to NewHashLinkListRepFactory(1000). 131 // - VectorRepFactory: 132 // Pass "vector:<count>" to config memtable to use VectorRepFactory, 133 // or simply "vector" to use the default Vector memtable. 134 // [Example]: 135 // * {"memtable", "vector:1024"} is equivalent to setting memtable 136 // to VectorRepFactory(1024). 137 // - HashCuckooRepFactory: 138 // Pass "cuckoo:<write_buffer_size>" to use HashCuckooRepFactory with the 139 // specified write buffer size, or simply "cuckoo" to use the default 140 // HashCuckooRepFactory. 141 // [Example]: 142 // * {"memtable", "cuckoo:1024"} is equivalent to setting memtable 143 // to NewHashCuckooRepFactory(1024). 144 // 145 // * compression_opts: 146 // Use "compression_opts" to config compression_opts. The value format 147 // is of the form "<window_bits>:<level>:<strategy>:<max_dict_bytes>". 148 // [Example]: 149 // * {"compression_opts", "4:5:6:7"} is equivalent to setting: 150 // ColumnFamilyOptions cf_opt; 151 // cf_opt.compression_opts.window_bits = 4; 152 // cf_opt.compression_opts.level = 5; 153 // cf_opt.compression_opts.strategy = 6; 154 // cf_opt.compression_opts.max_dict_bytes = 7; 155 // 156 // @param base_options the default options of the output "new_options". 157 // @param opts_map an option name to value map for specifying how "new_options" 158 // should be set. 159 // @param new_options the resulting options based on "base_options" with the 160 // change specified in "opts_map". 161 // @param input_strings_escaped when set to true, each escaped characters 162 // prefixed by '\' in the values of the opts_map will be further converted 163 // back to the raw string before assigning to the associated options. 164 // @param ignore_unknown_options when set to true, unknown options are ignored 165 // instead of resulting in an unknown-option error. 166 // @return Status::OK() on success. Otherwise, a non-ok status indicating 167 // error will be returned, and "new_options" will be set to "base_options". 168 Status GetColumnFamilyOptionsFromMap( 169 const ColumnFamilyOptions& base_options, 170 const std::unordered_map<std::string, std::string>& opts_map, 171 ColumnFamilyOptions* new_options, bool input_strings_escaped = false, 172 bool ignore_unknown_options = false); 173 174 // Take a default DBOptions "base_options" in addition to a 175 // map "opts_map" of option name to option value to construct the new 176 // DBOptions "new_options". 177 // 178 // Below are the instructions of how to config some non-primitive-typed 179 // options in DBOptions: 180 // 181 // * rate_limiter_bytes_per_sec: 182 // RateLimiter can be configured directly by specifying its bytes_per_sec. 183 // [Example]: 184 // - Passing {"rate_limiter_bytes_per_sec", "1024"} is equivalent to 185 // passing NewGenericRateLimiter(1024) to rate_limiter_bytes_per_sec. 186 // 187 // @param base_options the default options of the output "new_options". 188 // @param opts_map an option name to value map for specifying how "new_options" 189 // should be set. 190 // @param new_options the resulting options based on "base_options" with the 191 // change specified in "opts_map". 192 // @param input_strings_escaped when set to true, each escaped characters 193 // prefixed by '\' in the values of the opts_map will be further converted 194 // back to the raw string before assigning to the associated options. 195 // @param ignore_unknown_options when set to true, unknown options are ignored 196 // instead of resulting in an unknown-option error. 197 // @return Status::OK() on success. Otherwise, a non-ok status indicating 198 // error will be returned, and "new_options" will be set to "base_options". 199 Status GetDBOptionsFromMap( 200 const DBOptions& base_options, 201 const std::unordered_map<std::string, std::string>& opts_map, 202 DBOptions* new_options, bool input_strings_escaped = false, 203 bool ignore_unknown_options = false); 204 205 // Take a default BlockBasedTableOptions "table_options" in addition to a 206 // map "opts_map" of option name to option value to construct the new 207 // BlockBasedTableOptions "new_table_options". 208 // 209 // Below are the instructions of how to config some non-primitive-typed 210 // options in BlockBasedTableOptions: 211 // 212 // * filter_policy: 213 // We currently only support the following FilterPolicy in the convenience 214 // functions: 215 // - BloomFilter: use "bloomfilter:[bits_per_key]:[use_block_based_builder]" 216 // to specify BloomFilter. The above string is equivalent to calling 217 // NewBloomFilterPolicy(bits_per_key, use_block_based_builder). 218 // [Example]: 219 // - Pass {"filter_policy", "bloomfilter:4:true"} in 220 // GetBlockBasedTableOptionsFromMap to use a BloomFilter with 4-bits 221 // per key and use_block_based_builder enabled. 222 // 223 // * block_cache / block_cache_compressed: 224 // We currently only support LRU cache in the GetOptions API. The LRU 225 // cache can be set by directly specifying its size. 226 // [Example]: 227 // - Passing {"block_cache", "1M"} in GetBlockBasedTableOptionsFromMap is 228 // equivalent to setting block_cache using NewLRUCache(1024 * 1024). 229 // 230 // @param table_options the default options of the output "new_table_options". 231 // @param opts_map an option name to value map for specifying how 232 // "new_table_options" should be set. 233 // @param new_table_options the resulting options based on "table_options" 234 // with the change specified in "opts_map". 235 // @param input_strings_escaped when set to true, each escaped characters 236 // prefixed by '\' in the values of the opts_map will be further converted 237 // back to the raw string before assigning to the associated options. 238 // @param ignore_unknown_options when set to true, unknown options are ignored 239 // instead of resulting in an unknown-option error. 240 // @return Status::OK() on success. Otherwise, a non-ok status indicating 241 // error will be returned, and "new_table_options" will be set to 242 // "table_options". 243 Status GetBlockBasedTableOptionsFromMap( 244 const BlockBasedTableOptions& table_options, 245 const std::unordered_map<std::string, std::string>& opts_map, 246 BlockBasedTableOptions* new_table_options, 247 bool input_strings_escaped = false, bool ignore_unknown_options = false); 248 249 // Take a default PlainTableOptions "table_options" in addition to a 250 // map "opts_map" of option name to option value to construct the new 251 // PlainTableOptions "new_table_options". 252 // 253 // @param table_options the default options of the output "new_table_options". 254 // @param opts_map an option name to value map for specifying how 255 // "new_table_options" should be set. 256 // @param new_table_options the resulting options based on "table_options" 257 // with the change specified in "opts_map". 258 // @param input_strings_escaped when set to true, each escaped characters 259 // prefixed by '\' in the values of the opts_map will be further converted 260 // back to the raw string before assigning to the associated options. 261 // @param ignore_unknown_options when set to true, unknown options are ignored 262 // instead of resulting in an unknown-option error. 263 // @return Status::OK() on success. Otherwise, a non-ok status indicating 264 // error will be returned, and "new_table_options" will be set to 265 // "table_options". 266 Status GetPlainTableOptionsFromMap( 267 const PlainTableOptions& table_options, 268 const std::unordered_map<std::string, std::string>& opts_map, 269 PlainTableOptions* new_table_options, bool input_strings_escaped = false, 270 bool ignore_unknown_options = false); 271 272 // Take a string representation of option names and values, apply them into the 273 // base_options, and return the new options as a result. The string has the 274 // following format: 275 // "write_buffer_size=1024;max_write_buffer_number=2" 276 // Nested options config is also possible. For example, you can define 277 // BlockBasedTableOptions as part of the string for block-based table factory: 278 // "write_buffer_size=1024;block_based_table_factory={block_size=4k};" 279 // "max_write_buffer_num=2" 280 Status GetColumnFamilyOptionsFromString(const ColumnFamilyOptions& base_options, 281 const std::string& opts_str, 282 ColumnFamilyOptions* new_options); 283 284 Status GetDBOptionsFromString(const DBOptions& base_options, 285 const std::string& opts_str, 286 DBOptions* new_options); 287 288 Status GetStringFromDBOptions(std::string* opts_str, 289 const DBOptions& db_options, 290 const std::string& delimiter = "; "); 291 292 Status GetStringFromColumnFamilyOptions(std::string* opts_str, 293 const ColumnFamilyOptions& cf_options, 294 const std::string& delimiter = "; "); 295 296 Status GetStringFromCompressionType(std::string* compression_str, 297 CompressionType compression_type); 298 299 std::vector<CompressionType> GetSupportedCompressions(); 300 301 Status GetBlockBasedTableOptionsFromString( 302 const BlockBasedTableOptions& table_options, const std::string& opts_str, 303 BlockBasedTableOptions* new_table_options); 304 305 Status GetPlainTableOptionsFromString(const PlainTableOptions& table_options, 306 const std::string& opts_str, 307 PlainTableOptions* new_table_options); 308 309 Status GetMemTableRepFactoryFromString( 310 const std::string& opts_str, 311 std::unique_ptr<MemTableRepFactory>* new_mem_factory); 312 313 Status GetOptionsFromString(const Options& base_options, 314 const std::string& opts_str, Options* new_options); 315 316 Status StringToMap(const std::string& opts_str, 317 std::unordered_map<std::string, std::string>* opts_map); 318 319 // Request stopping background work, if wait is true wait until it's done 320 void CancelAllBackgroundWork(DB* db, bool wait = false); 321 322 // Delete files which are entirely in the given range 323 // Could leave some keys in the range which are in files which are not 324 // entirely in the range. Also leaves L0 files regardless of whether they're 325 // in the range. 326 // Snapshots before the delete might not see the data in the given range. 327 Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family, 328 const Slice* begin, const Slice* end, 329 bool include_end = true); 330 331 // Delete files in multiple ranges at once 332 // Delete files in a lot of ranges one at a time can be slow, use this API for 333 // better performance in that case. 334 Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family, 335 const RangePtr* ranges, size_t n, 336 bool include_end = true); 337 338 // Verify the checksum of file 339 Status VerifySstFileChecksum(const Options& options, 340 const EnvOptions& env_options, 341 const std::string& file_path); 342 343 // Verify the checksum of file 344 Status VerifySstFileChecksum(const Options& options, 345 const EnvOptions& env_options, 346 const ReadOptions& read_options, 347 const std::string& file_path); 348 349 #endif // ROCKSDB_LITE 350 351 } // namespace ROCKSDB_NAMESPACE 352