1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 // This source code is licensed under both the GPLv2 (found in the 3 // COPYING file in the root directory) and Apache 2.0 License 4 // (found in the LICENSE.Apache file in the root directory). 5 6 #pragma once 7 8 #include <string> 9 #include <vector> 10 11 #include "db/dbformat.h" 12 #include "options/db_options.h" 13 #include "rocksdb/options.h" 14 #include "util/compression.h" 15 16 namespace ROCKSDB_NAMESPACE { 17 18 // ImmutableCFOptions is a data struct used by RocksDB internal. It contains a 19 // subset of Options that should not be changed during the entire lifetime 20 // of DB. Raw pointers defined in this struct do not have ownership to the data 21 // they point to. Options contains std::shared_ptr to these data. 22 struct ImmutableCFOptions { 23 public: kNameImmutableCFOptions24 static const char* kName() { return "ImmutableCFOptions"; } 25 explicit ImmutableCFOptions(); 26 explicit ImmutableCFOptions(const ColumnFamilyOptions& cf_options); 27 28 CompactionStyle compaction_style; 29 30 CompactionPri compaction_pri; 31 32 const Comparator* user_comparator; 33 InternalKeyComparator internal_comparator; // Only in Immutable 34 35 std::shared_ptr<MergeOperator> merge_operator; 36 37 const CompactionFilter* compaction_filter; 38 39 std::shared_ptr<CompactionFilterFactory> compaction_filter_factory; 40 41 int min_write_buffer_number_to_merge; 42 43 int max_write_buffer_number_to_maintain; 44 45 int64_t max_write_buffer_size_to_maintain; 46 47 bool inplace_update_support; 48 49 UpdateStatus (*inplace_callback)(char* existing_value, 50 uint32_t* existing_value_size, 51 Slice delta_value, 52 std::string* merged_value); 53 54 std::shared_ptr<MemTableRepFactory> memtable_factory; 55 56 std::shared_ptr<TableFactory> table_factory; 57 58 Options::TablePropertiesCollectorFactories 59 table_properties_collector_factories; 60 61 // This options is required by PlainTableReader. May need to move it 62 // to PlainTableOptions just like bloom_bits_per_key 63 uint32_t bloom_locality; 64 65 bool purge_redundant_kvs_while_flush; 66 67 std::vector<CompressionType> compression_per_level; 68 69 bool level_compaction_dynamic_level_bytes; 70 71 int num_levels; 72 73 bool optimize_filters_for_hits; 74 75 bool force_consistency_checks; 76 77 std::shared_ptr<const SliceTransform> 78 memtable_insert_with_hint_prefix_extractor; 79 80 std::vector<DbPath> cf_paths; 81 82 std::shared_ptr<ConcurrentTaskLimiter> compaction_thread_limiter; 83 84 std::shared_ptr<SstPartitionerFactory> sst_partitioner_factory; 85 }; 86 87 struct ImmutableOptions : public ImmutableDBOptions, public ImmutableCFOptions { 88 explicit ImmutableOptions(); 89 explicit ImmutableOptions(const Options& options); 90 91 ImmutableOptions(const DBOptions& db_options, 92 const ColumnFamilyOptions& cf_options); 93 94 ImmutableOptions(const ImmutableDBOptions& db_options, 95 const ImmutableCFOptions& cf_options); 96 97 ImmutableOptions(const DBOptions& db_options, 98 const ImmutableCFOptions& cf_options); 99 100 ImmutableOptions(const ImmutableDBOptions& db_options, 101 const ColumnFamilyOptions& cf_options); 102 }; 103 104 struct MutableCFOptions { kNameMutableCFOptions105 static const char* kName() { return "MutableCFOptions"; } MutableCFOptionsMutableCFOptions106 explicit MutableCFOptions(const ColumnFamilyOptions& options) 107 : write_buffer_size(options.write_buffer_size), 108 max_write_buffer_number(options.max_write_buffer_number), 109 arena_block_size(options.arena_block_size), 110 memtable_prefix_bloom_size_ratio( 111 options.memtable_prefix_bloom_size_ratio), 112 memtable_whole_key_filtering(options.memtable_whole_key_filtering), 113 memtable_huge_page_size(options.memtable_huge_page_size), 114 max_successive_merges(options.max_successive_merges), 115 inplace_update_num_locks(options.inplace_update_num_locks), 116 prefix_extractor(options.prefix_extractor), 117 disable_auto_compactions(options.disable_auto_compactions), 118 soft_pending_compaction_bytes_limit( 119 options.soft_pending_compaction_bytes_limit), 120 hard_pending_compaction_bytes_limit( 121 options.hard_pending_compaction_bytes_limit), 122 level0_file_num_compaction_trigger( 123 options.level0_file_num_compaction_trigger), 124 level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger), 125 level0_stop_writes_trigger(options.level0_stop_writes_trigger), 126 max_compaction_bytes(options.max_compaction_bytes), 127 target_file_size_base(options.target_file_size_base), 128 target_file_size_multiplier(options.target_file_size_multiplier), 129 max_bytes_for_level_base(options.max_bytes_for_level_base), 130 max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier), 131 ttl(options.ttl), 132 periodic_compaction_seconds(options.periodic_compaction_seconds), 133 max_bytes_for_level_multiplier_additional( 134 options.max_bytes_for_level_multiplier_additional), 135 compaction_options_fifo(options.compaction_options_fifo), 136 compaction_options_universal(options.compaction_options_universal), 137 enable_blob_files(options.enable_blob_files), 138 min_blob_size(options.min_blob_size), 139 blob_file_size(options.blob_file_size), 140 blob_compression_type(options.blob_compression_type), 141 enable_blob_garbage_collection(options.enable_blob_garbage_collection), 142 blob_garbage_collection_age_cutoff( 143 options.blob_garbage_collection_age_cutoff), 144 blob_garbage_collection_force_threshold( 145 options.blob_garbage_collection_force_threshold), 146 max_sequential_skip_in_iterations( 147 options.max_sequential_skip_in_iterations), 148 check_flush_compaction_key_order( 149 options.check_flush_compaction_key_order), 150 paranoid_file_checks(options.paranoid_file_checks), 151 report_bg_io_stats(options.report_bg_io_stats), 152 compression(options.compression), 153 bottommost_compression(options.bottommost_compression), 154 compression_opts(options.compression_opts), 155 bottommost_compression_opts(options.bottommost_compression_opts), 156 bottommost_temperature(options.bottommost_temperature), 157 sample_for_compression( 158 options.sample_for_compression) { // TODO: is 0 fine here? 159 RefreshDerivedOptions(options.num_levels, options.compaction_style); 160 } 161 MutableCFOptionsMutableCFOptions162 MutableCFOptions() 163 : write_buffer_size(0), 164 max_write_buffer_number(0), 165 arena_block_size(0), 166 memtable_prefix_bloom_size_ratio(0), 167 memtable_whole_key_filtering(false), 168 memtable_huge_page_size(0), 169 max_successive_merges(0), 170 inplace_update_num_locks(0), 171 prefix_extractor(nullptr), 172 disable_auto_compactions(false), 173 soft_pending_compaction_bytes_limit(0), 174 hard_pending_compaction_bytes_limit(0), 175 level0_file_num_compaction_trigger(0), 176 level0_slowdown_writes_trigger(0), 177 level0_stop_writes_trigger(0), 178 max_compaction_bytes(0), 179 target_file_size_base(0), 180 target_file_size_multiplier(0), 181 max_bytes_for_level_base(0), 182 max_bytes_for_level_multiplier(0), 183 ttl(0), 184 periodic_compaction_seconds(0), 185 compaction_options_fifo(), 186 enable_blob_files(false), 187 min_blob_size(0), 188 blob_file_size(0), 189 blob_compression_type(kNoCompression), 190 enable_blob_garbage_collection(false), 191 blob_garbage_collection_age_cutoff(0.0), 192 blob_garbage_collection_force_threshold(0.0), 193 max_sequential_skip_in_iterations(0), 194 check_flush_compaction_key_order(true), 195 paranoid_file_checks(false), 196 report_bg_io_stats(false), 197 compression(Snappy_Supported() ? kSnappyCompression : kNoCompression), 198 bottommost_compression(kDisableCompressionOption), 199 bottommost_temperature(Temperature::kUnknown), 200 sample_for_compression(0) {} 201 202 explicit MutableCFOptions(const Options& options); 203 204 // Must be called after any change to MutableCFOptions 205 void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style); 206 RefreshDerivedOptionsMutableCFOptions207 void RefreshDerivedOptions(const ImmutableCFOptions& ioptions) { 208 RefreshDerivedOptions(ioptions.num_levels, ioptions.compaction_style); 209 } 210 MaxBytesMultiplerAdditionalMutableCFOptions211 int MaxBytesMultiplerAdditional(int level) const { 212 if (level >= 213 static_cast<int>(max_bytes_for_level_multiplier_additional.size())) { 214 return 1; 215 } 216 return max_bytes_for_level_multiplier_additional[level]; 217 } 218 219 void Dump(Logger* log) const; 220 221 // Memtable related options 222 size_t write_buffer_size; 223 int max_write_buffer_number; 224 size_t arena_block_size; 225 double memtable_prefix_bloom_size_ratio; 226 bool memtable_whole_key_filtering; 227 size_t memtable_huge_page_size; 228 size_t max_successive_merges; 229 size_t inplace_update_num_locks; 230 std::shared_ptr<const SliceTransform> prefix_extractor; 231 232 // Compaction related options 233 bool disable_auto_compactions; 234 uint64_t soft_pending_compaction_bytes_limit; 235 uint64_t hard_pending_compaction_bytes_limit; 236 int level0_file_num_compaction_trigger; 237 int level0_slowdown_writes_trigger; 238 int level0_stop_writes_trigger; 239 uint64_t max_compaction_bytes; 240 uint64_t target_file_size_base; 241 int target_file_size_multiplier; 242 uint64_t max_bytes_for_level_base; 243 double max_bytes_for_level_multiplier; 244 uint64_t ttl; 245 uint64_t periodic_compaction_seconds; 246 std::vector<int> max_bytes_for_level_multiplier_additional; 247 CompactionOptionsFIFO compaction_options_fifo; 248 CompactionOptionsUniversal compaction_options_universal; 249 250 // Blob file related options 251 bool enable_blob_files; 252 uint64_t min_blob_size; 253 uint64_t blob_file_size; 254 CompressionType blob_compression_type; 255 bool enable_blob_garbage_collection; 256 double blob_garbage_collection_age_cutoff; 257 double blob_garbage_collection_force_threshold; 258 259 // Misc options 260 uint64_t max_sequential_skip_in_iterations; 261 bool check_flush_compaction_key_order; 262 bool paranoid_file_checks; 263 bool report_bg_io_stats; 264 CompressionType compression; 265 CompressionType bottommost_compression; 266 CompressionOptions compression_opts; 267 CompressionOptions bottommost_compression_opts; 268 // TODO this experimental option isn't made configurable 269 // through strings yet. 270 Temperature bottommost_temperature; 271 272 uint64_t sample_for_compression; 273 274 // Derived options 275 // Per-level target file size. 276 std::vector<uint64_t> max_file_size; 277 }; 278 279 uint64_t MultiplyCheckOverflow(uint64_t op1, double op2); 280 281 // Get the max file size in a given level. 282 uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options, 283 int level, CompactionStyle compaction_style, int base_level = 1, 284 bool level_compaction_dynamic_level_bytes = false); 285 286 // Get the max size of an L0 file for which we will pin its meta-blocks when 287 // `pin_l0_filter_and_index_blocks_in_cache` is set. 288 size_t MaxFileSizeForL0MetaPin(const MutableCFOptions& cf_options); 289 290 #ifndef ROCKSDB_LITE 291 Status GetStringFromMutableCFOptions(const ConfigOptions& config_options, 292 const MutableCFOptions& mutable_opts, 293 std::string* opt_string); 294 295 Status GetMutableOptionsFromStrings( 296 const MutableCFOptions& base_options, 297 const std::unordered_map<std::string, std::string>& options_map, 298 Logger* info_log, MutableCFOptions* new_options); 299 #endif // ROCKSDB_LITE 300 301 } // namespace ROCKSDB_NAMESPACE 302