1 // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 5 #pragma once 6 7 #include <stdint.h> 8 #include <map> 9 #include <string> 10 #include "rocksdb/status.h" 11 #include "rocksdb/types.h" 12 13 namespace ROCKSDB_NAMESPACE { 14 15 // -- Table Properties 16 // Other than basic table properties, each table may also have the user 17 // collected properties. 18 // The value of the user-collected properties are encoded as raw bytes -- 19 // users have to interpret these values by themselves. 20 // Note: To do prefix seek/scan in `UserCollectedProperties`, you can do 21 // something similar to: 22 // 23 // UserCollectedProperties props = ...; 24 // for (auto pos = props.lower_bound(prefix); 25 // pos != props.end() && pos->first.compare(0, prefix.size(), prefix) == 0; 26 // ++pos) { 27 // ... 28 // } 29 typedef std::map<std::string, std::string> UserCollectedProperties; 30 31 // table properties' human-readable names in the property block. 32 struct TablePropertiesNames { 33 static const std::string kDataSize; 34 static const std::string kIndexSize; 35 static const std::string kIndexPartitions; 36 static const std::string kTopLevelIndexSize; 37 static const std::string kIndexKeyIsUserKey; 38 static const std::string kIndexValueIsDeltaEncoded; 39 static const std::string kFilterSize; 40 static const std::string kRawKeySize; 41 static const std::string kRawValueSize; 42 static const std::string kNumDataBlocks; 43 static const std::string kNumEntries; 44 static const std::string kDeletedKeys; 45 static const std::string kMergeOperands; 46 static const std::string kNumRangeDeletions; 47 static const std::string kFormatVersion; 48 static const std::string kFixedKeyLen; 49 static const std::string kFilterPolicy; 50 static const std::string kColumnFamilyName; 51 static const std::string kColumnFamilyId; 52 static const std::string kComparator; 53 static const std::string kMergeOperator; 54 static const std::string kPrefixExtractorName; 55 static const std::string kPropertyCollectors; 56 static const std::string kCompression; 57 static const std::string kCompressionOptions; 58 static const std::string kCreationTime; 59 static const std::string kOldestKeyTime; 60 static const std::string kFileCreationTime; 61 }; 62 63 extern const std::string kPropertiesBlock; 64 extern const std::string kCompressionDictBlock; 65 extern const std::string kRangeDelBlock; 66 67 // `TablePropertiesCollector` provides the mechanism for users to collect 68 // their own properties that they are interested in. This class is essentially 69 // a collection of callback functions that will be invoked during table 70 // building. It is constructed with TablePropertiesCollectorFactory. The methods 71 // don't need to be thread-safe, as we will create exactly one 72 // TablePropertiesCollector object per table and then call it sequentially 73 class TablePropertiesCollector { 74 public: ~TablePropertiesCollector()75 virtual ~TablePropertiesCollector() {} 76 77 // DEPRECATE User defined collector should implement AddUserKey(), though 78 // this old function still works for backward compatible reason. 79 // Add() will be called when a new key/value pair is inserted into the table. 80 // @params key the user key that is inserted into the table. 81 // @params value the value that is inserted into the table. Add(const Slice &,const Slice &)82 virtual Status Add(const Slice& /*key*/, const Slice& /*value*/) { 83 return Status::InvalidArgument( 84 "TablePropertiesCollector::Add() deprecated."); 85 } 86 87 // AddUserKey() will be called when a new key/value pair is inserted into the 88 // table. 89 // @params key the user key that is inserted into the table. 90 // @params value the value that is inserted into the table. AddUserKey(const Slice & key,const Slice & value,EntryType,SequenceNumber,uint64_t)91 virtual Status AddUserKey(const Slice& key, const Slice& value, 92 EntryType /*type*/, SequenceNumber /*seq*/, 93 uint64_t /*file_size*/) { 94 // For backwards-compatibility. 95 return Add(key, value); 96 } 97 98 // Called after each new block is cut BlockAdd(uint64_t,uint64_t,uint64_t)99 virtual void BlockAdd(uint64_t /* blockRawBytes */, 100 uint64_t /* blockCompressedBytesFast */, 101 uint64_t /* blockCompressedBytesSlow */) { 102 // Nothing to do here. Callback registers can override. 103 return; 104 } 105 106 // Finish() will be called when a table has already been built and is ready 107 // for writing the properties block. 108 // @params properties User will add their collected statistics to 109 // `properties`. 110 virtual Status Finish(UserCollectedProperties* properties) = 0; 111 112 // Return the human-readable properties, where the key is property name and 113 // the value is the human-readable form of value. 114 virtual UserCollectedProperties GetReadableProperties() const = 0; 115 116 // The name of the properties collector can be used for debugging purpose. 117 virtual const char* Name() const = 0; 118 119 // EXPERIMENTAL Return whether the output file should be further compacted NeedCompact()120 virtual bool NeedCompact() const { return false; } 121 }; 122 123 // Constructs TablePropertiesCollector. Internals create a new 124 // TablePropertiesCollector for each new table 125 class TablePropertiesCollectorFactory { 126 public: 127 struct Context { 128 uint32_t column_family_id; 129 static const uint32_t kUnknownColumnFamily; 130 }; 131 ~TablePropertiesCollectorFactory()132 virtual ~TablePropertiesCollectorFactory() {} 133 // has to be thread-safe 134 virtual TablePropertiesCollector* CreateTablePropertiesCollector( 135 TablePropertiesCollectorFactory::Context context) = 0; 136 137 // The name of the properties collector can be used for debugging purpose. 138 virtual const char* Name() const = 0; 139 }; 140 141 // TableProperties contains a bunch of read-only properties of its associated 142 // table. 143 struct TableProperties { 144 public: 145 // the total size of all data blocks. 146 uint64_t data_size = 0; 147 // the size of index block. 148 uint64_t index_size = 0; 149 // Total number of index partitions if kTwoLevelIndexSearch is used 150 uint64_t index_partitions = 0; 151 // Size of the top-level index if kTwoLevelIndexSearch is used 152 uint64_t top_level_index_size = 0; 153 // Whether the index key is user key. Otherwise it includes 8 byte of sequence 154 // number added by internal key format. 155 uint64_t index_key_is_user_key = 0; 156 // Whether delta encoding is used to encode the index values. 157 uint64_t index_value_is_delta_encoded = 0; 158 // the size of filter block. 159 uint64_t filter_size = 0; 160 // total raw key size 161 uint64_t raw_key_size = 0; 162 // total raw value size 163 uint64_t raw_value_size = 0; 164 // the number of blocks in this table 165 uint64_t num_data_blocks = 0; 166 // the number of entries in this table 167 uint64_t num_entries = 0; 168 // the number of deletions in the table 169 uint64_t num_deletions = 0; 170 // the number of merge operands in the table 171 uint64_t num_merge_operands = 0; 172 // the number of range deletions in this table 173 uint64_t num_range_deletions = 0; 174 // format version, reserved for backward compatibility 175 uint64_t format_version = 0; 176 // If 0, key is variable length. Otherwise number of bytes for each key. 177 uint64_t fixed_key_len = 0; 178 // ID of column family for this SST file, corresponding to the CF identified 179 // by column_family_name. 180 uint64_t column_family_id = ROCKSDB_NAMESPACE:: 181 TablePropertiesCollectorFactory::Context::kUnknownColumnFamily; 182 // Timestamp of the latest key. 0 means unknown. 183 // TODO(sagar0): Should be changed to latest_key_time ... but don't know the 184 // full implications of backward compatibility. Hence retaining for now. 185 uint64_t creation_time = 0; 186 // Timestamp of the earliest key. 0 means unknown. 187 uint64_t oldest_key_time = 0; 188 // Actual SST file creation time. 0 means unknown. 189 uint64_t file_creation_time = 0; 190 191 // Name of the column family with which this SST file is associated. 192 // If column family is unknown, `column_family_name` will be an empty string. 193 std::string column_family_name; 194 195 // The name of the filter policy used in this table. 196 // If no filter policy is used, `filter_policy_name` will be an empty string. 197 std::string filter_policy_name; 198 199 // The name of the comparator used in this table. 200 std::string comparator_name; 201 202 // The name of the merge operator used in this table. 203 // If no merge operator is used, `merge_operator_name` will be "nullptr". 204 std::string merge_operator_name; 205 206 // The name of the prefix extractor used in this table 207 // If no prefix extractor is used, `prefix_extractor_name` will be "nullptr". 208 std::string prefix_extractor_name; 209 210 // The names of the property collectors factories used in this table 211 // separated by commas 212 // {collector_name[1]},{collector_name[2]},{collector_name[3]} .. 213 std::string property_collectors_names; 214 215 // The compression algo used to compress the SST files. 216 std::string compression_name; 217 218 // Compression options used to compress the SST files. 219 std::string compression_options; 220 221 // user collected properties 222 UserCollectedProperties user_collected_properties; 223 UserCollectedProperties readable_properties; 224 225 // The offset of the value of each property in the file. 226 std::map<std::string, uint64_t> properties_offsets; 227 228 // convert this object to a human readable form 229 // @prop_delim: delimiter for each property. 230 std::string ToString(const std::string& prop_delim = "; ", 231 const std::string& kv_delim = "=") const; 232 233 // Aggregate the numerical member variables of the specified 234 // TableProperties. 235 void Add(const TableProperties& tp); 236 }; 237 238 // Extra properties 239 // Below is a list of non-basic properties that are collected by database 240 // itself. Especially some properties regarding to the internal keys (which 241 // is unknown to `table`). 242 // 243 // DEPRECATED: these properties now belong as TableProperties members. Please 244 // use TableProperties::num_deletions and TableProperties::num_merge_operands, 245 // respectively. 246 extern uint64_t GetDeletedKeys(const UserCollectedProperties& props); 247 extern uint64_t GetMergeOperands(const UserCollectedProperties& props, 248 bool* property_present); 249 250 } // namespace ROCKSDB_NAMESPACE 251