1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 // This source code is licensed under both the GPLv2 (found in the 3 // COPYING file in the root directory) and Apache 2.0 License 4 // (found in the LICENSE.Apache file in the root directory). 5 6 #ifdef GFLAGS 7 #pragma once 8 9 #include <mutex> 10 #include <unordered_set> 11 12 #include "file/filename.h" 13 #include "rocksdb/db.h" 14 #include "rocksdb/file_system.h" 15 #include "rocksdb/listener.h" 16 #include "rocksdb/table_properties.h" 17 #include "rocksdb/unique_id.h" 18 #include "util/gflags_compat.h" 19 #include "util/random.h" 20 21 DECLARE_int32(compact_files_one_in); 22 23 namespace ROCKSDB_NAMESPACE { 24 25 #ifndef ROCKSDB_LITE 26 // Verify across process executions that all seen IDs are unique 27 class UniqueIdVerifier { 28 public: 29 explicit UniqueIdVerifier(const std::string& db_name); 30 ~UniqueIdVerifier(); 31 32 void Verify(const std::string& id); 33 34 private: 35 void VerifyNoWrite(const std::string& id); 36 37 private: 38 std::mutex mutex_; 39 // IDs persisted to a hidden file inside DB dir 40 std::string path_; 41 std::unique_ptr<FSWritableFile> data_file_writer_; 42 // Starting byte for which 8 bytes to check in memory within 24 byte ID 43 size_t offset_; 44 // Working copy of the set of 8 byte pieces 45 std::unordered_set<uint64_t> id_set_; 46 }; 47 48 class DbStressListener : public EventListener { 49 public: DbStressListener(const std::string & db_name,const std::vector<DbPath> & db_paths,const std::vector<ColumnFamilyDescriptor> & column_families)50 DbStressListener(const std::string& db_name, 51 const std::vector<DbPath>& db_paths, 52 const std::vector<ColumnFamilyDescriptor>& column_families) 53 : db_name_(db_name), 54 db_paths_(db_paths), 55 column_families_(column_families), 56 num_pending_file_creations_(0), 57 unique_ids_(db_name) {} 58 Name()59 const char* Name() const override { return kClassName(); } kClassName()60 static const char* kClassName() { return "DBStressListener"; } 61 ~DbStressListener()62 ~DbStressListener() override { assert(num_pending_file_creations_ == 0); } OnFlushCompleted(DB *,const FlushJobInfo & info)63 void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override { 64 assert(IsValidColumnFamilyName(info.cf_name)); 65 VerifyFilePath(info.file_path); 66 // pretending doing some work here 67 RandomSleep(); 68 } 69 OnFlushBegin(DB *,const FlushJobInfo &)70 void OnFlushBegin(DB* /*db*/, 71 const FlushJobInfo& /*flush_job_info*/) override { 72 RandomSleep(); 73 } 74 OnTableFileDeleted(const TableFileDeletionInfo &)75 void OnTableFileDeleted(const TableFileDeletionInfo& /*info*/) override { 76 RandomSleep(); 77 } 78 OnCompactionBegin(DB *,const CompactionJobInfo &)79 void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& /*ci*/) override { 80 RandomSleep(); 81 } 82 OnCompactionCompleted(DB *,const CompactionJobInfo & ci)83 void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override { 84 assert(IsValidColumnFamilyName(ci.cf_name)); 85 assert(ci.input_files.size() + ci.output_files.size() > 0U); 86 for (const auto& file_path : ci.input_files) { 87 VerifyFilePath(file_path); 88 } 89 for (const auto& file_path : ci.output_files) { 90 VerifyFilePath(file_path); 91 } 92 // pretending doing some work here 93 RandomSleep(); 94 } 95 OnTableFileCreationStarted(const TableFileCreationBriefInfo &)96 void OnTableFileCreationStarted( 97 const TableFileCreationBriefInfo& /*info*/) override { 98 ++num_pending_file_creations_; 99 } 100 OnTableFileCreated(const TableFileCreationInfo & info)101 void OnTableFileCreated(const TableFileCreationInfo& info) override { 102 assert(info.db_name == db_name_); 103 assert(IsValidColumnFamilyName(info.cf_name)); 104 assert(info.job_id > 0 || FLAGS_compact_files_one_in > 0); 105 if (info.status.ok()) { 106 assert(info.file_size > 0); 107 VerifyFilePath(info.file_path); 108 assert(info.table_properties.data_size > 0 || 109 info.table_properties.num_range_deletions > 0); 110 assert(info.table_properties.raw_key_size > 0); 111 assert(info.table_properties.num_entries > 0); 112 VerifyTableFileUniqueId(info.table_properties, info.file_path); 113 } 114 --num_pending_file_creations_; 115 } 116 OnMemTableSealed(const MemTableInfo &)117 void OnMemTableSealed(const MemTableInfo& /*info*/) override { 118 RandomSleep(); 119 } 120 OnColumnFamilyHandleDeletionStarted(ColumnFamilyHandle *)121 void OnColumnFamilyHandleDeletionStarted( 122 ColumnFamilyHandle* /*handle*/) override { 123 RandomSleep(); 124 } 125 OnExternalFileIngested(DB *,const ExternalFileIngestionInfo & info)126 void OnExternalFileIngested(DB* /*db*/, 127 const ExternalFileIngestionInfo& info) override { 128 RandomSleep(); 129 // Here we assume that each generated external file is ingested 130 // exactly once (or thrown away in case of crash) 131 VerifyTableFileUniqueId(info.table_properties, info.internal_file_path); 132 } 133 OnBackgroundError(BackgroundErrorReason,Status *)134 void OnBackgroundError(BackgroundErrorReason /* reason */, 135 Status* /* bg_error */) override { 136 RandomSleep(); 137 } 138 OnStallConditionsChanged(const WriteStallInfo &)139 void OnStallConditionsChanged(const WriteStallInfo& /*info*/) override { 140 RandomSleep(); 141 } 142 OnFileReadFinish(const FileOperationInfo & info)143 void OnFileReadFinish(const FileOperationInfo& info) override { 144 // Even empty callback is valuable because sometimes some locks are 145 // released in order to make the callback. 146 147 // Sleep carefully here as it is a frequent operation and we don't want 148 // to slow down the tests. We always sleep when the read is large. 149 // When read is small, sleep in a small chance. 150 size_t length_read = info.length; 151 if (length_read >= 1000000 || Random::GetTLSInstance()->OneIn(1000)) { 152 RandomSleep(); 153 } 154 } 155 OnFileWriteFinish(const FileOperationInfo & info)156 void OnFileWriteFinish(const FileOperationInfo& info) override { 157 // Even empty callback is valuable because sometimes some locks are 158 // released in order to make the callback. 159 160 // Sleep carefully here as it is a frequent operation and we don't want 161 // to slow down the tests. When the write is large, always sleep. 162 // Otherwise, sleep in a relatively small chance. 163 size_t length_write = info.length; 164 if (length_write >= 1000000 || Random::GetTLSInstance()->OneIn(64)) { 165 RandomSleep(); 166 } 167 } 168 ShouldBeNotifiedOnFileIO()169 bool ShouldBeNotifiedOnFileIO() override { 170 RandomSleep(); 171 return static_cast<bool>(Random::GetTLSInstance()->OneIn(1)); 172 } 173 OnErrorRecoveryBegin(BackgroundErrorReason,Status,bool *)174 void OnErrorRecoveryBegin(BackgroundErrorReason /* reason */, 175 Status /* bg_error */, 176 bool* /* auto_recovery */) override { 177 RandomSleep(); 178 } 179 OnErrorRecoveryCompleted(Status)180 void OnErrorRecoveryCompleted(Status /* old_bg_error */) override { 181 RandomSleep(); 182 } 183 184 protected: IsValidColumnFamilyName(const std::string & cf_name)185 bool IsValidColumnFamilyName(const std::string& cf_name) const { 186 if (cf_name == kDefaultColumnFamilyName) { 187 return true; 188 } 189 // The column family names in the stress tests are numbers. 190 for (size_t i = 0; i < cf_name.size(); ++i) { 191 if (cf_name[i] < '0' || cf_name[i] > '9') { 192 return false; 193 } 194 } 195 return true; 196 } 197 VerifyFileDir(const std::string & file_dir)198 void VerifyFileDir(const std::string& file_dir) { 199 #ifndef NDEBUG 200 if (db_name_ == file_dir) { 201 return; 202 } 203 for (const auto& db_path : db_paths_) { 204 if (db_path.path == file_dir) { 205 return; 206 } 207 } 208 for (auto& cf : column_families_) { 209 for (const auto& cf_path : cf.options.cf_paths) { 210 if (cf_path.path == file_dir) { 211 return; 212 } 213 } 214 } 215 assert(false); 216 #else 217 (void)file_dir; 218 #endif // !NDEBUG 219 } 220 VerifyFileName(const std::string & file_name)221 void VerifyFileName(const std::string& file_name) { 222 #ifndef NDEBUG 223 uint64_t file_number; 224 FileType file_type; 225 bool result = ParseFileName(file_name, &file_number, &file_type); 226 assert(result); 227 assert(file_type == kTableFile); 228 #else 229 (void)file_name; 230 #endif // !NDEBUG 231 } 232 VerifyFilePath(const std::string & file_path)233 void VerifyFilePath(const std::string& file_path) { 234 #ifndef NDEBUG 235 size_t pos = file_path.find_last_of("/"); 236 if (pos == std::string::npos) { 237 VerifyFileName(file_path); 238 } else { 239 if (pos > 0) { 240 VerifyFileDir(file_path.substr(0, pos)); 241 } 242 VerifyFileName(file_path.substr(pos)); 243 } 244 #else 245 (void)file_path; 246 #endif // !NDEBUG 247 } 248 249 // Unique id is verified using the TableProperties. file_path is only used 250 // for reporting. 251 void VerifyTableFileUniqueId(const TableProperties& new_file_properties, 252 const std::string& file_path); 253 RandomSleep()254 void RandomSleep() { 255 std::this_thread::sleep_for( 256 std::chrono::microseconds(Random::GetTLSInstance()->Uniform(5000))); 257 } 258 259 private: 260 std::string db_name_; 261 std::vector<DbPath> db_paths_; 262 std::vector<ColumnFamilyDescriptor> column_families_; 263 std::atomic<int> num_pending_file_creations_; 264 UniqueIdVerifier unique_ids_; 265 }; 266 #endif // !ROCKSDB_LITE 267 } // namespace ROCKSDB_NAMESPACE 268 #endif // GFLAGS 269