1 // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 5 #ifndef THIRD_PARTY_LEVELDATABASE_ENV_CHROMIUM_H_ 6 #define THIRD_PARTY_LEVELDATABASE_ENV_CHROMIUM_H_ 7 8 #include <memory> 9 #include <set> 10 #include <string> 11 #include <vector> 12 13 #include "base/callback.h" 14 #include "base/containers/circular_deque.h" 15 #include "base/containers/flat_map.h" 16 #include "base/containers/linked_list.h" 17 #include "base/files/file.h" 18 #include "base/files/file_path.h" 19 #include "base/gtest_prod_util.h" 20 #include "base/macros.h" 21 #include "base/metrics/histogram.h" 22 #include "base/synchronization/condition_variable.h" 23 #include "build/build_config.h" 24 #include "leveldb/cache.h" 25 #include "leveldb/db.h" 26 #include "leveldb/env.h" 27 #include "leveldb/export.h" 28 #include "port/port_chromium.h" 29 #include "util/mutexlock.h" 30 31 namespace base { 32 namespace trace_event { 33 class MemoryAllocatorDump; 34 class ProcessMemoryDump; 35 } // namespace trace_event 36 } // namespace base 37 38 namespace storage { 39 class FilesystemProxy; 40 } 41 42 namespace leveldb_env { 43 44 // These entries map to values in tools/metrics/histograms/histograms.xml. New 45 // values should be appended at the end. 46 enum MethodID { 47 kSequentialFileRead, 48 kSequentialFileSkip, 49 kRandomAccessFileRead, 50 kWritableFileAppend, 51 kWritableFileClose, 52 kWritableFileFlush, 53 kWritableFileSync, 54 kNewSequentialFile, 55 kNewRandomAccessFile, 56 kNewWritableFile, 57 kObsoleteDeleteFile, 58 kCreateDir, 59 kObsoleteDeleteDir, 60 kGetFileSize, 61 kRenameFile, 62 kLockFile, 63 kUnlockFile, 64 kGetTestDirectory, 65 kNewLogger, 66 kSyncParent, 67 kGetChildren, 68 kNewAppendableFile, 69 kRemoveFile, 70 kRemoveDir, 71 kNumEntries 72 }; 73 74 // leveldb::Status::Code values are mapped to these values for UMA logging. 75 // Do not change/delete these values as you will break reporting for older 76 // copies of Chrome. Only add new values to the end. 77 enum LevelDBStatusValue { 78 LEVELDB_STATUS_OK = 0, 79 LEVELDB_STATUS_NOT_FOUND, 80 LEVELDB_STATUS_CORRUPTION, 81 LEVELDB_STATUS_NOT_SUPPORTED, 82 LEVELDB_STATUS_INVALID_ARGUMENT, 83 LEVELDB_STATUS_IO_ERROR, 84 LEVELDB_STATUS_MAX 85 }; 86 87 LEVELDB_EXPORT LevelDBStatusValue 88 GetLevelDBStatusUMAValue(const leveldb::Status& s); 89 90 using DatabaseErrorReportingCallback = 91 base::RepeatingCallback<void(const leveldb::Status&)>; 92 93 // Create the default leveldb options object suitable for leveldb operations. 94 struct LEVELDB_EXPORT Options : public leveldb::Options { 95 Options(); 96 97 // Called when there is a error during the Get() call. Intended for metrics 98 // reporting. 99 DatabaseErrorReportingCallback on_get_error; 100 // Called when there is a error during the Write() call, which is called for 101 // Write(), Put() and Delete(). Intended for metrics reporting. 102 DatabaseErrorReportingCallback on_write_error; 103 }; 104 105 LEVELDB_EXPORT const char* MethodIDToString(MethodID method); 106 107 leveldb::Status LEVELDB_EXPORT MakeIOError(leveldb::Slice filename, 108 const std::string& message, 109 MethodID method, 110 base::File::Error error); 111 leveldb::Status LEVELDB_EXPORT MakeIOError(leveldb::Slice filename, 112 const std::string& message, 113 MethodID method); 114 115 enum ErrorParsingResult { 116 METHOD_ONLY, 117 METHOD_AND_BFE, 118 NONE, 119 }; 120 121 ErrorParsingResult LEVELDB_EXPORT 122 ParseMethodAndError(const leveldb::Status& status, 123 MethodID* method, 124 base::File::Error* error); 125 LEVELDB_EXPORT int GetCorruptionCode(const leveldb::Status& status); 126 LEVELDB_EXPORT int GetNumCorruptionCodes(); 127 LEVELDB_EXPORT std::string GetCorruptionMessage(const leveldb::Status& status); 128 LEVELDB_EXPORT bool IndicatesDiskFull(const leveldb::Status& status); 129 130 // Returns the name for a temporary database copy during RewriteDB(). 131 LEVELDB_EXPORT std::string DatabaseNameForRewriteDB( 132 const std::string& original_name); 133 134 // Determine the appropriate leveldb write buffer size to use. The default size 135 // (4MB) may result in a log file too large to be compacted given the available 136 // storage space. This function will return smaller values for smaller disks, 137 // and the default leveldb value for larger disks. 138 // 139 // |disk_space| is the logical partition size (in bytes), and *not* available 140 // space. A value of -1 will return leveldb's default write buffer size. 141 LEVELDB_EXPORT extern size_t WriteBufferSize(int64_t disk_space); 142 143 class LEVELDB_EXPORT UMALogger { 144 public: 145 virtual void RecordErrorAt(MethodID method) const = 0; 146 virtual void RecordOSError(MethodID method, 147 base::File::Error error) const = 0; 148 virtual void RecordBytesRead(int amount) const = 0; 149 virtual void RecordBytesWritten(int amount) const = 0; 150 }; 151 152 class LEVELDB_EXPORT RetrierProvider { 153 public: 154 virtual int MaxRetryTimeMillis() const = 0; 155 virtual base::HistogramBase* GetRetryTimeHistogram(MethodID method) const = 0; 156 virtual base::HistogramBase* GetRecoveredFromErrorHistogram( 157 MethodID method) const = 0; 158 }; 159 160 class LEVELDB_EXPORT ChromiumEnv : public leveldb::Env, 161 public UMALogger, 162 public RetrierProvider { 163 public: 164 using ScheduleFunc = void(void*); 165 166 // Constructs a ChromiumEnv instance with an unrestricted FilesystemProxy 167 // instance that performs direct filesystem access. 168 ChromiumEnv(); 169 170 // Constructs a ChromiumEnv instance with a custom FilesystemProxy instance. 171 explicit ChromiumEnv(std::unique_ptr<storage::FilesystemProxy> filesystem); 172 173 ~ChromiumEnv() override; 174 175 bool FileExists(const std::string& fname) override; 176 leveldb::Status GetChildren(const std::string& dir, 177 std::vector<std::string>* result) override; 178 leveldb::Status RemoveFile(const std::string& fname) override; 179 leveldb::Status CreateDir(const std::string& name) override; 180 leveldb::Status RemoveDir(const std::string& name) override; 181 leveldb::Status GetFileSize(const std::string& fname, 182 uint64_t* size) override; 183 leveldb::Status RenameFile(const std::string& src, 184 const std::string& dst) override; 185 leveldb::Status LockFile(const std::string& fname, 186 leveldb::FileLock** lock) override; 187 leveldb::Status UnlockFile(leveldb::FileLock* lock) override; 188 void Schedule(ScheduleFunc*, void* arg) override; 189 void StartThread(void (*function)(void* arg), void* arg) override; 190 leveldb::Status GetTestDirectory(std::string* path) override; 191 uint64_t NowMicros() override; 192 void SleepForMicroseconds(int micros) override; 193 leveldb::Status NewSequentialFile(const std::string& fname, 194 leveldb::SequentialFile** result) override; 195 leveldb::Status NewRandomAccessFile( 196 const std::string& fname, 197 leveldb::RandomAccessFile** result) override; 198 leveldb::Status NewWritableFile(const std::string& fname, 199 leveldb::WritableFile** result) override; 200 leveldb::Status NewAppendableFile(const std::string& fname, 201 leveldb::WritableFile** result) override; 202 leveldb::Status NewLogger(const std::string& fname, 203 leveldb::Logger** result) override; 204 void SetReadOnlyFileLimitForTesting(int max_open_files); 205 206 protected: 207 // Constructs a ChromiumEnv instance with a local unrestricted FilesystemProxy 208 // instance that performs direct filesystem access. 209 explicit ChromiumEnv(const std::string& name); 210 211 // Constructs a ChromiumEnv instance with a custom FilesystemProxy instance. 212 ChromiumEnv(const std::string& name, 213 std::unique_ptr<storage::FilesystemProxy> filesystem); 214 215 static const char* FileErrorString(base::File::Error error); 216 217 private: 218 void RecordErrorAt(MethodID method) const override; 219 void RecordOSError(MethodID method, base::File::Error error) const override; 220 void RecordBytesRead(int amount) const override; 221 void RecordBytesWritten(int amount) const override; 222 base::HistogramBase* GetOSErrorHistogram(MethodID method, int limit) const; 223 void RemoveBackupFiles(const base::FilePath& dir); 224 225 const int kMaxRetryTimeMillis; 226 // BGThread() is the body of the background thread 227 void BGThread(); BGThreadWrapper(void * arg)228 static void BGThreadWrapper(void* arg) { 229 reinterpret_cast<ChromiumEnv*>(arg)->BGThread(); 230 } 231 232 base::HistogramBase* GetMethodIOErrorHistogram() const; 233 234 // RetrierProvider implementation. MaxRetryTimeMillis()235 int MaxRetryTimeMillis() const override { return kMaxRetryTimeMillis; } 236 base::HistogramBase* GetRetryTimeHistogram(MethodID method) const override; 237 base::HistogramBase* GetRecoveredFromErrorHistogram( 238 MethodID method) const override; 239 240 const std::unique_ptr<storage::FilesystemProxy> filesystem_; 241 242 base::FilePath test_directory_; 243 244 std::string name_; 245 std::string uma_ioerror_base_name_; 246 247 base::Lock mu_; 248 base::ConditionVariable bgsignal_; 249 bool started_bgthread_; 250 251 // Entry per Schedule() call 252 struct BGItem { 253 void* arg; 254 void (*function)(void*); 255 }; 256 using BGQueue = base::circular_deque<BGItem>; 257 BGQueue queue_; 258 std::unique_ptr<leveldb::Cache> file_cache_; 259 }; 260 261 // Tracks databases open via OpenDatabase() method and exposes them to 262 // memory-infra. The class is thread safe. 263 class LEVELDB_EXPORT DBTracker { 264 public: 265 enum SharedReadCacheUse : int { 266 // Use for databases whose access pattern is dictated by browser code. 267 SharedReadCacheUse_Browser = 0, 268 // Use for databases whose access pattern is directly influenced by Web 269 // APIs, like Indexed DB, etc. 270 SharedReadCacheUse_Web, 271 SharedReadCacheUse_Unified, // When Web == Browser. 272 SharedReadCacheUse_InMemory, // Shared by all in-memory databases. 273 SharedReadCacheUse_NumCacheUses 274 }; 275 276 // DBTracker singleton instance. 277 static DBTracker* GetInstance(); 278 279 // Returns the memory-infra dump for |tracked_db|. Can be used to attach 280 // additional info to the database dump, or to properly attribute memory 281 // usage in memory dump providers that also dump |tracked_db|. 282 // Note that |tracked_db| should be a live database instance produced by 283 // OpenDatabase() method or leveldb_env::OpenDB() function. 284 static base::trace_event::MemoryAllocatorDump* GetOrCreateAllocatorDump( 285 base::trace_event::ProcessMemoryDump* pmd, 286 leveldb::DB* tracked_db); 287 288 // Returns the memory-infra dump for |tracked_memenv|. Can be used to attach 289 // additional info to the database dump, or to properly attribute memory 290 // usage in memory dump providers that also dump |tracked_memenv|. 291 // Note that |tracked_memenv| should be a live Env instance produced by 292 // leveldb_chrome::NewMemEnv(). 293 static base::trace_event::MemoryAllocatorDump* GetOrCreateAllocatorDump( 294 base::trace_event::ProcessMemoryDump* pmd, 295 leveldb::Env* tracked_memenv); 296 297 // Report counts to UMA. 298 void UpdateHistograms(); 299 300 // Provides extra information about a tracked database. 301 class TrackedDB : public leveldb::DB { 302 public: 303 // Name that OpenDatabase() was called with. 304 virtual const std::string& name() const = 0; 305 306 // Options used when opening the database. 307 virtual SharedReadCacheUse block_cache_type() const = 0; 308 }; 309 310 // Opens a database and starts tracking it. As long as the opened database 311 // is alive (i.e. its instance is not destroyed) the database is exposed to 312 // memory-infra and is enumerated by VisitDatabases() method. 313 // This function is an implementation detail of leveldb_env::OpenDB(), and 314 // has similar guarantees regarding |dbptr| argument. 315 leveldb::Status OpenDatabase(const leveldb_env::Options& options, 316 const std::string& name, 317 TrackedDB** dbptr); 318 319 private: 320 class MemoryDumpProvider; 321 class TrackedDBImpl; 322 323 using DatabaseVisitor = base::RepeatingCallback<void(TrackedDB*)>; 324 325 friend class ChromiumEnvDBTrackerTest; 326 FRIEND_TEST_ALL_PREFIXES(ChromiumEnvDBTrackerTest, IsTrackedDB); 327 FRIEND_TEST_ALL_PREFIXES(ChromiumEnvDBTrackerTest, MemoryDumpCreation); 328 FRIEND_TEST_ALL_PREFIXES(ChromiumEnvDBTrackerTest, MemEnvMemoryDumpCreation); 329 330 DBTracker(); 331 ~DBTracker(); 332 333 // Calls |visitor| for each live database. The database is live from the 334 // point it was returned from OpenDatabase() and up until its instance is 335 // destroyed. 336 // The databases may be visited in an arbitrary order. 337 // This function takes a lock, preventing any database from being opened or 338 // destroyed (but doesn't lock the databases themselves). 339 void VisitDatabases(const DatabaseVisitor& visitor); 340 341 // Checks if |db| is tracked. 342 bool IsTrackedDB(const leveldb::DB* db) const; 343 344 void DatabaseOpened(TrackedDBImpl* database, SharedReadCacheUse cache_use); 345 void DatabaseDestroyed(TrackedDBImpl* database, SharedReadCacheUse cache_use); 346 347 // Protect databases_ and mdp_ members. 348 mutable base::Lock databases_lock_; 349 base::LinkedList<TrackedDBImpl> databases_; 350 std::unique_ptr<MemoryDumpProvider> mdp_; 351 352 DISALLOW_COPY_AND_ASSIGN(DBTracker); 353 }; 354 355 // Opens a database with the specified "name" and "options" (see note) and 356 // exposes it to Chrome's tracing (see DBTracker for details). The function 357 // guarantees that: 358 // 1. |dbptr| is not touched on failure 359 // 2. |dbptr| is not NULL on success 360 // 361 // Note: All |options| values are honored, except if options.env is an in-memory 362 // Env. In this case the block cache is disabled and a minimum write buffer size 363 // is used to conserve memory with all other values honored. 364 LEVELDB_EXPORT leveldb::Status OpenDB(const leveldb_env::Options& options, 365 const std::string& name, 366 std::unique_ptr<leveldb::DB>* dbptr); 367 368 // Copies the content of |dbptr| into a fresh database to remove traces of 369 // deleted data. |options| and |name| of the old database are required to create 370 // an identical copy. |dbptr| will be replaced with the new database on success. 371 // If the rewrite fails e.g. because we can't write to the temporary location, 372 // the old db is returned if possible, otherwise |*dbptr| can become NULL. 373 // The rewrite will only be performed if |kLevelDBRewriteFeature| is enabled. 374 LEVELDB_EXPORT leveldb::Status RewriteDB(const leveldb_env::Options& options, 375 const std::string& name, 376 std::unique_ptr<leveldb::DB>* dbptr); 377 378 LEVELDB_EXPORT base::StringPiece MakeStringPiece(const leveldb::Slice& s); 379 LEVELDB_EXPORT leveldb::Slice MakeSlice(const base::StringPiece& s); 380 LEVELDB_EXPORT leveldb::Slice MakeSlice(base::span<const uint8_t> s); 381 382 } // namespace leveldb_env 383 384 #endif // THIRD_PARTY_LEVELDATABASE_ENV_CHROMIUM_H_ 385