1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #ifndef CacheIndex__h__ 6 #define CacheIndex__h__ 7 8 #include "CacheLog.h" 9 #include "CacheFileIOManager.h" 10 #include "nsIRunnable.h" 11 #include "CacheHashUtils.h" 12 #include "nsICacheStorageService.h" 13 #include "nsICacheEntry.h" 14 #include "nsILoadContextInfo.h" 15 #include "nsTHashtable.h" 16 #include "nsThreadUtils.h" 17 #include "nsWeakReference.h" 18 #include "mozilla/SHA1.h" 19 #include "mozilla/StaticMutex.h" 20 #include "mozilla/StaticPtr.h" 21 #include "mozilla/EndianUtils.h" 22 #include "mozilla/TimeStamp.h" 23 24 class nsIFile; 25 class nsIDirectoryEnumerator; 26 class nsITimer; 27 28 29 #ifdef DEBUG 30 #define DEBUG_STATS 1 31 #endif 32 33 namespace mozilla { 34 namespace net { 35 36 class CacheFileMetadata; 37 class FileOpenHelper; 38 class CacheIndexIterator; 39 40 typedef struct { 41 // Version of the index. The index must be ignored and deleted when the file 42 // on disk was written with a newer version. 43 uint32_t mVersion; 44 45 // Timestamp of time when the last successful write of the index started. 46 // During update process we use this timestamp for a quick validation of entry 47 // files. If last modified time of the file is lower than this timestamp, we 48 // skip parsing of such file since the information in index should be up to 49 // date. 50 uint32_t mTimeStamp; 51 52 // We set this flag as soon as possible after parsing index during startup 53 // and clean it after we write journal to disk during shutdown. We ignore the 54 // journal and start update process whenever this flag is set during index 55 // parsing. 56 uint32_t mIsDirty; 57 } CacheIndexHeader; 58 59 static_assert( 60 sizeof(CacheIndexHeader::mVersion) + sizeof(CacheIndexHeader::mTimeStamp) + 61 sizeof(CacheIndexHeader::mIsDirty) == sizeof(CacheIndexHeader), 62 "Unexpected sizeof(CacheIndexHeader)!"); 63 64 struct CacheIndexRecord { 65 SHA1Sum::Hash mHash; 66 uint32_t mFrecency; 67 OriginAttrsHash mOriginAttrsHash; 68 uint32_t mExpirationTime; 69 70 /* 71 * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized 72 * 0100 0000 0000 0000 0000 0000 0000 0000 : anonymous 73 * 0010 0000 0000 0000 0000 0000 0000 0000 : removed 74 * 0001 0000 0000 0000 0000 0000 0000 0000 : dirty 75 * 0000 1000 0000 0000 0000 0000 0000 0000 : fresh 76 * 0000 0100 0000 0000 0000 0000 0000 0000 : pinned 77 * 0000 0011 0000 0000 0000 0000 0000 0000 : reserved 78 * 0000 0000 1111 1111 1111 1111 1111 1111 : file size (in kB) 79 */ 80 uint32_t mFlags; 81 CacheIndexRecordCacheIndexRecord82 CacheIndexRecord() 83 : mFrecency(0) 84 , mOriginAttrsHash(0) 85 , mExpirationTime(nsICacheEntry::NO_EXPIRATION_TIME) 86 , mFlags(0) 87 {} 88 }; 89 90 static_assert( 91 sizeof(CacheIndexRecord::mHash) + sizeof(CacheIndexRecord::mFrecency) + 92 sizeof(CacheIndexRecord::mOriginAttrsHash) + sizeof(CacheIndexRecord::mExpirationTime) + 93 sizeof(CacheIndexRecord::mFlags) == sizeof(CacheIndexRecord), 94 "Unexpected sizeof(CacheIndexRecord)!"); 95 96 class CacheIndexEntry : public PLDHashEntryHdr 97 { 98 public: 99 typedef const SHA1Sum::Hash& KeyType; 100 typedef const SHA1Sum::Hash* KeyTypePointer; 101 CacheIndexEntry(KeyTypePointer aKey)102 explicit CacheIndexEntry(KeyTypePointer aKey) 103 { 104 MOZ_COUNT_CTOR(CacheIndexEntry); 105 mRec = new CacheIndexRecord(); 106 LOG(("CacheIndexEntry::CacheIndexEntry() - Created record [rec=%p]", mRec.get())); 107 memcpy(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash)); 108 } CacheIndexEntry(const CacheIndexEntry & aOther)109 CacheIndexEntry(const CacheIndexEntry& aOther) 110 { 111 NS_NOTREACHED("CacheIndexEntry copy constructor is forbidden!"); 112 } ~CacheIndexEntry()113 ~CacheIndexEntry() 114 { 115 MOZ_COUNT_DTOR(CacheIndexEntry); 116 LOG(("CacheIndexEntry::~CacheIndexEntry() - Deleting record [rec=%p]", 117 mRec.get())); 118 } 119 120 // KeyEquals(): does this entry match this key? KeyEquals(KeyTypePointer aKey)121 bool KeyEquals(KeyTypePointer aKey) const 122 { 123 return memcmp(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash)) == 0; 124 } 125 126 // KeyToPointer(): Convert KeyType to KeyTypePointer KeyToPointer(KeyType aKey)127 static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; } 128 129 // HashKey(): calculate the hash number HashKey(KeyTypePointer aKey)130 static PLDHashNumber HashKey(KeyTypePointer aKey) 131 { 132 return (reinterpret_cast<const uint32_t *>(aKey))[0]; 133 } 134 135 // ALLOW_MEMMOVE can we move this class with memmove(), or do we have 136 // to use the copy constructor? 137 enum { ALLOW_MEMMOVE = true }; 138 139 bool operator==(const CacheIndexEntry& aOther) const 140 { 141 return KeyEquals(&aOther.mRec->mHash); 142 } 143 144 CacheIndexEntry& operator=(const CacheIndexEntry& aOther) 145 { 146 MOZ_ASSERT(memcmp(&mRec->mHash, &aOther.mRec->mHash, 147 sizeof(SHA1Sum::Hash)) == 0); 148 mRec->mFrecency = aOther.mRec->mFrecency; 149 mRec->mExpirationTime = aOther.mRec->mExpirationTime; 150 mRec->mOriginAttrsHash = aOther.mRec->mOriginAttrsHash; 151 mRec->mFlags = aOther.mRec->mFlags; 152 return *this; 153 } 154 InitNew()155 void InitNew() 156 { 157 mRec->mFrecency = 0; 158 mRec->mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME; 159 mRec->mOriginAttrsHash = 0; 160 mRec->mFlags = 0; 161 } 162 Init(OriginAttrsHash aOriginAttrsHash,bool aAnonymous,bool aPinned)163 void Init(OriginAttrsHash aOriginAttrsHash, bool aAnonymous, bool aPinned) 164 { 165 MOZ_ASSERT(mRec->mFrecency == 0); 166 MOZ_ASSERT(mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME); 167 MOZ_ASSERT(mRec->mOriginAttrsHash == 0); 168 // When we init the entry it must be fresh and may be dirty 169 MOZ_ASSERT((mRec->mFlags & ~kDirtyMask) == kFreshMask); 170 171 mRec->mOriginAttrsHash = aOriginAttrsHash; 172 mRec->mFlags |= kInitializedMask; 173 if (aAnonymous) { 174 mRec->mFlags |= kAnonymousMask; 175 } 176 if (aPinned) { 177 mRec->mFlags |= kPinnedMask; 178 } 179 } 180 Hash()181 const SHA1Sum::Hash * Hash() const { return &mRec->mHash; } 182 IsInitialized()183 bool IsInitialized() const { return !!(mRec->mFlags & kInitializedMask); } 184 OriginAttrsHash()185 mozilla::net::OriginAttrsHash OriginAttrsHash() const { return mRec->mOriginAttrsHash; } 186 Anonymous()187 bool Anonymous() const { return !!(mRec->mFlags & kAnonymousMask); } 188 IsRemoved()189 bool IsRemoved() const { return !!(mRec->mFlags & kRemovedMask); } MarkRemoved()190 void MarkRemoved() { mRec->mFlags |= kRemovedMask; } 191 IsDirty()192 bool IsDirty() const { return !!(mRec->mFlags & kDirtyMask); } MarkDirty()193 void MarkDirty() { mRec->mFlags |= kDirtyMask; } ClearDirty()194 void ClearDirty() { mRec->mFlags &= ~kDirtyMask; } 195 IsFresh()196 bool IsFresh() const { return !!(mRec->mFlags & kFreshMask); } MarkFresh()197 void MarkFresh() { mRec->mFlags |= kFreshMask; } 198 IsPinned()199 bool IsPinned() const { return !!(mRec->mFlags & kPinnedMask); } 200 SetFrecency(uint32_t aFrecency)201 void SetFrecency(uint32_t aFrecency) { mRec->mFrecency = aFrecency; } GetFrecency()202 uint32_t GetFrecency() const { return mRec->mFrecency; } 203 SetExpirationTime(uint32_t aExpirationTime)204 void SetExpirationTime(uint32_t aExpirationTime) 205 { 206 mRec->mExpirationTime = aExpirationTime; 207 } GetExpirationTime()208 uint32_t GetExpirationTime() const { return mRec->mExpirationTime; } 209 210 // Sets filesize in kilobytes. SetFileSize(uint32_t aFileSize)211 void SetFileSize(uint32_t aFileSize) 212 { 213 if (aFileSize > kFileSizeMask) { 214 LOG(("CacheIndexEntry::SetFileSize() - FileSize is too large, " 215 "truncating to %u", kFileSizeMask)); 216 aFileSize = kFileSizeMask; 217 } 218 mRec->mFlags &= ~kFileSizeMask; 219 mRec->mFlags |= aFileSize; 220 } 221 // Returns filesize in kilobytes. GetFileSize()222 uint32_t GetFileSize() const { return GetFileSize(mRec); } GetFileSize(CacheIndexRecord * aRec)223 static uint32_t GetFileSize(CacheIndexRecord *aRec) 224 { 225 return aRec->mFlags & kFileSizeMask; 226 } IsPinned(CacheIndexRecord * aRec)227 static uint32_t IsPinned(CacheIndexRecord *aRec) 228 { 229 return aRec->mFlags & kPinnedMask; 230 } IsFileEmpty()231 bool IsFileEmpty() const { return GetFileSize() == 0; } 232 WriteToBuf(void * aBuf)233 void WriteToBuf(void *aBuf) 234 { 235 uint8_t* ptr = static_cast<uint8_t*>(aBuf); 236 memcpy(ptr, mRec->mHash, sizeof(SHA1Sum::Hash)); ptr += sizeof(SHA1Sum::Hash); 237 NetworkEndian::writeUint32(ptr, mRec->mFrecency); ptr += sizeof(uint32_t); 238 NetworkEndian::writeUint64(ptr, mRec->mOriginAttrsHash); ptr += sizeof(uint64_t); 239 NetworkEndian::writeUint32(ptr, mRec->mExpirationTime); ptr += sizeof(uint32_t); 240 // Dirty and fresh flags should never go to disk, since they make sense only 241 // during current session. 242 NetworkEndian::writeUint32(ptr, mRec->mFlags & ~(kDirtyMask | kFreshMask)); 243 } 244 ReadFromBuf(void * aBuf)245 void ReadFromBuf(void *aBuf) 246 { 247 const uint8_t* ptr = static_cast<const uint8_t*>(aBuf); 248 MOZ_ASSERT(memcmp(&mRec->mHash, ptr, sizeof(SHA1Sum::Hash)) == 0); ptr += sizeof(SHA1Sum::Hash); 249 mRec->mFrecency = NetworkEndian::readUint32(ptr); ptr += sizeof(uint32_t); 250 mRec->mOriginAttrsHash = NetworkEndian::readUint64(ptr); ptr += sizeof(uint64_t); 251 mRec->mExpirationTime = NetworkEndian::readUint32(ptr); ptr += sizeof(uint32_t); 252 mRec->mFlags = NetworkEndian::readUint32(ptr); 253 } 254 Log()255 void Log() const { 256 LOG(("CacheIndexEntry::Log() [this=%p, hash=%08x%08x%08x%08x%08x, fresh=%u," 257 " initialized=%u, removed=%u, dirty=%u, anonymous=%u, " 258 "originAttrsHash=%llx, frecency=%u, expirationTime=%u, size=%u]", 259 this, LOGSHA1(mRec->mHash), IsFresh(), IsInitialized(), IsRemoved(), 260 IsDirty(), Anonymous(), OriginAttrsHash(), GetFrecency(), 261 GetExpirationTime(), GetFileSize())); 262 } 263 RecordMatchesLoadContextInfo(CacheIndexRecord * aRec,nsILoadContextInfo * aInfo)264 static bool RecordMatchesLoadContextInfo(CacheIndexRecord *aRec, 265 nsILoadContextInfo *aInfo) 266 { 267 if (!aInfo->IsPrivate() && 268 GetOriginAttrsHash(*aInfo->OriginAttributesPtr()) == aRec->mOriginAttrsHash && 269 aInfo->IsAnonymous() == !!(aRec->mFlags & kAnonymousMask)) { 270 return true; 271 } 272 273 return false; 274 } 275 276 // Memory reporting SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf)277 size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const 278 { 279 return mallocSizeOf(mRec.get()); 280 } 281 SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf)282 size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const 283 { 284 return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf); 285 } 286 287 private: 288 friend class CacheIndexEntryUpdate; 289 friend class CacheIndex; 290 friend class CacheIndexEntryAutoManage; 291 292 static const uint32_t kInitializedMask = 0x80000000; 293 static const uint32_t kAnonymousMask = 0x40000000; 294 295 // This flag is set when the entry was removed. We need to keep this 296 // information in memory until we write the index file. 297 static const uint32_t kRemovedMask = 0x20000000; 298 299 // This flag is set when the information in memory is not in sync with the 300 // information in index file on disk. 301 static const uint32_t kDirtyMask = 0x10000000; 302 303 // This flag is set when the information about the entry is fresh, i.e. 304 // we've created or opened this entry during this session, or we've seen 305 // this entry during update or build process. 306 static const uint32_t kFreshMask = 0x08000000; 307 308 // Indicates a pinned entry. 309 static const uint32_t kPinnedMask = 0x04000000; 310 311 static const uint32_t kReservedMask = 0x03000000; 312 313 // FileSize in kilobytes 314 static const uint32_t kFileSizeMask = 0x00FFFFFF; 315 316 nsAutoPtr<CacheIndexRecord> mRec; 317 }; 318 319 class CacheIndexEntryUpdate : public CacheIndexEntry 320 { 321 public: CacheIndexEntryUpdate(CacheIndexEntry::KeyTypePointer aKey)322 explicit CacheIndexEntryUpdate(CacheIndexEntry::KeyTypePointer aKey) 323 : CacheIndexEntry(aKey) 324 , mUpdateFlags(0) 325 { 326 MOZ_COUNT_CTOR(CacheIndexEntryUpdate); 327 LOG(("CacheIndexEntryUpdate::CacheIndexEntryUpdate()")); 328 } ~CacheIndexEntryUpdate()329 ~CacheIndexEntryUpdate() 330 { 331 MOZ_COUNT_DTOR(CacheIndexEntryUpdate); 332 LOG(("CacheIndexEntryUpdate::~CacheIndexEntryUpdate()")); 333 } 334 335 CacheIndexEntryUpdate& operator=(const CacheIndexEntry& aOther) 336 { 337 MOZ_ASSERT(memcmp(&mRec->mHash, &aOther.mRec->mHash, 338 sizeof(SHA1Sum::Hash)) == 0); 339 mUpdateFlags = 0; 340 *(static_cast<CacheIndexEntry *>(this)) = aOther; 341 return *this; 342 } 343 InitNew()344 void InitNew() 345 { 346 mUpdateFlags = kFrecencyUpdatedMask | kExpirationUpdatedMask | 347 kFileSizeUpdatedMask; 348 CacheIndexEntry::InitNew(); 349 } 350 SetFrecency(uint32_t aFrecency)351 void SetFrecency(uint32_t aFrecency) 352 { 353 mUpdateFlags |= kFrecencyUpdatedMask; 354 CacheIndexEntry::SetFrecency(aFrecency); 355 } 356 SetExpirationTime(uint32_t aExpirationTime)357 void SetExpirationTime(uint32_t aExpirationTime) 358 { 359 mUpdateFlags |= kExpirationUpdatedMask; 360 CacheIndexEntry::SetExpirationTime(aExpirationTime); 361 } 362 SetFileSize(uint32_t aFileSize)363 void SetFileSize(uint32_t aFileSize) 364 { 365 mUpdateFlags |= kFileSizeUpdatedMask; 366 CacheIndexEntry::SetFileSize(aFileSize); 367 } 368 ApplyUpdate(CacheIndexEntry * aDst)369 void ApplyUpdate(CacheIndexEntry *aDst) { 370 MOZ_ASSERT(memcmp(&mRec->mHash, &aDst->mRec->mHash, 371 sizeof(SHA1Sum::Hash)) == 0); 372 if (mUpdateFlags & kFrecencyUpdatedMask) { 373 aDst->mRec->mFrecency = mRec->mFrecency; 374 } 375 if (mUpdateFlags & kExpirationUpdatedMask) { 376 aDst->mRec->mExpirationTime = mRec->mExpirationTime; 377 } 378 aDst->mRec->mOriginAttrsHash = mRec->mOriginAttrsHash; 379 if (mUpdateFlags & kFileSizeUpdatedMask) { 380 aDst->mRec->mFlags = mRec->mFlags; 381 } else { 382 // Copy all flags except file size. 383 aDst->mRec->mFlags &= kFileSizeMask; 384 aDst->mRec->mFlags |= (mRec->mFlags & ~kFileSizeMask); 385 } 386 } 387 388 private: 389 static const uint32_t kFrecencyUpdatedMask = 0x00000001; 390 static const uint32_t kExpirationUpdatedMask = 0x00000002; 391 static const uint32_t kFileSizeUpdatedMask = 0x00000004; 392 393 uint32_t mUpdateFlags; 394 }; 395 396 class CacheIndexStats 397 { 398 public: CacheIndexStats()399 CacheIndexStats() 400 : mCount(0) 401 , mNotInitialized(0) 402 , mRemoved(0) 403 , mDirty(0) 404 , mFresh(0) 405 , mEmpty(0) 406 , mSize(0) 407 #ifdef DEBUG 408 , mStateLogged(false) 409 , mDisableLogging(false) 410 #endif 411 { 412 } 413 414 bool operator==(const CacheIndexStats& aOther) const 415 { 416 return 417 #ifdef DEBUG 418 aOther.mStateLogged == mStateLogged && 419 #endif 420 aOther.mCount == mCount && 421 aOther.mNotInitialized == mNotInitialized && 422 aOther.mRemoved == mRemoved && 423 aOther.mDirty == mDirty && 424 aOther.mFresh == mFresh && 425 aOther.mEmpty == mEmpty && 426 aOther.mSize == mSize; 427 } 428 429 #ifdef DEBUG DisableLogging()430 void DisableLogging() { 431 mDisableLogging = true; 432 } 433 #endif 434 Log()435 void Log() { 436 LOG(("CacheIndexStats::Log() [count=%u, notInitialized=%u, removed=%u, " 437 "dirty=%u, fresh=%u, empty=%u, size=%u]", mCount, mNotInitialized, 438 mRemoved, mDirty, mFresh, mEmpty, mSize)); 439 } 440 Clear()441 void Clear() { 442 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Clear() - state logged!"); 443 444 mCount = 0; 445 mNotInitialized = 0; 446 mRemoved = 0; 447 mDirty = 0; 448 mFresh = 0; 449 mEmpty = 0; 450 mSize = 0; 451 } 452 453 #ifdef DEBUG StateLogged()454 bool StateLogged() { 455 return mStateLogged; 456 } 457 #endif 458 Count()459 uint32_t Count() { 460 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Count() - state logged!"); 461 return mCount; 462 } 463 Dirty()464 uint32_t Dirty() { 465 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Dirty() - state logged!"); 466 return mDirty; 467 } 468 Fresh()469 uint32_t Fresh() { 470 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Fresh() - state logged!"); 471 return mFresh; 472 } 473 ActiveEntriesCount()474 uint32_t ActiveEntriesCount() { 475 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::ActiveEntriesCount() - state " 476 "logged!"); 477 return mCount - mRemoved - mNotInitialized - mEmpty; 478 } 479 Size()480 uint32_t Size() { 481 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Size() - state logged!"); 482 return mSize; 483 } 484 BeforeChange(const CacheIndexEntry * aEntry)485 void BeforeChange(const CacheIndexEntry *aEntry) { 486 #ifdef DEBUG_STATS 487 if (!mDisableLogging) { 488 LOG(("CacheIndexStats::BeforeChange()")); 489 Log(); 490 } 491 #endif 492 493 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::BeforeChange() - state " 494 "logged!"); 495 #ifdef DEBUG 496 mStateLogged = true; 497 #endif 498 if (aEntry) { 499 MOZ_ASSERT(mCount); 500 mCount--; 501 if (aEntry->IsDirty()) { 502 MOZ_ASSERT(mDirty); 503 mDirty--; 504 } 505 if (aEntry->IsFresh()) { 506 MOZ_ASSERT(mFresh); 507 mFresh--; 508 } 509 if (aEntry->IsRemoved()) { 510 MOZ_ASSERT(mRemoved); 511 mRemoved--; 512 } else { 513 if (!aEntry->IsInitialized()) { 514 MOZ_ASSERT(mNotInitialized); 515 mNotInitialized--; 516 } else { 517 if (aEntry->IsFileEmpty()) { 518 MOZ_ASSERT(mEmpty); 519 mEmpty--; 520 } else { 521 MOZ_ASSERT(mSize >= aEntry->GetFileSize()); 522 mSize -= aEntry->GetFileSize(); 523 } 524 } 525 } 526 } 527 } 528 AfterChange(const CacheIndexEntry * aEntry)529 void AfterChange(const CacheIndexEntry *aEntry) { 530 MOZ_ASSERT(mStateLogged, "CacheIndexStats::AfterChange() - state not " 531 "logged!"); 532 #ifdef DEBUG 533 mStateLogged = false; 534 #endif 535 if (aEntry) { 536 ++mCount; 537 if (aEntry->IsDirty()) { 538 mDirty++; 539 } 540 if (aEntry->IsFresh()) { 541 mFresh++; 542 } 543 if (aEntry->IsRemoved()) { 544 mRemoved++; 545 } else { 546 if (!aEntry->IsInitialized()) { 547 mNotInitialized++; 548 } else { 549 if (aEntry->IsFileEmpty()) { 550 mEmpty++; 551 } else { 552 mSize += aEntry->GetFileSize(); 553 } 554 } 555 } 556 } 557 558 #ifdef DEBUG_STATS 559 if (!mDisableLogging) { 560 LOG(("CacheIndexStats::AfterChange()")); 561 Log(); 562 } 563 #endif 564 } 565 566 private: 567 uint32_t mCount; 568 uint32_t mNotInitialized; 569 uint32_t mRemoved; 570 uint32_t mDirty; 571 uint32_t mFresh; 572 uint32_t mEmpty; 573 uint32_t mSize; 574 #ifdef DEBUG 575 // We completely remove the data about an entry from the stats in 576 // BeforeChange() and set this flag to true. The entry is then modified, 577 // deleted or created and the data is again put into the stats and this flag 578 // set to false. Statistics must not be read during this time since the 579 // information is not correct. 580 bool mStateLogged; 581 582 // Disables logging in this instance of CacheIndexStats 583 bool mDisableLogging; 584 #endif 585 }; 586 587 class CacheIndex : public CacheFileIOListener 588 , public nsIRunnable 589 { 590 public: 591 NS_DECL_THREADSAFE_ISUPPORTS 592 NS_DECL_NSIRUNNABLE 593 594 CacheIndex(); 595 596 static nsresult Init(nsIFile *aCacheDirectory); 597 static nsresult PreShutdown(); 598 static nsresult Shutdown(); 599 600 // Following methods can be called only on IO thread. 601 602 // Add entry to the index. The entry shouldn't be present in index. This 603 // method is called whenever a new handle for a new entry file is created. The 604 // newly created entry is not initialized and it must be either initialized 605 // with InitEntry() or removed with RemoveEntry(). 606 static nsresult AddEntry(const SHA1Sum::Hash *aHash); 607 608 // Inform index about an existing entry that should be present in index. This 609 // method is called whenever a new handle for an existing entry file is 610 // created. Like in case of AddEntry(), either InitEntry() or RemoveEntry() 611 // must be called on the entry, since the entry is not initizlized if the 612 // index is outdated. 613 static nsresult EnsureEntryExists(const SHA1Sum::Hash *aHash); 614 615 // Initialize the entry. It MUST be present in index. Call to AddEntry() or 616 // EnsureEntryExists() must precede the call to this method. 617 static nsresult InitEntry(const SHA1Sum::Hash *aHash, 618 OriginAttrsHash aOriginAttrsHash, 619 bool aAnonymous, 620 bool aPinned); 621 622 // Remove entry from index. The entry should be present in index. 623 static nsresult RemoveEntry(const SHA1Sum::Hash *aHash); 624 625 // Update some information in entry. The entry MUST be present in index and 626 // MUST be initialized. Call to AddEntry() or EnsureEntryExists() and to 627 // InitEntry() must precede the call to this method. 628 // Pass nullptr if the value didn't change. 629 static nsresult UpdateEntry(const SHA1Sum::Hash *aHash, 630 const uint32_t *aFrecency, 631 const uint32_t *aExpirationTime, 632 const uint32_t *aSize); 633 634 // Remove all entries from the index. Called when clearing the whole cache. 635 static nsresult RemoveAll(); 636 637 enum EntryStatus { 638 EXISTS = 0, 639 DOES_NOT_EXIST = 1, 640 DO_NOT_KNOW = 2 641 }; 642 643 // Returns status of the entry in index for the given key. It can be called 644 // on any thread. 645 // If _pinned is non-null, it's filled with pinning status of the entry. 646 static nsresult HasEntry(const nsACString &aKey, EntryStatus *_retval, 647 bool *_pinned = nullptr); 648 static nsresult HasEntry(const SHA1Sum::Hash &hash, EntryStatus *_retval, 649 bool *_pinned = nullptr); 650 651 // Returns a hash of the least important entry that should be evicted if the 652 // cache size is over limit and also returns a total number of all entries in 653 // the index minus the number of forced valid entries and unpinned entries 654 // that we encounter when searching (see below) 655 static nsresult GetEntryForEviction(bool aIgnoreEmptyEntries, SHA1Sum::Hash *aHash, uint32_t *aCnt); 656 657 // Checks if a cache entry is currently forced valid. Used to prevent an entry 658 // (that has been forced valid) from being evicted when the cache size reaches 659 // its limit. 660 static bool IsForcedValidEntry(const SHA1Sum::Hash *aHash); 661 662 // Returns cache size in kB. 663 static nsresult GetCacheSize(uint32_t *_retval); 664 665 // Returns number of entry files in the cache 666 static nsresult GetEntryFileCount(uint32_t *_retval); 667 668 // Synchronously returns the disk occupation and number of entries per-context. 669 // Callable on any thread. 670 static nsresult GetCacheStats(nsILoadContextInfo *aInfo, uint32_t *aSize, uint32_t *aCount); 671 672 // Asynchronously gets the disk cache size, used for display in the UI. 673 static nsresult AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver* aObserver); 674 675 // Returns an iterator that returns entries matching a given context that were 676 // present in the index at the time this method was called. If aAddNew is true 677 // then the iterator will also return entries created after this call. 678 // NOTE: When some entry is removed from index it is removed also from the 679 // iterator regardless what aAddNew was passed. 680 static nsresult GetIterator(nsILoadContextInfo *aInfo, bool aAddNew, 681 CacheIndexIterator **_retval); 682 683 // Returns true if we _think_ that the index is up to date. I.e. the state is 684 // READY or WRITING and mIndexNeedsUpdate as well as mShuttingDown is false. 685 static nsresult IsUpToDate(bool *_retval); 686 687 // Called from CacheStorageService::Clear() and CacheFileContextEvictor::EvictEntries(), 688 // sets a flag that blocks notification to AsyncGetDiskConsumption. 689 static void OnAsyncEviction(bool aEvicting); 690 691 // Memory reporting 692 static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf); 693 static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf); 694 695 private: 696 friend class CacheIndexEntryAutoManage; 697 friend class FileOpenHelper; 698 friend class CacheIndexIterator; 699 700 virtual ~CacheIndex(); 701 702 NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult) override; 703 nsresult OnFileOpenedInternal(FileOpenHelper *aOpener, 704 CacheFileHandle *aHandle, nsresult aResult); 705 NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf, 706 nsresult aResult) override; 707 NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult) override; 708 NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult) override; 709 NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) override; 710 NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) override; 711 712 nsresult InitInternal(nsIFile *aCacheDirectory); 713 void PreShutdownInternal(); 714 715 // This method returns false when index is not initialized or is shut down. 716 bool IsIndexUsable(); 717 718 // This method checks whether the entry has the same values of 719 // originAttributes and isAnonymous. We don't expect to find a collision 720 // since these values are part of the key that we hash and we use a strong 721 // hash function. 722 static bool IsCollision(CacheIndexEntry *aEntry, 723 OriginAttrsHash aOriginAttrsHash, 724 bool aAnonymous); 725 726 // Checks whether any of the information about the entry has changed. 727 static bool HasEntryChanged(CacheIndexEntry *aEntry, 728 const uint32_t *aFrecency, 729 const uint32_t *aExpirationTime, 730 const uint32_t *aSize); 731 732 // Merge all pending operations from mPendingUpdates into mIndex. 733 void ProcessPendingOperations(); 734 735 // Following methods perform writing of the index file. 736 // 737 // The index is written periodically, but not earlier than once in 738 // kMinDumpInterval and there must be at least kMinUnwrittenChanges 739 // differences between index on disk and in memory. Index is always first 740 // written to a temporary file and the old index file is replaced when the 741 // writing process succeeds. 742 // 743 // Starts writing of index when both limits (minimal delay between writes and 744 // minimum number of changes in index) were exceeded. 745 bool WriteIndexToDiskIfNeeded(); 746 // Starts writing of index file. 747 void WriteIndexToDisk(); 748 // Serializes part of mIndex hashtable to the write buffer a writes the buffer 749 // to the file. 750 void WriteRecords(); 751 // Finalizes writing process. 752 void FinishWrite(bool aSucceeded); 753 754 // Following methods perform writing of the journal during shutdown. All these 755 // methods must be called only during shutdown since they write/delete files 756 // directly on the main thread instead of using CacheFileIOManager that does 757 // it asynchronously on IO thread. Journal contains only entries that are 758 // dirty, i.e. changes that are not present in the index file on the disk. 759 // When the log is written successfully, the dirty flag in index file is 760 // cleared. 761 nsresult GetFile(const nsACString &aName, nsIFile **_retval); 762 nsresult RemoveFile(const nsACString &aName); 763 void RemoveAllIndexFiles(); 764 void RemoveJournalAndTempFile(); 765 // Writes journal to the disk and clears dirty flag in index header. 766 nsresult WriteLogToDisk(); 767 768 // Following methods perform reading of the index from the disk. 769 // 770 // Index is read at startup just after initializing the CacheIndex. There are 771 // 3 files used when manipulating with index: index file, journal file and 772 // a temporary file. All files contain the hash of the data, so we can check 773 // whether the content is valid and complete. Index file contains also a dirty 774 // flag in the index header which is unset on a clean shutdown. During opening 775 // and reading of the files we determine the status of the whole index from 776 // the states of the separate files. Following table shows all possible 777 // combinations: 778 // 779 // index, journal, tmpfile 780 // M * * - index is missing -> BUILD 781 // I * * - index is invalid -> BUILD 782 // D * * - index is dirty -> UPDATE 783 // C M * - index is dirty -> UPDATE 784 // C I * - unexpected state -> UPDATE 785 // C V E - unexpected state -> UPDATE 786 // C V M - index is up to date -> READY 787 // 788 // where the letters mean: 789 // * - any state 790 // E - file exists 791 // M - file is missing 792 // I - data is invalid (parsing failed or hash didn't match) 793 // D - dirty (data in index file is correct, but dirty flag is set) 794 // C - clean (index file is clean) 795 // V - valid (data in journal file is correct) 796 // 797 // Note: We accept the data from journal only when the index is up to date as 798 // a whole (i.e. C,V,M state). 799 // 800 // We rename the journal file to the temporary file as soon as possible after 801 // initial test to ensure that we start update process on the next startup if 802 // FF crashes during parsing of the index. 803 // 804 // Initiates reading index from disk. 805 void ReadIndexFromDisk(); 806 // Starts reading data from index file. 807 void StartReadingIndex(); 808 // Parses data read from index file. 809 void ParseRecords(); 810 // Starts reading data from journal file. 811 void StartReadingJournal(); 812 // Parses data read from journal file. 813 void ParseJournal(); 814 // Merges entries from journal into mIndex. 815 void MergeJournal(); 816 // In debug build this method checks that we have no fresh entry in mIndex 817 // after we finish reading index and before we process pending operations. 818 void EnsureNoFreshEntry(); 819 // In debug build this method is called after processing pending operations 820 // to make sure mIndexStats contains correct information. 821 void EnsureCorrectStats(); 822 // Finalizes reading process. 823 void FinishRead(bool aSucceeded); 824 825 // Following methods perform updating and building of the index. 826 // Timer callback that starts update or build process. 827 static void DelayedUpdate(nsITimer *aTimer, void *aClosure); 828 void DelayedUpdateLocked(); 829 // Posts timer event that start update or build process. 830 nsresult ScheduleUpdateTimer(uint32_t aDelay); 831 nsresult SetupDirectoryEnumerator(); 832 void InitEntryFromDiskData(CacheIndexEntry *aEntry, 833 CacheFileMetadata *aMetaData, 834 int64_t aFileSize); 835 // Returns true when either a timer is scheduled or event is posted. 836 bool IsUpdatePending(); 837 // Iterates through all files in entries directory that we didn't create/open 838 // during this session, parses them and adds the entries to the index. 839 void BuildIndex(); 840 841 bool StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState = false); 842 // Starts update or build process or fires a timer when it is too early after 843 // startup. 844 void StartUpdatingIndex(bool aRebuild); 845 // Iterates through all files in entries directory that we didn't create/open 846 // during this session and theirs last modified time is newer than timestamp 847 // in the index header. Parses the files and adds the entries to the index. 848 void UpdateIndex(); 849 // Finalizes update or build process. 850 void FinishUpdate(bool aSucceeded); 851 852 void RemoveNonFreshEntries(); 853 854 enum EState { 855 // Initial state in which the index is not usable 856 // Possible transitions: 857 // -> READING 858 INITIAL = 0, 859 860 // Index is being read from the disk. 861 // Possible transitions: 862 // -> INITIAL - We failed to dispatch a read event. 863 // -> BUILDING - No or corrupted index file was found. 864 // -> UPDATING - No or corrupted journal file was found. 865 // - Dirty flag was set in index header. 866 // -> READY - Index was read successfully or was interrupted by 867 // pre-shutdown. 868 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. 869 READING = 1, 870 871 // Index is being written to the disk. 872 // Possible transitions: 873 // -> READY - Writing of index finished or was interrupted by 874 // pre-shutdown.. 875 // -> UPDATING - Writing of index finished, but index was found outdated 876 // during writing. 877 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. 878 WRITING = 2, 879 880 // Index is being build. 881 // Possible transitions: 882 // -> READY - Building of index finished or was interrupted by 883 // pre-shutdown. 884 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. 885 BUILDING = 3, 886 887 // Index is being updated. 888 // Possible transitions: 889 // -> READY - Updating of index finished or was interrupted by 890 // pre-shutdown. 891 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. 892 UPDATING = 4, 893 894 // Index is ready. 895 // Possible transitions: 896 // -> UPDATING - Index was found outdated. 897 // -> SHUTDOWN - Index is shutting down. 898 READY = 5, 899 900 // Index is shutting down. 901 SHUTDOWN = 6 902 }; 903 904 static char const * StateString(EState aState); 905 void ChangeState(EState aNewState); 906 void NotifyAsyncGetDiskConsumptionCallbacks(); 907 908 // Allocates and releases buffer used for reading and writing index. 909 void AllocBuffer(); 910 void ReleaseBuffer(); 911 912 // Methods used by CacheIndexEntryAutoManage to keep the iterators up to date. 913 void AddRecordToIterators(CacheIndexRecord *aRecord); 914 void RemoveRecordFromIterators(CacheIndexRecord *aRecord); 915 void ReplaceRecordInIterators(CacheIndexRecord *aOldRecord, 916 CacheIndexRecord *aNewRecord); 917 918 // Memory reporting (private part) 919 size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const; 920 921 void ReportHashStats(); 922 923 static mozilla::StaticRefPtr<CacheIndex> gInstance; 924 static StaticMutex sLock; 925 926 nsCOMPtr<nsIFile> mCacheDirectory; 927 928 EState mState; 929 // Timestamp of time when the index was initialized. We use it to delay 930 // initial update or build of index. 931 TimeStamp mStartTime; 932 // Set to true in PreShutdown(), it is checked on variaous places to prevent 933 // starting any process (write, update, etc.) during shutdown. 934 bool mShuttingDown; 935 // When set to true, update process should start as soon as possible. This 936 // flag is set whenever we find some inconsistency which would be fixed by 937 // update process. The flag is checked always when switching to READY state. 938 // To make sure we start the update process as soon as possible, methods that 939 // set this flag should also call StartUpdatingIndexIfNeeded() to cover the 940 // case when we are currently in READY state. 941 bool mIndexNeedsUpdate; 942 // Set at the beginning of RemoveAll() which clears the whole index. When 943 // removing all entries we must stop any pending reading, writing, updating or 944 // building operation. This flag is checked at various places and it prevents 945 // we won't start another operation (e.g. canceling reading of the index would 946 // normally start update or build process) 947 bool mRemovingAll; 948 // Whether the index file on disk exists and is valid. 949 bool mIndexOnDiskIsValid; 950 // When something goes wrong during updating or building process, we don't 951 // mark index clean (and also don't write journal) to ensure that update or 952 // build will be initiated on the next start. 953 bool mDontMarkIndexClean; 954 // Timestamp value from index file. It is used during update process to skip 955 // entries that were last modified before this timestamp. 956 uint32_t mIndexTimeStamp; 957 // Timestamp of last time the index was dumped to disk. 958 // NOTE: The index might not be necessarily dumped at this time. The value 959 // is used to schedule next dump of the index. 960 TimeStamp mLastDumpTime; 961 962 // Timer of delayed update/build. 963 nsCOMPtr<nsITimer> mUpdateTimer; 964 // True when build or update event is posted 965 bool mUpdateEventPending; 966 967 // Helper members used when reading/writing index from/to disk. 968 // Contains number of entries that should be skipped: 969 // - in hashtable when writing index because they were already written 970 // - in index file when reading index because they were already read 971 uint32_t mSkipEntries; 972 // Number of entries that should be written to disk. This is number of entries 973 // in hashtable that are initialized and are not marked as removed when writing 974 // begins. 975 uint32_t mProcessEntries; 976 char *mRWBuf; 977 uint32_t mRWBufSize; 978 uint32_t mRWBufPos; 979 RefPtr<CacheHash> mRWHash; 980 981 // True if read or write operation is pending. It is used to ensure that 982 // mRWBuf is not freed until OnDataRead or OnDataWritten is called. 983 bool mRWPending; 984 985 // Reading of journal succeeded if true. 986 bool mJournalReadSuccessfully; 987 988 // Handle used for writing and reading index file. 989 RefPtr<CacheFileHandle> mIndexHandle; 990 // Handle used for reading journal file. 991 RefPtr<CacheFileHandle> mJournalHandle; 992 // Used to check the existence of the file during reading process. 993 RefPtr<CacheFileHandle> mTmpHandle; 994 995 RefPtr<FileOpenHelper> mIndexFileOpener; 996 RefPtr<FileOpenHelper> mJournalFileOpener; 997 RefPtr<FileOpenHelper> mTmpFileOpener; 998 999 // Directory enumerator used when building and updating index. 1000 nsCOMPtr<nsIDirectoryEnumerator> mDirEnumerator; 1001 1002 // Main index hashtable. 1003 nsTHashtable<CacheIndexEntry> mIndex; 1004 1005 // We cannot add, remove or change any entry in mIndex in states READING and 1006 // WRITING. We track all changes in mPendingUpdates during these states. 1007 nsTHashtable<CacheIndexEntryUpdate> mPendingUpdates; 1008 1009 // Contains information statistics for mIndex + mPendingUpdates. 1010 CacheIndexStats mIndexStats; 1011 1012 // When reading journal, we must first parse the whole file and apply the 1013 // changes iff the journal was read successfully. mTmpJournal is used to store 1014 // entries from the journal file. We throw away all these entries if parsing 1015 // of the journal fails or the hash does not match. 1016 nsTHashtable<CacheIndexEntry> mTmpJournal; 1017 1018 // FrecencyArray maintains order of entry records for eviction. Ideally, the 1019 // records would be ordered by frecency all the time, but since this would be 1020 // quite expensive, we allow certain amount of entries to be out of order. 1021 // When the frecency is updated the new value is always bigger than the old 1022 // one. Instead of keeping updated entries at the same position, we move them 1023 // at the end of the array. This protects recently updated entries from 1024 // eviction. The array is sorted once we hit the limit of maximum unsorted 1025 // entries. 1026 class FrecencyArray 1027 { 1028 class Iterator 1029 { 1030 public: Iterator(nsTArray<CacheIndexRecord * > * aRecs)1031 explicit Iterator(nsTArray<CacheIndexRecord *> *aRecs) 1032 : mRecs(aRecs) 1033 , mIdx(0) 1034 { 1035 while (!Done() && !(*mRecs)[mIdx]) { 1036 mIdx++; 1037 } 1038 } 1039 Done()1040 bool Done() const { return mIdx == mRecs->Length(); } 1041 Get()1042 CacheIndexRecord* Get() const 1043 { 1044 MOZ_ASSERT(!Done()); 1045 return (*mRecs)[mIdx]; 1046 } 1047 Next()1048 void Next() 1049 { 1050 MOZ_ASSERT(!Done()); 1051 ++mIdx; 1052 while (!Done() && !(*mRecs)[mIdx]) { 1053 mIdx++; 1054 } 1055 } 1056 1057 private: 1058 nsTArray<CacheIndexRecord *> *mRecs; 1059 uint32_t mIdx; 1060 }; 1061 1062 public: Iter()1063 Iterator Iter() { return Iterator(&mRecs); } 1064 FrecencyArray()1065 FrecencyArray() : mUnsortedElements(0) 1066 , mRemovedElements(0) {} 1067 1068 // Methods used by CacheIndexEntryAutoManage to keep the array up to date. 1069 void AppendRecord(CacheIndexRecord *aRecord); 1070 void RemoveRecord(CacheIndexRecord *aRecord); 1071 void ReplaceRecord(CacheIndexRecord *aOldRecord, 1072 CacheIndexRecord *aNewRecord); 1073 void SortIfNeeded(); 1074 Length()1075 size_t Length() const { return mRecs.Length() - mRemovedElements; } Clear()1076 void Clear() { mRecs.Clear(); } 1077 1078 private: 1079 friend class CacheIndex; 1080 1081 nsTArray<CacheIndexRecord *> mRecs; 1082 uint32_t mUnsortedElements; 1083 // Instead of removing elements from the array immediately, we null them out 1084 // and the iterator skips them when accessing the array. The null pointers 1085 // are placed at the end during sorting and we strip them out all at once. 1086 // This saves moving a lot of memory in nsTArray::RemoveElementsAt. 1087 uint32_t mRemovedElements; 1088 }; 1089 1090 FrecencyArray mFrecencyArray; 1091 1092 nsTArray<CacheIndexIterator *> mIterators; 1093 1094 // This flag is true iff we are between CacheStorageService:Clear() and processing 1095 // all contexts to be evicted. It will make UI to show "calculating" instead of 1096 // any intermediate cache size. 1097 bool mAsyncGetDiskConsumptionBlocked; 1098 1099 class DiskConsumptionObserver : public Runnable 1100 { 1101 public: Init(nsICacheStorageConsumptionObserver * aObserver)1102 static DiskConsumptionObserver* Init(nsICacheStorageConsumptionObserver* aObserver) 1103 { 1104 nsWeakPtr observer = do_GetWeakReference(aObserver); 1105 if (!observer) 1106 return nullptr; 1107 1108 return new DiskConsumptionObserver(observer); 1109 } 1110 OnDiskConsumption(int64_t aSize)1111 void OnDiskConsumption(int64_t aSize) 1112 { 1113 mSize = aSize; 1114 NS_DispatchToMainThread(this); 1115 } 1116 1117 private: DiskConsumptionObserver(nsWeakPtr const & aWeakObserver)1118 explicit DiskConsumptionObserver(nsWeakPtr const &aWeakObserver) 1119 : mObserver(aWeakObserver) { } ~DiskConsumptionObserver()1120 virtual ~DiskConsumptionObserver() { 1121 if (mObserver && !NS_IsMainThread()) { 1122 NS_ReleaseOnMainThread(mObserver.forget()); 1123 } 1124 } 1125 Run()1126 NS_IMETHOD Run() override 1127 { 1128 MOZ_ASSERT(NS_IsMainThread()); 1129 1130 nsCOMPtr<nsICacheStorageConsumptionObserver> observer = 1131 do_QueryReferent(mObserver); 1132 1133 mObserver = nullptr; 1134 1135 if (observer) { 1136 observer->OnNetworkCacheDiskConsumption(mSize); 1137 } 1138 1139 return NS_OK; 1140 } 1141 1142 nsWeakPtr mObserver; 1143 int64_t mSize; 1144 }; 1145 1146 // List of async observers that want to get disk consumption information 1147 nsTArray<RefPtr<DiskConsumptionObserver> > mDiskConsumptionObservers; 1148 }; 1149 1150 } // namespace net 1151 } // namespace mozilla 1152 1153 #endif 1154