1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5 #include "CacheIndex.h"
6
7 #include "CacheLog.h"
8 #include "CacheFileIOManager.h"
9 #include "CacheFileMetadata.h"
10 #include "CacheFileUtils.h"
11 #include "CacheIndexIterator.h"
12 #include "CacheIndexContextIterator.h"
13 #include "nsThreadUtils.h"
14 #include "nsISizeOf.h"
15 #include "nsPrintfCString.h"
16 #include "mozilla/DebugOnly.h"
17 #include "prinrval.h"
18 #include "nsIFile.h"
19 #include "nsITimer.h"
20 #include "mozilla/AutoRestore.h"
21 #include <algorithm>
22 #include "mozilla/StaticPrefs_network.h"
23 #include "mozilla/Telemetry.h"
24 #include "mozilla/Unused.h"
25
26 #define kMinUnwrittenChanges 300
27 #define kMinDumpInterval 20000 // in milliseconds
28 #define kMaxBufSize 16384
29 #define kIndexVersion 0x0000000A
30 #define kUpdateIndexStartDelay 50000 // in milliseconds
31 #define kTelemetryReportBytesLimit (2U * 1024U * 1024U * 1024U) // 2GB
32
33 #define INDEX_NAME "index"
34 #define TEMP_INDEX_NAME "index.tmp"
35 #define JOURNAL_NAME "index.log"
36
37 namespace mozilla::net {
38
39 namespace {
40
41 class FrecencyComparator {
42 public:
Equals(const RefPtr<CacheIndexRecordWrapper> & a,const RefPtr<CacheIndexRecordWrapper> & b) const43 bool Equals(const RefPtr<CacheIndexRecordWrapper>& a,
44 const RefPtr<CacheIndexRecordWrapper>& b) const {
45 if (!a || !b) {
46 return false;
47 }
48
49 return a->Get()->mFrecency == b->Get()->mFrecency;
50 }
LessThan(const RefPtr<CacheIndexRecordWrapper> & a,const RefPtr<CacheIndexRecordWrapper> & b) const51 bool LessThan(const RefPtr<CacheIndexRecordWrapper>& a,
52 const RefPtr<CacheIndexRecordWrapper>& b) const {
53 // Removed (=null) entries must be at the end of the array.
54 if (!a) {
55 return false;
56 }
57 if (!b) {
58 return true;
59 }
60
61 // Place entries with frecency 0 at the end of the non-removed entries.
62 if (a->Get()->mFrecency == 0) {
63 return false;
64 }
65 if (b->Get()->mFrecency == 0) {
66 return true;
67 }
68
69 return a->Get()->mFrecency < b->Get()->mFrecency;
70 }
71 };
72
73 } // namespace
74
75 /**
76 * This helper class is responsible for keeping CacheIndex::mIndexStats and
77 * CacheIndex::mFrecencyArray up to date.
78 */
79 class MOZ_RAII CacheIndexEntryAutoManage {
80 public:
CacheIndexEntryAutoManage(const SHA1Sum::Hash * aHash,CacheIndex * aIndex,const StaticMutexAutoLock & aProofOfLock)81 CacheIndexEntryAutoManage(const SHA1Sum::Hash* aHash, CacheIndex* aIndex,
82 const StaticMutexAutoLock& aProofOfLock)
83 : mIndex(aIndex), mProofOfLock(aProofOfLock) {
84 mHash = aHash;
85 const CacheIndexEntry* entry = FindEntry();
86 mIndex->mIndexStats.BeforeChange(entry);
87 if (entry && entry->IsInitialized() && !entry->IsRemoved()) {
88 mOldRecord = entry->mRec;
89 mOldFrecency = entry->mRec->Get()->mFrecency;
90 }
91 }
92
~CacheIndexEntryAutoManage()93 ~CacheIndexEntryAutoManage() {
94 const CacheIndexEntry* entry = FindEntry();
95 mIndex->mIndexStats.AfterChange(entry);
96 if (!entry || !entry->IsInitialized() || entry->IsRemoved()) {
97 entry = nullptr;
98 }
99
100 if (entry && !mOldRecord) {
101 mIndex->mFrecencyArray.AppendRecord(entry->mRec, mProofOfLock);
102 mIndex->AddRecordToIterators(entry->mRec, mProofOfLock);
103 } else if (!entry && mOldRecord) {
104 mIndex->mFrecencyArray.RemoveRecord(mOldRecord, mProofOfLock);
105 mIndex->RemoveRecordFromIterators(mOldRecord, mProofOfLock);
106 } else if (entry && mOldRecord) {
107 if (entry->mRec != mOldRecord) {
108 // record has a different address, we have to replace it
109 mIndex->ReplaceRecordInIterators(mOldRecord, entry->mRec, mProofOfLock);
110
111 if (entry->mRec->Get()->mFrecency == mOldFrecency) {
112 // If frecency hasn't changed simply replace the pointer
113 mIndex->mFrecencyArray.ReplaceRecord(mOldRecord, entry->mRec,
114 mProofOfLock);
115 } else {
116 // Remove old pointer and insert the new one at the end of the array
117 mIndex->mFrecencyArray.RemoveRecord(mOldRecord, mProofOfLock);
118 mIndex->mFrecencyArray.AppendRecord(entry->mRec, mProofOfLock);
119 }
120 } else if (entry->mRec->Get()->mFrecency != mOldFrecency) {
121 // Move the element at the end of the array
122 mIndex->mFrecencyArray.RemoveRecord(entry->mRec, mProofOfLock);
123 mIndex->mFrecencyArray.AppendRecord(entry->mRec, mProofOfLock);
124 }
125 } else {
126 // both entries were removed or not initialized, do nothing
127 }
128 }
129
130 // We cannot rely on nsTHashtable::GetEntry() in case we are removing entries
131 // while iterating. Destructor is called before the entry is removed. Caller
132 // must call one of following methods to skip lookup in the hashtable.
DoNotSearchInIndex()133 void DoNotSearchInIndex() { mDoNotSearchInIndex = true; }
DoNotSearchInUpdates()134 void DoNotSearchInUpdates() { mDoNotSearchInUpdates = true; }
135
136 private:
FindEntry()137 const CacheIndexEntry* FindEntry() {
138 const CacheIndexEntry* entry = nullptr;
139
140 switch (mIndex->mState) {
141 case CacheIndex::READING:
142 case CacheIndex::WRITING:
143 if (!mDoNotSearchInUpdates) {
144 entry = mIndex->mPendingUpdates.GetEntry(*mHash);
145 }
146 [[fallthrough]];
147 case CacheIndex::BUILDING:
148 case CacheIndex::UPDATING:
149 case CacheIndex::READY:
150 if (!entry && !mDoNotSearchInIndex) {
151 entry = mIndex->mIndex.GetEntry(*mHash);
152 }
153 break;
154 case CacheIndex::INITIAL:
155 case CacheIndex::SHUTDOWN:
156 default:
157 MOZ_ASSERT(false, "Unexpected state!");
158 }
159
160 return entry;
161 }
162
163 const SHA1Sum::Hash* mHash;
164 RefPtr<CacheIndex> mIndex;
165 RefPtr<CacheIndexRecordWrapper> mOldRecord;
166 uint32_t mOldFrecency{0};
167 bool mDoNotSearchInIndex{false};
168 bool mDoNotSearchInUpdates{false};
169 const StaticMutexAutoLock& mProofOfLock;
170 };
171
172 class FileOpenHelper final : public CacheFileIOListener {
173 public:
174 NS_DECL_THREADSAFE_ISUPPORTS
175
FileOpenHelper(CacheIndex * aIndex)176 explicit FileOpenHelper(CacheIndex* aIndex)
177 : mIndex(aIndex), mCanceled(false) {}
178
Cancel()179 void Cancel() {
180 CacheIndex::sLock.AssertCurrentThreadOwns();
181 mCanceled = true;
182 }
183
184 private:
185 virtual ~FileOpenHelper() = default;
186
187 NS_IMETHOD OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) override;
OnDataWritten(CacheFileHandle * aHandle,const char * aBuf,nsresult aResult)188 NS_IMETHOD OnDataWritten(CacheFileHandle* aHandle, const char* aBuf,
189 nsresult aResult) override {
190 MOZ_CRASH("FileOpenHelper::OnDataWritten should not be called!");
191 return NS_ERROR_UNEXPECTED;
192 }
OnDataRead(CacheFileHandle * aHandle,char * aBuf,nsresult aResult)193 NS_IMETHOD OnDataRead(CacheFileHandle* aHandle, char* aBuf,
194 nsresult aResult) override {
195 MOZ_CRASH("FileOpenHelper::OnDataRead should not be called!");
196 return NS_ERROR_UNEXPECTED;
197 }
OnFileDoomed(CacheFileHandle * aHandle,nsresult aResult)198 NS_IMETHOD OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) override {
199 MOZ_CRASH("FileOpenHelper::OnFileDoomed should not be called!");
200 return NS_ERROR_UNEXPECTED;
201 }
OnEOFSet(CacheFileHandle * aHandle,nsresult aResult)202 NS_IMETHOD OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) override {
203 MOZ_CRASH("FileOpenHelper::OnEOFSet should not be called!");
204 return NS_ERROR_UNEXPECTED;
205 }
OnFileRenamed(CacheFileHandle * aHandle,nsresult aResult)206 NS_IMETHOD OnFileRenamed(CacheFileHandle* aHandle,
207 nsresult aResult) override {
208 MOZ_CRASH("FileOpenHelper::OnFileRenamed should not be called!");
209 return NS_ERROR_UNEXPECTED;
210 }
211
212 RefPtr<CacheIndex> mIndex;
213 bool mCanceled;
214 };
215
OnFileOpened(CacheFileHandle * aHandle,nsresult aResult)216 NS_IMETHODIMP FileOpenHelper::OnFileOpened(CacheFileHandle* aHandle,
217 nsresult aResult) {
218 StaticMutexAutoLock lock(CacheIndex::sLock);
219
220 if (mCanceled) {
221 if (aHandle) {
222 CacheFileIOManager::DoomFile(aHandle, nullptr);
223 }
224
225 return NS_OK;
226 }
227
228 mIndex->OnFileOpenedInternal(this, aHandle, aResult, lock);
229
230 return NS_OK;
231 }
232
233 NS_IMPL_ISUPPORTS(FileOpenHelper, CacheFileIOListener);
234
235 StaticRefPtr<CacheIndex> CacheIndex::gInstance;
236 StaticMutex CacheIndex::sLock;
237
238 NS_IMPL_ADDREF(CacheIndex)
NS_IMPL_RELEASE(CacheIndex)239 NS_IMPL_RELEASE(CacheIndex)
240
241 NS_INTERFACE_MAP_BEGIN(CacheIndex)
242 NS_INTERFACE_MAP_ENTRY(mozilla::net::CacheFileIOListener)
243 NS_INTERFACE_MAP_ENTRY(nsIRunnable)
244 NS_INTERFACE_MAP_END
245
246 CacheIndex::CacheIndex()
247
248 {
249 sLock.AssertCurrentThreadOwns();
250 LOG(("CacheIndex::CacheIndex [this=%p]", this));
251 MOZ_ASSERT(!gInstance, "multiple CacheIndex instances!");
252 }
253
~CacheIndex()254 CacheIndex::~CacheIndex() {
255 sLock.AssertCurrentThreadOwns();
256 LOG(("CacheIndex::~CacheIndex [this=%p]", this));
257
258 ReleaseBuffer();
259 }
260
261 // static
Init(nsIFile * aCacheDirectory)262 nsresult CacheIndex::Init(nsIFile* aCacheDirectory) {
263 LOG(("CacheIndex::Init()"));
264
265 MOZ_ASSERT(NS_IsMainThread());
266
267 StaticMutexAutoLock lock(sLock);
268
269 if (gInstance) {
270 return NS_ERROR_ALREADY_INITIALIZED;
271 }
272
273 RefPtr<CacheIndex> idx = new CacheIndex();
274
275 nsresult rv = idx->InitInternal(aCacheDirectory, lock);
276 NS_ENSURE_SUCCESS(rv, rv);
277
278 gInstance = std::move(idx);
279 return NS_OK;
280 }
281
InitInternal(nsIFile * aCacheDirectory,const StaticMutexAutoLock & aProofOfLock)282 nsresult CacheIndex::InitInternal(nsIFile* aCacheDirectory,
283 const StaticMutexAutoLock& aProofOfLock) {
284 nsresult rv;
285
286 rv = aCacheDirectory->Clone(getter_AddRefs(mCacheDirectory));
287 NS_ENSURE_SUCCESS(rv, rv);
288
289 mStartTime = TimeStamp::NowLoRes();
290
291 ReadIndexFromDisk(aProofOfLock);
292
293 return NS_OK;
294 }
295
296 // static
PreShutdown()297 nsresult CacheIndex::PreShutdown() {
298 MOZ_ASSERT(NS_IsMainThread());
299
300 StaticMutexAutoLock lock(sLock);
301
302 LOG(("CacheIndex::PreShutdown() [gInstance=%p]", gInstance.get()));
303
304 nsresult rv;
305 RefPtr<CacheIndex> index = gInstance;
306
307 if (!index) {
308 return NS_ERROR_NOT_INITIALIZED;
309 }
310
311 LOG(
312 ("CacheIndex::PreShutdown() - [state=%d, indexOnDiskIsValid=%d, "
313 "dontMarkIndexClean=%d]",
314 index->mState, index->mIndexOnDiskIsValid, index->mDontMarkIndexClean));
315
316 LOG(("CacheIndex::PreShutdown() - Closing iterators."));
317 for (uint32_t i = 0; i < index->mIterators.Length();) {
318 rv = index->mIterators[i]->CloseInternal(NS_ERROR_FAILURE);
319 if (NS_FAILED(rv)) {
320 // CacheIndexIterator::CloseInternal() removes itself from mIteratos iff
321 // it returns success.
322 LOG(
323 ("CacheIndex::PreShutdown() - Failed to remove iterator %p. "
324 "[rv=0x%08" PRIx32 "]",
325 index->mIterators[i], static_cast<uint32_t>(rv)));
326 i++;
327 }
328 }
329
330 index->mShuttingDown = true;
331
332 if (index->mState == READY) {
333 return NS_OK; // nothing to do
334 }
335
336 nsCOMPtr<nsIRunnable> event;
337 event = NewRunnableMethod("net::CacheIndex::PreShutdownInternal", index,
338 &CacheIndex::PreShutdownInternal);
339
340 nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
341 MOZ_ASSERT(ioTarget);
342
343 // PreShutdownInternal() will be executed before any queued event on INDEX
344 // level. That's OK since we don't want to wait for any operation in progess.
345 rv = ioTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL);
346 if (NS_FAILED(rv)) {
347 NS_WARNING("CacheIndex::PreShutdown() - Can't dispatch event");
348 LOG(("CacheIndex::PreShutdown() - Can't dispatch event"));
349 return rv;
350 }
351
352 return NS_OK;
353 }
354
PreShutdownInternal()355 void CacheIndex::PreShutdownInternal() {
356 StaticMutexAutoLock lock(sLock);
357
358 LOG(
359 ("CacheIndex::PreShutdownInternal() - [state=%d, indexOnDiskIsValid=%d, "
360 "dontMarkIndexClean=%d]",
361 mState, mIndexOnDiskIsValid, mDontMarkIndexClean));
362
363 MOZ_ASSERT(mShuttingDown);
364
365 if (mUpdateTimer) {
366 mUpdateTimer->Cancel();
367 mUpdateTimer = nullptr;
368 }
369
370 switch (mState) {
371 case WRITING:
372 FinishWrite(false, lock);
373 break;
374 case READY:
375 // nothing to do, write the journal in Shutdown()
376 break;
377 case READING:
378 FinishRead(false, lock);
379 break;
380 case BUILDING:
381 case UPDATING:
382 FinishUpdate(false, lock);
383 break;
384 default:
385 MOZ_ASSERT(false, "Implement me!");
386 }
387
388 // We should end up in READY state
389 MOZ_ASSERT(mState == READY);
390 }
391
392 // static
Shutdown()393 nsresult CacheIndex::Shutdown() {
394 MOZ_ASSERT(NS_IsMainThread());
395
396 StaticMutexAutoLock lock(sLock);
397
398 LOG(("CacheIndex::Shutdown() [gInstance=%p]", gInstance.get()));
399
400 RefPtr<CacheIndex> index = gInstance.forget();
401
402 if (!index) {
403 return NS_ERROR_NOT_INITIALIZED;
404 }
405
406 bool sanitize = CacheObserver::ClearCacheOnShutdown();
407
408 LOG(
409 ("CacheIndex::Shutdown() - [state=%d, indexOnDiskIsValid=%d, "
410 "dontMarkIndexClean=%d, sanitize=%d]",
411 index->mState, index->mIndexOnDiskIsValid, index->mDontMarkIndexClean,
412 sanitize));
413
414 MOZ_ASSERT(index->mShuttingDown);
415
416 EState oldState = index->mState;
417 index->ChangeState(SHUTDOWN, lock);
418
419 if (oldState != READY) {
420 LOG(
421 ("CacheIndex::Shutdown() - Unexpected state. Did posting of "
422 "PreShutdownInternal() fail?"));
423 }
424
425 switch (oldState) {
426 case WRITING:
427 index->FinishWrite(false, lock);
428 [[fallthrough]];
429 case READY:
430 if (index->mIndexOnDiskIsValid && !index->mDontMarkIndexClean) {
431 if (!sanitize && NS_FAILED(index->WriteLogToDisk())) {
432 index->RemoveJournalAndTempFile();
433 }
434 } else {
435 index->RemoveJournalAndTempFile();
436 }
437 break;
438 case READING:
439 index->FinishRead(false, lock);
440 break;
441 case BUILDING:
442 case UPDATING:
443 index->FinishUpdate(false, lock);
444 break;
445 default:
446 MOZ_ASSERT(false, "Unexpected state!");
447 }
448
449 if (sanitize) {
450 index->RemoveAllIndexFiles();
451 }
452
453 return NS_OK;
454 }
455
456 // static
AddEntry(const SHA1Sum::Hash * aHash)457 nsresult CacheIndex::AddEntry(const SHA1Sum::Hash* aHash) {
458 LOG(("CacheIndex::AddEntry() [hash=%08x%08x%08x%08x%08x]", LOGSHA1(aHash)));
459
460 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
461
462 StaticMutexAutoLock lock(sLock);
463
464 RefPtr<CacheIndex> index = gInstance;
465
466 if (!index) {
467 return NS_ERROR_NOT_INITIALIZED;
468 }
469
470 if (!index->IsIndexUsable()) {
471 return NS_ERROR_NOT_AVAILABLE;
472 }
473
474 // Getters in CacheIndexStats assert when mStateLogged is true since the
475 // information is incomplete between calls to BeforeChange() and AfterChange()
476 // (i.e. while CacheIndexEntryAutoManage exists). We need to check whether
477 // non-fresh entries exists outside the scope of CacheIndexEntryAutoManage.
478 bool updateIfNonFreshEntriesExist = false;
479
480 {
481 CacheIndexEntryAutoManage entryMng(aHash, index, lock);
482
483 CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
484 bool entryRemoved = entry && entry->IsRemoved();
485 CacheIndexEntryUpdate* updated = nullptr;
486
487 if (index->mState == READY || index->mState == UPDATING ||
488 index->mState == BUILDING) {
489 MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
490
491 if (entry && !entryRemoved) {
492 // Found entry in index that shouldn't exist.
493
494 if (entry->IsFresh()) {
495 // Someone removed the file on disk while FF is running. Update
496 // process can fix only non-fresh entries (i.e. entries that were not
497 // added within this session). Start update only if we have such
498 // entries.
499 //
500 // TODO: This should be very rare problem. If it turns out not to be
501 // true, change the update process so that it also iterates all
502 // initialized non-empty entries and checks whether the file exists.
503
504 LOG(
505 ("CacheIndex::AddEntry() - Cache file was removed outside FF "
506 "process!"));
507
508 updateIfNonFreshEntriesExist = true;
509 } else if (index->mState == READY) {
510 // Index is outdated, update it.
511 LOG(
512 ("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
513 "update is needed"));
514 index->mIndexNeedsUpdate = true;
515 } else {
516 // We cannot be here when building index since all entries are fresh
517 // during building.
518 MOZ_ASSERT(index->mState == UPDATING);
519 }
520 }
521
522 if (!entry) {
523 entry = index->mIndex.PutEntry(*aHash);
524 }
525 } else { // WRITING, READING
526 updated = index->mPendingUpdates.GetEntry(*aHash);
527 bool updatedRemoved = updated && updated->IsRemoved();
528
529 if ((updated && !updatedRemoved) ||
530 (!updated && entry && !entryRemoved && entry->IsFresh())) {
531 // Fresh entry found, so the file was removed outside FF
532 LOG(
533 ("CacheIndex::AddEntry() - Cache file was removed outside FF "
534 "process!"));
535
536 updateIfNonFreshEntriesExist = true;
537 } else if (!updated && entry && !entryRemoved) {
538 if (index->mState == WRITING) {
539 LOG(
540 ("CacheIndex::AddEntry() - Found entry that shouldn't exist, "
541 "update is needed"));
542 index->mIndexNeedsUpdate = true;
543 }
544 // Ignore if state is READING since the index information is partial
545 }
546
547 updated = index->mPendingUpdates.PutEntry(*aHash);
548 }
549
550 if (updated) {
551 updated->InitNew();
552 updated->MarkDirty();
553 updated->MarkFresh();
554 } else {
555 entry->InitNew();
556 entry->MarkDirty();
557 entry->MarkFresh();
558 }
559 }
560
561 if (updateIfNonFreshEntriesExist &&
562 index->mIndexStats.Count() != index->mIndexStats.Fresh()) {
563 index->mIndexNeedsUpdate = true;
564 }
565
566 index->StartUpdatingIndexIfNeeded(lock);
567 index->WriteIndexToDiskIfNeeded(lock);
568
569 return NS_OK;
570 }
571
572 // static
EnsureEntryExists(const SHA1Sum::Hash * aHash)573 nsresult CacheIndex::EnsureEntryExists(const SHA1Sum::Hash* aHash) {
574 LOG(("CacheIndex::EnsureEntryExists() [hash=%08x%08x%08x%08x%08x]",
575 LOGSHA1(aHash)));
576
577 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
578
579 StaticMutexAutoLock lock(sLock);
580
581 RefPtr<CacheIndex> index = gInstance;
582
583 if (!index) {
584 return NS_ERROR_NOT_INITIALIZED;
585 }
586
587 if (!index->IsIndexUsable()) {
588 return NS_ERROR_NOT_AVAILABLE;
589 }
590
591 {
592 CacheIndexEntryAutoManage entryMng(aHash, index, lock);
593
594 CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
595 bool entryRemoved = entry && entry->IsRemoved();
596
597 if (index->mState == READY || index->mState == UPDATING ||
598 index->mState == BUILDING) {
599 MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
600
601 if (!entry || entryRemoved) {
602 if (entryRemoved && entry->IsFresh()) {
603 // This could happen only if somebody copies files to the entries
604 // directory while FF is running.
605 LOG(
606 ("CacheIndex::EnsureEntryExists() - Cache file was added outside "
607 "FF process! Update is needed."));
608 index->mIndexNeedsUpdate = true;
609 } else if (index->mState == READY ||
610 (entryRemoved && !entry->IsFresh())) {
611 // Removed non-fresh entries can be present as a result of
612 // MergeJournal()
613 LOG(
614 ("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
615 " exist, update is needed"));
616 index->mIndexNeedsUpdate = true;
617 }
618
619 if (!entry) {
620 entry = index->mIndex.PutEntry(*aHash);
621 }
622 entry->InitNew();
623 entry->MarkDirty();
624 }
625 entry->MarkFresh();
626 } else { // WRITING, READING
627 CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash);
628 bool updatedRemoved = updated && updated->IsRemoved();
629
630 if (updatedRemoved || (!updated && entryRemoved && entry->IsFresh())) {
631 // Fresh information about missing entry found. This could happen only
632 // if somebody copies files to the entries directory while FF is
633 // running.
634 LOG(
635 ("CacheIndex::EnsureEntryExists() - Cache file was added outside "
636 "FF process! Update is needed."));
637 index->mIndexNeedsUpdate = true;
638 } else if (!updated && (!entry || entryRemoved)) {
639 if (index->mState == WRITING) {
640 LOG(
641 ("CacheIndex::EnsureEntryExists() - Didn't find entry that should"
642 " exist, update is needed"));
643 index->mIndexNeedsUpdate = true;
644 }
645 // Ignore if state is READING since the index information is partial
646 }
647
648 // We don't need entryRemoved and updatedRemoved info anymore
649 if (entryRemoved) entry = nullptr;
650 if (updatedRemoved) updated = nullptr;
651
652 if (updated) {
653 updated->MarkFresh();
654 } else {
655 if (!entry) {
656 // Create a new entry
657 updated = index->mPendingUpdates.PutEntry(*aHash);
658 updated->InitNew();
659 updated->MarkFresh();
660 updated->MarkDirty();
661 } else {
662 if (!entry->IsFresh()) {
663 // To mark the entry fresh we must make a copy of index entry
664 // since the index is read-only.
665 updated = index->mPendingUpdates.PutEntry(*aHash);
666 *updated = *entry;
667 updated->MarkFresh();
668 }
669 }
670 }
671 }
672 }
673
674 index->StartUpdatingIndexIfNeeded(lock);
675 index->WriteIndexToDiskIfNeeded(lock);
676
677 return NS_OK;
678 }
679
680 // static
InitEntry(const SHA1Sum::Hash * aHash,OriginAttrsHash aOriginAttrsHash,bool aAnonymous,bool aPinned)681 nsresult CacheIndex::InitEntry(const SHA1Sum::Hash* aHash,
682 OriginAttrsHash aOriginAttrsHash,
683 bool aAnonymous, bool aPinned) {
684 LOG(
685 ("CacheIndex::InitEntry() [hash=%08x%08x%08x%08x%08x, "
686 "originAttrsHash=%" PRIx64 ", anonymous=%d, pinned=%d]",
687 LOGSHA1(aHash), aOriginAttrsHash, aAnonymous, aPinned));
688
689 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
690
691 StaticMutexAutoLock lock(sLock);
692
693 RefPtr<CacheIndex> index = gInstance;
694
695 if (!index) {
696 return NS_ERROR_NOT_INITIALIZED;
697 }
698
699 if (!index->IsIndexUsable()) {
700 return NS_ERROR_NOT_AVAILABLE;
701 }
702
703 {
704 CacheIndexEntryAutoManage entryMng(aHash, index, lock);
705
706 CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
707 CacheIndexEntryUpdate* updated = nullptr;
708 bool reinitEntry = false;
709
710 if (entry && entry->IsRemoved()) {
711 entry = nullptr;
712 }
713
714 if (index->mState == READY || index->mState == UPDATING ||
715 index->mState == BUILDING) {
716 MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
717 MOZ_ASSERT(entry);
718 MOZ_ASSERT(entry->IsFresh());
719
720 if (!entry) {
721 LOG(("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
722 NS_WARNING(
723 ("CacheIndex::InitEntry() - Entry was not found in mIndex!"));
724 return NS_ERROR_UNEXPECTED;
725 }
726
727 if (IsCollision(entry, aOriginAttrsHash, aAnonymous)) {
728 index->mIndexNeedsUpdate =
729 true; // TODO Does this really help in case of collision?
730 reinitEntry = true;
731 } else {
732 if (entry->IsInitialized()) {
733 return NS_OK;
734 }
735 }
736 } else {
737 updated = index->mPendingUpdates.GetEntry(*aHash);
738 DebugOnly<bool> removed = updated && updated->IsRemoved();
739
740 MOZ_ASSERT(updated || !removed);
741 MOZ_ASSERT(updated || entry);
742
743 if (!updated && !entry) {
744 LOG(
745 ("CacheIndex::InitEntry() - Entry was found neither in mIndex nor "
746 "in mPendingUpdates!"));
747 NS_WARNING(
748 ("CacheIndex::InitEntry() - Entry was found neither in "
749 "mIndex nor in mPendingUpdates!"));
750 return NS_ERROR_UNEXPECTED;
751 }
752
753 if (updated) {
754 MOZ_ASSERT(updated->IsFresh());
755
756 if (IsCollision(updated, aOriginAttrsHash, aAnonymous)) {
757 index->mIndexNeedsUpdate = true;
758 reinitEntry = true;
759 } else {
760 if (updated->IsInitialized()) {
761 return NS_OK;
762 }
763 }
764 } else {
765 MOZ_ASSERT(entry->IsFresh());
766
767 if (IsCollision(entry, aOriginAttrsHash, aAnonymous)) {
768 index->mIndexNeedsUpdate = true;
769 reinitEntry = true;
770 } else {
771 if (entry->IsInitialized()) {
772 return NS_OK;
773 }
774 }
775
776 // make a copy of a read-only entry
777 updated = index->mPendingUpdates.PutEntry(*aHash);
778 *updated = *entry;
779 }
780 }
781
782 if (reinitEntry) {
783 // There is a collision and we are going to rewrite this entry. Initialize
784 // it as a new entry.
785 if (updated) {
786 updated->InitNew();
787 updated->MarkFresh();
788 } else {
789 entry->InitNew();
790 entry->MarkFresh();
791 }
792 }
793
794 if (updated) {
795 updated->Init(aOriginAttrsHash, aAnonymous, aPinned);
796 updated->MarkDirty();
797 } else {
798 entry->Init(aOriginAttrsHash, aAnonymous, aPinned);
799 entry->MarkDirty();
800 }
801 }
802
803 index->StartUpdatingIndexIfNeeded(lock);
804 index->WriteIndexToDiskIfNeeded(lock);
805
806 return NS_OK;
807 }
808
809 // static
RemoveEntry(const SHA1Sum::Hash * aHash)810 nsresult CacheIndex::RemoveEntry(const SHA1Sum::Hash* aHash) {
811 LOG(("CacheIndex::RemoveEntry() [hash=%08x%08x%08x%08x%08x]",
812 LOGSHA1(aHash)));
813
814 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
815
816 StaticMutexAutoLock lock(sLock);
817
818 RefPtr<CacheIndex> index = gInstance;
819
820 if (!index) {
821 return NS_ERROR_NOT_INITIALIZED;
822 }
823
824 if (!index->IsIndexUsable()) {
825 return NS_ERROR_NOT_AVAILABLE;
826 }
827
828 {
829 CacheIndexEntryAutoManage entryMng(aHash, index, lock);
830
831 CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
832 bool entryRemoved = entry && entry->IsRemoved();
833
834 if (index->mState == READY || index->mState == UPDATING ||
835 index->mState == BUILDING) {
836 MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
837
838 if (!entry || entryRemoved) {
839 if (entryRemoved && entry->IsFresh()) {
840 // This could happen only if somebody copies files to the entries
841 // directory while FF is running.
842 LOG(
843 ("CacheIndex::RemoveEntry() - Cache file was added outside FF "
844 "process! Update is needed."));
845 index->mIndexNeedsUpdate = true;
846 } else if (index->mState == READY ||
847 (entryRemoved && !entry->IsFresh())) {
848 // Removed non-fresh entries can be present as a result of
849 // MergeJournal()
850 LOG(
851 ("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
852 ", update is needed"));
853 index->mIndexNeedsUpdate = true;
854 }
855 } else {
856 if (entry) {
857 if (!entry->IsDirty() && entry->IsFileEmpty()) {
858 index->mIndex.RemoveEntry(entry);
859 entry = nullptr;
860 } else {
861 entry->MarkRemoved();
862 entry->MarkDirty();
863 entry->MarkFresh();
864 }
865 }
866 }
867 } else { // WRITING, READING
868 CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash);
869 bool updatedRemoved = updated && updated->IsRemoved();
870
871 if (updatedRemoved || (!updated && entryRemoved && entry->IsFresh())) {
872 // Fresh information about missing entry found. This could happen only
873 // if somebody copies files to the entries directory while FF is
874 // running.
875 LOG(
876 ("CacheIndex::RemoveEntry() - Cache file was added outside FF "
877 "process! Update is needed."));
878 index->mIndexNeedsUpdate = true;
879 } else if (!updated && (!entry || entryRemoved)) {
880 if (index->mState == WRITING) {
881 LOG(
882 ("CacheIndex::RemoveEntry() - Didn't find entry that should exist"
883 ", update is needed"));
884 index->mIndexNeedsUpdate = true;
885 }
886 // Ignore if state is READING since the index information is partial
887 }
888
889 if (!updated) {
890 updated = index->mPendingUpdates.PutEntry(*aHash);
891 updated->InitNew();
892 }
893
894 updated->MarkRemoved();
895 updated->MarkDirty();
896 updated->MarkFresh();
897 }
898 }
899 index->StartUpdatingIndexIfNeeded(lock);
900 index->WriteIndexToDiskIfNeeded(lock);
901
902 return NS_OK;
903 }
904
905 // static
UpdateEntry(const SHA1Sum::Hash * aHash,const uint32_t * aFrecency,const bool * aHasAltData,const uint16_t * aOnStartTime,const uint16_t * aOnStopTime,const uint8_t * aContentType,const uint32_t * aSize)906 nsresult CacheIndex::UpdateEntry(const SHA1Sum::Hash* aHash,
907 const uint32_t* aFrecency,
908 const bool* aHasAltData,
909 const uint16_t* aOnStartTime,
910 const uint16_t* aOnStopTime,
911 const uint8_t* aContentType,
912 const uint32_t* aSize) {
913 LOG(
914 ("CacheIndex::UpdateEntry() [hash=%08x%08x%08x%08x%08x, "
915 "frecency=%s, hasAltData=%s, onStartTime=%s, onStopTime=%s, "
916 "contentType=%s, size=%s]",
917 LOGSHA1(aHash), aFrecency ? nsPrintfCString("%u", *aFrecency).get() : "",
918 aHasAltData ? (*aHasAltData ? "true" : "false") : "",
919 aOnStartTime ? nsPrintfCString("%u", *aOnStartTime).get() : "",
920 aOnStopTime ? nsPrintfCString("%u", *aOnStopTime).get() : "",
921 aContentType ? nsPrintfCString("%u", *aContentType).get() : "",
922 aSize ? nsPrintfCString("%u", *aSize).get() : ""));
923
924 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
925
926 StaticMutexAutoLock lock(sLock);
927
928 RefPtr<CacheIndex> index = gInstance;
929
930 if (!index) {
931 return NS_ERROR_NOT_INITIALIZED;
932 }
933
934 if (!index->IsIndexUsable()) {
935 return NS_ERROR_NOT_AVAILABLE;
936 }
937
938 {
939 CacheIndexEntryAutoManage entryMng(aHash, index, lock);
940
941 CacheIndexEntry* entry = index->mIndex.GetEntry(*aHash);
942
943 if (entry && entry->IsRemoved()) {
944 entry = nullptr;
945 }
946
947 if (index->mState == READY || index->mState == UPDATING ||
948 index->mState == BUILDING) {
949 MOZ_ASSERT(index->mPendingUpdates.Count() == 0);
950 MOZ_ASSERT(entry);
951
952 if (!entry) {
953 LOG(("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
954 NS_WARNING(
955 ("CacheIndex::UpdateEntry() - Entry was not found in mIndex!"));
956 return NS_ERROR_UNEXPECTED;
957 }
958
959 if (!HasEntryChanged(entry, aFrecency, aHasAltData, aOnStartTime,
960 aOnStopTime, aContentType, aSize)) {
961 return NS_OK;
962 }
963
964 MOZ_ASSERT(entry->IsFresh());
965 MOZ_ASSERT(entry->IsInitialized());
966 entry->MarkDirty();
967
968 if (aFrecency) {
969 entry->SetFrecency(*aFrecency);
970 }
971
972 if (aHasAltData) {
973 entry->SetHasAltData(*aHasAltData);
974 }
975
976 if (aOnStartTime) {
977 entry->SetOnStartTime(*aOnStartTime);
978 }
979
980 if (aOnStopTime) {
981 entry->SetOnStopTime(*aOnStopTime);
982 }
983
984 if (aContentType) {
985 entry->SetContentType(*aContentType);
986 }
987
988 if (aSize) {
989 entry->SetFileSize(*aSize);
990 }
991 } else {
992 CacheIndexEntryUpdate* updated = index->mPendingUpdates.GetEntry(*aHash);
993 DebugOnly<bool> removed = updated && updated->IsRemoved();
994
995 MOZ_ASSERT(updated || !removed);
996 MOZ_ASSERT(updated || entry);
997
998 if (!updated) {
999 if (!entry) {
1000 LOG(
1001 ("CacheIndex::UpdateEntry() - Entry was found neither in mIndex "
1002 "nor in mPendingUpdates!"));
1003 NS_WARNING(
1004 ("CacheIndex::UpdateEntry() - Entry was found neither in "
1005 "mIndex nor in mPendingUpdates!"));
1006 return NS_ERROR_UNEXPECTED;
1007 }
1008
1009 // make a copy of a read-only entry
1010 updated = index->mPendingUpdates.PutEntry(*aHash);
1011 *updated = *entry;
1012 }
1013
1014 MOZ_ASSERT(updated->IsFresh());
1015 MOZ_ASSERT(updated->IsInitialized());
1016 updated->MarkDirty();
1017
1018 if (aFrecency) {
1019 updated->SetFrecency(*aFrecency);
1020 }
1021
1022 if (aHasAltData) {
1023 updated->SetHasAltData(*aHasAltData);
1024 }
1025
1026 if (aOnStartTime) {
1027 updated->SetOnStartTime(*aOnStartTime);
1028 }
1029
1030 if (aOnStopTime) {
1031 updated->SetOnStopTime(*aOnStopTime);
1032 }
1033
1034 if (aContentType) {
1035 updated->SetContentType(*aContentType);
1036 }
1037
1038 if (aSize) {
1039 updated->SetFileSize(*aSize);
1040 }
1041 }
1042 }
1043
1044 index->WriteIndexToDiskIfNeeded(lock);
1045
1046 return NS_OK;
1047 }
1048
1049 // static
RemoveAll()1050 nsresult CacheIndex::RemoveAll() {
1051 LOG(("CacheIndex::RemoveAll()"));
1052
1053 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1054
1055 nsCOMPtr<nsIFile> file;
1056
1057 {
1058 StaticMutexAutoLock lock(sLock);
1059
1060 RefPtr<CacheIndex> index = gInstance;
1061
1062 if (!index) {
1063 return NS_ERROR_NOT_INITIALIZED;
1064 }
1065
1066 MOZ_ASSERT(!index->mRemovingAll);
1067
1068 if (!index->IsIndexUsable()) {
1069 return NS_ERROR_NOT_AVAILABLE;
1070 }
1071
1072 AutoRestore<bool> saveRemovingAll(index->mRemovingAll);
1073 index->mRemovingAll = true;
1074
1075 // Doom index and journal handles but don't null them out since this will be
1076 // done in FinishWrite/FinishRead methods.
1077 if (index->mIndexHandle) {
1078 CacheFileIOManager::DoomFile(index->mIndexHandle, nullptr);
1079 } else {
1080 // We don't have a handle to index file, so get the file here, but delete
1081 // it outside the lock. Ignore the result since this is not fatal.
1082 index->GetFile(nsLiteralCString(INDEX_NAME), getter_AddRefs(file));
1083 }
1084
1085 if (index->mJournalHandle) {
1086 CacheFileIOManager::DoomFile(index->mJournalHandle, nullptr);
1087 }
1088
1089 switch (index->mState) {
1090 case WRITING:
1091 index->FinishWrite(false, lock);
1092 break;
1093 case READY:
1094 // nothing to do
1095 break;
1096 case READING:
1097 index->FinishRead(false, lock);
1098 break;
1099 case BUILDING:
1100 case UPDATING:
1101 index->FinishUpdate(false, lock);
1102 break;
1103 default:
1104 MOZ_ASSERT(false, "Unexpected state!");
1105 }
1106
1107 // We should end up in READY state
1108 MOZ_ASSERT(index->mState == READY);
1109
1110 // There should not be any handle
1111 MOZ_ASSERT(!index->mIndexHandle);
1112 MOZ_ASSERT(!index->mJournalHandle);
1113
1114 index->mIndexOnDiskIsValid = false;
1115 index->mIndexNeedsUpdate = false;
1116
1117 index->mIndexStats.Clear();
1118 index->mFrecencyArray.Clear(lock);
1119 index->mIndex.Clear();
1120
1121 for (uint32_t i = 0; i < index->mIterators.Length();) {
1122 nsresult rv = index->mIterators[i]->CloseInternal(NS_ERROR_NOT_AVAILABLE);
1123 if (NS_FAILED(rv)) {
1124 // CacheIndexIterator::CloseInternal() removes itself from mIterators
1125 // iff it returns success.
1126 LOG(
1127 ("CacheIndex::RemoveAll() - Failed to remove iterator %p. "
1128 "[rv=0x%08" PRIx32 "]",
1129 index->mIterators[i], static_cast<uint32_t>(rv)));
1130 i++;
1131 }
1132 }
1133 }
1134
1135 if (file) {
1136 // Ignore the result. The file might not exist and the failure is not fatal.
1137 file->Remove(false);
1138 }
1139
1140 return NS_OK;
1141 }
1142
1143 // static
HasEntry(const nsACString & aKey,EntryStatus * _retval,const std::function<void (const CacheIndexEntry *)> & aCB)1144 nsresult CacheIndex::HasEntry(
1145 const nsACString& aKey, EntryStatus* _retval,
1146 const std::function<void(const CacheIndexEntry*)>& aCB) {
1147 LOG(("CacheIndex::HasEntry() [key=%s]", PromiseFlatCString(aKey).get()));
1148
1149 SHA1Sum sum;
1150 SHA1Sum::Hash hash;
1151 sum.update(aKey.BeginReading(), aKey.Length());
1152 sum.finish(hash);
1153
1154 return HasEntry(hash, _retval, aCB);
1155 }
1156
1157 // static
HasEntry(const SHA1Sum::Hash & hash,EntryStatus * _retval,const std::function<void (const CacheIndexEntry *)> & aCB)1158 nsresult CacheIndex::HasEntry(
1159 const SHA1Sum::Hash& hash, EntryStatus* _retval,
1160 const std::function<void(const CacheIndexEntry*)>& aCB) {
1161 StaticMutexAutoLock lock(sLock);
1162
1163 RefPtr<CacheIndex> index = gInstance;
1164
1165 if (!index) {
1166 return NS_ERROR_NOT_INITIALIZED;
1167 }
1168
1169 if (!index->IsIndexUsable()) {
1170 return NS_ERROR_NOT_AVAILABLE;
1171 }
1172
1173 const CacheIndexEntry* entry = nullptr;
1174
1175 switch (index->mState) {
1176 case READING:
1177 case WRITING:
1178 entry = index->mPendingUpdates.GetEntry(hash);
1179 [[fallthrough]];
1180 case BUILDING:
1181 case UPDATING:
1182 case READY:
1183 if (!entry) {
1184 entry = index->mIndex.GetEntry(hash);
1185 }
1186 break;
1187 case INITIAL:
1188 case SHUTDOWN:
1189 MOZ_ASSERT(false, "Unexpected state!");
1190 }
1191
1192 if (!entry) {
1193 if (index->mState == READY || index->mState == WRITING) {
1194 *_retval = DOES_NOT_EXIST;
1195 } else {
1196 *_retval = DO_NOT_KNOW;
1197 }
1198 } else {
1199 if (entry->IsRemoved()) {
1200 if (entry->IsFresh()) {
1201 *_retval = DOES_NOT_EXIST;
1202 } else {
1203 *_retval = DO_NOT_KNOW;
1204 }
1205 } else {
1206 *_retval = EXISTS;
1207 if (aCB) {
1208 aCB(entry);
1209 }
1210 }
1211 }
1212
1213 LOG(("CacheIndex::HasEntry() - result is %u", *_retval));
1214 return NS_OK;
1215 }
1216
1217 // static
GetEntryForEviction(bool aIgnoreEmptyEntries,SHA1Sum::Hash * aHash,uint32_t * aCnt)1218 nsresult CacheIndex::GetEntryForEviction(bool aIgnoreEmptyEntries,
1219 SHA1Sum::Hash* aHash, uint32_t* aCnt) {
1220 LOG(("CacheIndex::GetEntryForEviction()"));
1221
1222 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
1223
1224 StaticMutexAutoLock lock(sLock);
1225
1226 RefPtr<CacheIndex> index = gInstance;
1227
1228 if (!index) return NS_ERROR_NOT_INITIALIZED;
1229
1230 if (!index->IsIndexUsable()) {
1231 return NS_ERROR_NOT_AVAILABLE;
1232 }
1233
1234 if (index->mIndexStats.Size() == 0) {
1235 return NS_ERROR_NOT_AVAILABLE;
1236 }
1237
1238 int32_t mediaUsage =
1239 round(static_cast<double>(index->mIndexStats.SizeByType(
1240 nsICacheEntry::CONTENT_TYPE_MEDIA)) *
1241 100.0 / static_cast<double>(index->mIndexStats.Size()));
1242 int32_t mediaUsageLimit =
1243 StaticPrefs::browser_cache_disk_content_type_media_limit();
1244 bool evictMedia = false;
1245 if (mediaUsage > mediaUsageLimit) {
1246 LOG(
1247 ("CacheIndex::GetEntryForEviction() - media content type is over the "
1248 "limit [mediaUsage=%d, mediaUsageLimit=%d]",
1249 mediaUsage, mediaUsageLimit));
1250 evictMedia = true;
1251 }
1252
1253 SHA1Sum::Hash hash;
1254 CacheIndexRecord* foundRecord = nullptr;
1255 uint32_t skipped = 0;
1256
1257 // find first non-forced valid and unpinned entry with the lowest frecency
1258 index->mFrecencyArray.SortIfNeeded(lock);
1259
1260 for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1261 CacheIndexRecord* rec = iter.Get()->Get();
1262
1263 memcpy(&hash, rec->mHash, sizeof(SHA1Sum::Hash));
1264
1265 ++skipped;
1266
1267 if (evictMedia && CacheIndexEntry::GetContentType(rec) !=
1268 nsICacheEntry::CONTENT_TYPE_MEDIA) {
1269 continue;
1270 }
1271
1272 if (IsForcedValidEntry(&hash)) {
1273 continue;
1274 }
1275
1276 if (CacheIndexEntry::IsPinned(rec)) {
1277 continue;
1278 }
1279
1280 if (aIgnoreEmptyEntries && !CacheIndexEntry::GetFileSize(*rec)) {
1281 continue;
1282 }
1283
1284 --skipped;
1285 foundRecord = rec;
1286 break;
1287 }
1288
1289 if (!foundRecord) return NS_ERROR_NOT_AVAILABLE;
1290
1291 *aCnt = skipped;
1292
1293 LOG(
1294 ("CacheIndex::GetEntryForEviction() - returning entry "
1295 "[hash=%08x%08x%08x%08x%08x, cnt=%u, frecency=%u, contentType=%u]",
1296 LOGSHA1(&hash), *aCnt, foundRecord->mFrecency,
1297 CacheIndexEntry::GetContentType(foundRecord)));
1298
1299 memcpy(aHash, &hash, sizeof(SHA1Sum::Hash));
1300
1301 return NS_OK;
1302 }
1303
1304 // static
IsForcedValidEntry(const SHA1Sum::Hash * aHash)1305 bool CacheIndex::IsForcedValidEntry(const SHA1Sum::Hash* aHash) {
1306 RefPtr<CacheFileHandle> handle;
1307
1308 CacheFileIOManager::gInstance->mHandles.GetHandle(aHash,
1309 getter_AddRefs(handle));
1310
1311 if (!handle) return false;
1312
1313 nsCString hashKey = handle->Key();
1314 return CacheStorageService::Self()->IsForcedValidEntry(hashKey);
1315 }
1316
1317 // static
GetCacheSize(uint32_t * _retval)1318 nsresult CacheIndex::GetCacheSize(uint32_t* _retval) {
1319 LOG(("CacheIndex::GetCacheSize()"));
1320
1321 StaticMutexAutoLock lock(sLock);
1322
1323 RefPtr<CacheIndex> index = gInstance;
1324
1325 if (!index) return NS_ERROR_NOT_INITIALIZED;
1326
1327 if (!index->IsIndexUsable()) {
1328 return NS_ERROR_NOT_AVAILABLE;
1329 }
1330
1331 *_retval = index->mIndexStats.Size();
1332 LOG(("CacheIndex::GetCacheSize() - returning %u", *_retval));
1333 return NS_OK;
1334 }
1335
1336 // static
GetEntryFileCount(uint32_t * _retval)1337 nsresult CacheIndex::GetEntryFileCount(uint32_t* _retval) {
1338 LOG(("CacheIndex::GetEntryFileCount()"));
1339
1340 StaticMutexAutoLock lock(sLock);
1341
1342 RefPtr<CacheIndex> index = gInstance;
1343
1344 if (!index) {
1345 return NS_ERROR_NOT_INITIALIZED;
1346 }
1347
1348 if (!index->IsIndexUsable()) {
1349 return NS_ERROR_NOT_AVAILABLE;
1350 }
1351
1352 *_retval = index->mIndexStats.ActiveEntriesCount();
1353 LOG(("CacheIndex::GetEntryFileCount() - returning %u", *_retval));
1354 return NS_OK;
1355 }
1356
1357 // static
GetCacheStats(nsILoadContextInfo * aInfo,uint32_t * aSize,uint32_t * aCount)1358 nsresult CacheIndex::GetCacheStats(nsILoadContextInfo* aInfo, uint32_t* aSize,
1359 uint32_t* aCount) {
1360 LOG(("CacheIndex::GetCacheStats() [info=%p]", aInfo));
1361
1362 StaticMutexAutoLock lock(sLock);
1363
1364 RefPtr<CacheIndex> index = gInstance;
1365
1366 if (!index) {
1367 return NS_ERROR_NOT_INITIALIZED;
1368 }
1369
1370 if (!index->IsIndexUsable()) {
1371 return NS_ERROR_NOT_AVAILABLE;
1372 }
1373
1374 *aSize = 0;
1375 *aCount = 0;
1376
1377 for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1378 if (aInfo &&
1379 !CacheIndexEntry::RecordMatchesLoadContextInfo(iter.Get(), aInfo)) {
1380 continue;
1381 }
1382
1383 *aSize += CacheIndexEntry::GetFileSize(*(iter.Get()->Get()));
1384 ++*aCount;
1385 }
1386
1387 return NS_OK;
1388 }
1389
1390 // static
AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver * aObserver)1391 nsresult CacheIndex::AsyncGetDiskConsumption(
1392 nsICacheStorageConsumptionObserver* aObserver) {
1393 LOG(("CacheIndex::AsyncGetDiskConsumption()"));
1394
1395 StaticMutexAutoLock lock(sLock);
1396
1397 RefPtr<CacheIndex> index = gInstance;
1398
1399 if (!index) {
1400 return NS_ERROR_NOT_INITIALIZED;
1401 }
1402
1403 if (!index->IsIndexUsable()) {
1404 return NS_ERROR_NOT_AVAILABLE;
1405 }
1406
1407 RefPtr<DiskConsumptionObserver> observer =
1408 DiskConsumptionObserver::Init(aObserver);
1409
1410 NS_ENSURE_ARG(observer);
1411
1412 if ((index->mState == READY || index->mState == WRITING) &&
1413 !index->mAsyncGetDiskConsumptionBlocked) {
1414 LOG(("CacheIndex::AsyncGetDiskConsumption - calling immediately"));
1415 // Safe to call the callback under the lock,
1416 // we always post to the main thread.
1417 observer->OnDiskConsumption(index->mIndexStats.Size() << 10);
1418 return NS_OK;
1419 }
1420
1421 LOG(("CacheIndex::AsyncGetDiskConsumption - remembering callback"));
1422 // Will be called when the index get to the READY state.
1423 index->mDiskConsumptionObservers.AppendElement(observer);
1424
1425 // Move forward with index re/building if it is pending
1426 RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
1427 if (ioThread) {
1428 ioThread->Dispatch(
1429 NS_NewRunnableFunction("net::CacheIndex::AsyncGetDiskConsumption",
1430 []() -> void {
1431 StaticMutexAutoLock lock(sLock);
1432
1433 RefPtr<CacheIndex> index = gInstance;
1434 if (index && index->mUpdateTimer) {
1435 index->mUpdateTimer->Cancel();
1436 index->DelayedUpdateLocked(lock);
1437 }
1438 }),
1439 CacheIOThread::INDEX);
1440 }
1441
1442 return NS_OK;
1443 }
1444
1445 // static
GetIterator(nsILoadContextInfo * aInfo,bool aAddNew,CacheIndexIterator ** _retval)1446 nsresult CacheIndex::GetIterator(nsILoadContextInfo* aInfo, bool aAddNew,
1447 CacheIndexIterator** _retval) {
1448 LOG(("CacheIndex::GetIterator() [info=%p, addNew=%d]", aInfo, aAddNew));
1449
1450 StaticMutexAutoLock lock(sLock);
1451
1452 RefPtr<CacheIndex> index = gInstance;
1453
1454 if (!index) {
1455 return NS_ERROR_NOT_INITIALIZED;
1456 }
1457
1458 if (!index->IsIndexUsable()) {
1459 return NS_ERROR_NOT_AVAILABLE;
1460 }
1461
1462 RefPtr<CacheIndexIterator> idxIter;
1463 if (aInfo) {
1464 idxIter = new CacheIndexContextIterator(index, aAddNew, aInfo);
1465 } else {
1466 idxIter = new CacheIndexIterator(index, aAddNew);
1467 }
1468
1469 index->mFrecencyArray.SortIfNeeded(lock);
1470
1471 for (auto iter = index->mFrecencyArray.Iter(); !iter.Done(); iter.Next()) {
1472 idxIter->AddRecord(iter.Get(), lock);
1473 }
1474
1475 index->mIterators.AppendElement(idxIter);
1476 idxIter.swap(*_retval);
1477 return NS_OK;
1478 }
1479
1480 // static
IsUpToDate(bool * _retval)1481 nsresult CacheIndex::IsUpToDate(bool* _retval) {
1482 LOG(("CacheIndex::IsUpToDate()"));
1483
1484 StaticMutexAutoLock lock(sLock);
1485
1486 RefPtr<CacheIndex> index = gInstance;
1487
1488 if (!index) {
1489 return NS_ERROR_NOT_INITIALIZED;
1490 }
1491
1492 if (!index->IsIndexUsable()) {
1493 return NS_ERROR_NOT_AVAILABLE;
1494 }
1495
1496 *_retval = (index->mState == READY || index->mState == WRITING) &&
1497 !index->mIndexNeedsUpdate && !index->mShuttingDown;
1498
1499 LOG(("CacheIndex::IsUpToDate() - returning %d", *_retval));
1500 return NS_OK;
1501 }
1502
IsIndexUsable()1503 bool CacheIndex::IsIndexUsable() {
1504 MOZ_ASSERT(mState != INITIAL);
1505
1506 switch (mState) {
1507 case INITIAL:
1508 case SHUTDOWN:
1509 return false;
1510
1511 case READING:
1512 case WRITING:
1513 case BUILDING:
1514 case UPDATING:
1515 case READY:
1516 break;
1517 }
1518
1519 return true;
1520 }
1521
1522 // static
IsCollision(CacheIndexEntry * aEntry,OriginAttrsHash aOriginAttrsHash,bool aAnonymous)1523 bool CacheIndex::IsCollision(CacheIndexEntry* aEntry,
1524 OriginAttrsHash aOriginAttrsHash,
1525 bool aAnonymous) {
1526 if (!aEntry->IsInitialized()) {
1527 return false;
1528 }
1529
1530 if (aEntry->Anonymous() != aAnonymous ||
1531 aEntry->OriginAttrsHash() != aOriginAttrsHash) {
1532 LOG(
1533 ("CacheIndex::IsCollision() - Collision detected for entry hash=%08x"
1534 "%08x%08x%08x%08x, expected values: originAttrsHash=%" PRIu64 ", "
1535 "anonymous=%d; actual values: originAttrsHash=%" PRIu64
1536 ", anonymous=%d]",
1537 LOGSHA1(aEntry->Hash()), aOriginAttrsHash, aAnonymous,
1538 aEntry->OriginAttrsHash(), aEntry->Anonymous()));
1539 return true;
1540 }
1541
1542 return false;
1543 }
1544
1545 // static
HasEntryChanged(CacheIndexEntry * aEntry,const uint32_t * aFrecency,const bool * aHasAltData,const uint16_t * aOnStartTime,const uint16_t * aOnStopTime,const uint8_t * aContentType,const uint32_t * aSize)1546 bool CacheIndex::HasEntryChanged(
1547 CacheIndexEntry* aEntry, const uint32_t* aFrecency, const bool* aHasAltData,
1548 const uint16_t* aOnStartTime, const uint16_t* aOnStopTime,
1549 const uint8_t* aContentType, const uint32_t* aSize) {
1550 if (aFrecency && *aFrecency != aEntry->GetFrecency()) {
1551 return true;
1552 }
1553
1554 if (aHasAltData && *aHasAltData != aEntry->GetHasAltData()) {
1555 return true;
1556 }
1557
1558 if (aOnStartTime && *aOnStartTime != aEntry->GetOnStartTime()) {
1559 return true;
1560 }
1561
1562 if (aOnStopTime && *aOnStopTime != aEntry->GetOnStopTime()) {
1563 return true;
1564 }
1565
1566 if (aContentType && *aContentType != aEntry->GetContentType()) {
1567 return true;
1568 }
1569
1570 if (aSize &&
1571 (*aSize & CacheIndexEntry::kFileSizeMask) != aEntry->GetFileSize()) {
1572 return true;
1573 }
1574
1575 return false;
1576 }
1577
ProcessPendingOperations(const StaticMutexAutoLock & aProofOfLock)1578 void CacheIndex::ProcessPendingOperations(
1579 const StaticMutexAutoLock& aProofOfLock) {
1580 LOG(("CacheIndex::ProcessPendingOperations()"));
1581
1582 for (auto iter = mPendingUpdates.Iter(); !iter.Done(); iter.Next()) {
1583 CacheIndexEntryUpdate* update = iter.Get();
1584
1585 LOG(("CacheIndex::ProcessPendingOperations() [hash=%08x%08x%08x%08x%08x]",
1586 LOGSHA1(update->Hash())));
1587
1588 MOZ_ASSERT(update->IsFresh());
1589
1590 CacheIndexEntry* entry = mIndex.GetEntry(*update->Hash());
1591 {
1592 CacheIndexEntryAutoManage emng(update->Hash(), this, aProofOfLock);
1593 emng.DoNotSearchInUpdates();
1594
1595 if (update->IsRemoved()) {
1596 if (entry) {
1597 if (entry->IsRemoved()) {
1598 MOZ_ASSERT(entry->IsFresh());
1599 MOZ_ASSERT(entry->IsDirty());
1600 } else if (!entry->IsDirty() && entry->IsFileEmpty()) {
1601 // Entries with empty file are not stored in index on disk. Just
1602 // remove the entry, but only in case the entry is not dirty, i.e.
1603 // the entry file was empty when we wrote the index.
1604 mIndex.RemoveEntry(entry);
1605 entry = nullptr;
1606 } else {
1607 entry->MarkRemoved();
1608 entry->MarkDirty();
1609 entry->MarkFresh();
1610 }
1611 }
1612 } else if (entry) {
1613 // Some information in mIndex can be newer than in mPendingUpdates (see
1614 // bug 1074832). This will copy just those values that were really
1615 // updated.
1616 update->ApplyUpdate(entry);
1617 } else {
1618 // There is no entry in mIndex, copy all information from
1619 // mPendingUpdates to mIndex.
1620 entry = mIndex.PutEntry(*update->Hash());
1621 *entry = *update;
1622 }
1623 }
1624 iter.Remove();
1625 }
1626
1627 MOZ_ASSERT(mPendingUpdates.Count() == 0);
1628
1629 EnsureCorrectStats();
1630 }
1631
WriteIndexToDiskIfNeeded(const StaticMutexAutoLock & aProofOfLock)1632 bool CacheIndex::WriteIndexToDiskIfNeeded(
1633 const StaticMutexAutoLock& aProofOfLock) {
1634 if (mState != READY || mShuttingDown || mRWPending) {
1635 return false;
1636 }
1637
1638 if (!mLastDumpTime.IsNull() &&
1639 (TimeStamp::NowLoRes() - mLastDumpTime).ToMilliseconds() <
1640 kMinDumpInterval) {
1641 return false;
1642 }
1643
1644 if (mIndexStats.Dirty() < kMinUnwrittenChanges) {
1645 return false;
1646 }
1647
1648 WriteIndexToDisk(aProofOfLock);
1649 return true;
1650 }
1651
WriteIndexToDisk(const StaticMutexAutoLock & aProofOfLock)1652 void CacheIndex::WriteIndexToDisk(const StaticMutexAutoLock& aProofOfLock) {
1653 LOG(("CacheIndex::WriteIndexToDisk()"));
1654 mIndexStats.Log();
1655
1656 nsresult rv;
1657
1658 MOZ_ASSERT(mState == READY);
1659 MOZ_ASSERT(!mRWBuf);
1660 MOZ_ASSERT(!mRWHash);
1661 MOZ_ASSERT(!mRWPending);
1662
1663 ChangeState(WRITING, aProofOfLock);
1664
1665 mProcessEntries = mIndexStats.ActiveEntriesCount();
1666
1667 mIndexFileOpener = new FileOpenHelper(this);
1668 rv = CacheFileIOManager::OpenFile(
1669 nsLiteralCString(TEMP_INDEX_NAME),
1670 CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::CREATE,
1671 mIndexFileOpener);
1672 if (NS_FAILED(rv)) {
1673 LOG(("CacheIndex::WriteIndexToDisk() - Can't open file [rv=0x%08" PRIx32
1674 "]",
1675 static_cast<uint32_t>(rv)));
1676 FinishWrite(false, aProofOfLock);
1677 return;
1678 }
1679
1680 // Write index header to a buffer, it will be written to disk together with
1681 // records in WriteRecords() once we open the file successfully.
1682 AllocBuffer();
1683 mRWHash = new CacheHash();
1684
1685 mRWBufPos = 0;
1686 // index version
1687 NetworkEndian::writeUint32(mRWBuf + mRWBufPos, kIndexVersion);
1688 mRWBufPos += sizeof(uint32_t);
1689 // timestamp
1690 NetworkEndian::writeUint32(mRWBuf + mRWBufPos,
1691 static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC));
1692 mRWBufPos += sizeof(uint32_t);
1693 // dirty flag
1694 NetworkEndian::writeUint32(mRWBuf + mRWBufPos, 1);
1695 mRWBufPos += sizeof(uint32_t);
1696 // amount of data written to the cache
1697 NetworkEndian::writeUint32(mRWBuf + mRWBufPos,
1698 static_cast<uint32_t>(mTotalBytesWritten >> 10));
1699 mRWBufPos += sizeof(uint32_t);
1700
1701 mSkipEntries = 0;
1702 }
1703
WriteRecords(const StaticMutexAutoLock & aProofOfLock)1704 void CacheIndex::WriteRecords(const StaticMutexAutoLock& aProofOfLock) {
1705 LOG(("CacheIndex::WriteRecords()"));
1706
1707 nsresult rv;
1708
1709 MOZ_ASSERT(mState == WRITING);
1710 MOZ_ASSERT(!mRWPending);
1711
1712 int64_t fileOffset;
1713
1714 if (mSkipEntries) {
1715 MOZ_ASSERT(mRWBufPos == 0);
1716 fileOffset = sizeof(CacheIndexHeader);
1717 fileOffset += sizeof(CacheIndexRecord) * mSkipEntries;
1718 } else {
1719 MOZ_ASSERT(mRWBufPos == sizeof(CacheIndexHeader));
1720 fileOffset = 0;
1721 }
1722 uint32_t hashOffset = mRWBufPos;
1723
1724 char* buf = mRWBuf + mRWBufPos;
1725 uint32_t skip = mSkipEntries;
1726 uint32_t processMax = (mRWBufSize - mRWBufPos) / sizeof(CacheIndexRecord);
1727 MOZ_ASSERT(processMax != 0 ||
1728 mProcessEntries ==
1729 0); // TODO make sure we can write an empty index
1730 uint32_t processed = 0;
1731 #ifdef DEBUG
1732 bool hasMore = false;
1733 #endif
1734 for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
1735 CacheIndexEntry* entry = iter.Get();
1736 if (entry->IsRemoved() || !entry->IsInitialized() || entry->IsFileEmpty()) {
1737 continue;
1738 }
1739
1740 if (skip) {
1741 skip--;
1742 continue;
1743 }
1744
1745 if (processed == processMax) {
1746 #ifdef DEBUG
1747 hasMore = true;
1748 #endif
1749 break;
1750 }
1751
1752 entry->WriteToBuf(buf);
1753 buf += sizeof(CacheIndexRecord);
1754 processed++;
1755 }
1756
1757 MOZ_ASSERT(mRWBufPos != static_cast<uint32_t>(buf - mRWBuf) ||
1758 mProcessEntries == 0);
1759 mRWBufPos = buf - mRWBuf;
1760 mSkipEntries += processed;
1761 MOZ_ASSERT(mSkipEntries <= mProcessEntries);
1762
1763 mRWHash->Update(mRWBuf + hashOffset, mRWBufPos - hashOffset);
1764
1765 if (mSkipEntries == mProcessEntries) {
1766 MOZ_ASSERT(!hasMore);
1767
1768 // We've processed all records
1769 if (mRWBufPos + sizeof(CacheHash::Hash32_t) > mRWBufSize) {
1770 // realloc buffer to spare another write cycle
1771 mRWBufSize = mRWBufPos + sizeof(CacheHash::Hash32_t);
1772 mRWBuf = static_cast<char*>(moz_xrealloc(mRWBuf, mRWBufSize));
1773 }
1774
1775 NetworkEndian::writeUint32(mRWBuf + mRWBufPos, mRWHash->GetHash());
1776 mRWBufPos += sizeof(CacheHash::Hash32_t);
1777 } else {
1778 MOZ_ASSERT(hasMore);
1779 }
1780
1781 rv = CacheFileIOManager::Write(mIndexHandle, fileOffset, mRWBuf, mRWBufPos,
1782 mSkipEntries == mProcessEntries, false, this);
1783 if (NS_FAILED(rv)) {
1784 LOG(
1785 ("CacheIndex::WriteRecords() - CacheFileIOManager::Write() failed "
1786 "synchronously [rv=0x%08" PRIx32 "]",
1787 static_cast<uint32_t>(rv)));
1788 FinishWrite(false, aProofOfLock);
1789 } else {
1790 mRWPending = true;
1791 }
1792
1793 mRWBufPos = 0;
1794 }
1795
FinishWrite(bool aSucceeded,const StaticMutexAutoLock & aProofOfLock)1796 void CacheIndex::FinishWrite(bool aSucceeded,
1797 const StaticMutexAutoLock& aProofOfLock) {
1798 LOG(("CacheIndex::FinishWrite() [succeeded=%d]", aSucceeded));
1799
1800 MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == WRITING);
1801
1802 // If there is write operation pending we must be cancelling writing of the
1803 // index when shutting down or removing the whole index.
1804 MOZ_ASSERT(!mRWPending || (!aSucceeded && (mShuttingDown || mRemovingAll)));
1805
1806 mIndexHandle = nullptr;
1807 mRWHash = nullptr;
1808 ReleaseBuffer();
1809
1810 if (aSucceeded) {
1811 // Opening of the file must not be in progress if writing succeeded.
1812 MOZ_ASSERT(!mIndexFileOpener);
1813
1814 for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
1815 CacheIndexEntry* entry = iter.Get();
1816
1817 bool remove = false;
1818 {
1819 CacheIndexEntryAutoManage emng(entry->Hash(), this, aProofOfLock);
1820
1821 if (entry->IsRemoved()) {
1822 emng.DoNotSearchInIndex();
1823 remove = true;
1824 } else if (entry->IsDirty()) {
1825 entry->ClearDirty();
1826 }
1827 }
1828 if (remove) {
1829 iter.Remove();
1830 }
1831 }
1832
1833 mIndexOnDiskIsValid = true;
1834 } else {
1835 if (mIndexFileOpener) {
1836 // If opening of the file is still in progress (e.g. WRITE process was
1837 // canceled by RemoveAll()) then we need to cancel the opener to make sure
1838 // that OnFileOpenedInternal() won't be called.
1839 mIndexFileOpener->Cancel();
1840 mIndexFileOpener = nullptr;
1841 }
1842 }
1843
1844 ProcessPendingOperations(aProofOfLock);
1845 mIndexStats.Log();
1846
1847 if (mState == WRITING) {
1848 ChangeState(READY, aProofOfLock);
1849 mLastDumpTime = TimeStamp::NowLoRes();
1850 }
1851 }
1852
GetFile(const nsACString & aName,nsIFile ** _retval)1853 nsresult CacheIndex::GetFile(const nsACString& aName, nsIFile** _retval) {
1854 nsresult rv;
1855
1856 nsCOMPtr<nsIFile> file;
1857 rv = mCacheDirectory->Clone(getter_AddRefs(file));
1858 NS_ENSURE_SUCCESS(rv, rv);
1859
1860 rv = file->AppendNative(aName);
1861 NS_ENSURE_SUCCESS(rv, rv);
1862
1863 file.swap(*_retval);
1864 return NS_OK;
1865 }
1866
RemoveFile(const nsACString & aName)1867 void CacheIndex::RemoveFile(const nsACString& aName) {
1868 MOZ_ASSERT(mState == SHUTDOWN);
1869
1870 nsresult rv;
1871
1872 nsCOMPtr<nsIFile> file;
1873 rv = GetFile(aName, getter_AddRefs(file));
1874 NS_ENSURE_SUCCESS_VOID(rv);
1875
1876 rv = file->Remove(false);
1877 if (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND) {
1878 LOG(
1879 ("CacheIndex::RemoveFile() - Cannot remove old entry file from disk "
1880 "[rv=0x%08" PRIx32 ", name=%s]",
1881 static_cast<uint32_t>(rv), PromiseFlatCString(aName).get()));
1882 }
1883 }
1884
RemoveAllIndexFiles()1885 void CacheIndex::RemoveAllIndexFiles() {
1886 LOG(("CacheIndex::RemoveAllIndexFiles()"));
1887 RemoveFile(nsLiteralCString(INDEX_NAME));
1888 RemoveJournalAndTempFile();
1889 }
1890
RemoveJournalAndTempFile()1891 void CacheIndex::RemoveJournalAndTempFile() {
1892 LOG(("CacheIndex::RemoveJournalAndTempFile()"));
1893 RemoveFile(nsLiteralCString(TEMP_INDEX_NAME));
1894 RemoveFile(nsLiteralCString(JOURNAL_NAME));
1895 }
1896
1897 class WriteLogHelper {
1898 public:
WriteLogHelper(PRFileDesc * aFD)1899 explicit WriteLogHelper(PRFileDesc* aFD)
1900 : mFD(aFD), mBufSize(kMaxBufSize), mBufPos(0) {
1901 mHash = new CacheHash();
1902 mBuf = static_cast<char*>(moz_xmalloc(mBufSize));
1903 }
1904
~WriteLogHelper()1905 ~WriteLogHelper() { free(mBuf); }
1906
1907 nsresult AddEntry(CacheIndexEntry* aEntry);
1908 nsresult Finish();
1909
1910 private:
1911 nsresult FlushBuffer();
1912
1913 PRFileDesc* mFD;
1914 char* mBuf;
1915 uint32_t mBufSize;
1916 int32_t mBufPos;
1917 RefPtr<CacheHash> mHash;
1918 };
1919
AddEntry(CacheIndexEntry * aEntry)1920 nsresult WriteLogHelper::AddEntry(CacheIndexEntry* aEntry) {
1921 nsresult rv;
1922
1923 if (mBufPos + sizeof(CacheIndexRecord) > mBufSize) {
1924 mHash->Update(mBuf, mBufPos);
1925
1926 rv = FlushBuffer();
1927 NS_ENSURE_SUCCESS(rv, rv);
1928 MOZ_ASSERT(mBufPos + sizeof(CacheIndexRecord) <= mBufSize);
1929 }
1930
1931 aEntry->WriteToBuf(mBuf + mBufPos);
1932 mBufPos += sizeof(CacheIndexRecord);
1933
1934 return NS_OK;
1935 }
1936
Finish()1937 nsresult WriteLogHelper::Finish() {
1938 nsresult rv;
1939
1940 mHash->Update(mBuf, mBufPos);
1941 if (mBufPos + sizeof(CacheHash::Hash32_t) > mBufSize) {
1942 rv = FlushBuffer();
1943 NS_ENSURE_SUCCESS(rv, rv);
1944 MOZ_ASSERT(mBufPos + sizeof(CacheHash::Hash32_t) <= mBufSize);
1945 }
1946
1947 NetworkEndian::writeUint32(mBuf + mBufPos, mHash->GetHash());
1948 mBufPos += sizeof(CacheHash::Hash32_t);
1949
1950 rv = FlushBuffer();
1951 NS_ENSURE_SUCCESS(rv, rv);
1952
1953 return NS_OK;
1954 }
1955
FlushBuffer()1956 nsresult WriteLogHelper::FlushBuffer() {
1957 if (CacheObserver::IsPastShutdownIOLag()) {
1958 LOG(("WriteLogHelper::FlushBuffer() - Interrupting writing journal."));
1959 return NS_ERROR_FAILURE;
1960 }
1961
1962 int32_t bytesWritten = PR_Write(mFD, mBuf, mBufPos);
1963
1964 if (bytesWritten != mBufPos) {
1965 return NS_ERROR_FAILURE;
1966 }
1967
1968 mBufPos = 0;
1969 return NS_OK;
1970 }
1971
WriteLogToDisk()1972 nsresult CacheIndex::WriteLogToDisk() {
1973 LOG(("CacheIndex::WriteLogToDisk()"));
1974
1975 nsresult rv;
1976
1977 MOZ_ASSERT(mPendingUpdates.Count() == 0);
1978 MOZ_ASSERT(mState == SHUTDOWN);
1979
1980 if (CacheObserver::IsPastShutdownIOLag()) {
1981 LOG(("CacheIndex::WriteLogToDisk() - Skipping writing journal."));
1982 return NS_ERROR_FAILURE;
1983 }
1984
1985 RemoveFile(nsLiteralCString(TEMP_INDEX_NAME));
1986
1987 nsCOMPtr<nsIFile> indexFile;
1988 rv = GetFile(nsLiteralCString(INDEX_NAME), getter_AddRefs(indexFile));
1989 NS_ENSURE_SUCCESS(rv, rv);
1990
1991 nsCOMPtr<nsIFile> logFile;
1992 rv = GetFile(nsLiteralCString(JOURNAL_NAME), getter_AddRefs(logFile));
1993 NS_ENSURE_SUCCESS(rv, rv);
1994
1995 mIndexStats.Log();
1996
1997 PRFileDesc* fd = nullptr;
1998 rv = logFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, 0600,
1999 &fd);
2000 NS_ENSURE_SUCCESS(rv, rv);
2001
2002 WriteLogHelper wlh(fd);
2003 for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2004 CacheIndexEntry* entry = iter.Get();
2005 if (entry->IsRemoved() || entry->IsDirty()) {
2006 rv = wlh.AddEntry(entry);
2007 if (NS_WARN_IF(NS_FAILED(rv))) {
2008 return rv;
2009 }
2010 }
2011 }
2012
2013 rv = wlh.Finish();
2014 PR_Close(fd);
2015 NS_ENSURE_SUCCESS(rv, rv);
2016
2017 rv = indexFile->OpenNSPRFileDesc(PR_RDWR, 0600, &fd);
2018 NS_ENSURE_SUCCESS(rv, rv);
2019
2020 // Seek to dirty flag in the index header and clear it.
2021 static_assert(2 * sizeof(uint32_t) == offsetof(CacheIndexHeader, mIsDirty),
2022 "Unexpected offset of CacheIndexHeader::mIsDirty");
2023 int64_t offset = PR_Seek64(fd, 2 * sizeof(uint32_t), PR_SEEK_SET);
2024 if (offset == -1) {
2025 PR_Close(fd);
2026 return NS_ERROR_FAILURE;
2027 }
2028
2029 uint32_t isDirty = 0;
2030 int32_t bytesWritten = PR_Write(fd, &isDirty, sizeof(isDirty));
2031 PR_Close(fd);
2032 if (bytesWritten != sizeof(isDirty)) {
2033 return NS_ERROR_FAILURE;
2034 }
2035
2036 return NS_OK;
2037 }
2038
ReadIndexFromDisk(const StaticMutexAutoLock & aProofOfLock)2039 void CacheIndex::ReadIndexFromDisk(const StaticMutexAutoLock& aProofOfLock) {
2040 LOG(("CacheIndex::ReadIndexFromDisk()"));
2041
2042 nsresult rv;
2043
2044 MOZ_ASSERT(mState == INITIAL);
2045
2046 ChangeState(READING, aProofOfLock);
2047
2048 mIndexFileOpener = new FileOpenHelper(this);
2049 rv = CacheFileIOManager::OpenFile(
2050 nsLiteralCString(INDEX_NAME),
2051 CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN,
2052 mIndexFileOpener);
2053 if (NS_FAILED(rv)) {
2054 LOG(
2055 ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2056 "failed [rv=0x%08" PRIx32 ", file=%s]",
2057 static_cast<uint32_t>(rv), INDEX_NAME));
2058 FinishRead(false, aProofOfLock);
2059 return;
2060 }
2061
2062 mJournalFileOpener = new FileOpenHelper(this);
2063 rv = CacheFileIOManager::OpenFile(
2064 nsLiteralCString(JOURNAL_NAME),
2065 CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN,
2066 mJournalFileOpener);
2067 if (NS_FAILED(rv)) {
2068 LOG(
2069 ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2070 "failed [rv=0x%08" PRIx32 ", file=%s]",
2071 static_cast<uint32_t>(rv), JOURNAL_NAME));
2072 FinishRead(false, aProofOfLock);
2073 }
2074
2075 mTmpFileOpener = new FileOpenHelper(this);
2076 rv = CacheFileIOManager::OpenFile(
2077 nsLiteralCString(TEMP_INDEX_NAME),
2078 CacheFileIOManager::SPECIAL_FILE | CacheFileIOManager::OPEN,
2079 mTmpFileOpener);
2080 if (NS_FAILED(rv)) {
2081 LOG(
2082 ("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() "
2083 "failed [rv=0x%08" PRIx32 ", file=%s]",
2084 static_cast<uint32_t>(rv), TEMP_INDEX_NAME));
2085 FinishRead(false, aProofOfLock);
2086 }
2087 }
2088
StartReadingIndex(const StaticMutexAutoLock & aProofOfLock)2089 void CacheIndex::StartReadingIndex(const StaticMutexAutoLock& aProofOfLock) {
2090 LOG(("CacheIndex::StartReadingIndex()"));
2091
2092 nsresult rv;
2093
2094 MOZ_ASSERT(mIndexHandle);
2095 MOZ_ASSERT(mState == READING);
2096 MOZ_ASSERT(!mIndexOnDiskIsValid);
2097 MOZ_ASSERT(!mDontMarkIndexClean);
2098 MOZ_ASSERT(!mJournalReadSuccessfully);
2099 MOZ_ASSERT(mIndexHandle->FileSize() >= 0);
2100 MOZ_ASSERT(!mRWPending);
2101
2102 int64_t entriesSize = mIndexHandle->FileSize() - sizeof(CacheIndexHeader) -
2103 sizeof(CacheHash::Hash32_t);
2104
2105 if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) {
2106 LOG(("CacheIndex::StartReadingIndex() - Index is corrupted"));
2107 FinishRead(false, aProofOfLock);
2108 return;
2109 }
2110
2111 AllocBuffer();
2112 mSkipEntries = 0;
2113 mRWHash = new CacheHash();
2114
2115 mRWBufPos =
2116 std::min(mRWBufSize, static_cast<uint32_t>(mIndexHandle->FileSize()));
2117
2118 rv = CacheFileIOManager::Read(mIndexHandle, 0, mRWBuf, mRWBufPos, this);
2119 if (NS_FAILED(rv)) {
2120 LOG(
2121 ("CacheIndex::StartReadingIndex() - CacheFileIOManager::Read() failed "
2122 "synchronously [rv=0x%08" PRIx32 "]",
2123 static_cast<uint32_t>(rv)));
2124 FinishRead(false, aProofOfLock);
2125 } else {
2126 mRWPending = true;
2127 }
2128 }
2129
ParseRecords(const StaticMutexAutoLock & aProofOfLock)2130 void CacheIndex::ParseRecords(const StaticMutexAutoLock& aProofOfLock) {
2131 LOG(("CacheIndex::ParseRecords()"));
2132
2133 nsresult rv;
2134
2135 MOZ_ASSERT(!mRWPending);
2136
2137 uint32_t entryCnt = (mIndexHandle->FileSize() - sizeof(CacheIndexHeader) -
2138 sizeof(CacheHash::Hash32_t)) /
2139 sizeof(CacheIndexRecord);
2140 uint32_t pos = 0;
2141
2142 if (!mSkipEntries) {
2143 if (NetworkEndian::readUint32(mRWBuf + pos) != kIndexVersion) {
2144 FinishRead(false, aProofOfLock);
2145 return;
2146 }
2147 pos += sizeof(uint32_t);
2148
2149 mIndexTimeStamp = NetworkEndian::readUint32(mRWBuf + pos);
2150 pos += sizeof(uint32_t);
2151
2152 if (NetworkEndian::readUint32(mRWBuf + pos)) {
2153 if (mJournalHandle) {
2154 CacheFileIOManager::DoomFile(mJournalHandle, nullptr);
2155 mJournalHandle = nullptr;
2156 }
2157 } else {
2158 uint32_t* isDirty =
2159 reinterpret_cast<uint32_t*>(moz_xmalloc(sizeof(uint32_t)));
2160 NetworkEndian::writeUint32(isDirty, 1);
2161
2162 // Mark index dirty. The buffer is freed by CacheFileIOManager when
2163 // nullptr is passed as the listener and the call doesn't fail
2164 // synchronously.
2165 rv = CacheFileIOManager::Write(mIndexHandle, 2 * sizeof(uint32_t),
2166 reinterpret_cast<char*>(isDirty),
2167 sizeof(uint32_t), true, false, nullptr);
2168 if (NS_FAILED(rv)) {
2169 // This is not fatal, just free the memory
2170 free(isDirty);
2171 }
2172 }
2173 pos += sizeof(uint32_t);
2174
2175 uint64_t dataWritten = NetworkEndian::readUint32(mRWBuf + pos);
2176 pos += sizeof(uint32_t);
2177 dataWritten <<= 10;
2178 mTotalBytesWritten += dataWritten;
2179 }
2180
2181 uint32_t hashOffset = pos;
2182
2183 while (pos + sizeof(CacheIndexRecord) <= mRWBufPos &&
2184 mSkipEntries != entryCnt) {
2185 CacheIndexRecord* rec = reinterpret_cast<CacheIndexRecord*>(mRWBuf + pos);
2186 CacheIndexEntry tmpEntry(&rec->mHash);
2187 tmpEntry.ReadFromBuf(mRWBuf + pos);
2188
2189 if (tmpEntry.IsDirty() || !tmpEntry.IsInitialized() ||
2190 tmpEntry.IsFileEmpty() || tmpEntry.IsFresh() || tmpEntry.IsRemoved()) {
2191 LOG(
2192 ("CacheIndex::ParseRecords() - Invalid entry found in index, removing"
2193 " whole index [dirty=%d, initialized=%d, fileEmpty=%d, fresh=%d, "
2194 "removed=%d]",
2195 tmpEntry.IsDirty(), tmpEntry.IsInitialized(), tmpEntry.IsFileEmpty(),
2196 tmpEntry.IsFresh(), tmpEntry.IsRemoved()));
2197 FinishRead(false, aProofOfLock);
2198 return;
2199 }
2200
2201 CacheIndexEntryAutoManage emng(tmpEntry.Hash(), this, aProofOfLock);
2202
2203 CacheIndexEntry* entry = mIndex.PutEntry(*tmpEntry.Hash());
2204 *entry = tmpEntry;
2205
2206 pos += sizeof(CacheIndexRecord);
2207 mSkipEntries++;
2208 }
2209
2210 mRWHash->Update(mRWBuf + hashOffset, pos - hashOffset);
2211
2212 if (pos != mRWBufPos) {
2213 memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos);
2214 }
2215
2216 mRWBufPos -= pos;
2217 pos = 0;
2218
2219 int64_t fileOffset = sizeof(CacheIndexHeader) +
2220 mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos;
2221
2222 MOZ_ASSERT(fileOffset <= mIndexHandle->FileSize());
2223 if (fileOffset == mIndexHandle->FileSize()) {
2224 uint32_t expectedHash = NetworkEndian::readUint32(mRWBuf);
2225 if (mRWHash->GetHash() != expectedHash) {
2226 LOG(("CacheIndex::ParseRecords() - Hash mismatch, [is %x, should be %x]",
2227 mRWHash->GetHash(), expectedHash));
2228 FinishRead(false, aProofOfLock);
2229 return;
2230 }
2231
2232 mIndexOnDiskIsValid = true;
2233 mJournalReadSuccessfully = false;
2234
2235 if (mJournalHandle) {
2236 StartReadingJournal(aProofOfLock);
2237 } else {
2238 FinishRead(false, aProofOfLock);
2239 }
2240
2241 return;
2242 }
2243
2244 pos = mRWBufPos;
2245 uint32_t toRead =
2246 std::min(mRWBufSize - pos,
2247 static_cast<uint32_t>(mIndexHandle->FileSize() - fileOffset));
2248 mRWBufPos = pos + toRead;
2249
2250 rv = CacheFileIOManager::Read(mIndexHandle, fileOffset, mRWBuf + pos, toRead,
2251 this);
2252 if (NS_FAILED(rv)) {
2253 LOG(
2254 ("CacheIndex::ParseRecords() - CacheFileIOManager::Read() failed "
2255 "synchronously [rv=0x%08" PRIx32 "]",
2256 static_cast<uint32_t>(rv)));
2257 FinishRead(false, aProofOfLock);
2258 return;
2259 }
2260 mRWPending = true;
2261 }
2262
StartReadingJournal(const StaticMutexAutoLock & aProofOfLock)2263 void CacheIndex::StartReadingJournal(const StaticMutexAutoLock& aProofOfLock) {
2264 LOG(("CacheIndex::StartReadingJournal()"));
2265
2266 nsresult rv;
2267
2268 MOZ_ASSERT(mJournalHandle);
2269 MOZ_ASSERT(mIndexOnDiskIsValid);
2270 MOZ_ASSERT(mTmpJournal.Count() == 0);
2271 MOZ_ASSERT(mJournalHandle->FileSize() >= 0);
2272 MOZ_ASSERT(!mRWPending);
2273
2274 int64_t entriesSize =
2275 mJournalHandle->FileSize() - sizeof(CacheHash::Hash32_t);
2276
2277 if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) {
2278 LOG(("CacheIndex::StartReadingJournal() - Journal is corrupted"));
2279 FinishRead(false, aProofOfLock);
2280 return;
2281 }
2282
2283 mSkipEntries = 0;
2284 mRWHash = new CacheHash();
2285
2286 mRWBufPos =
2287 std::min(mRWBufSize, static_cast<uint32_t>(mJournalHandle->FileSize()));
2288
2289 rv = CacheFileIOManager::Read(mJournalHandle, 0, mRWBuf, mRWBufPos, this);
2290 if (NS_FAILED(rv)) {
2291 LOG(
2292 ("CacheIndex::StartReadingJournal() - CacheFileIOManager::Read() failed"
2293 " synchronously [rv=0x%08" PRIx32 "]",
2294 static_cast<uint32_t>(rv)));
2295 FinishRead(false, aProofOfLock);
2296 } else {
2297 mRWPending = true;
2298 }
2299 }
2300
ParseJournal(const StaticMutexAutoLock & aProofOfLock)2301 void CacheIndex::ParseJournal(const StaticMutexAutoLock& aProofOfLock) {
2302 LOG(("CacheIndex::ParseJournal()"));
2303
2304 nsresult rv;
2305
2306 MOZ_ASSERT(!mRWPending);
2307
2308 uint32_t entryCnt =
2309 (mJournalHandle->FileSize() - sizeof(CacheHash::Hash32_t)) /
2310 sizeof(CacheIndexRecord);
2311
2312 uint32_t pos = 0;
2313
2314 while (pos + sizeof(CacheIndexRecord) <= mRWBufPos &&
2315 mSkipEntries != entryCnt) {
2316 CacheIndexEntry tmpEntry(reinterpret_cast<SHA1Sum::Hash*>(mRWBuf + pos));
2317 tmpEntry.ReadFromBuf(mRWBuf + pos);
2318
2319 CacheIndexEntry* entry = mTmpJournal.PutEntry(*tmpEntry.Hash());
2320 *entry = tmpEntry;
2321
2322 if (entry->IsDirty() || entry->IsFresh()) {
2323 LOG(
2324 ("CacheIndex::ParseJournal() - Invalid entry found in journal, "
2325 "ignoring whole journal [dirty=%d, fresh=%d]",
2326 entry->IsDirty(), entry->IsFresh()));
2327 FinishRead(false, aProofOfLock);
2328 return;
2329 }
2330
2331 pos += sizeof(CacheIndexRecord);
2332 mSkipEntries++;
2333 }
2334
2335 mRWHash->Update(mRWBuf, pos);
2336
2337 if (pos != mRWBufPos) {
2338 memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos);
2339 }
2340
2341 mRWBufPos -= pos;
2342 pos = 0;
2343
2344 int64_t fileOffset = mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos;
2345
2346 MOZ_ASSERT(fileOffset <= mJournalHandle->FileSize());
2347 if (fileOffset == mJournalHandle->FileSize()) {
2348 uint32_t expectedHash = NetworkEndian::readUint32(mRWBuf);
2349 if (mRWHash->GetHash() != expectedHash) {
2350 LOG(("CacheIndex::ParseJournal() - Hash mismatch, [is %x, should be %x]",
2351 mRWHash->GetHash(), expectedHash));
2352 FinishRead(false, aProofOfLock);
2353 return;
2354 }
2355
2356 mJournalReadSuccessfully = true;
2357 FinishRead(true, aProofOfLock);
2358 return;
2359 }
2360
2361 pos = mRWBufPos;
2362 uint32_t toRead =
2363 std::min(mRWBufSize - pos,
2364 static_cast<uint32_t>(mJournalHandle->FileSize() - fileOffset));
2365 mRWBufPos = pos + toRead;
2366
2367 rv = CacheFileIOManager::Read(mJournalHandle, fileOffset, mRWBuf + pos,
2368 toRead, this);
2369 if (NS_FAILED(rv)) {
2370 LOG(
2371 ("CacheIndex::ParseJournal() - CacheFileIOManager::Read() failed "
2372 "synchronously [rv=0x%08" PRIx32 "]",
2373 static_cast<uint32_t>(rv)));
2374 FinishRead(false, aProofOfLock);
2375 return;
2376 }
2377 mRWPending = true;
2378 }
2379
MergeJournal(const StaticMutexAutoLock & aProofOfLock)2380 void CacheIndex::MergeJournal(const StaticMutexAutoLock& aProofOfLock) {
2381 LOG(("CacheIndex::MergeJournal()"));
2382
2383 for (auto iter = mTmpJournal.Iter(); !iter.Done(); iter.Next()) {
2384 CacheIndexEntry* entry = iter.Get();
2385
2386 LOG(("CacheIndex::MergeJournal() [hash=%08x%08x%08x%08x%08x]",
2387 LOGSHA1(entry->Hash())));
2388
2389 CacheIndexEntry* entry2 = mIndex.GetEntry(*entry->Hash());
2390 {
2391 CacheIndexEntryAutoManage emng(entry->Hash(), this, aProofOfLock);
2392 if (entry->IsRemoved()) {
2393 if (entry2) {
2394 entry2->MarkRemoved();
2395 entry2->MarkDirty();
2396 }
2397 } else {
2398 if (!entry2) {
2399 entry2 = mIndex.PutEntry(*entry->Hash());
2400 }
2401
2402 *entry2 = *entry;
2403 entry2->MarkDirty();
2404 }
2405 }
2406 iter.Remove();
2407 }
2408
2409 MOZ_ASSERT(mTmpJournal.Count() == 0);
2410 }
2411
EnsureNoFreshEntry()2412 void CacheIndex::EnsureNoFreshEntry() {
2413 #ifdef DEBUG_STATS
2414 CacheIndexStats debugStats;
2415 debugStats.DisableLogging();
2416 for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2417 debugStats.BeforeChange(nullptr);
2418 debugStats.AfterChange(iter.Get());
2419 }
2420 MOZ_ASSERT(debugStats.Fresh() == 0);
2421 #endif
2422 }
2423
EnsureCorrectStats()2424 void CacheIndex::EnsureCorrectStats() {
2425 #ifdef DEBUG_STATS
2426 MOZ_ASSERT(mPendingUpdates.Count() == 0);
2427 CacheIndexStats debugStats;
2428 debugStats.DisableLogging();
2429 for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
2430 debugStats.BeforeChange(nullptr);
2431 debugStats.AfterChange(iter.Get());
2432 }
2433 MOZ_ASSERT(debugStats == mIndexStats);
2434 #endif
2435 }
2436
FinishRead(bool aSucceeded,const StaticMutexAutoLock & aProofOfLock)2437 void CacheIndex::FinishRead(bool aSucceeded,
2438 const StaticMutexAutoLock& aProofOfLock) {
2439 LOG(("CacheIndex::FinishRead() [succeeded=%d]", aSucceeded));
2440
2441 MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == READING);
2442
2443 MOZ_ASSERT(
2444 // -> rebuild
2445 (!aSucceeded && !mIndexOnDiskIsValid && !mJournalReadSuccessfully) ||
2446 // -> update
2447 (!aSucceeded && mIndexOnDiskIsValid && !mJournalReadSuccessfully) ||
2448 // -> ready
2449 (aSucceeded && mIndexOnDiskIsValid && mJournalReadSuccessfully));
2450
2451 // If there is read operation pending we must be cancelling reading of the
2452 // index when shutting down or removing the whole index.
2453 MOZ_ASSERT(!mRWPending || (!aSucceeded && (mShuttingDown || mRemovingAll)));
2454
2455 if (mState == SHUTDOWN) {
2456 RemoveFile(nsLiteralCString(TEMP_INDEX_NAME));
2457 RemoveFile(nsLiteralCString(JOURNAL_NAME));
2458 } else {
2459 if (mIndexHandle && !mIndexOnDiskIsValid) {
2460 CacheFileIOManager::DoomFile(mIndexHandle, nullptr);
2461 }
2462
2463 if (mJournalHandle) {
2464 CacheFileIOManager::DoomFile(mJournalHandle, nullptr);
2465 }
2466 }
2467
2468 if (mIndexFileOpener) {
2469 mIndexFileOpener->Cancel();
2470 mIndexFileOpener = nullptr;
2471 }
2472 if (mJournalFileOpener) {
2473 mJournalFileOpener->Cancel();
2474 mJournalFileOpener = nullptr;
2475 }
2476 if (mTmpFileOpener) {
2477 mTmpFileOpener->Cancel();
2478 mTmpFileOpener = nullptr;
2479 }
2480
2481 mIndexHandle = nullptr;
2482 mJournalHandle = nullptr;
2483 mRWHash = nullptr;
2484 ReleaseBuffer();
2485
2486 if (mState == SHUTDOWN) {
2487 return;
2488 }
2489
2490 if (!mIndexOnDiskIsValid) {
2491 MOZ_ASSERT(mTmpJournal.Count() == 0);
2492 EnsureNoFreshEntry();
2493 ProcessPendingOperations(aProofOfLock);
2494 // Remove all entries that we haven't seen during this session
2495 RemoveNonFreshEntries(aProofOfLock);
2496 StartUpdatingIndex(true, aProofOfLock);
2497 return;
2498 }
2499
2500 if (!mJournalReadSuccessfully) {
2501 mTmpJournal.Clear();
2502 EnsureNoFreshEntry();
2503 ProcessPendingOperations(aProofOfLock);
2504 StartUpdatingIndex(false, aProofOfLock);
2505 return;
2506 }
2507
2508 MergeJournal(aProofOfLock);
2509 EnsureNoFreshEntry();
2510 ProcessPendingOperations(aProofOfLock);
2511 mIndexStats.Log();
2512
2513 ChangeState(READY, aProofOfLock);
2514 mLastDumpTime = TimeStamp::NowLoRes(); // Do not dump new index immediately
2515 }
2516
2517 // static
DelayedUpdate(nsITimer * aTimer,void * aClosure)2518 void CacheIndex::DelayedUpdate(nsITimer* aTimer, void* aClosure) {
2519 LOG(("CacheIndex::DelayedUpdate()"));
2520
2521 StaticMutexAutoLock lock(sLock);
2522 RefPtr<CacheIndex> index = gInstance;
2523
2524 if (!index) {
2525 return;
2526 }
2527
2528 index->DelayedUpdateLocked(lock);
2529 }
2530
2531 // static
DelayedUpdateLocked(const StaticMutexAutoLock & aProofOfLock)2532 void CacheIndex::DelayedUpdateLocked(const StaticMutexAutoLock& aProofOfLock) {
2533 LOG(("CacheIndex::DelayedUpdateLocked()"));
2534
2535 nsresult rv;
2536
2537 mUpdateTimer = nullptr;
2538
2539 if (!IsIndexUsable()) {
2540 return;
2541 }
2542
2543 if (mState == READY && mShuttingDown) {
2544 return;
2545 }
2546
2547 // mUpdateEventPending must be false here since StartUpdatingIndex() won't
2548 // schedule timer if it is true.
2549 MOZ_ASSERT(!mUpdateEventPending);
2550 if (mState != BUILDING && mState != UPDATING) {
2551 LOG(("CacheIndex::DelayedUpdateLocked() - Update was canceled"));
2552 return;
2553 }
2554
2555 // We need to redispatch to run with lower priority
2556 RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
2557 MOZ_ASSERT(ioThread);
2558
2559 mUpdateEventPending = true;
2560 rv = ioThread->Dispatch(this, CacheIOThread::INDEX);
2561 if (NS_FAILED(rv)) {
2562 mUpdateEventPending = false;
2563 NS_WARNING("CacheIndex::DelayedUpdateLocked() - Can't dispatch event");
2564 LOG(("CacheIndex::DelayedUpdate() - Can't dispatch event"));
2565 FinishUpdate(false, aProofOfLock);
2566 }
2567 }
2568
ScheduleUpdateTimer(uint32_t aDelay)2569 nsresult CacheIndex::ScheduleUpdateTimer(uint32_t aDelay) {
2570 LOG(("CacheIndex::ScheduleUpdateTimer() [delay=%u]", aDelay));
2571
2572 MOZ_ASSERT(!mUpdateTimer);
2573
2574 nsCOMPtr<nsIEventTarget> ioTarget = CacheFileIOManager::IOTarget();
2575 MOZ_ASSERT(ioTarget);
2576
2577 return NS_NewTimerWithFuncCallback(
2578 getter_AddRefs(mUpdateTimer), CacheIndex::DelayedUpdate, nullptr, aDelay,
2579 nsITimer::TYPE_ONE_SHOT, "net::CacheIndex::ScheduleUpdateTimer",
2580 ioTarget);
2581 }
2582
SetupDirectoryEnumerator()2583 nsresult CacheIndex::SetupDirectoryEnumerator() {
2584 MOZ_ASSERT(!NS_IsMainThread());
2585 MOZ_ASSERT(!mDirEnumerator);
2586
2587 nsresult rv;
2588 nsCOMPtr<nsIFile> file;
2589
2590 rv = mCacheDirectory->Clone(getter_AddRefs(file));
2591 NS_ENSURE_SUCCESS(rv, rv);
2592
2593 rv = file->AppendNative(nsLiteralCString(ENTRIES_DIR));
2594 NS_ENSURE_SUCCESS(rv, rv);
2595
2596 bool exists;
2597 rv = file->Exists(&exists);
2598 NS_ENSURE_SUCCESS(rv, rv);
2599
2600 if (!exists) {
2601 NS_WARNING(
2602 "CacheIndex::SetupDirectoryEnumerator() - Entries directory "
2603 "doesn't exist!");
2604 LOG(
2605 ("CacheIndex::SetupDirectoryEnumerator() - Entries directory doesn't "
2606 "exist!"));
2607 return NS_ERROR_UNEXPECTED;
2608 }
2609
2610 rv = file->GetDirectoryEntries(getter_AddRefs(mDirEnumerator));
2611 NS_ENSURE_SUCCESS(rv, rv);
2612
2613 return NS_OK;
2614 }
2615
InitEntryFromDiskData(CacheIndexEntry * aEntry,CacheFileMetadata * aMetaData,int64_t aFileSize)2616 nsresult CacheIndex::InitEntryFromDiskData(CacheIndexEntry* aEntry,
2617 CacheFileMetadata* aMetaData,
2618 int64_t aFileSize) {
2619 nsresult rv;
2620
2621 aEntry->InitNew();
2622 aEntry->MarkDirty();
2623 aEntry->MarkFresh();
2624
2625 aEntry->Init(GetOriginAttrsHash(aMetaData->OriginAttributes()),
2626 aMetaData->IsAnonymous(), aMetaData->Pinned());
2627
2628 aEntry->SetFrecency(aMetaData->GetFrecency());
2629
2630 const char* altData = aMetaData->GetElement(CacheFileUtils::kAltDataKey);
2631 bool hasAltData = altData != nullptr;
2632 if (hasAltData && NS_FAILED(CacheFileUtils::ParseAlternativeDataInfo(
2633 altData, nullptr, nullptr))) {
2634 return NS_ERROR_FAILURE;
2635 }
2636 aEntry->SetHasAltData(hasAltData);
2637
2638 static auto toUint16 = [](const char* aUint16String) -> uint16_t {
2639 if (!aUint16String) {
2640 return kIndexTimeNotAvailable;
2641 }
2642 nsresult rv;
2643 uint64_t n64 = nsDependentCString(aUint16String).ToInteger64(&rv);
2644 MOZ_ASSERT(NS_SUCCEEDED(rv));
2645 return n64 <= kIndexTimeOutOfBound ? n64 : kIndexTimeOutOfBound;
2646 };
2647
2648 aEntry->SetOnStartTime(
2649 toUint16(aMetaData->GetElement("net-response-time-onstart")));
2650 aEntry->SetOnStopTime(
2651 toUint16(aMetaData->GetElement("net-response-time-onstop")));
2652
2653 const char* contentTypeStr = aMetaData->GetElement("ctid");
2654 uint8_t contentType = nsICacheEntry::CONTENT_TYPE_UNKNOWN;
2655 if (contentTypeStr) {
2656 int64_t n64 = nsDependentCString(contentTypeStr).ToInteger64(&rv);
2657 if (NS_FAILED(rv) || n64 < nsICacheEntry::CONTENT_TYPE_UNKNOWN ||
2658 n64 >= nsICacheEntry::CONTENT_TYPE_LAST) {
2659 n64 = nsICacheEntry::CONTENT_TYPE_UNKNOWN;
2660 }
2661 contentType = n64;
2662 }
2663 aEntry->SetContentType(contentType);
2664
2665 aEntry->SetFileSize(static_cast<uint32_t>(std::min(
2666 static_cast<int64_t>(PR_UINT32_MAX), (aFileSize + 0x3FF) >> 10)));
2667 return NS_OK;
2668 }
2669
IsUpdatePending()2670 bool CacheIndex::IsUpdatePending() {
2671 sLock.AssertCurrentThreadOwns();
2672
2673 return mUpdateTimer || mUpdateEventPending;
2674 }
2675
BuildIndex(const StaticMutexAutoLock & aProofOfLock)2676 void CacheIndex::BuildIndex(const StaticMutexAutoLock& aProofOfLock) {
2677 LOG(("CacheIndex::BuildIndex()"));
2678
2679 MOZ_ASSERT(mPendingUpdates.Count() == 0);
2680
2681 nsresult rv;
2682
2683 if (!mDirEnumerator) {
2684 {
2685 // Do not do IO under the lock.
2686 StaticMutexAutoUnlock unlock(sLock);
2687 rv = SetupDirectoryEnumerator();
2688 }
2689 if (mState == SHUTDOWN) {
2690 // The index was shut down while we released the lock. FinishUpdate() was
2691 // already called from Shutdown(), so just simply return here.
2692 return;
2693 }
2694
2695 if (NS_FAILED(rv)) {
2696 FinishUpdate(false, aProofOfLock);
2697 return;
2698 }
2699 }
2700
2701 while (true) {
2702 if (CacheIOThread::YieldAndRerun()) {
2703 LOG((
2704 "CacheIndex::BuildIndex() - Breaking loop for higher level events."));
2705 mUpdateEventPending = true;
2706 return;
2707 }
2708
2709 bool fileExists = false;
2710 nsCOMPtr<nsIFile> file;
2711 {
2712 // Do not do IO under the lock.
2713 StaticMutexAutoUnlock unlock(sLock);
2714 rv = mDirEnumerator->GetNextFile(getter_AddRefs(file));
2715
2716 if (file) {
2717 file->Exists(&fileExists);
2718 }
2719 }
2720 if (mState == SHUTDOWN) {
2721 return;
2722 }
2723 if (!file) {
2724 FinishUpdate(NS_SUCCEEDED(rv), aProofOfLock);
2725 return;
2726 }
2727
2728 nsAutoCString leaf;
2729 rv = file->GetNativeLeafName(leaf);
2730 if (NS_FAILED(rv)) {
2731 LOG(
2732 ("CacheIndex::BuildIndex() - GetNativeLeafName() failed! Skipping "
2733 "file."));
2734 mDontMarkIndexClean = true;
2735 continue;
2736 }
2737
2738 if (!fileExists) {
2739 LOG(
2740 ("CacheIndex::BuildIndex() - File returned by the iterator was "
2741 "removed in the meantime [name=%s]",
2742 leaf.get()));
2743 continue;
2744 }
2745
2746 SHA1Sum::Hash hash;
2747 rv = CacheFileIOManager::StrToHash(leaf, &hash);
2748 if (NS_FAILED(rv)) {
2749 LOG(
2750 ("CacheIndex::BuildIndex() - Filename is not a hash, removing file. "
2751 "[name=%s]",
2752 leaf.get()));
2753 file->Remove(false);
2754 continue;
2755 }
2756
2757 CacheIndexEntry* entry = mIndex.GetEntry(hash);
2758 if (entry && entry->IsRemoved()) {
2759 LOG(
2760 ("CacheIndex::BuildIndex() - Found file that should not exist. "
2761 "[name=%s]",
2762 leaf.get()));
2763 entry->Log();
2764 MOZ_ASSERT(entry->IsFresh());
2765 entry = nullptr;
2766 }
2767
2768 #ifdef DEBUG
2769 RefPtr<CacheFileHandle> handle;
2770 CacheFileIOManager::gInstance->mHandles.GetHandle(&hash,
2771 getter_AddRefs(handle));
2772 #endif
2773
2774 if (entry) {
2775 // the entry is up to date
2776 LOG(
2777 ("CacheIndex::BuildIndex() - Skipping file because the entry is up to"
2778 " date. [name=%s]",
2779 leaf.get()));
2780 entry->Log();
2781 MOZ_ASSERT(entry->IsFresh()); // The entry must be from this session
2782 // there must be an active CacheFile if the entry is not initialized
2783 MOZ_ASSERT(entry->IsInitialized() || handle);
2784 continue;
2785 }
2786
2787 MOZ_ASSERT(!handle);
2788
2789 RefPtr<CacheFileMetadata> meta = new CacheFileMetadata();
2790 int64_t size = 0;
2791
2792 {
2793 // Do not do IO under the lock.
2794 StaticMutexAutoUnlock unlock(sLock);
2795 rv = meta->SyncReadMetadata(file);
2796
2797 if (NS_SUCCEEDED(rv)) {
2798 rv = file->GetFileSize(&size);
2799 if (NS_FAILED(rv)) {
2800 LOG(
2801 ("CacheIndex::BuildIndex() - Cannot get filesize of file that was"
2802 " successfully parsed. [name=%s]",
2803 leaf.get()));
2804 }
2805 }
2806 }
2807 if (mState == SHUTDOWN) {
2808 return;
2809 }
2810
2811 // Nobody could add the entry while the lock was released since we modify
2812 // the index only on IO thread and this loop is executed on IO thread too.
2813 entry = mIndex.GetEntry(hash);
2814 MOZ_ASSERT(!entry || entry->IsRemoved());
2815
2816 if (NS_FAILED(rv)) {
2817 LOG(
2818 ("CacheIndex::BuildIndex() - CacheFileMetadata::SyncReadMetadata() "
2819 "failed, removing file. [name=%s]",
2820 leaf.get()));
2821 file->Remove(false);
2822 } else {
2823 CacheIndexEntryAutoManage entryMng(&hash, this, aProofOfLock);
2824 entry = mIndex.PutEntry(hash);
2825 if (NS_FAILED(InitEntryFromDiskData(entry, meta, size))) {
2826 LOG(
2827 ("CacheIndex::BuildIndex() - CacheFile::InitEntryFromDiskData() "
2828 "failed, removing file. [name=%s]",
2829 leaf.get()));
2830 file->Remove(false);
2831 entry->MarkRemoved();
2832 } else {
2833 LOG(("CacheIndex::BuildIndex() - Added entry to index. [name=%s]",
2834 leaf.get()));
2835 entry->Log();
2836 }
2837 }
2838 }
2839
2840 MOZ_ASSERT_UNREACHABLE("We should never get here");
2841 }
2842
StartUpdatingIndexIfNeeded(const StaticMutexAutoLock & aProofOfLock,bool aSwitchingToReadyState)2843 bool CacheIndex::StartUpdatingIndexIfNeeded(
2844 const StaticMutexAutoLock& aProofOfLock, bool aSwitchingToReadyState) {
2845 // Start updating process when we are in or we are switching to READY state
2846 // and index needs update, but not during shutdown or when removing all
2847 // entries.
2848 if ((mState == READY || aSwitchingToReadyState) && mIndexNeedsUpdate &&
2849 !mShuttingDown && !mRemovingAll) {
2850 LOG(("CacheIndex::StartUpdatingIndexIfNeeded() - starting update process"));
2851 mIndexNeedsUpdate = false;
2852 StartUpdatingIndex(false, aProofOfLock);
2853 return true;
2854 }
2855
2856 return false;
2857 }
2858
StartUpdatingIndex(bool aRebuild,const StaticMutexAutoLock & aProofOfLock)2859 void CacheIndex::StartUpdatingIndex(bool aRebuild,
2860 const StaticMutexAutoLock& aProofOfLock) {
2861 LOG(("CacheIndex::StartUpdatingIndex() [rebuild=%d]", aRebuild));
2862
2863 nsresult rv;
2864
2865 mIndexStats.Log();
2866
2867 ChangeState(aRebuild ? BUILDING : UPDATING, aProofOfLock);
2868 mDontMarkIndexClean = false;
2869
2870 if (mShuttingDown || mRemovingAll) {
2871 FinishUpdate(false, aProofOfLock);
2872 return;
2873 }
2874
2875 if (IsUpdatePending()) {
2876 LOG(("CacheIndex::StartUpdatingIndex() - Update is already pending"));
2877 return;
2878 }
2879
2880 uint32_t elapsed = (TimeStamp::NowLoRes() - mStartTime).ToMilliseconds();
2881 if (elapsed < kUpdateIndexStartDelay) {
2882 LOG(
2883 ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2884 "scheduling timer to fire in %u ms.",
2885 elapsed, kUpdateIndexStartDelay - elapsed));
2886 rv = ScheduleUpdateTimer(kUpdateIndexStartDelay - elapsed);
2887 if (NS_SUCCEEDED(rv)) {
2888 return;
2889 }
2890
2891 LOG(
2892 ("CacheIndex::StartUpdatingIndex() - ScheduleUpdateTimer() failed. "
2893 "Starting update immediately."));
2894 } else {
2895 LOG(
2896 ("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, "
2897 "starting update now.",
2898 elapsed));
2899 }
2900
2901 RefPtr<CacheIOThread> ioThread = CacheFileIOManager::IOThread();
2902 MOZ_ASSERT(ioThread);
2903
2904 // We need to dispatch an event even if we are on IO thread since we need to
2905 // update the index with the correct priority.
2906 mUpdateEventPending = true;
2907 rv = ioThread->Dispatch(this, CacheIOThread::INDEX);
2908 if (NS_FAILED(rv)) {
2909 mUpdateEventPending = false;
2910 NS_WARNING("CacheIndex::StartUpdatingIndex() - Can't dispatch event");
2911 LOG(("CacheIndex::StartUpdatingIndex() - Can't dispatch event"));
2912 FinishUpdate(false, aProofOfLock);
2913 }
2914 }
2915
UpdateIndex(const StaticMutexAutoLock & aProofOfLock)2916 void CacheIndex::UpdateIndex(const StaticMutexAutoLock& aProofOfLock) {
2917 LOG(("CacheIndex::UpdateIndex()"));
2918
2919 MOZ_ASSERT(mPendingUpdates.Count() == 0);
2920
2921 nsresult rv;
2922
2923 if (!mDirEnumerator) {
2924 {
2925 // Do not do IO under the lock.
2926 StaticMutexAutoUnlock unlock(sLock);
2927 rv = SetupDirectoryEnumerator();
2928 }
2929 if (mState == SHUTDOWN) {
2930 // The index was shut down while we released the lock. FinishUpdate() was
2931 // already called from Shutdown(), so just simply return here.
2932 return;
2933 }
2934
2935 if (NS_FAILED(rv)) {
2936 FinishUpdate(false, aProofOfLock);
2937 return;
2938 }
2939 }
2940
2941 while (true) {
2942 if (CacheIOThread::YieldAndRerun()) {
2943 LOG(
2944 ("CacheIndex::UpdateIndex() - Breaking loop for higher level "
2945 "events."));
2946 mUpdateEventPending = true;
2947 return;
2948 }
2949
2950 bool fileExists = false;
2951 nsCOMPtr<nsIFile> file;
2952 {
2953 // Do not do IO under the lock.
2954 StaticMutexAutoUnlock unlock(sLock);
2955 rv = mDirEnumerator->GetNextFile(getter_AddRefs(file));
2956
2957 if (file) {
2958 file->Exists(&fileExists);
2959 }
2960 }
2961 if (mState == SHUTDOWN) {
2962 return;
2963 }
2964 if (!file) {
2965 FinishUpdate(NS_SUCCEEDED(rv), aProofOfLock);
2966 return;
2967 }
2968
2969 nsAutoCString leaf;
2970 rv = file->GetNativeLeafName(leaf);
2971 if (NS_FAILED(rv)) {
2972 LOG(
2973 ("CacheIndex::UpdateIndex() - GetNativeLeafName() failed! Skipping "
2974 "file."));
2975 mDontMarkIndexClean = true;
2976 continue;
2977 }
2978
2979 if (!fileExists) {
2980 LOG(
2981 ("CacheIndex::UpdateIndex() - File returned by the iterator was "
2982 "removed in the meantime [name=%s]",
2983 leaf.get()));
2984 continue;
2985 }
2986
2987 SHA1Sum::Hash hash;
2988 rv = CacheFileIOManager::StrToHash(leaf, &hash);
2989 if (NS_FAILED(rv)) {
2990 LOG(
2991 ("CacheIndex::UpdateIndex() - Filename is not a hash, removing file. "
2992 "[name=%s]",
2993 leaf.get()));
2994 file->Remove(false);
2995 continue;
2996 }
2997
2998 CacheIndexEntry* entry = mIndex.GetEntry(hash);
2999 if (entry && entry->IsRemoved()) {
3000 if (entry->IsFresh()) {
3001 LOG(
3002 ("CacheIndex::UpdateIndex() - Found file that should not exist. "
3003 "[name=%s]",
3004 leaf.get()));
3005 entry->Log();
3006 }
3007 entry = nullptr;
3008 }
3009
3010 #ifdef DEBUG
3011 RefPtr<CacheFileHandle> handle;
3012 CacheFileIOManager::gInstance->mHandles.GetHandle(&hash,
3013 getter_AddRefs(handle));
3014 #endif
3015
3016 if (entry && entry->IsFresh()) {
3017 // the entry is up to date
3018 LOG(
3019 ("CacheIndex::UpdateIndex() - Skipping file because the entry is up "
3020 " to date. [name=%s]",
3021 leaf.get()));
3022 entry->Log();
3023 // there must be an active CacheFile if the entry is not initialized
3024 MOZ_ASSERT(entry->IsInitialized() || handle);
3025 continue;
3026 }
3027
3028 MOZ_ASSERT(!handle);
3029
3030 if (entry) {
3031 PRTime lastModifiedTime;
3032 {
3033 // Do not do IO under the lock.
3034 StaticMutexAutoUnlock unlock(sLock);
3035 rv = file->GetLastModifiedTime(&lastModifiedTime);
3036 }
3037 if (mState == SHUTDOWN) {
3038 return;
3039 }
3040 if (NS_FAILED(rv)) {
3041 LOG(
3042 ("CacheIndex::UpdateIndex() - Cannot get lastModifiedTime. "
3043 "[name=%s]",
3044 leaf.get()));
3045 // Assume the file is newer than index
3046 } else {
3047 if (mIndexTimeStamp > (lastModifiedTime / PR_MSEC_PER_SEC)) {
3048 LOG(
3049 ("CacheIndex::UpdateIndex() - Skipping file because of last "
3050 "modified time. [name=%s, indexTimeStamp=%" PRIu32 ", "
3051 "lastModifiedTime=%" PRId64 "]",
3052 leaf.get(), mIndexTimeStamp,
3053 lastModifiedTime / PR_MSEC_PER_SEC));
3054
3055 CacheIndexEntryAutoManage entryMng(&hash, this, aProofOfLock);
3056 entry->MarkFresh();
3057 continue;
3058 }
3059 }
3060 }
3061
3062 RefPtr<CacheFileMetadata> meta = new CacheFileMetadata();
3063 int64_t size = 0;
3064
3065 {
3066 // Do not do IO under the lock.
3067 StaticMutexAutoUnlock unlock(sLock);
3068 rv = meta->SyncReadMetadata(file);
3069
3070 if (NS_SUCCEEDED(rv)) {
3071 rv = file->GetFileSize(&size);
3072 if (NS_FAILED(rv)) {
3073 LOG(
3074 ("CacheIndex::UpdateIndex() - Cannot get filesize of file that "
3075 "was successfully parsed. [name=%s]",
3076 leaf.get()));
3077 }
3078 }
3079 }
3080 if (mState == SHUTDOWN) {
3081 return;
3082 }
3083
3084 // Nobody could add the entry while the lock was released since we modify
3085 // the index only on IO thread and this loop is executed on IO thread too.
3086 entry = mIndex.GetEntry(hash);
3087 MOZ_ASSERT(!entry || !entry->IsFresh());
3088
3089 CacheIndexEntryAutoManage entryMng(&hash, this, aProofOfLock);
3090
3091 if (NS_FAILED(rv)) {
3092 LOG(
3093 ("CacheIndex::UpdateIndex() - CacheFileMetadata::SyncReadMetadata() "
3094 "failed, removing file. [name=%s]",
3095 leaf.get()));
3096 } else {
3097 entry = mIndex.PutEntry(hash);
3098 rv = InitEntryFromDiskData(entry, meta, size);
3099 if (NS_FAILED(rv)) {
3100 LOG(
3101 ("CacheIndex::UpdateIndex() - CacheIndex::InitEntryFromDiskData "
3102 "failed, removing file. [name=%s]",
3103 leaf.get()));
3104 }
3105 }
3106
3107 if (NS_FAILED(rv)) {
3108 file->Remove(false);
3109 if (entry) {
3110 entry->MarkRemoved();
3111 entry->MarkFresh();
3112 entry->MarkDirty();
3113 }
3114 } else {
3115 LOG(
3116 ("CacheIndex::UpdateIndex() - Added/updated entry to/in index. "
3117 "[name=%s]",
3118 leaf.get()));
3119 entry->Log();
3120 }
3121 }
3122
3123 MOZ_ASSERT_UNREACHABLE("We should never get here");
3124 }
3125
FinishUpdate(bool aSucceeded,const StaticMutexAutoLock & aProofOfLock)3126 void CacheIndex::FinishUpdate(bool aSucceeded,
3127 const StaticMutexAutoLock& aProofOfLock) {
3128 LOG(("CacheIndex::FinishUpdate() [succeeded=%d]", aSucceeded));
3129
3130 MOZ_ASSERT(mState == UPDATING || mState == BUILDING ||
3131 (!aSucceeded && mState == SHUTDOWN));
3132
3133 if (mDirEnumerator) {
3134 if (NS_IsMainThread()) {
3135 LOG(
3136 ("CacheIndex::FinishUpdate() - posting of PreShutdownInternal failed?"
3137 " Cannot safely release mDirEnumerator, leaking it!"));
3138 NS_WARNING(("CacheIndex::FinishUpdate() - Leaking mDirEnumerator!"));
3139 // This can happen only in case dispatching event to IO thread failed in
3140 // CacheIndex::PreShutdown().
3141 Unused << mDirEnumerator.forget(); // Leak it since dir enumerator is not
3142 // threadsafe
3143 } else {
3144 mDirEnumerator->Close();
3145 mDirEnumerator = nullptr;
3146 }
3147 }
3148
3149 if (!aSucceeded) {
3150 mDontMarkIndexClean = true;
3151 }
3152
3153 if (mState == SHUTDOWN) {
3154 return;
3155 }
3156
3157 if (mState == UPDATING && aSucceeded) {
3158 // If we've iterated over all entries successfully then all entries that
3159 // really exist on the disk are now marked as fresh. All non-fresh entries
3160 // don't exist anymore and must be removed from the index.
3161 RemoveNonFreshEntries(aProofOfLock);
3162 }
3163
3164 // Make sure we won't start update. If the build or update failed, there is no
3165 // reason to believe that it will succeed next time.
3166 mIndexNeedsUpdate = false;
3167
3168 ChangeState(READY, aProofOfLock);
3169 mLastDumpTime = TimeStamp::NowLoRes(); // Do not dump new index immediately
3170 }
3171
RemoveNonFreshEntries(const StaticMutexAutoLock & aProofOfLock)3172 void CacheIndex::RemoveNonFreshEntries(
3173 const StaticMutexAutoLock& aProofOfLock) {
3174 for (auto iter = mIndex.Iter(); !iter.Done(); iter.Next()) {
3175 CacheIndexEntry* entry = iter.Get();
3176 if (entry->IsFresh()) {
3177 continue;
3178 }
3179
3180 LOG(
3181 ("CacheIndex::RemoveNonFreshEntries() - Removing entry. "
3182 "[hash=%08x%08x%08x%08x%08x]",
3183 LOGSHA1(entry->Hash())));
3184
3185 {
3186 CacheIndexEntryAutoManage emng(entry->Hash(), this, aProofOfLock);
3187 emng.DoNotSearchInIndex();
3188 }
3189
3190 iter.Remove();
3191 }
3192 }
3193
3194 // static
StateString(EState aState)3195 char const* CacheIndex::StateString(EState aState) {
3196 switch (aState) {
3197 case INITIAL:
3198 return "INITIAL";
3199 case READING:
3200 return "READING";
3201 case WRITING:
3202 return "WRITING";
3203 case BUILDING:
3204 return "BUILDING";
3205 case UPDATING:
3206 return "UPDATING";
3207 case READY:
3208 return "READY";
3209 case SHUTDOWN:
3210 return "SHUTDOWN";
3211 }
3212
3213 MOZ_ASSERT(false, "Unexpected state!");
3214 return "?";
3215 }
3216
ChangeState(EState aNewState,const StaticMutexAutoLock & aProofOfLock)3217 void CacheIndex::ChangeState(EState aNewState,
3218 const StaticMutexAutoLock& aProofOfLock) {
3219 LOG(("CacheIndex::ChangeState() changing state %s -> %s", StateString(mState),
3220 StateString(aNewState)));
3221
3222 // All pending updates should be processed before changing state
3223 MOZ_ASSERT(mPendingUpdates.Count() == 0);
3224
3225 // PreShutdownInternal() should change the state to READY from every state. It
3226 // may go through different states, but once we are in READY state the only
3227 // possible transition is to SHUTDOWN state.
3228 MOZ_ASSERT(!mShuttingDown || mState != READY || aNewState == SHUTDOWN);
3229
3230 // Start updating process when switching to READY state if needed
3231 if (aNewState == READY && StartUpdatingIndexIfNeeded(aProofOfLock, true)) {
3232 return;
3233 }
3234
3235 // Try to evict entries over limit everytime we're leaving state READING,
3236 // BUILDING or UPDATING, but not during shutdown or when removing all
3237 // entries.
3238 if (!mShuttingDown && !mRemovingAll && aNewState != SHUTDOWN &&
3239 (mState == READING || mState == BUILDING || mState == UPDATING)) {
3240 CacheFileIOManager::EvictIfOverLimit();
3241 }
3242
3243 mState = aNewState;
3244
3245 if (mState != SHUTDOWN) {
3246 CacheFileIOManager::CacheIndexStateChanged();
3247 }
3248
3249 NotifyAsyncGetDiskConsumptionCallbacks();
3250 }
3251
NotifyAsyncGetDiskConsumptionCallbacks()3252 void CacheIndex::NotifyAsyncGetDiskConsumptionCallbacks() {
3253 if ((mState == READY || mState == WRITING) &&
3254 !mAsyncGetDiskConsumptionBlocked && mDiskConsumptionObservers.Length()) {
3255 for (uint32_t i = 0; i < mDiskConsumptionObservers.Length(); ++i) {
3256 DiskConsumptionObserver* o = mDiskConsumptionObservers[i];
3257 // Safe to call under the lock. We always post to the main thread.
3258 o->OnDiskConsumption(mIndexStats.Size() << 10);
3259 }
3260
3261 mDiskConsumptionObservers.Clear();
3262 }
3263 }
3264
AllocBuffer()3265 void CacheIndex::AllocBuffer() {
3266 switch (mState) {
3267 case WRITING:
3268 mRWBufSize = sizeof(CacheIndexHeader) + sizeof(CacheHash::Hash32_t) +
3269 mProcessEntries * sizeof(CacheIndexRecord);
3270 if (mRWBufSize > kMaxBufSize) {
3271 mRWBufSize = kMaxBufSize;
3272 }
3273 break;
3274 case READING:
3275 mRWBufSize = kMaxBufSize;
3276 break;
3277 default:
3278 MOZ_ASSERT(false, "Unexpected state!");
3279 }
3280
3281 mRWBuf = static_cast<char*>(moz_xmalloc(mRWBufSize));
3282 }
3283
ReleaseBuffer()3284 void CacheIndex::ReleaseBuffer() {
3285 sLock.AssertCurrentThreadOwns();
3286
3287 if (!mRWBuf || mRWPending) {
3288 return;
3289 }
3290
3291 LOG(("CacheIndex::ReleaseBuffer() releasing buffer"));
3292
3293 free(mRWBuf);
3294 mRWBuf = nullptr;
3295 mRWBufSize = 0;
3296 mRWBufPos = 0;
3297 }
3298
AppendRecord(CacheIndexRecordWrapper * aRecord,const StaticMutexAutoLock & aProofOfLock)3299 void CacheIndex::FrecencyArray::AppendRecord(
3300 CacheIndexRecordWrapper* aRecord, const StaticMutexAutoLock& aProofOfLock) {
3301 LOG(
3302 ("CacheIndex::FrecencyArray::AppendRecord() [record=%p, hash=%08x%08x%08x"
3303 "%08x%08x]",
3304 aRecord, LOGSHA1(aRecord->Get()->mHash)));
3305
3306 MOZ_ASSERT(!mRecs.Contains(aRecord));
3307 mRecs.AppendElement(aRecord);
3308
3309 // If the new frecency is 0, the element should be at the end of the array,
3310 // i.e. this change doesn't affect order of the array
3311 if (aRecord->Get()->mFrecency != 0) {
3312 ++mUnsortedElements;
3313 }
3314 }
3315
RemoveRecord(CacheIndexRecordWrapper * aRecord,const StaticMutexAutoLock & aProofOfLock)3316 void CacheIndex::FrecencyArray::RemoveRecord(
3317 CacheIndexRecordWrapper* aRecord, const StaticMutexAutoLock& aProofOfLock) {
3318 LOG(("CacheIndex::FrecencyArray::RemoveRecord() [record=%p]", aRecord));
3319
3320 decltype(mRecs)::index_type idx;
3321 idx = mRecs.IndexOf(aRecord);
3322 MOZ_RELEASE_ASSERT(idx != mRecs.NoIndex);
3323 mRecs[idx] = nullptr;
3324 ++mRemovedElements;
3325
3326 // Calling SortIfNeeded ensures that we get rid of removed elements in the
3327 // array once we hit the limit.
3328 SortIfNeeded(aProofOfLock);
3329 }
3330
ReplaceRecord(CacheIndexRecordWrapper * aOldRecord,CacheIndexRecordWrapper * aNewRecord,const StaticMutexAutoLock & aProofOfLock)3331 void CacheIndex::FrecencyArray::ReplaceRecord(
3332 CacheIndexRecordWrapper* aOldRecord, CacheIndexRecordWrapper* aNewRecord,
3333 const StaticMutexAutoLock& aProofOfLock) {
3334 LOG(
3335 ("CacheIndex::FrecencyArray::ReplaceRecord() [oldRecord=%p, "
3336 "newRecord=%p]",
3337 aOldRecord, aNewRecord));
3338
3339 decltype(mRecs)::index_type idx;
3340 idx = mRecs.IndexOf(aOldRecord);
3341 MOZ_RELEASE_ASSERT(idx != mRecs.NoIndex);
3342 mRecs[idx] = aNewRecord;
3343 }
3344
SortIfNeeded(const StaticMutexAutoLock & aProofOfLock)3345 void CacheIndex::FrecencyArray::SortIfNeeded(
3346 const StaticMutexAutoLock& aProofOfLock) {
3347 const uint32_t kMaxUnsortedCount = 512;
3348 const uint32_t kMaxUnsortedPercent = 10;
3349 const uint32_t kMaxRemovedCount = 512;
3350
3351 uint32_t unsortedLimit = std::min<uint32_t>(
3352 kMaxUnsortedCount, Length() * kMaxUnsortedPercent / 100);
3353
3354 if (mUnsortedElements > unsortedLimit ||
3355 mRemovedElements > kMaxRemovedCount) {
3356 LOG(
3357 ("CacheIndex::FrecencyArray::SortIfNeeded() - Sorting array "
3358 "[unsortedElements=%u, unsortedLimit=%u, removedElements=%u, "
3359 "maxRemovedCount=%u]",
3360 mUnsortedElements, unsortedLimit, mRemovedElements, kMaxRemovedCount));
3361
3362 mRecs.Sort(FrecencyComparator());
3363 mUnsortedElements = 0;
3364 if (mRemovedElements) {
3365 #ifdef DEBUG
3366 for (uint32_t i = Length(); i < mRecs.Length(); ++i) {
3367 MOZ_ASSERT(!mRecs[i]);
3368 }
3369 #endif
3370 // Removed elements are at the end after sorting.
3371 mRecs.RemoveElementsAt(Length(), mRemovedElements);
3372 mRemovedElements = 0;
3373 }
3374 }
3375 }
3376
AddRecordToIterators(CacheIndexRecordWrapper * aRecord,const StaticMutexAutoLock & aProofOfLock)3377 void CacheIndex::AddRecordToIterators(CacheIndexRecordWrapper* aRecord,
3378 const StaticMutexAutoLock& aProofOfLock) {
3379 for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3380 // Add a new record only when iterator is supposed to be updated.
3381 if (mIterators[i]->ShouldBeNewAdded()) {
3382 mIterators[i]->AddRecord(aRecord, aProofOfLock);
3383 }
3384 }
3385 }
3386
RemoveRecordFromIterators(CacheIndexRecordWrapper * aRecord,const StaticMutexAutoLock & aProofOfLock)3387 void CacheIndex::RemoveRecordFromIterators(
3388 CacheIndexRecordWrapper* aRecord, const StaticMutexAutoLock& aProofOfLock) {
3389 for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3390 // Remove the record from iterator always, it makes no sence to return
3391 // non-existing entries. Also the pointer to the record is no longer valid
3392 // once the entry is removed from index.
3393 mIterators[i]->RemoveRecord(aRecord, aProofOfLock);
3394 }
3395 }
3396
ReplaceRecordInIterators(CacheIndexRecordWrapper * aOldRecord,CacheIndexRecordWrapper * aNewRecord,const StaticMutexAutoLock & aProofOfLock)3397 void CacheIndex::ReplaceRecordInIterators(
3398 CacheIndexRecordWrapper* aOldRecord, CacheIndexRecordWrapper* aNewRecord,
3399 const StaticMutexAutoLock& aProofOfLock) {
3400 for (uint32_t i = 0; i < mIterators.Length(); ++i) {
3401 // We have to replace the record always since the pointer is no longer
3402 // valid after this point. NOTE: Replacing the record doesn't mean that
3403 // a new entry was added, it just means that the data in the entry was
3404 // changed (e.g. a file size) and we had to track this change in
3405 // mPendingUpdates since mIndex was read-only.
3406 mIterators[i]->ReplaceRecord(aOldRecord, aNewRecord, aProofOfLock);
3407 }
3408 }
3409
Run()3410 nsresult CacheIndex::Run() {
3411 LOG(("CacheIndex::Run()"));
3412
3413 StaticMutexAutoLock lock(sLock);
3414
3415 if (!IsIndexUsable()) {
3416 return NS_ERROR_NOT_AVAILABLE;
3417 }
3418
3419 if (mState == READY && mShuttingDown) {
3420 return NS_OK;
3421 }
3422
3423 mUpdateEventPending = false;
3424
3425 switch (mState) {
3426 case BUILDING:
3427 BuildIndex(lock);
3428 break;
3429 case UPDATING:
3430 UpdateIndex(lock);
3431 break;
3432 default:
3433 LOG(("CacheIndex::Run() - Update/Build was canceled"));
3434 }
3435
3436 return NS_OK;
3437 }
3438
OnFileOpenedInternal(FileOpenHelper * aOpener,CacheFileHandle * aHandle,nsresult aResult,const StaticMutexAutoLock & aProofOfLock)3439 void CacheIndex::OnFileOpenedInternal(FileOpenHelper* aOpener,
3440 CacheFileHandle* aHandle,
3441 nsresult aResult,
3442 const StaticMutexAutoLock& aProofOfLock) {
3443 LOG(
3444 ("CacheIndex::OnFileOpenedInternal() [opener=%p, handle=%p, "
3445 "result=0x%08" PRIx32 "]",
3446 aOpener, aHandle, static_cast<uint32_t>(aResult)));
3447 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3448
3449 nsresult rv;
3450
3451 MOZ_RELEASE_ASSERT(IsIndexUsable());
3452
3453 if (mState == READY && mShuttingDown) {
3454 return;
3455 }
3456
3457 switch (mState) {
3458 case WRITING:
3459 MOZ_ASSERT(aOpener == mIndexFileOpener);
3460 mIndexFileOpener = nullptr;
3461
3462 if (NS_FAILED(aResult)) {
3463 LOG(
3464 ("CacheIndex::OnFileOpenedInternal() - Can't open index file for "
3465 "writing [rv=0x%08" PRIx32 "]",
3466 static_cast<uint32_t>(aResult)));
3467 FinishWrite(false, aProofOfLock);
3468 } else {
3469 mIndexHandle = aHandle;
3470 WriteRecords(aProofOfLock);
3471 }
3472 break;
3473 case READING:
3474 if (aOpener == mIndexFileOpener) {
3475 mIndexFileOpener = nullptr;
3476
3477 if (NS_SUCCEEDED(aResult)) {
3478 if (aHandle->FileSize() == 0) {
3479 FinishRead(false, aProofOfLock);
3480 CacheFileIOManager::DoomFile(aHandle, nullptr);
3481 break;
3482 }
3483 mIndexHandle = aHandle;
3484 } else {
3485 FinishRead(false, aProofOfLock);
3486 break;
3487 }
3488 } else if (aOpener == mJournalFileOpener) {
3489 mJournalFileOpener = nullptr;
3490 mJournalHandle = aHandle;
3491 } else if (aOpener == mTmpFileOpener) {
3492 mTmpFileOpener = nullptr;
3493 mTmpHandle = aHandle;
3494 } else {
3495 MOZ_ASSERT(false, "Unexpected state!");
3496 }
3497
3498 if (mIndexFileOpener || mJournalFileOpener || mTmpFileOpener) {
3499 // Some opener still didn't finish
3500 break;
3501 }
3502
3503 // We fail and cancel all other openers when we opening index file fails.
3504 MOZ_ASSERT(mIndexHandle);
3505
3506 if (mTmpHandle) {
3507 CacheFileIOManager::DoomFile(mTmpHandle, nullptr);
3508 mTmpHandle = nullptr;
3509
3510 if (mJournalHandle) { // this shouldn't normally happen
3511 LOG(
3512 ("CacheIndex::OnFileOpenedInternal() - Unexpected state, all "
3513 "files [%s, %s, %s] should never exist. Removing whole index.",
3514 INDEX_NAME, JOURNAL_NAME, TEMP_INDEX_NAME));
3515 FinishRead(false, aProofOfLock);
3516 break;
3517 }
3518 }
3519
3520 if (mJournalHandle) {
3521 // Rename journal to make sure we update index on next start in case
3522 // firefox crashes
3523 rv = CacheFileIOManager::RenameFile(
3524 mJournalHandle, nsLiteralCString(TEMP_INDEX_NAME), this);
3525 if (NS_FAILED(rv)) {
3526 LOG(
3527 ("CacheIndex::OnFileOpenedInternal() - CacheFileIOManager::"
3528 "RenameFile() failed synchronously [rv=0x%08" PRIx32 "]",
3529 static_cast<uint32_t>(rv)));
3530 FinishRead(false, aProofOfLock);
3531 break;
3532 }
3533 } else {
3534 StartReadingIndex(aProofOfLock);
3535 }
3536
3537 break;
3538 default:
3539 MOZ_ASSERT(false, "Unexpected state!");
3540 }
3541 }
3542
OnFileOpened(CacheFileHandle * aHandle,nsresult aResult)3543 nsresult CacheIndex::OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) {
3544 MOZ_CRASH("CacheIndex::OnFileOpened should not be called!");
3545 return NS_ERROR_UNEXPECTED;
3546 }
3547
OnDataWritten(CacheFileHandle * aHandle,const char * aBuf,nsresult aResult)3548 nsresult CacheIndex::OnDataWritten(CacheFileHandle* aHandle, const char* aBuf,
3549 nsresult aResult) {
3550 LOG(("CacheIndex::OnDataWritten() [handle=%p, result=0x%08" PRIx32 "]",
3551 aHandle, static_cast<uint32_t>(aResult)));
3552
3553 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3554
3555 nsresult rv;
3556
3557 StaticMutexAutoLock lock(sLock);
3558
3559 MOZ_RELEASE_ASSERT(IsIndexUsable());
3560 MOZ_RELEASE_ASSERT(mRWPending);
3561 mRWPending = false;
3562
3563 if (mState == READY && mShuttingDown) {
3564 return NS_OK;
3565 }
3566
3567 switch (mState) {
3568 case WRITING:
3569 MOZ_ASSERT(mIndexHandle == aHandle);
3570
3571 if (NS_FAILED(aResult)) {
3572 FinishWrite(false, lock);
3573 } else {
3574 if (mSkipEntries == mProcessEntries) {
3575 rv = CacheFileIOManager::RenameFile(
3576 mIndexHandle, nsLiteralCString(INDEX_NAME), this);
3577 if (NS_FAILED(rv)) {
3578 LOG(
3579 ("CacheIndex::OnDataWritten() - CacheFileIOManager::"
3580 "RenameFile() failed synchronously [rv=0x%08" PRIx32 "]",
3581 static_cast<uint32_t>(rv)));
3582 FinishWrite(false, lock);
3583 }
3584 } else {
3585 WriteRecords(lock);
3586 }
3587 }
3588 break;
3589 default:
3590 // Writing was canceled.
3591 LOG(
3592 ("CacheIndex::OnDataWritten() - ignoring notification since the "
3593 "operation was previously canceled [state=%d]",
3594 mState));
3595 ReleaseBuffer();
3596 }
3597
3598 return NS_OK;
3599 }
3600
OnDataRead(CacheFileHandle * aHandle,char * aBuf,nsresult aResult)3601 nsresult CacheIndex::OnDataRead(CacheFileHandle* aHandle, char* aBuf,
3602 nsresult aResult) {
3603 LOG(("CacheIndex::OnDataRead() [handle=%p, result=0x%08" PRIx32 "]", aHandle,
3604 static_cast<uint32_t>(aResult)));
3605
3606 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3607
3608 StaticMutexAutoLock lock(sLock);
3609
3610 MOZ_RELEASE_ASSERT(IsIndexUsable());
3611 MOZ_RELEASE_ASSERT(mRWPending);
3612 mRWPending = false;
3613
3614 switch (mState) {
3615 case READING:
3616 MOZ_ASSERT(mIndexHandle == aHandle || mJournalHandle == aHandle);
3617
3618 if (NS_FAILED(aResult)) {
3619 FinishRead(false, lock);
3620 } else {
3621 if (!mIndexOnDiskIsValid) {
3622 ParseRecords(lock);
3623 } else {
3624 ParseJournal(lock);
3625 }
3626 }
3627 break;
3628 default:
3629 // Reading was canceled.
3630 LOG(
3631 ("CacheIndex::OnDataRead() - ignoring notification since the "
3632 "operation was previously canceled [state=%d]",
3633 mState));
3634 ReleaseBuffer();
3635 }
3636
3637 return NS_OK;
3638 }
3639
OnFileDoomed(CacheFileHandle * aHandle,nsresult aResult)3640 nsresult CacheIndex::OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) {
3641 MOZ_CRASH("CacheIndex::OnFileDoomed should not be called!");
3642 return NS_ERROR_UNEXPECTED;
3643 }
3644
OnEOFSet(CacheFileHandle * aHandle,nsresult aResult)3645 nsresult CacheIndex::OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) {
3646 MOZ_CRASH("CacheIndex::OnEOFSet should not be called!");
3647 return NS_ERROR_UNEXPECTED;
3648 }
3649
OnFileRenamed(CacheFileHandle * aHandle,nsresult aResult)3650 nsresult CacheIndex::OnFileRenamed(CacheFileHandle* aHandle, nsresult aResult) {
3651 LOG(("CacheIndex::OnFileRenamed() [handle=%p, result=0x%08" PRIx32 "]",
3652 aHandle, static_cast<uint32_t>(aResult)));
3653
3654 MOZ_ASSERT(CacheFileIOManager::IsOnIOThread());
3655
3656 StaticMutexAutoLock lock(sLock);
3657
3658 MOZ_RELEASE_ASSERT(IsIndexUsable());
3659
3660 if (mState == READY && mShuttingDown) {
3661 return NS_OK;
3662 }
3663
3664 switch (mState) {
3665 case WRITING:
3666 // This is a result of renaming the new index written to tmpfile to index
3667 // file. This is the last step when writing the index and the whole
3668 // writing process is successful iff renaming was successful.
3669
3670 if (mIndexHandle != aHandle) {
3671 LOG(
3672 ("CacheIndex::OnFileRenamed() - ignoring notification since it "
3673 "belongs to previously canceled operation [state=%d]",
3674 mState));
3675 break;
3676 }
3677
3678 FinishWrite(NS_SUCCEEDED(aResult), lock);
3679 break;
3680 case READING:
3681 // This is a result of renaming journal file to tmpfile. It is renamed
3682 // before we start reading index and journal file and it should normally
3683 // succeed. If it fails give up reading of index.
3684
3685 if (mJournalHandle != aHandle) {
3686 LOG(
3687 ("CacheIndex::OnFileRenamed() - ignoring notification since it "
3688 "belongs to previously canceled operation [state=%d]",
3689 mState));
3690 break;
3691 }
3692
3693 if (NS_FAILED(aResult)) {
3694 FinishRead(false, lock);
3695 } else {
3696 StartReadingIndex(lock);
3697 }
3698 break;
3699 default:
3700 // Reading/writing was canceled.
3701 LOG(
3702 ("CacheIndex::OnFileRenamed() - ignoring notification since the "
3703 "operation was previously canceled [state=%d]",
3704 mState));
3705 }
3706
3707 return NS_OK;
3708 }
3709
3710 // Memory reporting
3711
SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const3712 size_t CacheIndex::SizeOfExcludingThisInternal(
3713 mozilla::MallocSizeOf mallocSizeOf) const {
3714 sLock.AssertCurrentThreadOwns();
3715
3716 size_t n = 0;
3717 nsCOMPtr<nsISizeOf> sizeOf;
3718
3719 // mIndexHandle and mJournalHandle are reported via SizeOfHandlesRunnable
3720 // in CacheFileIOManager::SizeOfExcludingThisInternal as part of special
3721 // handles array.
3722
3723 sizeOf = do_QueryInterface(mCacheDirectory);
3724 if (sizeOf) {
3725 n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
3726 }
3727
3728 sizeOf = do_QueryInterface(mUpdateTimer);
3729 if (sizeOf) {
3730 n += sizeOf->SizeOfIncludingThis(mallocSizeOf);
3731 }
3732
3733 n += mallocSizeOf(mRWBuf);
3734 n += mallocSizeOf(mRWHash);
3735
3736 n += mIndex.SizeOfExcludingThis(mallocSizeOf);
3737 n += mPendingUpdates.SizeOfExcludingThis(mallocSizeOf);
3738 n += mTmpJournal.SizeOfExcludingThis(mallocSizeOf);
3739
3740 // mFrecencyArray items are reported by mIndex/mPendingUpdates
3741 n += mFrecencyArray.mRecs.ShallowSizeOfExcludingThis(mallocSizeOf);
3742 n += mDiskConsumptionObservers.ShallowSizeOfExcludingThis(mallocSizeOf);
3743
3744 return n;
3745 }
3746
3747 // static
SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf)3748 size_t CacheIndex::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) {
3749 sLock.AssertCurrentThreadOwns();
3750
3751 if (!gInstance) return 0;
3752
3753 return gInstance->SizeOfExcludingThisInternal(mallocSizeOf);
3754 }
3755
3756 // static
SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf)3757 size_t CacheIndex::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) {
3758 StaticMutexAutoLock lock(sLock);
3759
3760 return mallocSizeOf(gInstance) + SizeOfExcludingThis(mallocSizeOf);
3761 }
3762
3763 // static
UpdateTotalBytesWritten(uint32_t aBytesWritten)3764 void CacheIndex::UpdateTotalBytesWritten(uint32_t aBytesWritten) {
3765 StaticMutexAutoLock lock(sLock);
3766
3767 RefPtr<CacheIndex> index = gInstance;
3768 if (!index) {
3769 return;
3770 }
3771
3772 index->mTotalBytesWritten += aBytesWritten;
3773
3774 // Do telemetry report if enough data has been written and the index is
3775 // in READY state. The data is available also in WRITING state, but we would
3776 // need to deal with pending updates.
3777 if (index->mTotalBytesWritten >= kTelemetryReportBytesLimit &&
3778 index->mState == READY && !index->mIndexNeedsUpdate &&
3779 !index->mShuttingDown) {
3780 index->DoTelemetryReport();
3781 index->mTotalBytesWritten = 0;
3782 return;
3783 }
3784 }
3785
DoTelemetryReport()3786 void CacheIndex::DoTelemetryReport() {
3787 static const nsLiteralCString
3788 contentTypeNames[nsICacheEntry::CONTENT_TYPE_LAST] = {
3789 "UNKNOWN"_ns, "OTHER"_ns, "JAVASCRIPT"_ns, "IMAGE"_ns,
3790 "MEDIA"_ns, "STYLESHEET"_ns, "WASM"_ns};
3791
3792 for (uint32_t i = 0; i < nsICacheEntry::CONTENT_TYPE_LAST; ++i) {
3793 if (mIndexStats.Size() > 0) {
3794 Telemetry::Accumulate(
3795 Telemetry::NETWORK_CACHE_SIZE_SHARE, contentTypeNames[i],
3796 round(static_cast<double>(mIndexStats.SizeByType(i)) * 100.0 /
3797 static_cast<double>(mIndexStats.Size())));
3798 }
3799
3800 if (mIndexStats.Count() > 0) {
3801 Telemetry::Accumulate(
3802 Telemetry::NETWORK_CACHE_ENTRY_COUNT_SHARE, contentTypeNames[i],
3803 round(static_cast<double>(mIndexStats.CountByType(i)) * 100.0 /
3804 static_cast<double>(mIndexStats.Count())));
3805 }
3806 }
3807
3808 nsCString probeKey;
3809 if (CacheObserver::SmartCacheSizeEnabled()) {
3810 probeKey = "SMARTSIZE"_ns;
3811 } else {
3812 probeKey = "USERDEFINEDSIZE"_ns;
3813 }
3814 Telemetry::Accumulate(Telemetry::NETWORK_CACHE_ENTRY_COUNT, probeKey,
3815 mIndexStats.Count());
3816 Telemetry::Accumulate(Telemetry::NETWORK_CACHE_SIZE, probeKey,
3817 mIndexStats.Size() >> 10);
3818 }
3819
3820 // static
OnAsyncEviction(bool aEvicting)3821 void CacheIndex::OnAsyncEviction(bool aEvicting) {
3822 StaticMutexAutoLock lock(sLock);
3823
3824 RefPtr<CacheIndex> index = gInstance;
3825 if (!index) {
3826 return;
3827 }
3828
3829 index->mAsyncGetDiskConsumptionBlocked = aEvicting;
3830 if (!aEvicting) {
3831 index->NotifyAsyncGetDiskConsumptionCallbacks();
3832 }
3833 }
3834
3835 } // namespace mozilla::net
3836