1 //* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 // Originally based on Chrome sources:
3 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 #include "HashStore.h"
32 #include "nsICryptoHash.h"
33 #include "nsISeekableStream.h"
34 #include "nsNetUtil.h"
35 #include "nsCheckSummedOutputStream.h"
36 #include "prio.h"
37 #include "mozilla/Logging.h"
38 #include "mozilla/IntegerPrintfMacros.h"
39 #include "zlib.h"
40 #include "Classifier.h"
41 #include "nsUrlClassifierDBService.h"
42 #include "mozilla/Telemetry.h"
43
44 // Main store for SafeBrowsing protocol data. We store
45 // known add/sub chunks, prefixes and completions in memory
46 // during an update, and serialize to disk.
47 // We do not store the add prefixes, those are retrieved by
48 // decompressing the PrefixSet cache whenever we need to apply
49 // an update.
50 //
51 // byte slicing: Many of the 4-byte values stored here are strongly
52 // correlated in the upper bytes, and uncorrelated in the lower
53 // bytes. Because zlib/DEFLATE requires match lengths of at least
54 // 3 to achieve good compression, and we don't get those if only
55 // the upper 16-bits are correlated, it is worthwhile to slice 32-bit
56 // values into 4 1-byte slices and compress the slices individually.
57 // The slices corresponding to MSBs will compress very well, and the
58 // slice corresponding to LSB almost nothing. Because of this, we
59 // only apply DEFLATE to the 3 most significant bytes, and store the
60 // LSB uncompressed.
61 //
62 // byte sliced (numValues) data format:
63 // uint32_t compressed-size
64 // compressed-size bytes zlib DEFLATE data
65 // 0...numValues byte MSB of 4-byte numValues data
66 // uint32_t compressed-size
67 // compressed-size bytes zlib DEFLATE data
68 // 0...numValues byte 2nd byte of 4-byte numValues data
69 // uint32_t compressed-size
70 // compressed-size bytes zlib DEFLATE data
71 // 0...numValues byte 3rd byte of 4-byte numValues data
72 // 0...numValues byte LSB of 4-byte numValues data
73 //
74 // Store data format:
75 // uint32_t magic
76 // uint32_t version
77 // uint32_t numAddChunks
78 // uint32_t numSubChunks
79 // uint32_t numAddPrefixes
80 // uint32_t numSubPrefixes
81 // uint32_t numAddCompletes
82 // uint32_t numSubCompletes
83 // 0...numAddChunks uint32_t addChunk
84 // 0...numSubChunks uint32_t subChunk
85 // byte sliced (numAddPrefixes) uint32_t add chunk of AddPrefixes
86 // byte sliced (numSubPrefixes) uint32_t add chunk of SubPrefixes
87 // byte sliced (numSubPrefixes) uint32_t sub chunk of SubPrefixes
88 // byte sliced (numSubPrefixes) uint32_t SubPrefixes
89 // byte sliced (numAddCompletes) uint32_t add chunk of AddCompletes
90 // 0...numSubCompletes 32-byte Completions + uint32_t addChunk
91 // + uint32_t subChunk
92 // 16-byte MD5 of all preceding data
93
94 // Name of the SafeBrowsing store
95 #define STORE_SUFFIX ".sbstore"
96
97 // MOZ_LOG=UrlClassifierDbService:5
98 extern mozilla::LazyLogModule gUrlClassifierDbServiceLog;
99 #define LOG(args) \
100 MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args)
101 #define LOG_ENABLED() \
102 MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug)
103
104 namespace mozilla::safebrowsing {
105
106 const uint32_t STORE_MAGIC = 0x1231af3b;
107 const uint32_t CURRENT_VERSION = 4;
108
NewAddPrefix(uint32_t aAddChunk,const Prefix & aHash)109 nsresult TableUpdateV2::NewAddPrefix(uint32_t aAddChunk, const Prefix& aHash) {
110 AddPrefix* add = mAddPrefixes.AppendElement(fallible);
111 if (!add) return NS_ERROR_OUT_OF_MEMORY;
112 add->addChunk = aAddChunk;
113 add->prefix = aHash;
114 return NS_OK;
115 }
116
NewSubPrefix(uint32_t aAddChunk,const Prefix & aHash,uint32_t aSubChunk)117 nsresult TableUpdateV2::NewSubPrefix(uint32_t aAddChunk, const Prefix& aHash,
118 uint32_t aSubChunk) {
119 SubPrefix* sub = mSubPrefixes.AppendElement(fallible);
120 if (!sub) return NS_ERROR_OUT_OF_MEMORY;
121 sub->addChunk = aAddChunk;
122 sub->prefix = aHash;
123 sub->subChunk = aSubChunk;
124 return NS_OK;
125 }
126
NewAddComplete(uint32_t aAddChunk,const Completion & aHash)127 nsresult TableUpdateV2::NewAddComplete(uint32_t aAddChunk,
128 const Completion& aHash) {
129 AddComplete* add = mAddCompletes.AppendElement(fallible);
130 if (!add) return NS_ERROR_OUT_OF_MEMORY;
131 add->addChunk = aAddChunk;
132 add->complete = aHash;
133 return NS_OK;
134 }
135
NewSubComplete(uint32_t aAddChunk,const Completion & aHash,uint32_t aSubChunk)136 nsresult TableUpdateV2::NewSubComplete(uint32_t aAddChunk,
137 const Completion& aHash,
138 uint32_t aSubChunk) {
139 SubComplete* sub = mSubCompletes.AppendElement(fallible);
140 if (!sub) return NS_ERROR_OUT_OF_MEMORY;
141 sub->addChunk = aAddChunk;
142 sub->complete = aHash;
143 sub->subChunk = aSubChunk;
144 return NS_OK;
145 }
146
NewMissPrefix(const Prefix & aPrefix)147 nsresult TableUpdateV2::NewMissPrefix(const Prefix& aPrefix) {
148 Prefix* prefix = mMissPrefixes.AppendElement(aPrefix, fallible);
149 if (!prefix) return NS_ERROR_OUT_OF_MEMORY;
150 return NS_OK;
151 }
152
NewPrefixes(int32_t aSize,const nsACString & aPrefixes)153 void TableUpdateV4::NewPrefixes(int32_t aSize, const nsACString& aPrefixes) {
154 NS_ENSURE_TRUE_VOID(aSize >= 4 && aSize <= COMPLETE_SIZE);
155 NS_ENSURE_TRUE_VOID(aPrefixes.Length() % aSize == 0);
156 NS_ENSURE_TRUE_VOID(!mPrefixesMap.Contains(aSize));
157
158 int numOfPrefixes = aPrefixes.Length() / aSize;
159
160 if (aSize > 4) {
161 // TODO Bug 1364043 we may have a better API to record multiple samples into
162 // histograms with one call
163 #ifdef NIGHTLY_BUILD
164 for (int i = 0; i < std::min(20, numOfPrefixes); i++) {
165 Telemetry::Accumulate(Telemetry::URLCLASSIFIER_VLPS_LONG_PREFIXES, aSize);
166 }
167 #endif
168 } else if (LOG_ENABLED()) {
169 const uint32_t* p =
170 reinterpret_cast<const uint32_t*>(ToNewCString(aPrefixes));
171
172 // Dump the first/last 10 fixed-length prefixes for debugging.
173 LOG(("* The first 10 (maximum) fixed-length prefixes: "));
174 for (int i = 0; i < std::min(10, numOfPrefixes); i++) {
175 const uint8_t* c = reinterpret_cast<const uint8_t*>(&p[i]);
176 LOG(("%.2X%.2X%.2X%.2X", c[0], c[1], c[2], c[3]));
177 }
178
179 LOG(("* The last 10 (maximum) fixed-length prefixes: "));
180 for (int i = std::max(0, numOfPrefixes - 10); i < numOfPrefixes; i++) {
181 const uint8_t* c = reinterpret_cast<const uint8_t*>(&p[i]);
182 LOG(("%.2X%.2X%.2X%.2X", c[0], c[1], c[2], c[3]));
183 }
184
185 LOG(("---- %zu fixed-length prefixes in total.",
186 aPrefixes.Length() / aSize));
187 }
188
189 mPrefixesMap.InsertOrUpdate(aSize, MakeUnique<nsCString>(aPrefixes));
190 }
191
NewRemovalIndices(const uint32_t * aIndices,size_t aNumOfIndices)192 nsresult TableUpdateV4::NewRemovalIndices(const uint32_t* aIndices,
193 size_t aNumOfIndices) {
194 MOZ_ASSERT(mRemovalIndiceArray.IsEmpty(),
195 "mRemovalIndiceArray must be empty");
196
197 if (!mRemovalIndiceArray.SetCapacity(aNumOfIndices, fallible)) {
198 return NS_ERROR_OUT_OF_MEMORY;
199 }
200
201 for (size_t i = 0; i < aNumOfIndices; i++) {
202 mRemovalIndiceArray.AppendElement(aIndices[i]);
203 }
204 return NS_OK;
205 }
206
SetSHA256(const std::string & aSHA256)207 void TableUpdateV4::SetSHA256(const std::string& aSHA256) {
208 mSHA256.Assign(aSHA256.data(), aSHA256.size());
209 }
210
NewFullHashResponse(const Prefix & aPrefix,const CachedFullHashResponse & aResponse)211 nsresult TableUpdateV4::NewFullHashResponse(
212 const Prefix& aPrefix, const CachedFullHashResponse& aResponse) {
213 CachedFullHashResponse* response =
214 mFullHashResponseMap.GetOrInsertNew(aPrefix.ToUint32());
215 if (!response) {
216 return NS_ERROR_OUT_OF_MEMORY;
217 }
218 *response = aResponse;
219 return NS_OK;
220 }
221
Clear()222 void TableUpdateV4::Clear() {
223 mPrefixesMap.Clear();
224 mRemovalIndiceArray.Clear();
225 }
226
HashStore(const nsACString & aTableName,const nsACString & aProvider,nsIFile * aRootStoreDir)227 HashStore::HashStore(const nsACString& aTableName, const nsACString& aProvider,
228 nsIFile* aRootStoreDir)
229 : mTableName(aTableName), mInUpdate(false), mFileSize(0) {
230 nsresult rv = Classifier::GetPrivateStoreDirectory(
231 aRootStoreDir, aTableName, aProvider, getter_AddRefs(mStoreDirectory));
232 if (NS_FAILED(rv)) {
233 LOG(("Failed to get private store directory for %s", mTableName.get()));
234 mStoreDirectory = aRootStoreDir;
235 }
236 }
237
238 HashStore::~HashStore() = default;
239
Reset()240 nsresult HashStore::Reset() {
241 LOG(("HashStore resetting"));
242
243 // Close InputStream before removing the file
244 if (mInputStream) {
245 nsresult rv = mInputStream->Close();
246 NS_ENSURE_SUCCESS(rv, rv);
247
248 mInputStream = nullptr;
249 }
250
251 nsCOMPtr<nsIFile> storeFile;
252 nsresult rv = mStoreDirectory->Clone(getter_AddRefs(storeFile));
253 NS_ENSURE_SUCCESS(rv, rv);
254
255 rv = storeFile->AppendNative(mTableName + nsLiteralCString(STORE_SUFFIX));
256 NS_ENSURE_SUCCESS(rv, rv);
257
258 rv = storeFile->Remove(false);
259 NS_ENSURE_SUCCESS(rv, rv);
260
261 mFileSize = 0;
262
263 return NS_OK;
264 }
265
CheckChecksum(uint32_t aFileSize)266 nsresult HashStore::CheckChecksum(uint32_t aFileSize) {
267 if (!mInputStream) {
268 return NS_OK;
269 }
270
271 // Check for file corruption by
272 // comparing the stored checksum to actual checksum of data
273 nsAutoCString hash;
274 nsAutoCString compareHash;
275 uint32_t read;
276
277 nsresult rv = CalculateChecksum(hash, aFileSize, true);
278 NS_ENSURE_SUCCESS(rv, rv);
279
280 compareHash.SetLength(hash.Length());
281
282 if (hash.Length() > aFileSize) {
283 NS_WARNING("SafeBrowsing file not long enough to store its hash");
284 return NS_ERROR_FAILURE;
285 }
286 nsCOMPtr<nsISeekableStream> seekIn = do_QueryInterface(mInputStream);
287 rv = seekIn->Seek(nsISeekableStream::NS_SEEK_SET, aFileSize - hash.Length());
288 NS_ENSURE_SUCCESS(rv, rv);
289
290 rv = mInputStream->Read(compareHash.BeginWriting(), hash.Length(), &read);
291 NS_ENSURE_SUCCESS(rv, rv);
292 NS_ASSERTION(read == hash.Length(), "Could not read hash bytes");
293
294 if (!hash.Equals(compareHash)) {
295 NS_WARNING("SafeBrowsing file failed checksum.");
296 return NS_ERROR_FAILURE;
297 }
298
299 return NS_OK;
300 }
301
Open(uint32_t aVersion)302 nsresult HashStore::Open(uint32_t aVersion) {
303 nsCOMPtr<nsIFile> storeFile;
304 nsresult rv = mStoreDirectory->Clone(getter_AddRefs(storeFile));
305 NS_ENSURE_SUCCESS(rv, rv);
306
307 rv = storeFile->AppendNative(mTableName + ".sbstore"_ns);
308 NS_ENSURE_SUCCESS(rv, rv);
309
310 nsCOMPtr<nsIInputStream> origStream;
311 rv = NS_NewLocalFileInputStream(getter_AddRefs(origStream), storeFile,
312 PR_RDONLY | nsIFile::OS_READAHEAD);
313
314 if (rv == NS_ERROR_FILE_NOT_FOUND) {
315 UpdateHeader();
316 return NS_OK;
317 }
318 NS_ENSURE_SUCCESS(rv, rv);
319
320 int64_t fileSize;
321 rv = storeFile->GetFileSize(&fileSize);
322 NS_ENSURE_SUCCESS(rv, rv);
323
324 if (fileSize < 0 || fileSize > UINT32_MAX) {
325 return NS_ERROR_FAILURE;
326 }
327
328 mFileSize = static_cast<uint32_t>(fileSize);
329 rv = NS_NewBufferedInputStream(getter_AddRefs(mInputStream),
330 origStream.forget(), mFileSize);
331 NS_ENSURE_SUCCESS(rv, rv);
332
333 rv = ReadHeader();
334 if (NS_WARN_IF(NS_FAILED(rv))) {
335 LOG(("Failed to read header for %s", mTableName.get()));
336 return NS_ERROR_FILE_CORRUPTED;
337 }
338
339 rv = SanityCheck(aVersion);
340 NS_ENSURE_SUCCESS(rv, rv);
341
342 return NS_OK;
343 }
344
ReadHeader()345 nsresult HashStore::ReadHeader() {
346 if (!mInputStream) {
347 UpdateHeader();
348 return NS_OK;
349 }
350
351 nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(mInputStream);
352 nsresult rv = seekable->Seek(nsISeekableStream::NS_SEEK_SET, 0);
353 NS_ENSURE_SUCCESS(rv, rv);
354
355 void* buffer = &mHeader;
356 rv = NS_ReadInputStreamToBuffer(mInputStream, &buffer, sizeof(Header));
357 NS_ENSURE_SUCCESS(rv, rv);
358
359 return NS_OK;
360 }
361
SanityCheck(uint32_t aVersion) const362 nsresult HashStore::SanityCheck(uint32_t aVersion) const {
363 const uint32_t VER = aVersion == 0 ? CURRENT_VERSION : aVersion;
364 if (mHeader.magic != STORE_MAGIC || mHeader.version != VER) {
365 NS_WARNING("Unexpected header data in the store.");
366 // Version mismatch is also considered file corrupted,
367 // We need this error code to know if we should remove the file.
368 return NS_ERROR_FILE_CORRUPTED;
369 }
370
371 return NS_OK;
372 }
373
CalculateChecksum(nsAutoCString & aChecksum,uint32_t aFileSize,bool aChecksumPresent)374 nsresult HashStore::CalculateChecksum(nsAutoCString& aChecksum,
375 uint32_t aFileSize,
376 bool aChecksumPresent) {
377 aChecksum.Truncate();
378
379 // Reset mInputStream to start
380 nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(mInputStream);
381 nsresult rv = seekable->Seek(nsISeekableStream::NS_SEEK_SET, 0);
382
383 nsCOMPtr<nsICryptoHash> hash =
384 do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
385 NS_ENSURE_SUCCESS(rv, rv);
386
387 // Size of MD5 hash in bytes
388 const uint32_t CHECKSUM_SIZE = 16;
389
390 // MD5 is not a secure hash function, but since this is a filesystem integrity
391 // check, this usage is ok.
392 rv = hash->Init(nsICryptoHash::MD5);
393 NS_ENSURE_SUCCESS(rv, rv);
394
395 if (!aChecksumPresent) {
396 // Hash entire file
397 rv = hash->UpdateFromStream(mInputStream, UINT32_MAX);
398 } else {
399 // Hash everything but last checksum bytes
400 if (aFileSize < CHECKSUM_SIZE) {
401 NS_WARNING("SafeBrowsing file isn't long enough to store its checksum");
402 return NS_ERROR_FAILURE;
403 }
404 rv = hash->UpdateFromStream(mInputStream, aFileSize - CHECKSUM_SIZE);
405 }
406 NS_ENSURE_SUCCESS(rv, rv);
407
408 rv = hash->Finish(false, aChecksum);
409 NS_ENSURE_SUCCESS(rv, rv);
410
411 return NS_OK;
412 }
413
UpdateHeader()414 void HashStore::UpdateHeader() {
415 mHeader.magic = STORE_MAGIC;
416 mHeader.version = CURRENT_VERSION;
417
418 mHeader.numAddChunks = mAddChunks.Length();
419 mHeader.numSubChunks = mSubChunks.Length();
420 mHeader.numAddPrefixes = mAddPrefixes.Length();
421 mHeader.numSubPrefixes = mSubPrefixes.Length();
422 mHeader.numAddCompletes = mAddCompletes.Length();
423 mHeader.numSubCompletes = mSubCompletes.Length();
424 }
425
ReadChunkNumbers()426 nsresult HashStore::ReadChunkNumbers() {
427 if (!mInputStream) {
428 return NS_OK;
429 }
430
431 nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(mInputStream);
432 nsresult rv = seekable->Seek(nsISeekableStream::NS_SEEK_SET, sizeof(Header));
433 NS_ENSURE_SUCCESS(rv, rv);
434
435 rv = mAddChunks.Read(mInputStream, mHeader.numAddChunks);
436 NS_ENSURE_SUCCESS(rv, rv);
437 NS_ASSERTION(mAddChunks.Length() == mHeader.numAddChunks,
438 "Read the right amount of add chunks.");
439
440 rv = mSubChunks.Read(mInputStream, mHeader.numSubChunks);
441 NS_ENSURE_SUCCESS(rv, rv);
442 NS_ASSERTION(mSubChunks.Length() == mHeader.numSubChunks,
443 "Read the right amount of sub chunks.");
444
445 return NS_OK;
446 }
447
ReadHashes()448 nsresult HashStore::ReadHashes() {
449 if (!mInputStream) {
450 // BeginUpdate has been called but Open hasn't initialized mInputStream,
451 // because the existing HashStore is empty.
452 return NS_OK;
453 }
454
455 nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(mInputStream);
456
457 uint32_t offset = sizeof(Header);
458 offset += (mHeader.numAddChunks + mHeader.numSubChunks) * sizeof(uint32_t);
459 nsresult rv = seekable->Seek(nsISeekableStream::NS_SEEK_SET, offset);
460 NS_ENSURE_SUCCESS(rv, rv);
461
462 rv = ReadAddPrefixes();
463 NS_ENSURE_SUCCESS(rv, rv);
464
465 rv = ReadSubPrefixes();
466 NS_ENSURE_SUCCESS(rv, rv);
467
468 rv = ReadAddCompletes();
469 NS_ENSURE_SUCCESS(rv, rv);
470
471 rv = ReadTArray(mInputStream, &mSubCompletes, mHeader.numSubCompletes);
472 NS_ENSURE_SUCCESS(rv, rv);
473
474 return NS_OK;
475 }
476
PrepareForUpdate()477 nsresult HashStore::PrepareForUpdate() {
478 nsresult rv = CheckChecksum(mFileSize);
479 NS_ENSURE_SUCCESS(rv, rv);
480
481 rv = ReadChunkNumbers();
482 NS_ENSURE_SUCCESS(rv, rv);
483
484 rv = ReadHashes();
485 NS_ENSURE_SUCCESS(rv, rv);
486
487 return NS_OK;
488 }
489
BeginUpdate()490 nsresult HashStore::BeginUpdate() {
491 // Check wether the file is corrupted and read the rest of the store
492 // in memory.
493 nsresult rv = PrepareForUpdate();
494 NS_ENSURE_SUCCESS(rv, rv);
495
496 // Close input stream, won't be needed any more and
497 // we will rewrite ourselves.
498 if (mInputStream) {
499 rv = mInputStream->Close();
500 NS_ENSURE_SUCCESS(rv, rv);
501 }
502
503 mInUpdate = true;
504
505 return NS_OK;
506 }
507
508 template <class T>
Merge(ChunkSet * aStoreChunks,FallibleTArray<T> * aStorePrefixes,const ChunkSet & aUpdateChunks,FallibleTArray<T> & aUpdatePrefixes,bool aAllowMerging=false)509 static nsresult Merge(ChunkSet* aStoreChunks, FallibleTArray<T>* aStorePrefixes,
510 const ChunkSet& aUpdateChunks,
511 FallibleTArray<T>& aUpdatePrefixes,
512 bool aAllowMerging = false) {
513 EntrySort(aUpdatePrefixes);
514
515 auto storeIter = aStorePrefixes->begin();
516 auto storeEnd = aStorePrefixes->end();
517
518 // use a separate array so we can keep the iterators valid
519 // if the nsTArray grows
520 nsTArray<T> adds;
521
522 for (const auto& updatePrefix : aUpdatePrefixes) {
523 // skip this chunk if we already have it, unless we're
524 // merging completions, in which case we'll always already
525 // have the chunk from the original prefix
526 if (aStoreChunks->Has(updatePrefix.Chunk()))
527 if (!aAllowMerging) continue;
528 // XXX: binary search for insertion point might be faster in common
529 // case?
530 while (storeIter < storeEnd && (storeIter->Compare(updatePrefix) < 0)) {
531 // skip forward to matching element (or not...)
532 storeIter++;
533 }
534 // no match, add
535 if (storeIter == storeEnd || storeIter->Compare(updatePrefix) != 0) {
536 if (!adds.AppendElement(updatePrefix, fallible)) {
537 return NS_ERROR_OUT_OF_MEMORY;
538 }
539 }
540 }
541
542 // Chunks can be empty, but we should still report we have them
543 // to make the chunkranges continuous.
544 aStoreChunks->Merge(aUpdateChunks);
545
546 if (!aStorePrefixes->AppendElements(adds, fallible))
547 return NS_ERROR_OUT_OF_MEMORY;
548
549 EntrySort(*aStorePrefixes);
550
551 return NS_OK;
552 }
553
ApplyUpdate(RefPtr<TableUpdateV2> aUpdate)554 nsresult HashStore::ApplyUpdate(RefPtr<TableUpdateV2> aUpdate) {
555 MOZ_ASSERT(mTableName.Equals(aUpdate->TableName()));
556
557 nsresult rv = mAddExpirations.Merge(aUpdate->AddExpirations());
558 NS_ENSURE_SUCCESS(rv, rv);
559
560 rv = mSubExpirations.Merge(aUpdate->SubExpirations());
561 NS_ENSURE_SUCCESS(rv, rv);
562
563 rv = Expire();
564 NS_ENSURE_SUCCESS(rv, rv);
565
566 rv = Merge(&mAddChunks, &mAddPrefixes, aUpdate->AddChunks(),
567 aUpdate->AddPrefixes());
568 NS_ENSURE_SUCCESS(rv, rv);
569
570 rv = Merge(&mAddChunks, &mAddCompletes, aUpdate->AddChunks(),
571 aUpdate->AddCompletes(), true);
572 NS_ENSURE_SUCCESS(rv, rv);
573
574 rv = Merge(&mSubChunks, &mSubPrefixes, aUpdate->SubChunks(),
575 aUpdate->SubPrefixes());
576 NS_ENSURE_SUCCESS(rv, rv);
577
578 rv = Merge(&mSubChunks, &mSubCompletes, aUpdate->SubChunks(),
579 aUpdate->SubCompletes(), true);
580 NS_ENSURE_SUCCESS(rv, rv);
581
582 return NS_OK;
583 }
584
Rebuild()585 nsresult HashStore::Rebuild() {
586 NS_ASSERTION(mInUpdate, "Must be in update to rebuild.");
587
588 nsresult rv = ProcessSubs();
589 NS_ENSURE_SUCCESS(rv, rv);
590
591 UpdateHeader();
592
593 return NS_OK;
594 }
595
596 template <class T>
ExpireEntries(FallibleTArray<T> * aEntries,ChunkSet & aExpirations)597 static void ExpireEntries(FallibleTArray<T>* aEntries, ChunkSet& aExpirations) {
598 auto addIter = aEntries->begin();
599
600 for (const auto& entry : *aEntries) {
601 if (!aExpirations.Has(entry.Chunk())) {
602 *addIter = entry;
603 addIter++;
604 }
605 }
606
607 aEntries->TruncateLength(addIter - aEntries->begin());
608 }
609
Expire()610 nsresult HashStore::Expire() {
611 ExpireEntries(&mAddPrefixes, mAddExpirations);
612 ExpireEntries(&mAddCompletes, mAddExpirations);
613 ExpireEntries(&mSubPrefixes, mSubExpirations);
614 ExpireEntries(&mSubCompletes, mSubExpirations);
615
616 mAddChunks.Remove(mAddExpirations);
617 mSubChunks.Remove(mSubExpirations);
618
619 mAddExpirations.Clear();
620 mSubExpirations.Clear();
621
622 return NS_OK;
623 }
624
625 template <class T>
DeflateWriteTArray(nsIOutputStream * aStream,nsTArray<T> & aIn)626 nsresult DeflateWriteTArray(nsIOutputStream* aStream, nsTArray<T>& aIn) {
627 uLongf insize = aIn.Length() * sizeof(T);
628 uLongf outsize = compressBound(insize);
629 FallibleTArray<char> outBuff;
630 if (!outBuff.SetLength(outsize, fallible)) {
631 return NS_ERROR_OUT_OF_MEMORY;
632 }
633
634 int zerr = compress(reinterpret_cast<Bytef*>(outBuff.Elements()), &outsize,
635 reinterpret_cast<const Bytef*>(aIn.Elements()), insize);
636 if (zerr != Z_OK) {
637 return NS_ERROR_FAILURE;
638 }
639 LOG(("DeflateWriteTArray: %lu in %lu out", insize, outsize));
640
641 outBuff.TruncateLength(outsize);
642
643 // Length of compressed data stream
644 uint32_t dataLen = outBuff.Length();
645 uint32_t written;
646 nsresult rv = aStream->Write(reinterpret_cast<char*>(&dataLen),
647 sizeof(dataLen), &written);
648 NS_ENSURE_SUCCESS(rv, rv);
649
650 NS_ASSERTION(written == sizeof(dataLen), "Error writing deflate length");
651
652 // Store to stream
653 rv = WriteTArray(aStream, outBuff);
654 NS_ENSURE_SUCCESS(rv, rv);
655
656 return NS_OK;
657 }
658
659 template <class T>
InflateReadTArray(nsIInputStream * aStream,FallibleTArray<T> * aOut,uint32_t aExpectedSize)660 nsresult InflateReadTArray(nsIInputStream* aStream, FallibleTArray<T>* aOut,
661 uint32_t aExpectedSize) {
662 uint32_t inLen;
663 uint32_t read;
664 nsresult rv =
665 aStream->Read(reinterpret_cast<char*>(&inLen), sizeof(inLen), &read);
666 NS_ENSURE_SUCCESS(rv, rv);
667
668 NS_ASSERTION(read == sizeof(inLen), "Error reading inflate length");
669
670 FallibleTArray<char> inBuff;
671 if (!inBuff.SetLength(inLen, fallible)) {
672 return NS_ERROR_OUT_OF_MEMORY;
673 }
674
675 rv = ReadTArray(aStream, &inBuff, inLen);
676 NS_ENSURE_SUCCESS(rv, rv);
677
678 uLongf insize = inLen;
679 uLongf outsize = aExpectedSize * sizeof(T);
680 if (!aOut->SetLength(aExpectedSize, fallible)) {
681 return NS_ERROR_OUT_OF_MEMORY;
682 }
683
684 int zerr =
685 uncompress(reinterpret_cast<Bytef*>(aOut->Elements()), &outsize,
686 reinterpret_cast<const Bytef*>(inBuff.Elements()), insize);
687 if (zerr != Z_OK) {
688 return NS_ERROR_FAILURE;
689 }
690 LOG(("InflateReadTArray: %lu in %lu out", insize, outsize));
691
692 NS_ASSERTION(outsize == aExpectedSize * sizeof(T),
693 "Decompression size mismatch");
694
695 return NS_OK;
696 }
697
ByteSliceWrite(nsIOutputStream * aOut,nsTArray<uint32_t> & aData)698 static nsresult ByteSliceWrite(nsIOutputStream* aOut,
699 nsTArray<uint32_t>& aData) {
700 nsTArray<uint8_t> slice;
701 uint32_t count = aData.Length();
702
703 // Only process one slice at a time to avoid using too much memory.
704 if (!slice.SetLength(count, fallible)) {
705 return NS_ERROR_OUT_OF_MEMORY;
706 }
707
708 // Process slice 1.
709 for (uint32_t i = 0; i < count; i++) {
710 slice[i] = (aData[i] >> 24);
711 }
712
713 nsresult rv = DeflateWriteTArray(aOut, slice);
714 NS_ENSURE_SUCCESS(rv, rv);
715
716 // Process slice 2.
717 for (uint32_t i = 0; i < count; i++) {
718 slice[i] = ((aData[i] >> 16) & 0xFF);
719 }
720
721 rv = DeflateWriteTArray(aOut, slice);
722 NS_ENSURE_SUCCESS(rv, rv);
723
724 // Process slice 3.
725 for (uint32_t i = 0; i < count; i++) {
726 slice[i] = ((aData[i] >> 8) & 0xFF);
727 }
728
729 rv = DeflateWriteTArray(aOut, slice);
730 NS_ENSURE_SUCCESS(rv, rv);
731
732 // Process slice 4.
733 for (uint32_t i = 0; i < count; i++) {
734 slice[i] = (aData[i] & 0xFF);
735 }
736
737 // The LSB slice is generally uncompressible, don't bother
738 // compressing it.
739 rv = WriteTArray(aOut, slice);
740 NS_ENSURE_SUCCESS(rv, rv);
741
742 return NS_OK;
743 }
744
ByteSliceRead(nsIInputStream * aInStream,FallibleTArray<uint32_t> * aData,uint32_t count)745 static nsresult ByteSliceRead(nsIInputStream* aInStream,
746 FallibleTArray<uint32_t>* aData, uint32_t count) {
747 FallibleTArray<uint8_t> slice1;
748 FallibleTArray<uint8_t> slice2;
749 FallibleTArray<uint8_t> slice3;
750 FallibleTArray<uint8_t> slice4;
751
752 nsresult rv = InflateReadTArray(aInStream, &slice1, count);
753 NS_ENSURE_SUCCESS(rv, rv);
754
755 rv = InflateReadTArray(aInStream, &slice2, count);
756 NS_ENSURE_SUCCESS(rv, rv);
757
758 rv = InflateReadTArray(aInStream, &slice3, count);
759 NS_ENSURE_SUCCESS(rv, rv);
760
761 rv = ReadTArray(aInStream, &slice4, count);
762 NS_ENSURE_SUCCESS(rv, rv);
763
764 if (!aData->SetCapacity(count, fallible)) {
765 return NS_ERROR_OUT_OF_MEMORY;
766 }
767
768 for (uint32_t i = 0; i < count; i++) {
769 // SetCapacity was just called, these cannot fail.
770 (void)aData->AppendElement(
771 (slice1[i] << 24) | (slice2[i] << 16) | (slice3[i] << 8) | (slice4[i]),
772 fallible);
773 }
774
775 return NS_OK;
776 }
777
ReadAddPrefixes()778 nsresult HashStore::ReadAddPrefixes() {
779 FallibleTArray<uint32_t> chunks;
780 uint32_t count = mHeader.numAddPrefixes;
781
782 nsresult rv = ByteSliceRead(mInputStream, &chunks, count);
783 NS_ENSURE_SUCCESS(rv, rv);
784
785 if (!mAddPrefixes.SetCapacity(count, fallible)) {
786 return NS_ERROR_OUT_OF_MEMORY;
787 }
788 for (uint32_t i = 0; i < count; i++) {
789 AddPrefix* add = mAddPrefixes.AppendElement(fallible);
790 add->prefix.FromUint32(0);
791 add->addChunk = chunks[i];
792 }
793
794 return NS_OK;
795 }
796
ReadAddCompletes()797 nsresult HashStore::ReadAddCompletes() {
798 FallibleTArray<uint32_t> chunks;
799 uint32_t count = mHeader.numAddCompletes;
800
801 nsresult rv = ByteSliceRead(mInputStream, &chunks, count);
802 NS_ENSURE_SUCCESS(rv, rv);
803
804 if (!mAddCompletes.SetCapacity(count, fallible)) {
805 return NS_ERROR_OUT_OF_MEMORY;
806 }
807 for (uint32_t i = 0; i < count; i++) {
808 AddComplete* add = mAddCompletes.AppendElement(fallible);
809 add->addChunk = chunks[i];
810 }
811
812 return NS_OK;
813 }
814
ReadSubPrefixes()815 nsresult HashStore::ReadSubPrefixes() {
816 FallibleTArray<uint32_t> addchunks;
817 FallibleTArray<uint32_t> subchunks;
818 FallibleTArray<uint32_t> prefixes;
819 uint32_t count = mHeader.numSubPrefixes;
820
821 nsresult rv = ByteSliceRead(mInputStream, &addchunks, count);
822 NS_ENSURE_SUCCESS(rv, rv);
823
824 rv = ByteSliceRead(mInputStream, &subchunks, count);
825 NS_ENSURE_SUCCESS(rv, rv);
826
827 rv = ByteSliceRead(mInputStream, &prefixes, count);
828 NS_ENSURE_SUCCESS(rv, rv);
829
830 if (!mSubPrefixes.SetCapacity(count, fallible)) {
831 return NS_ERROR_OUT_OF_MEMORY;
832 }
833 for (uint32_t i = 0; i < count; i++) {
834 SubPrefix* sub = mSubPrefixes.AppendElement(fallible);
835 sub->addChunk = addchunks[i];
836 sub->prefix.FromUint32(prefixes[i]);
837 sub->subChunk = subchunks[i];
838 }
839
840 return NS_OK;
841 }
842
843 // Split up PrefixArray back into the constituents
WriteAddPrefixChunks(nsIOutputStream * aOut)844 nsresult HashStore::WriteAddPrefixChunks(nsIOutputStream* aOut) {
845 nsTArray<uint32_t> chunks;
846 uint32_t count = mAddPrefixes.Length();
847 if (!chunks.SetCapacity(count, fallible)) {
848 return NS_ERROR_OUT_OF_MEMORY;
849 }
850
851 for (uint32_t i = 0; i < count; i++) {
852 chunks.AppendElement(mAddPrefixes[i].Chunk());
853 }
854
855 nsresult rv = ByteSliceWrite(aOut, chunks);
856 NS_ENSURE_SUCCESS(rv, rv);
857
858 return NS_OK;
859 }
860
WriteAddCompleteChunks(nsIOutputStream * aOut)861 nsresult HashStore::WriteAddCompleteChunks(nsIOutputStream* aOut) {
862 nsTArray<uint32_t> chunks;
863 uint32_t count = mAddCompletes.Length();
864 if (!chunks.SetCapacity(count, fallible)) {
865 return NS_ERROR_OUT_OF_MEMORY;
866 }
867
868 for (uint32_t i = 0; i < count; i++) {
869 chunks.AppendElement(mAddCompletes[i].Chunk());
870 }
871
872 nsresult rv = ByteSliceWrite(aOut, chunks);
873 NS_ENSURE_SUCCESS(rv, rv);
874
875 return NS_OK;
876 }
877
WriteSubPrefixes(nsIOutputStream * aOut)878 nsresult HashStore::WriteSubPrefixes(nsIOutputStream* aOut) {
879 nsTArray<uint32_t> addchunks;
880 nsTArray<uint32_t> subchunks;
881 nsTArray<uint32_t> prefixes;
882 uint32_t count = mSubPrefixes.Length();
883 if (!addchunks.SetCapacity(count, fallible) ||
884 !subchunks.SetCapacity(count, fallible) ||
885 !prefixes.SetCapacity(count, fallible)) {
886 return NS_ERROR_OUT_OF_MEMORY;
887 }
888
889 for (uint32_t i = 0; i < count; i++) {
890 addchunks.AppendElement(mSubPrefixes[i].AddChunk());
891 prefixes.AppendElement(mSubPrefixes[i].PrefixHash().ToUint32());
892 subchunks.AppendElement(mSubPrefixes[i].Chunk());
893 }
894
895 nsresult rv = ByteSliceWrite(aOut, addchunks);
896 NS_ENSURE_SUCCESS(rv, rv);
897
898 rv = ByteSliceWrite(aOut, subchunks);
899 NS_ENSURE_SUCCESS(rv, rv);
900
901 rv = ByteSliceWrite(aOut, prefixes);
902 NS_ENSURE_SUCCESS(rv, rv);
903
904 return NS_OK;
905 }
906
WriteFile()907 nsresult HashStore::WriteFile() {
908 NS_ASSERTION(mInUpdate, "Must be in update to write database.");
909 if (nsUrlClassifierDBService::ShutdownHasStarted()) {
910 return NS_ERROR_ABORT;
911 }
912
913 nsCOMPtr<nsIFile> storeFile;
914 nsresult rv = mStoreDirectory->Clone(getter_AddRefs(storeFile));
915 NS_ENSURE_SUCCESS(rv, rv);
916 rv = storeFile->AppendNative(mTableName + ".sbstore"_ns);
917 NS_ENSURE_SUCCESS(rv, rv);
918
919 nsCOMPtr<nsIOutputStream> out;
920 rv = NS_NewCheckSummedOutputStream(getter_AddRefs(out), storeFile);
921 NS_ENSURE_SUCCESS(rv, rv);
922
923 uint32_t written;
924 rv = out->Write(reinterpret_cast<char*>(&mHeader), sizeof(mHeader), &written);
925 NS_ENSURE_SUCCESS(rv, rv);
926
927 // Write chunk numbers.
928 rv = mAddChunks.Write(out);
929 NS_ENSURE_SUCCESS(rv, rv);
930
931 rv = mSubChunks.Write(out);
932 NS_ENSURE_SUCCESS(rv, rv);
933
934 // Write hashes.
935 rv = WriteAddPrefixChunks(out);
936 NS_ENSURE_SUCCESS(rv, rv);
937
938 rv = WriteSubPrefixes(out);
939 NS_ENSURE_SUCCESS(rv, rv);
940
941 rv = WriteAddCompleteChunks(out);
942 NS_ENSURE_SUCCESS(rv, rv);
943
944 rv = WriteTArray(out, mSubCompletes);
945 NS_ENSURE_SUCCESS(rv, rv);
946
947 nsCOMPtr<nsISafeOutputStream> safeOut = do_QueryInterface(out, &rv);
948 NS_ENSURE_SUCCESS(rv, rv);
949
950 rv = safeOut->Finish();
951 NS_ENSURE_SUCCESS(rv, rv);
952
953 return NS_OK;
954 }
955
ReadCompletionsLegacyV3(AddCompleteArray & aCompletes)956 nsresult HashStore::ReadCompletionsLegacyV3(AddCompleteArray& aCompletes) {
957 if (mHeader.version != 3) {
958 return NS_ERROR_FAILURE;
959 }
960
961 nsCOMPtr<nsIFile> storeFile;
962 nsresult rv = mStoreDirectory->Clone(getter_AddRefs(storeFile));
963 NS_ENSURE_SUCCESS(rv, rv);
964
965 rv = storeFile->AppendNative(mTableName + nsLiteralCString(STORE_SUFFIX));
966 NS_ENSURE_SUCCESS(rv, rv);
967
968 uint32_t offset = mFileSize -
969 sizeof(struct AddComplete) * mHeader.numAddCompletes -
970 sizeof(struct SubComplete) * mHeader.numSubCompletes -
971 nsCheckSummedOutputStream::CHECKSUM_SIZE;
972
973 nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(mInputStream);
974
975 rv = seekable->Seek(nsISeekableStream::NS_SEEK_SET, offset);
976 NS_ENSURE_SUCCESS(rv, rv);
977
978 rv = ReadTArray(mInputStream, &aCompletes, mHeader.numAddCompletes);
979 NS_ENSURE_SUCCESS(rv, rv);
980
981 return NS_OK;
982 }
983
984 template <class T>
Erase(FallibleTArray<T> * array,typename FallibleTArray<T>::iterator & iterStart,typename FallibleTArray<T>::iterator & iterEnd)985 static void Erase(FallibleTArray<T>* array,
986 typename FallibleTArray<T>::iterator& iterStart,
987 typename FallibleTArray<T>::iterator& iterEnd) {
988 array->RemoveElementsRange(iterStart, iterEnd);
989 }
990
991 // Find items matching between |subs| and |adds|, and remove them,
992 // recording the item from |adds| in |adds_removed|. To minimize
993 // copies, the inputs are processing in parallel, so |subs| and |adds|
994 // should be compatibly ordered (either by SBAddPrefixLess or
995 // SBAddPrefixHashLess).
996 //
997 // |predAS| provides add < sub, |predSA| provides sub < add, for the
998 // tightest compare appropriate (see calls in SBProcessSubs).
999 template <class TSub, class TAdd>
KnockoutSubs(FallibleTArray<TSub> * aSubs,FallibleTArray<TAdd> * aAdds)1000 static void KnockoutSubs(FallibleTArray<TSub>* aSubs,
1001 FallibleTArray<TAdd>* aAdds) {
1002 // Keep a pair of output iterators for writing kept items. Due to
1003 // deletions, these may lag the main iterators. Using erase() on
1004 // individual items would result in O(N^2) copies. Using a list
1005 // would work around that, at double or triple the memory cost.
1006 auto addOut = aAdds->begin();
1007 auto addIter = aAdds->begin();
1008
1009 auto subOut = aSubs->begin();
1010 auto subIter = aSubs->begin();
1011
1012 auto addEnd = aAdds->end();
1013 auto subEnd = aSubs->end();
1014
1015 while (addIter != addEnd && subIter != subEnd) {
1016 // additer compare, so it compares on add chunk
1017 int32_t cmp = addIter->Compare(*subIter);
1018 if (cmp > 0) {
1019 // If |*sub_iter| < |*add_iter|, retain the sub.
1020 *subOut = *subIter;
1021 ++subOut;
1022 ++subIter;
1023 } else if (cmp < 0) {
1024 // If |*add_iter| < |*sub_iter|, retain the add.
1025 *addOut = *addIter;
1026 ++addOut;
1027 ++addIter;
1028 } else {
1029 // Drop equal items
1030 ++addIter;
1031 ++subIter;
1032 }
1033 }
1034
1035 Erase(aAdds, addOut, addIter);
1036 Erase(aSubs, subOut, subIter);
1037 }
1038
1039 // Remove items in |removes| from |fullHashes|. |fullHashes| and
1040 // |removes| should be ordered by SBAddPrefix component.
1041 template <class T>
RemoveMatchingPrefixes(const SubPrefixArray & aSubs,FallibleTArray<T> * aFullHashes)1042 static void RemoveMatchingPrefixes(const SubPrefixArray& aSubs,
1043 FallibleTArray<T>* aFullHashes) {
1044 // Where to store kept items.
1045 auto out = aFullHashes->begin();
1046 auto hashIter = aFullHashes->begin();
1047 auto hashEnd = aFullHashes->end();
1048
1049 auto removeIter = aSubs.begin();
1050 auto removeEnd = aSubs.end();
1051
1052 while (hashIter != hashEnd && removeIter != removeEnd) {
1053 int32_t cmp = removeIter->CompareAlt(*hashIter);
1054 if (cmp > 0) {
1055 // Keep items less than |*removeIter|.
1056 *out = *hashIter;
1057 ++out;
1058 ++hashIter;
1059 } else if (cmp < 0) {
1060 // No hit for |*removeIter|, bump it forward.
1061 ++removeIter;
1062 } else {
1063 // Drop equal items, there may be multiple hits.
1064 do {
1065 ++hashIter;
1066 } while (hashIter != hashEnd && !(removeIter->CompareAlt(*hashIter) < 0));
1067 ++removeIter;
1068 }
1069 }
1070 Erase(aFullHashes, out, hashIter);
1071 }
1072
RemoveDeadSubPrefixes(SubPrefixArray & aSubs,ChunkSet & aAddChunks)1073 static void RemoveDeadSubPrefixes(SubPrefixArray& aSubs, ChunkSet& aAddChunks) {
1074 auto subIter = aSubs.begin();
1075
1076 for (const auto& sub : aSubs) {
1077 bool hasChunk = aAddChunks.Has(sub.AddChunk());
1078 // Keep the subprefix if the chunk it refers to is one
1079 // we haven't seen it yet.
1080 if (!hasChunk) {
1081 *subIter = sub;
1082 subIter++;
1083 }
1084 }
1085
1086 LOG(("Removed %" PRId64 " dead SubPrefix entries.",
1087 static_cast<int64_t>(aSubs.end() - subIter)));
1088 aSubs.TruncateLength(subIter - aSubs.begin());
1089 }
1090
1091 #ifdef DEBUG
1092 template <class T>
EnsureSorted(FallibleTArray<T> * aArray)1093 static void EnsureSorted(FallibleTArray<T>* aArray) {
1094 auto start = aArray->begin();
1095 auto end = aArray->end();
1096 auto iter = start;
1097 auto previous = start;
1098
1099 while (iter != end) {
1100 previous = iter;
1101 ++iter;
1102 if (iter != end) {
1103 MOZ_ASSERT(iter->Compare(*previous) >= 0);
1104 }
1105 }
1106 }
1107 #endif
1108
ProcessSubs()1109 nsresult HashStore::ProcessSubs() {
1110 #ifdef DEBUG
1111 EnsureSorted(&mAddPrefixes);
1112 EnsureSorted(&mSubPrefixes);
1113 EnsureSorted(&mAddCompletes);
1114 EnsureSorted(&mSubCompletes);
1115 LOG(("All databases seem to have a consistent sort order."));
1116 #endif
1117
1118 RemoveMatchingPrefixes(mSubPrefixes, &mAddCompletes);
1119 RemoveMatchingPrefixes(mSubPrefixes, &mSubCompletes);
1120
1121 // Remove any remaining subbed prefixes from both addprefixes
1122 // and addcompletes.
1123 KnockoutSubs(&mSubPrefixes, &mAddPrefixes);
1124 KnockoutSubs(&mSubCompletes, &mAddCompletes);
1125
1126 // Remove any remaining subprefixes referring to addchunks that
1127 // we have (and hence have been processed above).
1128 RemoveDeadSubPrefixes(mSubPrefixes, mAddChunks);
1129
1130 #ifdef DEBUG
1131 EnsureSorted(&mAddPrefixes);
1132 EnsureSorted(&mSubPrefixes);
1133 EnsureSorted(&mAddCompletes);
1134 EnsureSorted(&mSubCompletes);
1135 LOG(("All databases seem to have a consistent sort order."));
1136 #endif
1137
1138 return NS_OK;
1139 }
1140
AugmentAdds(const nsTArray<uint32_t> & aPrefixes,const nsTArray<nsCString> & aCompletes)1141 nsresult HashStore::AugmentAdds(const nsTArray<uint32_t>& aPrefixes,
1142 const nsTArray<nsCString>& aCompletes) {
1143 if (aPrefixes.Length() != mAddPrefixes.Length() ||
1144 aCompletes.Length() != mAddCompletes.Length()) {
1145 LOG((
1146 "Amount of prefixes/completes in cache not consistent with store prefixes(%zu vs %zu), \
1147 store completes(%zu vs %zu)",
1148 aPrefixes.Length(), mAddPrefixes.Length(), aCompletes.Length(),
1149 mAddCompletes.Length()));
1150 return NS_ERROR_FAILURE;
1151 }
1152
1153 for (size_t i = 0; i < mAddPrefixes.Length(); i++) {
1154 mAddPrefixes[i].prefix.FromUint32(aPrefixes[i]);
1155 }
1156
1157 for (size_t i = 0; i < mAddCompletes.Length(); i++) {
1158 mAddCompletes[i].complete.Assign(aCompletes[i]);
1159 }
1160
1161 return NS_OK;
1162 }
1163
AddChunks()1164 ChunkSet& HashStore::AddChunks() {
1165 ReadChunkNumbers();
1166
1167 return mAddChunks;
1168 }
1169
SubChunks()1170 ChunkSet& HashStore::SubChunks() {
1171 ReadChunkNumbers();
1172
1173 return mSubChunks;
1174 }
1175
1176 } // namespace mozilla::safebrowsing
1177