1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6 #include "nsCOMPtr.h"
7 #include "nsAppDirectoryServiceDefs.h"
8 #include "nsArrayUtils.h"
9 #include "nsCRT.h"
10 #include "nsIObserverService.h"
11 #include "nsIPermissionManager.h"
12 #include "nsIPrefBranch.h"
13 #include "nsIXULRuntime.h"
14 #include "nsToolkitCompsCID.h"
15 #include "nsUrlClassifierDBService.h"
16 #include "nsUrlClassifierUtils.h"
17 #include "nsUrlClassifierProxies.h"
18 #include "nsURILoader.h"
19 #include "nsString.h"
20 #include "nsReadableUtils.h"
21 #include "nsTArray.h"
22 #include "nsNetCID.h"
23 #include "nsThreadUtils.h"
24 #include "nsProxyRelease.h"
25 #include "nsString.h"
26 #include "mozilla/Atomics.h"
27 #include "mozilla/BasePrincipal.h"
28 #include "mozilla/Components.h"
29 #include "mozilla/DebugOnly.h"
30 #include "mozilla/ErrorNames.h"
31 #include "mozilla/Mutex.h"
32 #include "mozilla/Preferences.h"
33 #include "mozilla/ScopeExit.h"
34 #include "mozilla/TimeStamp.h"
35 #include "mozilla/Telemetry.h"
36 #include "mozilla/Unused.h"
37 #include "mozilla/Logging.h"
38 #include "prnetdb.h"
39 #include "Entries.h"
40 #include "Classifier.h"
41 #include "ProtocolParser.h"
42 #include "mozilla/Attributes.h"
43 #include "nsIHttpChannel.h"
44 #include "nsIPrincipal.h"
45 #include "nsIUrlListManager.h"
46 #include "Classifier.h"
47 #include "ProtocolParser.h"
48 #include "nsContentUtils.h"
49 #include "mozilla/Components.h"
50 #include "mozilla/dom/ContentChild.h"
51 #include "mozilla/dom/PermissionMessageUtils.h"
52 #include "mozilla/dom/URLClassifierChild.h"
53 #include "mozilla/net/UrlClassifierFeatureFactory.h"
54 #include "mozilla/net/UrlClassifierFeatureResult.h"
55 #include "mozilla/ipc/URIUtils.h"
56 #include "mozilla/SyncRunnable.h"
57 #include "UrlClassifierTelemetryUtils.h"
58 #include "nsIURLFormatter.h"
59 #include "nsIUploadChannel.h"
60 #include "nsStringStream.h"
61 #include "nsNetUtil.h"
62 #include "nsToolkitCompsCID.h"
63
64 namespace mozilla {
65 namespace safebrowsing {
66
TablesToResponse(const nsACString & tables)67 nsresult TablesToResponse(const nsACString& tables) {
68 if (tables.IsEmpty()) {
69 return NS_OK;
70 }
71
72 // We don't check mCheckMalware and friends because disabled tables are
73 // never included
74 if (FindInReadable("-malware-"_ns, tables)) {
75 return NS_ERROR_MALWARE_URI;
76 }
77 if (FindInReadable("-harmful-"_ns, tables)) {
78 return NS_ERROR_HARMFUL_URI;
79 }
80 if (FindInReadable("-phish-"_ns, tables)) {
81 return NS_ERROR_PHISHING_URI;
82 }
83 if (FindInReadable("-unwanted-"_ns, tables)) {
84 return NS_ERROR_UNWANTED_URI;
85 }
86 if (FindInReadable("-track-"_ns, tables)) {
87 return NS_ERROR_TRACKING_URI;
88 }
89 if (FindInReadable("-block-"_ns, tables)) {
90 return NS_ERROR_BLOCKED_URI;
91 }
92 return NS_OK;
93 }
94
95 } // namespace safebrowsing
96 } // namespace mozilla
97
98 // This class holds a list of features, their tables, and it stores the lookup
99 // results.
100 class nsUrlClassifierDBService::FeatureHolder final {
101 public:
102 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(FeatureHolder);
103
104 // In order to avoid multiple lookup for the same table, we have a special
105 // array for tables and their results. The Features are stored in a separate
106 // array together with the references to their tables.
107
108 class TableData {
109 public:
110 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(
111 nsUrlClassifierDBService::FeatureHolder::TableData);
112
TableData(const nsACString & aTable)113 explicit TableData(const nsACString& aTable) : mTable(aTable) {}
114
115 nsCString mTable;
116 LookupResultArray mResults;
117
118 private:
119 ~TableData() = default;
120 };
121
122 struct FeatureData {
123 RefPtr<nsIUrlClassifierFeature> mFeature;
124 nsTArray<RefPtr<TableData>> mTables;
125 };
126
Create(nsIURI * aURI,const nsTArray<RefPtr<nsIUrlClassifierFeature>> & aFeatures,nsIUrlClassifierFeature::listType aListType)127 static already_AddRefed<FeatureHolder> Create(
128 nsIURI* aURI, const nsTArray<RefPtr<nsIUrlClassifierFeature>>& aFeatures,
129 nsIUrlClassifierFeature::listType aListType) {
130 MOZ_ASSERT(NS_IsMainThread());
131 MOZ_ASSERT(aURI);
132
133 RefPtr<FeatureHolder> holder = new FeatureHolder(aURI);
134
135 for (nsIUrlClassifierFeature* feature : aFeatures) {
136 FeatureData* featureData = holder->mFeatureData.AppendElement();
137 MOZ_ASSERT(featureData);
138
139 featureData->mFeature = feature;
140 nsTArray<nsCString> tables;
141 nsresult rv = feature->GetTables(aListType, tables);
142 if (NS_WARN_IF(NS_FAILED(rv))) {
143 return nullptr;
144 }
145
146 for (const nsCString& table : tables) {
147 TableData* tableData = holder->GetOrCreateTableData(table);
148 MOZ_ASSERT(tableData);
149
150 featureData->mTables.AppendElement(tableData);
151 }
152 }
153
154 return holder.forget();
155 }
156
DoLocalLookup(const nsACString & aSpec,nsUrlClassifierDBServiceWorker * aWorker)157 nsresult DoLocalLookup(const nsACString& aSpec,
158 nsUrlClassifierDBServiceWorker* aWorker) {
159 MOZ_ASSERT(!NS_IsMainThread());
160 MOZ_ASSERT(aWorker);
161
162 mozilla::Telemetry::AutoTimer<
163 mozilla::Telemetry::URLCLASSIFIER_CL_CHECK_TIME>
164 timer;
165
166 // Get the set of fragments based on the url. This is necessary because we
167 // only look up at most 5 URLs per aSpec, even if aSpec has more than 5
168 // components.
169 nsTArray<nsCString> fragments;
170 nsresult rv = LookupCache::GetLookupFragments(aSpec, &fragments);
171 NS_ENSURE_SUCCESS(rv, rv);
172
173 for (TableData* tableData : mTableData) {
174 rv = aWorker->DoSingleLocalLookupWithURIFragments(
175 fragments, tableData->mTable, tableData->mResults);
176 if (NS_WARN_IF(NS_FAILED(rv))) {
177 return rv;
178 }
179 }
180
181 return NS_OK;
182 }
183
184 // This method is used to convert the LookupResultArray from
185 // ::DoSingleLocalLookupWithURIFragments to nsIUrlClassifierFeatureResult
GetResults(nsTArray<RefPtr<nsIUrlClassifierFeatureResult>> & aResults)186 void GetResults(nsTArray<RefPtr<nsIUrlClassifierFeatureResult>>& aResults) {
187 MOZ_ASSERT(NS_IsMainThread());
188
189 // For each table, we must concatenate the results of the corresponding
190 // tables.
191
192 for (FeatureData& featureData : mFeatureData) {
193 nsAutoCString list;
194 for (TableData* tableData : featureData.mTables) {
195 for (uint32_t i = 0; i < tableData->mResults.Length(); ++i) {
196 if (!list.IsEmpty()) {
197 list.AppendLiteral(",");
198 }
199 list.Append(tableData->mResults[i]->mTableName);
200 }
201 }
202
203 if (list.IsEmpty()) {
204 continue;
205 }
206
207 RefPtr<mozilla::net::UrlClassifierFeatureResult> result =
208 new mozilla::net::UrlClassifierFeatureResult(
209 mURI, featureData.mFeature, list);
210 aResults.AppendElement(result);
211 }
212 }
213
GetTableResults() const214 mozilla::UniquePtr<LookupResultArray> GetTableResults() const {
215 mozilla::UniquePtr<LookupResultArray> results =
216 mozilla::MakeUnique<LookupResultArray>();
217 if (NS_WARN_IF(!results)) {
218 return nullptr;
219 }
220
221 for (TableData* tableData : mTableData) {
222 results->AppendElements(tableData->mResults);
223 }
224
225 return results;
226 }
227
228 private:
FeatureHolder(nsIURI * aURI)229 explicit FeatureHolder(nsIURI* aURI) : mURI(aURI) {
230 MOZ_ASSERT(NS_IsMainThread());
231 }
232
~FeatureHolder()233 ~FeatureHolder() {
234 for (FeatureData& featureData : mFeatureData) {
235 NS_ReleaseOnMainThread("FeatureHolder:mFeatureData",
236 featureData.mFeature.forget());
237 }
238
239 NS_ReleaseOnMainThread("FeatureHolder:mURI", mURI.forget());
240 }
241
GetOrCreateTableData(const nsACString & aTable)242 TableData* GetOrCreateTableData(const nsACString& aTable) {
243 for (TableData* tableData : mTableData) {
244 if (tableData->mTable == aTable) {
245 return tableData;
246 }
247 }
248
249 RefPtr<TableData> tableData = new TableData(aTable);
250 mTableData.AppendElement(tableData);
251 return tableData;
252 }
253
254 nsCOMPtr<nsIURI> mURI;
255 nsTArray<FeatureData> mFeatureData;
256 nsTArray<RefPtr<TableData>> mTableData;
257 };
258
259 using namespace mozilla;
260 using namespace mozilla::safebrowsing;
261
262 // MOZ_LOG=UrlClassifierDbService:5
263 LazyLogModule gUrlClassifierDbServiceLog("UrlClassifierDbService");
264 #define LOG(args) \
265 MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args)
266 #define LOG_ENABLED() \
267 MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug)
268
269 #define GETHASH_NOISE_PREF "urlclassifier.gethashnoise"
270 #define GETHASH_NOISE_DEFAULT 4
271
272 // 30 minutes as the maximum negative cache duration.
273 #define MAXIMUM_NEGATIVE_CACHE_DURATION_SEC (30 * 60 * 1000)
274
275 class nsUrlClassifierDBServiceWorker;
276
277 // Singleton instance.
278 static nsUrlClassifierDBService* sUrlClassifierDBService;
279
280 nsIThread* nsUrlClassifierDBService::gDbBackgroundThread = nullptr;
281
282 // Once we've committed to shutting down, don't do work in the background
283 // thread.
284 static Atomic<bool> gShuttingDownThread(false);
285
NS_IMPL_ISUPPORTS(nsUrlClassifierDBServiceWorker,nsIUrlClassifierDBService)286 NS_IMPL_ISUPPORTS(nsUrlClassifierDBServiceWorker, nsIUrlClassifierDBService)
287
288 nsUrlClassifierDBServiceWorker::nsUrlClassifierDBServiceWorker()
289 : mInStream(false),
290 mGethashNoise(0),
291 mPendingLookupLock("nsUrlClassifierDBServerWorker.mPendingLookupLock") {}
292
~nsUrlClassifierDBServiceWorker()293 nsUrlClassifierDBServiceWorker::~nsUrlClassifierDBServiceWorker() {
294 NS_ASSERTION(!mClassifier,
295 "Db connection not closed, leaking memory! Call CloseDb "
296 "to close the connection.");
297 }
298
Init(uint32_t aGethashNoise,nsCOMPtr<nsIFile> aCacheDir,nsUrlClassifierDBService * aDBService)299 nsresult nsUrlClassifierDBServiceWorker::Init(
300 uint32_t aGethashNoise, nsCOMPtr<nsIFile> aCacheDir,
301 nsUrlClassifierDBService* aDBService) {
302 mGethashNoise = aGethashNoise;
303 mCacheDir = aCacheDir;
304 mDBService = aDBService;
305
306 ResetUpdate();
307
308 return NS_OK;
309 }
310
QueueLookup(const nsACString & aKey,nsUrlClassifierDBService::FeatureHolder * aFeatureHolder,nsIUrlClassifierLookupCallback * aCallback)311 nsresult nsUrlClassifierDBServiceWorker::QueueLookup(
312 const nsACString& aKey,
313 nsUrlClassifierDBService::FeatureHolder* aFeatureHolder,
314 nsIUrlClassifierLookupCallback* aCallback) {
315 MOZ_ASSERT(aFeatureHolder);
316 MOZ_ASSERT(aCallback);
317
318 MutexAutoLock lock(mPendingLookupLock);
319 if (gShuttingDownThread) {
320 return NS_ERROR_ABORT;
321 }
322
323 PendingLookup* lookup = mPendingLookups.AppendElement(fallible);
324 if (NS_WARN_IF(!lookup)) return NS_ERROR_OUT_OF_MEMORY;
325
326 lookup->mStartTime = TimeStamp::Now();
327 lookup->mKey = aKey;
328 lookup->mCallback = aCallback;
329 lookup->mFeatureHolder = aFeatureHolder;
330
331 return NS_OK;
332 }
333
DoSingleLocalLookupWithURIFragments(const nsTArray<nsCString> & aSpecFragments,const nsACString & aTable,LookupResultArray & aResults)334 nsresult nsUrlClassifierDBServiceWorker::DoSingleLocalLookupWithURIFragments(
335 const nsTArray<nsCString>& aSpecFragments, const nsACString& aTable,
336 LookupResultArray& aResults) {
337 if (gShuttingDownThread) {
338 return NS_ERROR_ABORT;
339 }
340
341 MOZ_ASSERT(
342 !NS_IsMainThread(),
343 "DoSingleLocalLookupWithURIFragments must be on background thread");
344
345 // Bail if we haven't been initialized on the background thread.
346 if (!mClassifier) {
347 return NS_ERROR_NOT_AVAILABLE;
348 }
349
350 nsresult rv =
351 mClassifier->CheckURIFragments(aSpecFragments, aTable, aResults);
352 if (NS_WARN_IF(NS_FAILED(rv))) {
353 return rv;
354 }
355
356 LOG(("Found %zu results.", aResults.Length()));
357 return NS_OK;
358 }
359
360 /**
361 * Lookup up a key in the database is a two step process:
362 *
363 * a) First we look for any Entries in the database that might apply to this
364 * url. For each URL there are one or two possible domain names to check:
365 * the two-part domain name (example.com) and the three-part name
366 * (www.example.com). We check the database for both of these.
367 * b) If we find any entries, we check the list of fragments for that entry
368 * against the possible subfragments of the URL as described in the
369 * "Simplified Regular Expression Lookup" section of the protocol doc.
370 */
DoLookup(const nsACString & spec,nsUrlClassifierDBService::FeatureHolder * aFeatureHolder,nsIUrlClassifierLookupCallback * c)371 nsresult nsUrlClassifierDBServiceWorker::DoLookup(
372 const nsACString& spec,
373 nsUrlClassifierDBService::FeatureHolder* aFeatureHolder,
374 nsIUrlClassifierLookupCallback* c) {
375 // Make sure the callback is invoked when a failure occurs,
376 // otherwise we will not be able to load any url.
377 auto scopeExit = MakeScopeExit([&c]() { c->LookupComplete(nullptr); });
378
379 if (gShuttingDownThread) {
380 return NS_ERROR_NOT_INITIALIZED;
381 }
382
383 PRIntervalTime clockStart = 0;
384 if (LOG_ENABLED()) {
385 clockStart = PR_IntervalNow();
386 }
387
388 nsresult rv = aFeatureHolder->DoLocalLookup(spec, this);
389 if (NS_WARN_IF(NS_FAILED(rv))) {
390 return rv;
391 }
392
393 if (LOG_ENABLED()) {
394 PRIntervalTime clockEnd = PR_IntervalNow();
395 LOG(("query took %dms\n",
396 PR_IntervalToMilliseconds(clockEnd - clockStart)));
397 }
398
399 UniquePtr<LookupResultArray> results = aFeatureHolder->GetTableResults();
400 if (NS_WARN_IF(!results)) {
401 return NS_ERROR_OUT_OF_MEMORY;
402 }
403
404 LOG(("Found %zu results.", results->Length()));
405
406 for (const RefPtr<const LookupResult> lookupResult : *results) {
407 if (!lookupResult->Confirmed() &&
408 mDBService->CanComplete(lookupResult->mTableName)) {
409 // We're going to be doing a gethash request, add some extra entries.
410 // Note that we cannot pass the first two by reference, because we
411 // add to completes, which can cause completes to reallocate and move.
412 AddNoise(lookupResult->hash.fixedLengthPrefix, lookupResult->mTableName,
413 mGethashNoise, *results);
414 break;
415 }
416 }
417
418 // At this point ownership of 'results' is handed to the callback.
419 scopeExit.release();
420 c->LookupComplete(std::move(results));
421
422 return NS_OK;
423 }
424
HandlePendingLookups()425 nsresult nsUrlClassifierDBServiceWorker::HandlePendingLookups() {
426 if (gShuttingDownThread) {
427 return NS_ERROR_ABORT;
428 }
429
430 MutexAutoLock lock(mPendingLookupLock);
431 while (mPendingLookups.Length() > 0) {
432 PendingLookup lookup = mPendingLookups[0];
433 mPendingLookups.RemoveElementAt(0);
434 {
435 MutexAutoUnlock unlock(mPendingLookupLock);
436 DoLookup(lookup.mKey, lookup.mFeatureHolder, lookup.mCallback);
437 }
438 double lookupTime = (TimeStamp::Now() - lookup.mStartTime).ToMilliseconds();
439 Telemetry::Accumulate(Telemetry::URLCLASSIFIER_LOOKUP_TIME_2,
440 static_cast<uint32_t>(lookupTime));
441 }
442
443 return NS_OK;
444 }
445
AddNoise(const Prefix aPrefix,const nsCString tableName,uint32_t aCount,LookupResultArray & results)446 nsresult nsUrlClassifierDBServiceWorker::AddNoise(const Prefix aPrefix,
447 const nsCString tableName,
448 uint32_t aCount,
449 LookupResultArray& results) {
450 if (gShuttingDownThread) {
451 return NS_ERROR_ABORT;
452 }
453
454 if (aCount < 1) {
455 return NS_OK;
456 }
457
458 PrefixArray noiseEntries;
459 nsresult rv =
460 mClassifier->ReadNoiseEntries(aPrefix, tableName, aCount, noiseEntries);
461 NS_ENSURE_SUCCESS(rv, rv);
462
463 for (const auto noiseEntry : noiseEntries) {
464 RefPtr<LookupResult> result = new LookupResult;
465 results.AppendElement(result);
466
467 result->hash.fixedLengthPrefix = noiseEntry;
468 result->mNoise = true;
469 result->mPartialHashLength = PREFIX_SIZE; // Noise is always 4-byte,
470 result->mTableName.Assign(tableName);
471 }
472
473 return NS_OK;
474 }
475
476 // Lookup a key in the db.
477 NS_IMETHODIMP
Lookup(nsIPrincipal * aPrincipal,const nsACString & aTables,nsIUrlClassifierCallback * c)478 nsUrlClassifierDBServiceWorker::Lookup(nsIPrincipal* aPrincipal,
479 const nsACString& aTables,
480 nsIUrlClassifierCallback* c) {
481 if (gShuttingDownThread) {
482 return NS_ERROR_ABORT;
483 }
484
485 return HandlePendingLookups();
486 }
487
488 NS_IMETHODIMP
GetTables(nsIUrlClassifierCallback * c)489 nsUrlClassifierDBServiceWorker::GetTables(nsIUrlClassifierCallback* c) {
490 if (gShuttingDownThread) {
491 return NS_ERROR_NOT_INITIALIZED;
492 }
493
494 nsresult rv = OpenDb();
495 if (NS_FAILED(rv)) {
496 NS_ERROR("Unable to open SafeBrowsing database");
497 return NS_ERROR_FAILURE;
498 }
499
500 NS_ENSURE_SUCCESS(rv, rv);
501
502 nsAutoCString response;
503 mClassifier->TableRequest(response);
504 LOG(("GetTables: %s", response.get()));
505 c->HandleEvent(response);
506
507 return rv;
508 }
509
ResetStream()510 void nsUrlClassifierDBServiceWorker::ResetStream() {
511 LOG(("ResetStream"));
512 mInStream = false;
513 mProtocolParser = nullptr;
514 }
515
ResetUpdate()516 void nsUrlClassifierDBServiceWorker::ResetUpdate() {
517 LOG(("ResetUpdate"));
518 mUpdateWaitSec = 0;
519 mUpdateStatus = NS_OK;
520 mUpdateObserver = nullptr;
521 }
522
523 NS_IMETHODIMP
SetHashCompleter(const nsACString & tableName,nsIUrlClassifierHashCompleter * completer)524 nsUrlClassifierDBServiceWorker::SetHashCompleter(
525 const nsACString& tableName, nsIUrlClassifierHashCompleter* completer) {
526 return NS_ERROR_NOT_IMPLEMENTED;
527 }
528
529 NS_IMETHODIMP
BeginUpdate(nsIUrlClassifierUpdateObserver * observer,const nsACString & tables)530 nsUrlClassifierDBServiceWorker::BeginUpdate(
531 nsIUrlClassifierUpdateObserver* observer, const nsACString& tables) {
532 LOG(("nsUrlClassifierDBServiceWorker::BeginUpdate [%s]",
533 PromiseFlatCString(tables).get()));
534
535 if (gShuttingDownThread) {
536 return NS_ERROR_NOT_INITIALIZED;
537 }
538
539 NS_ENSURE_STATE(!mUpdateObserver);
540
541 nsresult rv = OpenDb();
542 if (NS_FAILED(rv)) {
543 NS_ERROR("Unable to open SafeBrowsing database");
544 return NS_ERROR_FAILURE;
545 }
546
547 mUpdateStatus = NS_OK;
548 MOZ_ASSERT(mTableUpdates.IsEmpty(),
549 "mTableUpdates should have been cleared in FinishUpdate()");
550 mUpdateObserver = observer;
551 Classifier::SplitTables(tables, mUpdateTables);
552
553 return NS_OK;
554 }
555
556 // Called from the stream updater.
557 NS_IMETHODIMP
BeginStream(const nsACString & table)558 nsUrlClassifierDBServiceWorker::BeginStream(const nsACString& table) {
559 LOG(("nsUrlClassifierDBServiceWorker::BeginStream"));
560 MOZ_ASSERT(!NS_IsMainThread(), "Streaming must be on the background thread");
561
562 if (gShuttingDownThread) {
563 return NS_ERROR_NOT_INITIALIZED;
564 }
565
566 NS_ENSURE_STATE(mUpdateObserver);
567 NS_ENSURE_STATE(!mInStream);
568
569 mInStream = true;
570
571 NS_ASSERTION(!mProtocolParser, "Should not have a protocol parser.");
572
573 // Check if we should use protobuf to parse the update.
574 bool useProtobuf = false;
575 for (size_t i = 0; i < mUpdateTables.Length(); i++) {
576 bool isCurProtobuf = StringEndsWith(mUpdateTables[i], "-proto"_ns);
577
578 if (0 == i) {
579 // Use the first table name to decice if all the subsequent tables
580 // should be '-proto'.
581 useProtobuf = isCurProtobuf;
582 continue;
583 }
584
585 if (useProtobuf != isCurProtobuf) {
586 NS_WARNING(
587 "Cannot mix 'proto' tables with other types "
588 "within the same provider.");
589 break;
590 }
591 }
592
593 if (useProtobuf) {
594 mProtocolParser.reset(new (fallible) ProtocolParserProtobuf());
595 } else {
596 mProtocolParser.reset(new (fallible) ProtocolParserV2());
597 }
598 if (!mProtocolParser) {
599 return NS_ERROR_OUT_OF_MEMORY;
600 }
601
602 return mProtocolParser->Begin(table, mUpdateTables);
603 }
604
605 /**
606 * Updating the database:
607 *
608 * The Update() method takes a series of chunks separated with control data,
609 * as described in
610 * http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec
611 *
612 * It will iterate through the control data until it reaches a chunk. By
613 * the time it reaches a chunk, it should have received
614 * a) the table to which this chunk applies
615 * b) the type of chunk (add, delete, expire add, expire delete).
616 * c) the chunk ID
617 * d) the length of the chunk.
618 *
619 * For add and subtract chunks, it needs to read the chunk data (expires
620 * don't have any data). Chunk data is a list of URI fragments whose
621 * encoding depends on the type of table (which is indicated by the end
622 * of the table name):
623 * a) tables ending with -exp are a zlib-compressed list of URI fragments
624 * separated by newlines.
625 * b) tables ending with -sha128 have the form
626 * [domain][N][frag0]...[fragN]
627 * 16 1 16 16
628 * If N is 0, the domain is reused as a fragment.
629 * c) any other tables are assumed to be a plaintext list of URI fragments
630 * separated by newlines.
631 *
632 * Update() can be fed partial data; It will accumulate data until there is
633 * enough to act on. Finish() should be called when there will be no more
634 * data.
635 */
636 NS_IMETHODIMP
UpdateStream(const nsACString & chunk)637 nsUrlClassifierDBServiceWorker::UpdateStream(const nsACString& chunk) {
638 if (gShuttingDownThread) {
639 return NS_ERROR_NOT_INITIALIZED;
640 }
641
642 MOZ_ASSERT(mProtocolParser);
643
644 NS_ENSURE_STATE(mInStream);
645
646 HandlePendingLookups();
647
648 // Feed the chunk to the parser.
649 return mProtocolParser->AppendStream(chunk);
650 }
651
652 NS_IMETHODIMP
FinishStream()653 nsUrlClassifierDBServiceWorker::FinishStream() {
654 if (gShuttingDownThread) {
655 LOG(("shutting down"));
656 return NS_ERROR_NOT_INITIALIZED;
657 }
658
659 MOZ_ASSERT(mProtocolParser);
660
661 NS_ENSURE_STATE(mInStream);
662 NS_ENSURE_STATE(mUpdateObserver);
663
664 mInStream = false;
665
666 mProtocolParser->End();
667
668 if (NS_SUCCEEDED(mProtocolParser->Status())) {
669 if (mProtocolParser->UpdateWaitSec()) {
670 mUpdateWaitSec = mProtocolParser->UpdateWaitSec();
671 }
672 // XXX: Only allow forwards from the initial update?
673 const nsTArray<ProtocolParser::ForwardedUpdate>& forwards =
674 mProtocolParser->Forwards();
675 for (uint32_t i = 0; i < forwards.Length(); i++) {
676 const ProtocolParser::ForwardedUpdate& forward = forwards[i];
677 mUpdateObserver->UpdateUrlRequested(forward.url, forward.table);
678 }
679 // Hold on to any TableUpdate objects that were created by the
680 // parser.
681 mTableUpdates.AppendElements(mProtocolParser->GetTableUpdates());
682 mProtocolParser->ForgetTableUpdates();
683
684 #ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
685 // The assignment involves no string copy since the source string is
686 // sharable.
687 mRawTableUpdates = mProtocolParser->GetRawTableUpdates();
688 #endif
689 } else {
690 LOG(
691 ("nsUrlClassifierDBService::FinishStream Failed to parse the stream "
692 "using mProtocolParser."));
693 mUpdateStatus = mProtocolParser->Status();
694 }
695 mUpdateObserver->StreamFinished(mProtocolParser->Status(), 0);
696
697 if (NS_SUCCEEDED(mUpdateStatus)) {
698 if (mProtocolParser->ResetRequested()) {
699 mClassifier->ResetTables(Classifier::Clear_All,
700 mProtocolParser->TablesToReset());
701 }
702 }
703
704 mProtocolParser = nullptr;
705
706 return mUpdateStatus;
707 }
708
709 NS_IMETHODIMP
FinishUpdate()710 nsUrlClassifierDBServiceWorker::FinishUpdate() {
711 LOG(("nsUrlClassifierDBServiceWorker::FinishUpdate"));
712
713 MOZ_ASSERT(!NS_IsMainThread(),
714 "nsUrlClassifierDBServiceWorker::FinishUpdate "
715 "NUST NOT be on the main thread.");
716
717 if (gShuttingDownThread) {
718 return NS_ERROR_NOT_INITIALIZED;
719 }
720
721 MOZ_ASSERT(!mProtocolParser,
722 "Should have been nulled out in FinishStream() "
723 "or never created.");
724
725 NS_ENSURE_STATE(mUpdateObserver);
726
727 if (NS_FAILED(mUpdateStatus)) {
728 LOG(
729 ("nsUrlClassifierDBServiceWorker::FinishUpdate() Not running "
730 "ApplyUpdate() since the update has already failed."));
731 mTableUpdates.Clear();
732 return NotifyUpdateObserver(mUpdateStatus);
733 }
734
735 if (mTableUpdates.IsEmpty()) {
736 LOG(("Nothing to update. Just notify update observer."));
737 return NotifyUpdateObserver(NS_OK);
738 }
739
740 RefPtr<nsUrlClassifierDBServiceWorker> self = this;
741 nsresult rv = mClassifier->AsyncApplyUpdates(
742 mTableUpdates, [self](nsresult aRv) -> void {
743 #ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
744 if (NS_FAILED(aRv) && NS_ERROR_OUT_OF_MEMORY != aRv &&
745 NS_ERROR_UC_UPDATE_SHUTDOWNING != aRv) {
746 self->mClassifier->DumpRawTableUpdates(self->mRawTableUpdates);
747 }
748 // Invalidate the raw table updates.
749 self->mRawTableUpdates.Truncate();
750 #endif
751
752 self->NotifyUpdateObserver(aRv);
753 });
754 mTableUpdates.Clear(); // Classifier is working on its copy.
755
756 if (NS_FAILED(rv)) {
757 LOG(("Failed to start async update. Notify immediately."));
758 NotifyUpdateObserver(rv);
759 }
760
761 return rv;
762 }
763
NotifyUpdateObserver(nsresult aUpdateStatus)764 nsresult nsUrlClassifierDBServiceWorker::NotifyUpdateObserver(
765 nsresult aUpdateStatus) {
766 MOZ_ASSERT(!NS_IsMainThread(),
767 "nsUrlClassifierDBServiceWorker::NotifyUpdateObserver "
768 "NUST NOT be on the main thread.");
769
770 LOG(("nsUrlClassifierDBServiceWorker::NotifyUpdateObserver"));
771
772 // We've either
773 // 1) failed starting a download stream
774 // 2) succeeded in starting a download stream but failed to obtain
775 // table updates
776 // 3) succeeded in obtaining table updates but failed to build new
777 // tables.
778 // 4) succeeded in building new tables but failed to take them.
779 // 5) succeeded in taking new tables.
780
781 mUpdateStatus = aUpdateStatus;
782
783 nsUrlClassifierUtils* urlUtil = nsUrlClassifierUtils::GetInstance();
784 if (NS_WARN_IF(!urlUtil)) {
785 return NS_ERROR_FAILURE;
786 }
787
788 nsCString provider;
789 // Assume that all the tables in update should have the same provider.
790 urlUtil->GetTelemetryProvider(mUpdateTables.SafeElementAt(0, ""_ns),
791 provider);
792
793 nsresult updateStatus = mUpdateStatus;
794 if (NS_FAILED(mUpdateStatus)) {
795 updateStatus =
796 NS_ERROR_GET_MODULE(mUpdateStatus) == NS_ERROR_MODULE_URL_CLASSIFIER
797 ? mUpdateStatus
798 : NS_ERROR_UC_UPDATE_UNKNOWN;
799 }
800
801 // Do not record telemetry for testing tables.
802 if (!provider.EqualsLiteral(TESTING_TABLE_PROVIDER_NAME)) {
803 Telemetry::Accumulate(Telemetry::URLCLASSIFIER_UPDATE_ERROR, provider,
804 NS_ERROR_GET_CODE(updateStatus));
805 }
806
807 if (!mUpdateObserver) {
808 // In the normal shutdown process, CancelUpdate() would NOT be
809 // called prior to NotifyUpdateObserver(). However, CancelUpdate()
810 // is a public API which can be called in the test case at any point.
811 // If the call sequence is FinishUpdate() then CancelUpdate(), the later
812 // might be executed before NotifyUpdateObserver() which is triggered
813 // by the update thread. In this case, we will get null mUpdateObserver.
814 NS_WARNING(
815 "CancelUpdate() is called before we asynchronously call "
816 "NotifyUpdateObserver() in FinishUpdate().");
817
818 // The DB cleanup will be done in CancelUpdate() so we can just return.
819 return NS_OK;
820 }
821
822 // Null out mUpdateObserver before notifying so that BeginUpdate()
823 // becomes available prior to callback.
824 nsCOMPtr<nsIUrlClassifierUpdateObserver> updateObserver = nullptr;
825 updateObserver.swap(mUpdateObserver);
826
827 if (NS_SUCCEEDED(mUpdateStatus)) {
828 LOG(("Notifying success: %d", mUpdateWaitSec));
829 updateObserver->UpdateSuccess(mUpdateWaitSec);
830 } else {
831 if (LOG_ENABLED()) {
832 nsAutoCString errorName;
833 mozilla::GetErrorName(mUpdateStatus, errorName);
834 LOG(("Notifying error: %s (%" PRIu32 ")", errorName.get(),
835 static_cast<uint32_t>(mUpdateStatus)));
836 }
837
838 updateObserver->UpdateError(mUpdateStatus);
839 /*
840 * mark the tables as spoiled(clear cache in LookupCache), we don't want to
841 * block hosts longer than normal because our update failed
842 */
843 mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables);
844 }
845
846 return NS_OK;
847 }
848
849 NS_IMETHODIMP
ResetDatabase()850 nsUrlClassifierDBServiceWorker::ResetDatabase() {
851 nsresult rv = OpenDb();
852
853 if (NS_SUCCEEDED(rv)) {
854 mClassifier->Reset();
855 }
856
857 rv = CloseDb();
858 NS_ENSURE_SUCCESS(rv, rv);
859
860 return NS_OK;
861 }
862
863 NS_IMETHODIMP
ReloadDatabase()864 nsUrlClassifierDBServiceWorker::ReloadDatabase() {
865 // This will null out mClassifier
866 nsresult rv = CloseDb();
867 NS_ENSURE_SUCCESS(rv, rv);
868
869 // Create new mClassifier and load prefixset and completions from disk.
870 rv = OpenDb();
871 NS_ENSURE_SUCCESS(rv, rv);
872
873 return NS_OK;
874 }
875
876 NS_IMETHODIMP
ClearCache()877 nsUrlClassifierDBServiceWorker::ClearCache() {
878 nsTArray<nsCString> tables;
879 nsresult rv = mClassifier->ActiveTables(tables);
880 NS_ENSURE_SUCCESS(rv, rv);
881
882 mClassifier->ResetTables(Classifier::Clear_Cache, tables);
883
884 return NS_OK;
885 }
886
887 NS_IMETHODIMP
CancelUpdate()888 nsUrlClassifierDBServiceWorker::CancelUpdate() {
889 LOG(("nsUrlClassifierDBServiceWorker::CancelUpdate"));
890
891 if (mUpdateObserver) {
892 LOG(("UpdateObserver exists, cancelling"));
893
894 mUpdateStatus = NS_BINDING_ABORTED;
895
896 mUpdateObserver->UpdateError(mUpdateStatus);
897
898 /*
899 * mark the tables as spoiled(clear cache in LookupCache), we don't want to
900 * block hosts longer than normal because our update failed
901 */
902 mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables);
903
904 ResetStream();
905 ResetUpdate();
906 } else {
907 LOG(("No UpdateObserver, nothing to cancel"));
908 }
909
910 return NS_OK;
911 }
912
FlushAndDisableAsyncUpdate()913 void nsUrlClassifierDBServiceWorker::FlushAndDisableAsyncUpdate() {
914 LOG(("nsUrlClassifierDBServiceWorker::FlushAndDisableAsyncUpdate()"));
915
916 if (mClassifier) {
917 mClassifier->FlushAndDisableAsyncUpdate();
918 }
919 }
920
921 // Allows the main thread to delete the connection which may be in
922 // a background thread.
923 // XXX This could be turned into a single shutdown event so the logic
924 // is simpler in nsUrlClassifierDBService::Shutdown.
CloseDb()925 nsresult nsUrlClassifierDBServiceWorker::CloseDb() {
926 if (mClassifier) {
927 mClassifier->Close();
928 mClassifier = nullptr;
929 }
930
931 // Clear last completion result when close db so we will still cache
932 // completion result next time we re-open it.
933 mLastResults.Clear();
934
935 LOG(("urlclassifier db closed\n"));
936
937 return NS_OK;
938 }
939
PreShutdown()940 nsresult nsUrlClassifierDBServiceWorker::PreShutdown() {
941 if (mClassifier) {
942 // Classifier close will release all lookup caches which may be a
943 // time-consuming job. See Bug 1408631.
944 mClassifier->Close();
945 }
946
947 // WARNING: nothing we put here should affect an ongoing update thread. When
948 // in doubt, put things in Shutdown() instead.
949 return NS_OK;
950 }
951
CacheCompletions(const ConstCacheResultArray & aResults)952 nsresult nsUrlClassifierDBServiceWorker::CacheCompletions(
953 const ConstCacheResultArray& aResults) {
954 if (gShuttingDownThread) {
955 return NS_ERROR_ABORT;
956 }
957
958 LOG(("nsUrlClassifierDBServiceWorker::CacheCompletions [%p]", this));
959 if (!mClassifier) {
960 return NS_OK;
961 }
962
963 if (aResults.Length() == 0) {
964 return NS_OK;
965 }
966
967 if (IsSameAsLastResults(aResults)) {
968 LOG(("Skipping completions that have just been cached already."));
969 return NS_OK;
970 }
971
972 // Only cache results for tables that we have, don't take
973 // in tables we might accidentally have hit during a completion.
974 // This happens due to goog vs googpub lists existing.
975 nsTArray<nsCString> tables;
976 nsresult rv = mClassifier->ActiveTables(tables);
977 NS_ENSURE_SUCCESS(rv, rv);
978 if (LOG_ENABLED()) {
979 nsCString s;
980 for (size_t i = 0; i < tables.Length(); i++) {
981 if (!s.IsEmpty()) {
982 s += ",";
983 }
984 s += tables[i];
985 }
986 LOG(("Active tables: %s", s.get()));
987 }
988
989 ConstTableUpdateArray updates;
990
991 for (const auto& result : aResults) {
992 bool activeTable = false;
993
994 for (uint32_t table = 0; table < tables.Length(); table++) {
995 if (tables[table].Equals(result->table)) {
996 activeTable = true;
997 break;
998 }
999 }
1000 if (activeTable) {
1001 UniquePtr<ProtocolParser> pParse;
1002 if (result->Ver() == CacheResult::V2) {
1003 pParse.reset(new ProtocolParserV2());
1004 } else {
1005 pParse.reset(new ProtocolParserProtobuf());
1006 }
1007
1008 RefPtr<TableUpdate> tu = pParse->GetTableUpdate(result->table);
1009
1010 rv = CacheResultToTableUpdate(result, tu);
1011 if (NS_FAILED(rv)) {
1012 // We can bail without leaking here because ForgetTableUpdates
1013 // hasn't been called yet.
1014 return rv;
1015 }
1016 updates.AppendElement(tu);
1017 pParse->ForgetTableUpdates();
1018 } else {
1019 LOG(("Completion received, but table %s is not active, so not caching.",
1020 result->table.get()));
1021 }
1022 }
1023
1024 rv = mClassifier->ApplyFullHashes(updates);
1025 if (NS_SUCCEEDED(rv)) {
1026 mLastResults = aResults.Clone();
1027 }
1028 return rv;
1029 }
1030
CacheResultToTableUpdate(RefPtr<const CacheResult> aCacheResult,RefPtr<TableUpdate> aUpdate)1031 nsresult nsUrlClassifierDBServiceWorker::CacheResultToTableUpdate(
1032 RefPtr<const CacheResult> aCacheResult, RefPtr<TableUpdate> aUpdate) {
1033 RefPtr<TableUpdateV2> tuV2 = TableUpdate::Cast<TableUpdateV2>(aUpdate);
1034 if (tuV2) {
1035 RefPtr<const CacheResultV2> result =
1036 CacheResult::Cast<const CacheResultV2>(aCacheResult);
1037 MOZ_ASSERT(result);
1038
1039 if (result->miss) {
1040 return tuV2->NewMissPrefix(result->prefix);
1041 } else {
1042 LOG(("CacheCompletion hash %X, Addchunk %d",
1043 result->completion.ToUint32(), result->addChunk));
1044
1045 nsresult rv = tuV2->NewAddComplete(result->addChunk, result->completion);
1046 if (NS_FAILED(rv)) {
1047 return rv;
1048 }
1049 return tuV2->NewAddChunk(result->addChunk);
1050 }
1051 }
1052
1053 RefPtr<TableUpdateV4> tuV4 = TableUpdate::Cast<TableUpdateV4>(aUpdate);
1054 if (tuV4) {
1055 RefPtr<const CacheResultV4> result =
1056 CacheResult::Cast<const CacheResultV4>(aCacheResult);
1057 MOZ_ASSERT(result);
1058
1059 if (LOG_ENABLED()) {
1060 const FullHashExpiryCache& fullHashes = result->response.fullHashes;
1061 for (const auto& entry : fullHashes) {
1062 Completion completion;
1063 completion.Assign(entry.GetKey());
1064 LOG(("CacheCompletion(v4) hash %X, CacheExpireTime %" PRId64,
1065 completion.ToUint32(), entry.GetData()));
1066 }
1067 }
1068
1069 tuV4->NewFullHashResponse(result->prefix, result->response);
1070 return NS_OK;
1071 }
1072
1073 // tableUpdate object should be either V2 or V4.
1074 return NS_ERROR_FAILURE;
1075 }
1076
OpenDb()1077 nsresult nsUrlClassifierDBServiceWorker::OpenDb() {
1078 if (gShuttingDownThread) {
1079 return NS_ERROR_ABORT;
1080 }
1081
1082 MOZ_ASSERT(!NS_IsMainThread(), "Must initialize DB on background thread");
1083 // Connection already open, don't do anything.
1084 if (mClassifier) {
1085 return NS_OK;
1086 }
1087
1088 nsresult rv;
1089 RefPtr<Classifier> classifier = new (fallible) Classifier();
1090 if (!classifier) {
1091 return NS_ERROR_OUT_OF_MEMORY;
1092 }
1093
1094 rv = classifier->Open(*mCacheDir);
1095 NS_ENSURE_SUCCESS(rv, rv);
1096
1097 mClassifier = classifier;
1098
1099 return NS_OK;
1100 }
1101
1102 NS_IMETHODIMP
ClearLastResults()1103 nsUrlClassifierDBServiceWorker::ClearLastResults() {
1104 MOZ_ASSERT(!NS_IsMainThread(), "Must be on the background thread");
1105 mLastResults.Clear();
1106 return NS_OK;
1107 }
1108
GetCacheInfo(const nsACString & aTable,nsIUrlClassifierCacheInfo ** aCache)1109 nsresult nsUrlClassifierDBServiceWorker::GetCacheInfo(
1110 const nsACString& aTable, nsIUrlClassifierCacheInfo** aCache) {
1111 MOZ_ASSERT(!NS_IsMainThread(), "Must be on the background thread");
1112 if (!mClassifier) {
1113 return NS_ERROR_NOT_AVAILABLE;
1114 }
1115
1116 mClassifier->GetCacheInfo(aTable, aCache);
1117 return NS_OK;
1118 }
1119
IsSameAsLastResults(const ConstCacheResultArray & aResult) const1120 bool nsUrlClassifierDBServiceWorker::IsSameAsLastResults(
1121 const ConstCacheResultArray& aResult) const {
1122 if (mLastResults.Length() != aResult.Length()) {
1123 return false;
1124 }
1125
1126 bool equal = true;
1127 for (uint32_t i = 0; i < mLastResults.Length() && equal; i++) {
1128 RefPtr<const CacheResult> lhs = mLastResults[i];
1129 RefPtr<const CacheResult> rhs = aResult[i];
1130
1131 if (lhs->Ver() != rhs->Ver()) {
1132 return false;
1133 }
1134
1135 if (lhs->Ver() == CacheResult::V2) {
1136 equal = *(CacheResult::Cast<const CacheResultV2>(lhs)) ==
1137 *(CacheResult::Cast<const CacheResultV2>(rhs));
1138 } else if (lhs->Ver() == CacheResult::V4) {
1139 equal = *(CacheResult::Cast<const CacheResultV4>(lhs)) ==
1140 *(CacheResult::Cast<const CacheResultV4>(rhs));
1141 }
1142 }
1143
1144 return equal;
1145 }
1146
1147 // -------------------------------------------------------------------------
1148 // nsUrlClassifierLookupCallback
1149 //
1150 // This class takes the results of a lookup found on the worker thread
1151 // and handles any necessary partial hash expansions before calling
1152 // the client callback.
1153
1154 class nsUrlClassifierLookupCallback final
1155 : public nsIUrlClassifierLookupCallback,
1156 public nsIUrlClassifierHashCompleterCallback {
1157 public:
1158 NS_DECL_THREADSAFE_ISUPPORTS
1159 NS_DECL_NSIURLCLASSIFIERLOOKUPCALLBACK
1160 NS_DECL_NSIURLCLASSIFIERHASHCOMPLETERCALLBACK
1161
nsUrlClassifierLookupCallback(nsUrlClassifierDBService * dbservice,nsIUrlClassifierCallback * c)1162 nsUrlClassifierLookupCallback(nsUrlClassifierDBService* dbservice,
1163 nsIUrlClassifierCallback* c)
1164 : mDBService(dbservice),
1165 mResults(nullptr),
1166 mPendingCompletions(0),
1167 mCallback(c) {}
1168
1169 private:
1170 ~nsUrlClassifierLookupCallback();
1171
1172 nsresult HandleResults();
1173 nsresult ProcessComplete(RefPtr<CacheResult> aCacheResult);
1174 nsresult CacheMisses();
1175
1176 RefPtr<nsUrlClassifierDBService> mDBService;
1177 UniquePtr<LookupResultArray> mResults;
1178
1179 // Completed results to send back to the worker for caching.
1180 ConstCacheResultArray mCacheResults;
1181
1182 uint32_t mPendingCompletions;
1183 nsCOMPtr<nsIUrlClassifierCallback> mCallback;
1184 };
1185
NS_IMPL_ISUPPORTS(nsUrlClassifierLookupCallback,nsIUrlClassifierLookupCallback,nsIUrlClassifierHashCompleterCallback)1186 NS_IMPL_ISUPPORTS(nsUrlClassifierLookupCallback, nsIUrlClassifierLookupCallback,
1187 nsIUrlClassifierHashCompleterCallback)
1188
1189 nsUrlClassifierLookupCallback::~nsUrlClassifierLookupCallback() {
1190 if (mCallback) {
1191 NS_ReleaseOnMainThread("nsUrlClassifierLookupCallback::mCallback",
1192 mCallback.forget());
1193 }
1194 }
1195
1196 NS_IMETHODIMP
LookupComplete(UniquePtr<LookupResultArray> results)1197 nsUrlClassifierLookupCallback::LookupComplete(
1198 UniquePtr<LookupResultArray> results) {
1199 NS_ASSERTION(
1200 mResults == nullptr,
1201 "Should only get one set of results per nsUrlClassifierLookupCallback!");
1202
1203 if (!results) {
1204 HandleResults();
1205 return NS_OK;
1206 }
1207
1208 mResults = std::move(results);
1209
1210 // Check the results entries that need to be completed.
1211 for (const auto& result : *mResults) {
1212 // We will complete partial matches and matches that are stale.
1213 if (!result->Confirmed()) {
1214 nsCOMPtr<nsIUrlClassifierHashCompleter> completer;
1215 nsCString gethashUrl;
1216 nsresult rv;
1217 nsCOMPtr<nsIUrlListManager> listManager =
1218 do_GetService("@mozilla.org/url-classifier/listmanager;1", &rv);
1219 NS_ENSURE_SUCCESS(rv, rv);
1220 rv = listManager->GetGethashUrl(result->mTableName, gethashUrl);
1221 NS_ENSURE_SUCCESS(rv, rv);
1222 LOG(("The match from %s needs to be completed at %s",
1223 result->mTableName.get(), gethashUrl.get()));
1224 // gethashUrls may be empty in 2 cases: test tables, and on startup where
1225 // we may have found a prefix in an existing table before the listmanager
1226 // has registered the table. In the second case we should not call
1227 // complete.
1228 if ((!gethashUrl.IsEmpty() ||
1229 nsUrlClassifierUtils::IsTestTable(result->mTableName)) &&
1230 mDBService->GetCompleter(result->mTableName,
1231 getter_AddRefs(completer))) {
1232 // Bug 1323953 - Send the first 4 bytes for completion no matter how
1233 // long we matched the prefix.
1234 nsresult rv = completer->Complete(result->PartialHash(), gethashUrl,
1235 result->mTableName, this);
1236 if (NS_SUCCEEDED(rv)) {
1237 mPendingCompletions++;
1238 }
1239 } else {
1240 // For tables with no hash completer, a complete hash match is
1241 // good enough, we'll consider it is valid.
1242 if (result->Complete()) {
1243 result->mConfirmed = true;
1244 LOG(("Skipping completion in a table without a valid completer (%s).",
1245 result->mTableName.get()));
1246 } else {
1247 NS_WARNING(
1248 "Partial match in a table without a valid completer, ignoring "
1249 "partial match.");
1250 }
1251 }
1252 }
1253 }
1254
1255 LOG(
1256 ("nsUrlClassifierLookupCallback::LookupComplete [%p] "
1257 "%u pending completions",
1258 this, mPendingCompletions));
1259 if (mPendingCompletions == 0) {
1260 // All results were complete, we're ready!
1261 HandleResults();
1262 }
1263
1264 return NS_OK;
1265 }
1266
1267 NS_IMETHODIMP
CompletionFinished(nsresult status)1268 nsUrlClassifierLookupCallback::CompletionFinished(nsresult status) {
1269 if (LOG_ENABLED()) {
1270 nsAutoCString errorName;
1271 mozilla::GetErrorName(status, errorName);
1272 LOG(("nsUrlClassifierLookupCallback::CompletionFinished [%p, %s]", this,
1273 errorName.get()));
1274 }
1275
1276 mPendingCompletions--;
1277 if (mPendingCompletions == 0) {
1278 HandleResults();
1279 }
1280
1281 return NS_OK;
1282 }
1283
1284 NS_IMETHODIMP
CompletionV2(const nsACString & aCompleteHash,const nsACString & aTableName,uint32_t aChunkId)1285 nsUrlClassifierLookupCallback::CompletionV2(const nsACString& aCompleteHash,
1286 const nsACString& aTableName,
1287 uint32_t aChunkId) {
1288 LOG(("nsUrlClassifierLookupCallback::Completion [%p, %s, %d]", this,
1289 PromiseFlatCString(aTableName).get(), aChunkId));
1290
1291 MOZ_ASSERT(!StringEndsWith(aTableName, "-proto"_ns));
1292
1293 RefPtr<CacheResultV2> result = new CacheResultV2();
1294
1295 result->table = aTableName;
1296 result->prefix.Assign(aCompleteHash);
1297 result->completion.Assign(aCompleteHash);
1298 result->addChunk = aChunkId;
1299
1300 return ProcessComplete(result);
1301 }
1302
1303 NS_IMETHODIMP
CompletionV4(const nsACString & aPartialHash,const nsACString & aTableName,uint32_t aNegativeCacheDuration,nsIArray * aFullHashes)1304 nsUrlClassifierLookupCallback::CompletionV4(const nsACString& aPartialHash,
1305 const nsACString& aTableName,
1306 uint32_t aNegativeCacheDuration,
1307 nsIArray* aFullHashes) {
1308 LOG(("nsUrlClassifierLookupCallback::CompletionV4 [%p, %s, %d]", this,
1309 PromiseFlatCString(aTableName).get(), aNegativeCacheDuration));
1310
1311 MOZ_ASSERT(StringEndsWith(aTableName, "-proto"_ns));
1312
1313 if (!aFullHashes) {
1314 return NS_ERROR_INVALID_ARG;
1315 }
1316
1317 if (aNegativeCacheDuration > MAXIMUM_NEGATIVE_CACHE_DURATION_SEC) {
1318 LOG(
1319 ("Negative cache duration too large, clamping it down to"
1320 "a reasonable value."));
1321 aNegativeCacheDuration = MAXIMUM_NEGATIVE_CACHE_DURATION_SEC;
1322 }
1323
1324 RefPtr<CacheResultV4> result = new CacheResultV4();
1325
1326 int64_t nowSec = PR_Now() / PR_USEC_PER_SEC;
1327
1328 result->table = aTableName;
1329 result->prefix.Assign(aPartialHash);
1330 result->response.negativeCacheExpirySec = nowSec + aNegativeCacheDuration;
1331
1332 // Fill in positive cache entries.
1333 uint32_t fullHashCount = 0;
1334 nsresult rv = aFullHashes->GetLength(&fullHashCount);
1335 if (NS_FAILED(rv)) {
1336 return rv;
1337 }
1338
1339 for (uint32_t i = 0; i < fullHashCount; i++) {
1340 nsCOMPtr<nsIFullHashMatch> match = do_QueryElementAt(aFullHashes, i);
1341
1342 nsCString fullHash;
1343 match->GetFullHash(fullHash);
1344
1345 uint32_t duration;
1346 match->GetCacheDuration(&duration);
1347
1348 result->response.fullHashes.InsertOrUpdate(fullHash, nowSec + duration);
1349 }
1350
1351 return ProcessComplete(result);
1352 }
1353
ProcessComplete(RefPtr<CacheResult> aCacheResult)1354 nsresult nsUrlClassifierLookupCallback::ProcessComplete(
1355 RefPtr<CacheResult> aCacheResult) {
1356 NS_ENSURE_ARG_POINTER(mResults);
1357
1358 if (!mCacheResults.AppendElement(aCacheResult, fallible)) {
1359 // OK if this failed, we just won't cache the item.
1360 }
1361
1362 // Check if this matched any of our results.
1363 for (const auto& result : *mResults) {
1364 // Now, see if it verifies a lookup
1365 if (!result->mNoise && result->mTableName.Equals(aCacheResult->table) &&
1366 aCacheResult->findCompletion(result->CompleteHash())) {
1367 result->mProtocolConfirmed = true;
1368 }
1369 }
1370
1371 return NS_OK;
1372 }
1373
HandleResults()1374 nsresult nsUrlClassifierLookupCallback::HandleResults() {
1375 if (!mResults) {
1376 // No results, this URI is clean.
1377 LOG(("nsUrlClassifierLookupCallback::HandleResults [%p, no results]",
1378 this));
1379 return mCallback->HandleEvent(""_ns);
1380 }
1381 MOZ_ASSERT(mPendingCompletions == 0,
1382 "HandleResults() should never be "
1383 "called while there are pending completions");
1384
1385 LOG(("nsUrlClassifierLookupCallback::HandleResults [%p, %zu results]", this,
1386 mResults->Length()));
1387
1388 nsCOMPtr<nsIUrlClassifierClassifyCallback> classifyCallback =
1389 do_QueryInterface(mCallback);
1390
1391 nsTArray<nsCString> tables;
1392 // Build a stringified list of result tables.
1393 for (const auto& result : *mResults) {
1394 // Leave out results that weren't confirmed, as their existence on
1395 // the list can't be verified. Also leave out randomly-generated
1396 // noise.
1397 if (result->mNoise) {
1398 LOG(("Skipping result %s from table %s (noise)",
1399 result->PartialHashHex().get(), result->mTableName.get()));
1400 continue;
1401 }
1402
1403 if (!result->Confirmed()) {
1404 LOG(("Skipping result %s from table %s (not confirmed)",
1405 result->PartialHashHex().get(), result->mTableName.get()));
1406 continue;
1407 }
1408
1409 LOG(("Confirmed result %s from table %s", result->PartialHashHex().get(),
1410 result->mTableName.get()));
1411
1412 if (tables.IndexOf(result->mTableName) == nsTArray<nsCString>::NoIndex) {
1413 tables.AppendElement(result->mTableName);
1414 }
1415
1416 if (classifyCallback) {
1417 nsCString fullHashString;
1418 result->hash.complete.ToString(fullHashString);
1419 classifyCallback->HandleResult(result->mTableName, fullHashString);
1420 }
1421 }
1422
1423 // Some parts of this gethash request generated no hits at all.
1424 // Save the prefixes we checked to prevent repeated requests.
1425 CacheMisses();
1426
1427 // This hands ownership of the cache results array back to the worker
1428 // thread.
1429 mDBService->CacheCompletions(mCacheResults);
1430 mCacheResults.Clear();
1431
1432 return mCallback->HandleEvent(StringJoin(","_ns, tables));
1433 }
1434
CacheMisses()1435 nsresult nsUrlClassifierLookupCallback::CacheMisses() {
1436 MOZ_ASSERT(mResults);
1437
1438 for (const RefPtr<const LookupResult> result : *mResults) {
1439 // Skip V4 because cache information is already included in the
1440 // fullhash response so we don't need to manually add it here.
1441 if (!result->mProtocolV2 || result->Confirmed() || result->mNoise) {
1442 continue;
1443 }
1444
1445 RefPtr<CacheResultV2> cacheResult = new CacheResultV2();
1446
1447 cacheResult->table = result->mTableName;
1448 cacheResult->prefix = result->hash.fixedLengthPrefix;
1449 cacheResult->miss = true;
1450 if (!mCacheResults.AppendElement(cacheResult, fallible)) {
1451 return NS_ERROR_OUT_OF_MEMORY;
1452 }
1453 }
1454 return NS_OK;
1455 }
1456
1457 struct Provider {
1458 nsCString name;
1459 uint8_t priority;
1460 };
1461
1462 // Order matters
1463 // Provider which is not included in this table has the lowest priority 0
1464 static const Provider kBuiltInProviders[] = {
1465 {"mozilla"_ns, 1},
1466 {"google4"_ns, 2},
1467 {"google"_ns, 3},
1468 };
1469
1470 // -------------------------------------------------------------------------
1471 // Helper class for nsIURIClassifier implementation, handle classify result and
1472 // send back to nsIURIClassifier
1473
1474 class nsUrlClassifierClassifyCallback final
1475 : public nsIUrlClassifierCallback,
1476 public nsIUrlClassifierClassifyCallback {
1477 public:
1478 NS_DECL_THREADSAFE_ISUPPORTS
1479 NS_DECL_NSIURLCLASSIFIERCALLBACK
1480 NS_DECL_NSIURLCLASSIFIERCLASSIFYCALLBACK
1481
nsUrlClassifierClassifyCallback(nsIURIClassifierCallback * c)1482 explicit nsUrlClassifierClassifyCallback(nsIURIClassifierCallback* c)
1483 : mCallback(c) {}
1484
1485 private:
1486 struct ClassifyMatchedInfo {
1487 nsCString table;
1488 nsCString fullhash;
1489 Provider provider;
1490 nsresult errorCode;
1491 };
1492
1493 ~nsUrlClassifierClassifyCallback() = default;
1494
1495 nsCOMPtr<nsIURIClassifierCallback> mCallback;
1496 nsTArray<ClassifyMatchedInfo> mMatchedArray;
1497 };
1498
NS_IMPL_ISUPPORTS(nsUrlClassifierClassifyCallback,nsIUrlClassifierCallback,nsIUrlClassifierClassifyCallback)1499 NS_IMPL_ISUPPORTS(nsUrlClassifierClassifyCallback, nsIUrlClassifierCallback,
1500 nsIUrlClassifierClassifyCallback)
1501
1502 NS_IMETHODIMP
1503 nsUrlClassifierClassifyCallback::HandleEvent(const nsACString& tables) {
1504 nsresult response = TablesToResponse(tables);
1505 ClassifyMatchedInfo* matchedInfo = nullptr;
1506
1507 if (NS_FAILED(response)) {
1508 // Filter all matched info which has correct response
1509 // In the case multiple tables found, use the higher priority provider
1510 nsTArray<ClassifyMatchedInfo> matches;
1511 for (uint32_t i = 0; i < mMatchedArray.Length(); i++) {
1512 if (mMatchedArray[i].errorCode == response &&
1513 (!matchedInfo || matchedInfo->provider.priority <
1514 mMatchedArray[i].provider.priority)) {
1515 matchedInfo = &mMatchedArray[i];
1516 }
1517 }
1518 }
1519
1520 nsCString provider = matchedInfo ? matchedInfo->provider.name : ""_ns;
1521 nsCString fullhash = matchedInfo ? matchedInfo->fullhash : ""_ns;
1522 nsCString table = matchedInfo ? matchedInfo->table : ""_ns;
1523
1524 mCallback->OnClassifyComplete(response, table, provider, fullhash);
1525 return NS_OK;
1526 }
1527
1528 NS_IMETHODIMP
HandleResult(const nsACString & aTable,const nsACString & aFullHash)1529 nsUrlClassifierClassifyCallback::HandleResult(const nsACString& aTable,
1530 const nsACString& aFullHash) {
1531 LOG(
1532 ("nsUrlClassifierClassifyCallback::HandleResult [%p, table %s full hash "
1533 "%s]",
1534 this, PromiseFlatCString(aTable).get(),
1535 PromiseFlatCString(aFullHash).get()));
1536
1537 if (NS_WARN_IF(aTable.IsEmpty()) || NS_WARN_IF(aFullHash.IsEmpty())) {
1538 return NS_ERROR_INVALID_ARG;
1539 }
1540
1541 ClassifyMatchedInfo* matchedInfo = mMatchedArray.AppendElement();
1542 matchedInfo->table = aTable;
1543 matchedInfo->fullhash = aFullHash;
1544
1545 nsUrlClassifierUtils* urlUtil = nsUrlClassifierUtils::GetInstance();
1546 if (NS_WARN_IF(!urlUtil)) {
1547 return NS_ERROR_FAILURE;
1548 }
1549
1550 nsCString provider;
1551 nsresult rv = urlUtil->GetProvider(aTable, provider);
1552
1553 matchedInfo->provider.name = NS_SUCCEEDED(rv) ? provider : ""_ns;
1554 matchedInfo->provider.priority = 0;
1555 for (uint8_t i = 0; i < ArrayLength(kBuiltInProviders); i++) {
1556 if (kBuiltInProviders[i].name.Equals(matchedInfo->provider.name)) {
1557 matchedInfo->provider.priority = kBuiltInProviders[i].priority;
1558 }
1559 }
1560 matchedInfo->errorCode = TablesToResponse(aTable);
1561
1562 return NS_OK;
1563 }
1564
1565 // -------------------------------------------------------------------------
1566 // Proxy class implementation
1567
1568 NS_IMPL_ADDREF(nsUrlClassifierDBService)
NS_IMPL_RELEASE(nsUrlClassifierDBService)1569 NS_IMPL_RELEASE(nsUrlClassifierDBService)
1570 NS_INTERFACE_MAP_BEGIN(nsUrlClassifierDBService)
1571 // Only nsIURIClassifier is supported in the content process!
1572 NS_INTERFACE_MAP_ENTRY_CONDITIONAL(nsIUrlClassifierDBService,
1573 XRE_IsParentProcess())
1574 NS_INTERFACE_MAP_ENTRY(nsIURIClassifier)
1575 NS_INTERFACE_MAP_ENTRY(nsIUrlClassifierInfo)
1576 NS_INTERFACE_MAP_ENTRY_CONDITIONAL(nsIObserver, XRE_IsParentProcess())
1577 NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIURIClassifier)
1578 NS_INTERFACE_MAP_END
1579
1580 /* static */
1581 already_AddRefed<nsUrlClassifierDBService>
1582 nsUrlClassifierDBService::GetInstance(nsresult* result) {
1583 *result = NS_OK;
1584 if (!sUrlClassifierDBService) {
1585 sUrlClassifierDBService = new (fallible) nsUrlClassifierDBService();
1586 if (!sUrlClassifierDBService) {
1587 *result = NS_ERROR_OUT_OF_MEMORY;
1588 return nullptr;
1589 }
1590
1591 *result = sUrlClassifierDBService->Init();
1592 if (NS_FAILED(*result)) {
1593 return nullptr;
1594 }
1595 }
1596 return do_AddRef(sUrlClassifierDBService);
1597 }
1598
nsUrlClassifierDBService()1599 nsUrlClassifierDBService::nsUrlClassifierDBService() : mInUpdate(false) {}
1600
~nsUrlClassifierDBService()1601 nsUrlClassifierDBService::~nsUrlClassifierDBService() {
1602 sUrlClassifierDBService = nullptr;
1603 }
1604
ReadDisallowCompletionsTablesFromPrefs()1605 nsresult nsUrlClassifierDBService::ReadDisallowCompletionsTablesFromPrefs() {
1606 nsAutoCString tables;
1607
1608 Preferences::GetCString(DISALLOW_COMPLETION_TABLE_PREF, tables);
1609 Classifier::SplitTables(tables, mDisallowCompletionsTables);
1610
1611 return NS_OK;
1612 }
1613
Init()1614 nsresult nsUrlClassifierDBService::Init() {
1615 MOZ_ASSERT(NS_IsMainThread(), "Must initialize DB service on main thread");
1616
1617 switch (XRE_GetProcessType()) {
1618 case GeckoProcessType_Default:
1619 // The parent process is supported.
1620 break;
1621 case GeckoProcessType_Content:
1622 // In a content process, we simply forward all requests to the parent
1623 // process, so we can skip the initialization steps here. Note that since
1624 // we never register an observer, Shutdown() will also never be called in
1625 // the content process.
1626 return NS_OK;
1627 default:
1628 // No other process type is supported!
1629 return NS_ERROR_NOT_AVAILABLE;
1630 }
1631
1632 uint32_t hashNoise =
1633 Preferences::GetUint(GETHASH_NOISE_PREF, GETHASH_NOISE_DEFAULT);
1634 ReadDisallowCompletionsTablesFromPrefs();
1635
1636 // Force nsUrlClassifierUtils loading on main thread.
1637 if (NS_WARN_IF(!nsUrlClassifierUtils::GetInstance())) {
1638 return NS_ERROR_FAILURE;
1639 }
1640
1641 // Directory providers must also be accessed on the main thread.
1642 nsresult rv;
1643 nsCOMPtr<nsIFile> cacheDir;
1644 rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR,
1645 getter_AddRefs(cacheDir));
1646 if (NS_FAILED(rv)) {
1647 rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
1648 getter_AddRefs(cacheDir));
1649 if (NS_FAILED(rv)) {
1650 return rv;
1651 }
1652 }
1653
1654 // Start the background thread.
1655 rv = NS_NewNamedThread("URL Classifier", &gDbBackgroundThread);
1656 if (NS_FAILED(rv)) return rv;
1657
1658 mWorker = new (fallible) nsUrlClassifierDBServiceWorker();
1659 if (!mWorker) {
1660 return NS_ERROR_OUT_OF_MEMORY;
1661 }
1662
1663 rv = mWorker->Init(hashNoise, cacheDir, this);
1664 if (NS_FAILED(rv)) {
1665 mWorker = nullptr;
1666 return rv;
1667 }
1668
1669 // Proxy for calling the worker on the background thread
1670 mWorkerProxy = new UrlClassifierDBServiceWorkerProxy(mWorker);
1671 rv = mWorkerProxy->OpenDb();
1672 if (NS_FAILED(rv)) {
1673 return rv;
1674 }
1675
1676 // Add an observer for shutdown
1677 nsCOMPtr<nsIObserverService> observerService =
1678 mozilla::services::GetObserverService();
1679 if (!observerService) return NS_ERROR_FAILURE;
1680
1681 // The application is about to quit
1682 observerService->AddObserver(this, "quit-application", false);
1683 observerService->AddObserver(this, "profile-before-change", false);
1684
1685 Preferences::AddStrongObserver(this, DISALLOW_COMPLETION_TABLE_PREF);
1686
1687 return NS_OK;
1688 }
1689
1690 // nsChannelClassifier is the only consumer of this interface.
1691 NS_IMETHODIMP
Classify(nsIPrincipal * aPrincipal,nsISerialEventTarget * aEventTarget,nsIURIClassifierCallback * c,bool * aResult)1692 nsUrlClassifierDBService::Classify(nsIPrincipal* aPrincipal,
1693 nsISerialEventTarget* aEventTarget,
1694 nsIURIClassifierCallback* c, bool* aResult) {
1695 NS_ENSURE_ARG(aPrincipal);
1696 NS_ENSURE_ARG(aResult);
1697
1698 if (aPrincipal->IsSystemPrincipal()) {
1699 *aResult = false;
1700 return NS_OK;
1701 }
1702
1703 if (XRE_IsContentProcess()) {
1704 using namespace mozilla::dom;
1705
1706 ContentChild* content = ContentChild::GetSingleton();
1707 MOZ_ASSERT(content);
1708
1709 auto actor = static_cast<URLClassifierChild*>(
1710 content->AllocPURLClassifierChild(IPC::Principal(aPrincipal), aResult));
1711 MOZ_ASSERT(actor);
1712
1713 if (aEventTarget) {
1714 content->SetEventTargetForActor(actor, aEventTarget);
1715 }
1716
1717 if (!content->SendPURLClassifierConstructor(
1718 actor, IPC::Principal(aPrincipal), aResult)) {
1719 *aResult = false;
1720 return NS_ERROR_FAILURE;
1721 }
1722
1723 actor->SetCallback(c);
1724 return NS_OK;
1725 }
1726
1727 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
1728
1729 nsCOMPtr<nsIPermissionManager> permissionManager =
1730 components::PermissionManager::Service();
1731 if (NS_WARN_IF(!permissionManager)) {
1732 return NS_ERROR_FAILURE;
1733 }
1734
1735 uint32_t perm;
1736 nsresult rv = permissionManager->TestPermissionFromPrincipal(
1737 aPrincipal, "safe-browsing"_ns, &perm);
1738 NS_ENSURE_SUCCESS(rv, rv);
1739
1740 if (perm == nsIPermissionManager::ALLOW_ACTION) {
1741 *aResult = false;
1742 return NS_OK;
1743 }
1744
1745 nsTArray<RefPtr<nsIUrlClassifierFeature>> features;
1746 mozilla::net::UrlClassifierFeatureFactory::GetPhishingProtectionFeatures(
1747 features);
1748 if (features.IsEmpty()) {
1749 *aResult = false;
1750 return NS_OK;
1751 }
1752
1753 nsCOMPtr<nsIURI> uri;
1754 // Casting to BasePrincipal, as we can't get InnerMost URI otherwise
1755 auto* basePrincipal = BasePrincipal::Cast(aPrincipal);
1756 rv = basePrincipal->GetURI(getter_AddRefs(uri));
1757 NS_ENSURE_SUCCESS(rv, rv);
1758 NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE);
1759
1760 // Let's keep the features alive and release them on the correct thread.
1761 RefPtr<FeatureHolder> holder =
1762 FeatureHolder::Create(uri, features, nsIUrlClassifierFeature::blocklist);
1763 if (NS_WARN_IF(!holder)) {
1764 return NS_ERROR_FAILURE;
1765 }
1766
1767 uri = NS_GetInnermostURI(uri);
1768 NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE);
1769
1770 nsUrlClassifierUtils* utilsService = nsUrlClassifierUtils::GetInstance();
1771 if (NS_WARN_IF(!utilsService)) {
1772 return NS_ERROR_FAILURE;
1773 }
1774
1775 // Canonicalize the url
1776 nsAutoCString key;
1777 rv = utilsService->GetKeyForURI(uri, key);
1778 NS_ENSURE_SUCCESS(rv, rv);
1779
1780 RefPtr<nsUrlClassifierClassifyCallback> callback =
1781 new (fallible) nsUrlClassifierClassifyCallback(c);
1782 if (NS_WARN_IF(!callback)) {
1783 return NS_ERROR_OUT_OF_MEMORY;
1784 }
1785
1786 // The rest is done async.
1787 rv = LookupURI(key, holder, callback);
1788 NS_ENSURE_SUCCESS(rv, rv);
1789
1790 *aResult = true;
1791 return NS_OK;
1792 }
1793
1794 class ThreatHitReportListener final : public nsIStreamListener {
1795 public:
1796 NS_DECL_ISUPPORTS
1797 NS_DECL_NSIREQUESTOBSERVER
1798 NS_DECL_NSISTREAMLISTENER
1799
1800 ThreatHitReportListener() = default;
1801
1802 private:
1803 ~ThreatHitReportListener() = default;
1804 };
1805
NS_IMPL_ISUPPORTS(ThreatHitReportListener,nsIStreamListener,nsIRequestObserver)1806 NS_IMPL_ISUPPORTS(ThreatHitReportListener, nsIStreamListener,
1807 nsIRequestObserver)
1808
1809 NS_IMETHODIMP
1810 ThreatHitReportListener::OnStartRequest(nsIRequest* aRequest) {
1811 if (!LOG_ENABLED()) {
1812 return NS_OK; // Nothing to do!
1813 }
1814
1815 nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aRequest);
1816 NS_ENSURE_TRUE(httpChannel, NS_OK);
1817
1818 nsresult rv, status;
1819 rv = httpChannel->GetStatus(&status);
1820 NS_ENSURE_SUCCESS(rv, NS_OK);
1821 nsAutoCString errorName;
1822 mozilla::GetErrorName(status, errorName);
1823
1824 uint32_t requestStatus;
1825 rv = httpChannel->GetResponseStatus(&requestStatus);
1826 NS_ENSURE_SUCCESS(rv, NS_OK);
1827
1828 nsAutoCString spec;
1829 nsCOMPtr<nsIURI> uri;
1830 rv = httpChannel->GetURI(getter_AddRefs(uri));
1831 if (NS_SUCCEEDED(rv) && uri) {
1832 uri->GetAsciiSpec(spec);
1833 }
1834 nsCOMPtr<nsIURLFormatter> urlFormatter =
1835 do_GetService("@mozilla.org/toolkit/URLFormatterService;1");
1836 nsAutoString trimmed;
1837 rv = urlFormatter->TrimSensitiveURLs(NS_ConvertUTF8toUTF16(spec), trimmed);
1838 NS_ENSURE_SUCCESS(rv, NS_OK);
1839
1840 LOG(
1841 ("ThreatHitReportListener::OnStartRequest "
1842 "(status=%s, code=%d, uri=%s, this=%p)",
1843 errorName.get(), requestStatus, NS_ConvertUTF16toUTF8(trimmed).get(),
1844 this));
1845
1846 return NS_OK;
1847 }
1848
1849 NS_IMETHODIMP
OnDataAvailable(nsIRequest * aRequest,nsIInputStream * aInputStream,uint64_t aOffset,uint32_t aCount)1850 ThreatHitReportListener::OnDataAvailable(nsIRequest* aRequest,
1851 nsIInputStream* aInputStream,
1852 uint64_t aOffset, uint32_t aCount) {
1853 return NS_OK;
1854 }
1855
1856 NS_IMETHODIMP
OnStopRequest(nsIRequest * aRequest,nsresult aStatus)1857 ThreatHitReportListener::OnStopRequest(nsIRequest* aRequest, nsresult aStatus) {
1858 nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aRequest);
1859 NS_ENSURE_TRUE(httpChannel, aStatus);
1860
1861 uint8_t netErrCode =
1862 NS_FAILED(aStatus) ? mozilla::safebrowsing::NetworkErrorToBucket(aStatus)
1863 : 0;
1864 mozilla::Telemetry::Accumulate(
1865 mozilla::Telemetry::URLCLASSIFIER_THREATHIT_NETWORK_ERROR, netErrCode);
1866
1867 uint32_t requestStatus;
1868 nsresult rv = httpChannel->GetResponseStatus(&requestStatus);
1869 NS_ENSURE_SUCCESS(rv, aStatus);
1870 mozilla::Telemetry::Accumulate(
1871 mozilla::Telemetry::URLCLASSIFIER_THREATHIT_REMOTE_STATUS,
1872 mozilla::safebrowsing::HTTPStatusToBucket(requestStatus));
1873
1874 if (LOG_ENABLED()) {
1875 nsAutoCString errorName;
1876 mozilla::GetErrorName(aStatus, errorName);
1877
1878 nsAutoCString spec;
1879 nsCOMPtr<nsIURI> uri;
1880 rv = httpChannel->GetURI(getter_AddRefs(uri));
1881 if (NS_SUCCEEDED(rv) && uri) {
1882 uri->GetAsciiSpec(spec);
1883 }
1884 nsCOMPtr<nsIURLFormatter> urlFormatter =
1885 do_GetService("@mozilla.org/toolkit/URLFormatterService;1");
1886 nsString trimmed;
1887 rv = urlFormatter->TrimSensitiveURLs(NS_ConvertUTF8toUTF16(spec), trimmed);
1888 NS_ENSURE_SUCCESS(rv, aStatus);
1889
1890 LOG(
1891 ("ThreatHitReportListener::OnStopRequest "
1892 "(status=%s, code=%d, uri=%s, this=%p)",
1893 errorName.get(), requestStatus, NS_ConvertUTF16toUTF8(trimmed).get(),
1894 this));
1895 }
1896
1897 return aStatus;
1898 }
1899
1900 NS_IMETHODIMP
SendThreatHitReport(nsIChannel * aChannel,const nsACString & aProvider,const nsACString & aList,const nsACString & aFullHash)1901 nsUrlClassifierDBService::SendThreatHitReport(nsIChannel* aChannel,
1902 const nsACString& aProvider,
1903 const nsACString& aList,
1904 const nsACString& aFullHash) {
1905 NS_ENSURE_ARG_POINTER(aChannel);
1906
1907 if (aProvider.IsEmpty()) {
1908 LOG(("nsUrlClassifierDBService::SendThreatHitReport missing provider"));
1909 return NS_ERROR_FAILURE;
1910 }
1911 if (aList.IsEmpty()) {
1912 LOG(("nsUrlClassifierDBService::SendThreatHitReport missing list"));
1913 return NS_ERROR_FAILURE;
1914 }
1915 if (aFullHash.IsEmpty()) {
1916 LOG(("nsUrlClassifierDBService::SendThreatHitReport missing fullhash"));
1917 return NS_ERROR_FAILURE;
1918 }
1919
1920 nsPrintfCString reportUrlPref(
1921 "browser.safebrowsing.provider.%s.dataSharingURL",
1922 PromiseFlatCString(aProvider).get());
1923
1924 nsCOMPtr<nsIURLFormatter> formatter(
1925 do_GetService("@mozilla.org/toolkit/URLFormatterService;1"));
1926 if (!formatter) {
1927 return NS_ERROR_UNEXPECTED;
1928 }
1929
1930 nsString urlStr;
1931 nsresult rv =
1932 formatter->FormatURLPref(NS_ConvertUTF8toUTF16(reportUrlPref), urlStr);
1933 NS_ENSURE_SUCCESS(rv, rv);
1934
1935 if (urlStr.IsEmpty() || u"about:blank"_ns.Equals(urlStr)) {
1936 LOG(("%s is missing a ThreatHit data reporting URL.",
1937 PromiseFlatCString(aProvider).get()));
1938 return NS_OK;
1939 }
1940
1941 nsUrlClassifierUtils* utilsService = nsUrlClassifierUtils::GetInstance();
1942 if (NS_WARN_IF(!utilsService)) {
1943 return NS_ERROR_FAILURE;
1944 }
1945
1946 nsAutoCString reportBody;
1947 rv =
1948 utilsService->MakeThreatHitReport(aChannel, aList, aFullHash, reportBody);
1949 NS_ENSURE_SUCCESS(rv, rv);
1950 nsCOMPtr<nsIStringInputStream> sis(
1951 do_CreateInstance(NS_STRINGINPUTSTREAM_CONTRACTID));
1952 rv = sis->SetData(reportBody.get(), reportBody.Length());
1953 NS_ENSURE_SUCCESS(rv, rv);
1954
1955 LOG(("Sending the following ThreatHit report to %s about %s: %s",
1956 PromiseFlatCString(aProvider).get(), PromiseFlatCString(aList).get(),
1957 reportBody.get()));
1958
1959 nsCOMPtr<nsIURI> reportURI;
1960 rv = NS_NewURI(getter_AddRefs(reportURI), urlStr);
1961 NS_ENSURE_SUCCESS(rv, rv);
1962
1963 uint32_t loadFlags = nsIRequest::LOAD_ANONYMOUS | // no cookies
1964 nsIChannel::INHIBIT_CACHING |
1965 nsIChannel::LOAD_BYPASS_CACHE;
1966
1967 nsCOMPtr<nsIChannel> reportChannel;
1968 rv = NS_NewChannel(getter_AddRefs(reportChannel), reportURI,
1969 nsContentUtils::GetSystemPrincipal(),
1970 nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL,
1971 nsIContentPolicy::TYPE_OTHER,
1972 nullptr, // nsICookieJarSettings
1973 nullptr, // aPerformanceStorage
1974 nullptr, // aLoadGroup
1975 nullptr, loadFlags);
1976 NS_ENSURE_SUCCESS(rv, rv);
1977
1978 nsCOMPtr<nsILoadInfo> loadInfo = reportChannel->LoadInfo();
1979 mozilla::OriginAttributes attrs;
1980 attrs.mFirstPartyDomain.AssignLiteral(NECKO_SAFEBROWSING_FIRST_PARTY_DOMAIN);
1981 loadInfo->SetOriginAttributes(attrs);
1982
1983 nsCOMPtr<nsIUploadChannel> uploadChannel(do_QueryInterface(reportChannel));
1984 NS_ENSURE_TRUE(uploadChannel, NS_ERROR_FAILURE);
1985 rv = uploadChannel->SetUploadStream(sis, "application/x-protobuf"_ns, -1);
1986 NS_ENSURE_SUCCESS(rv, rv);
1987
1988 nsCOMPtr<nsIHttpChannel> httpChannel(do_QueryInterface(reportChannel));
1989 NS_ENSURE_TRUE(httpChannel, NS_ERROR_FAILURE);
1990 rv = httpChannel->SetRequestMethod("POST"_ns);
1991 NS_ENSURE_SUCCESS(rv, rv);
1992 // Disable keepalive.
1993 rv = httpChannel->SetRequestHeader("Connection"_ns, "close"_ns, false);
1994 NS_ENSURE_SUCCESS(rv, rv);
1995
1996 RefPtr<ThreatHitReportListener> listener = new ThreatHitReportListener();
1997 rv = reportChannel->AsyncOpen(listener);
1998 if (NS_FAILED(rv)) {
1999 LOG(("Failure to send Safe Browsing ThreatHit report"));
2000 return rv;
2001 }
2002
2003 return NS_OK;
2004 }
2005
2006 NS_IMETHODIMP
Lookup(nsIPrincipal * aPrincipal,const nsACString & aTables,nsIUrlClassifierCallback * aCallback)2007 nsUrlClassifierDBService::Lookup(nsIPrincipal* aPrincipal,
2008 const nsACString& aTables,
2009 nsIUrlClassifierCallback* aCallback) {
2010 // We don't expect someone with SystemPrincipal calls this API(See Bug
2011 // 813897).
2012 MOZ_ASSERT(!aPrincipal->IsSystemPrincipal());
2013
2014 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2015
2016 nsTArray<nsCString> tableArray;
2017 Classifier::SplitTables(aTables, tableArray);
2018
2019 nsCOMPtr<nsIUrlClassifierFeature> feature;
2020 nsresult rv = CreateFeatureWithTables(
2021 "lookup"_ns, tableArray, nsTArray<nsCString>(), getter_AddRefs(feature));
2022 NS_ENSURE_SUCCESS(rv, rv);
2023
2024 nsCOMPtr<nsIURI> uri;
2025 // Casting to BasePrincipal, as we can't get InnerMost URI otherwise
2026 auto* basePrincipal = BasePrincipal::Cast(aPrincipal);
2027 rv = basePrincipal->GetURI(getter_AddRefs(uri));
2028 NS_ENSURE_SUCCESS(rv, rv);
2029 NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE);
2030
2031 nsTArray<RefPtr<nsIUrlClassifierFeature>> features;
2032 features.AppendElement(feature.get());
2033
2034 // Let's keep the features alive and release them on the correct thread.
2035 RefPtr<FeatureHolder> holder =
2036 FeatureHolder::Create(uri, features, nsIUrlClassifierFeature::blocklist);
2037 if (NS_WARN_IF(!holder)) {
2038 return NS_ERROR_FAILURE;
2039 }
2040
2041 uri = NS_GetInnermostURI(uri);
2042 NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE);
2043
2044 nsUrlClassifierUtils* utilsService = nsUrlClassifierUtils::GetInstance();
2045 if (NS_WARN_IF(!utilsService)) {
2046 return NS_ERROR_FAILURE;
2047 }
2048
2049 nsAutoCString key;
2050 // Canonicalize the url
2051 rv = utilsService->GetKeyForURI(uri, key);
2052 NS_ENSURE_SUCCESS(rv, rv);
2053
2054 return LookupURI(key, holder, aCallback);
2055 }
2056
LookupURI(const nsACString & aKey,FeatureHolder * aHolder,nsIUrlClassifierCallback * aCallback)2057 nsresult nsUrlClassifierDBService::LookupURI(
2058 const nsACString& aKey, FeatureHolder* aHolder,
2059 nsIUrlClassifierCallback* aCallback) {
2060 MOZ_ASSERT(aHolder);
2061 MOZ_ASSERT(aCallback);
2062
2063 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2064
2065 // Create an nsUrlClassifierLookupCallback object. This object will
2066 // take care of confirming partial hash matches if necessary before
2067 // calling the client's callback.
2068 nsCOMPtr<nsIUrlClassifierLookupCallback> callback =
2069 new nsUrlClassifierLookupCallback(this, aCallback);
2070
2071 nsCOMPtr<nsIUrlClassifierLookupCallback> proxyCallback =
2072 new UrlClassifierLookupCallbackProxy(callback);
2073
2074 // Queue this lookup and call the lookup function to flush the queue if
2075 // necessary.
2076 nsresult rv = mWorker->QueueLookup(aKey, aHolder, proxyCallback);
2077 NS_ENSURE_SUCCESS(rv, rv);
2078
2079 // This seems to just call HandlePendingLookups.
2080 nsAutoCString dummy;
2081 return mWorkerProxy->Lookup(nullptr, dummy, nullptr);
2082 }
2083
2084 NS_IMETHODIMP
GetTables(nsIUrlClassifierCallback * c)2085 nsUrlClassifierDBService::GetTables(nsIUrlClassifierCallback* c) {
2086 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2087
2088 // The proxy callback uses the current thread.
2089 nsCOMPtr<nsIUrlClassifierCallback> proxyCallback =
2090 new UrlClassifierCallbackProxy(c);
2091
2092 return mWorkerProxy->GetTables(proxyCallback);
2093 }
2094
2095 NS_IMETHODIMP
SetHashCompleter(const nsACString & tableName,nsIUrlClassifierHashCompleter * completer)2096 nsUrlClassifierDBService::SetHashCompleter(
2097 const nsACString& tableName, nsIUrlClassifierHashCompleter* completer) {
2098 if (completer) {
2099 mCompleters.InsertOrUpdate(tableName, completer);
2100 } else {
2101 mCompleters.Remove(tableName);
2102 }
2103 ClearLastResults();
2104 return NS_OK;
2105 }
2106
2107 NS_IMETHODIMP
ClearLastResults()2108 nsUrlClassifierDBService::ClearLastResults() {
2109 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2110
2111 return mWorkerProxy->ClearLastResults();
2112 }
2113
2114 NS_IMETHODIMP
BeginUpdate(nsIUrlClassifierUpdateObserver * observer,const nsACString & updateTables)2115 nsUrlClassifierDBService::BeginUpdate(nsIUrlClassifierUpdateObserver* observer,
2116 const nsACString& updateTables) {
2117 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2118
2119 if (mInUpdate) {
2120 LOG(("Already updating, not available"));
2121 return NS_ERROR_NOT_AVAILABLE;
2122 }
2123 if (mWorker->IsBusyUpdating()) {
2124 // |mInUpdate| used to work well because "notifying update observer"
2125 // is synchronously done in Worker::FinishUpdate(). Even if the
2126 // update observer hasn't been notified at this point, we can still
2127 // dispatch BeginUpdate() since it will NOT be run until the
2128 // previous Worker::FinishUpdate() returns.
2129 //
2130 // However, some tasks in Worker::FinishUpdate() have been moved to
2131 // another thread. The update observer will NOT be notified when
2132 // Worker::FinishUpdate() returns. If we only check |mInUpdate|,
2133 // the following sequence might happen on worker thread:
2134 //
2135 // Worker::FinishUpdate() // for update 1
2136 // Worker::BeginUpdate() // for update 2
2137 // Worker::NotifyUpdateObserver() // for update 1
2138 //
2139 // So, we have to find out a way to reject BeginUpdate() right here
2140 // if the previous update observer hasn't been notified.
2141 //
2142 // Directly probing the worker's state is the most lightweight solution.
2143 // No lock is required since Worker::BeginUpdate() and
2144 // Worker::NotifyUpdateObserver() are by nature mutual exclusive.
2145 // (both run on worker thread.)
2146 LOG(("The previous update observer hasn't been notified."));
2147 return NS_ERROR_NOT_AVAILABLE;
2148 }
2149
2150 mInUpdate = true;
2151
2152 // The proxy observer uses the current thread
2153 nsCOMPtr<nsIUrlClassifierUpdateObserver> proxyObserver =
2154 new UrlClassifierUpdateObserverProxy(observer);
2155
2156 return mWorkerProxy->BeginUpdate(proxyObserver, updateTables);
2157 }
2158
2159 NS_IMETHODIMP
BeginStream(const nsACString & table)2160 nsUrlClassifierDBService::BeginStream(const nsACString& table) {
2161 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2162
2163 return mWorkerProxy->BeginStream(table);
2164 }
2165
2166 NS_IMETHODIMP
UpdateStream(const nsACString & aUpdateChunk)2167 nsUrlClassifierDBService::UpdateStream(const nsACString& aUpdateChunk) {
2168 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2169
2170 return mWorkerProxy->UpdateStream(aUpdateChunk);
2171 }
2172
2173 NS_IMETHODIMP
FinishStream()2174 nsUrlClassifierDBService::FinishStream() {
2175 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2176
2177 return mWorkerProxy->FinishStream();
2178 }
2179
2180 NS_IMETHODIMP
FinishUpdate()2181 nsUrlClassifierDBService::FinishUpdate() {
2182 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2183
2184 mInUpdate = false;
2185
2186 return mWorkerProxy->FinishUpdate();
2187 }
2188
2189 NS_IMETHODIMP
CancelUpdate()2190 nsUrlClassifierDBService::CancelUpdate() {
2191 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2192
2193 mInUpdate = false;
2194
2195 return mWorkerProxy->CancelUpdate();
2196 }
2197
2198 NS_IMETHODIMP
ResetDatabase()2199 nsUrlClassifierDBService::ResetDatabase() {
2200 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2201
2202 if (mWorker->IsBusyUpdating()) {
2203 LOG(("Failed to ResetDatabase because of the unfinished update."));
2204 return NS_ERROR_FAILURE;
2205 }
2206
2207 return mWorkerProxy->ResetDatabase();
2208 }
2209
2210 NS_IMETHODIMP
ReloadDatabase()2211 nsUrlClassifierDBService::ReloadDatabase() {
2212 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2213
2214 if (mWorker->IsBusyUpdating()) {
2215 LOG(("Failed to ReloadDatabase because of the unfinished update."));
2216 return NS_ERROR_FAILURE;
2217 }
2218
2219 return mWorkerProxy->ReloadDatabase();
2220 }
2221
2222 NS_IMETHODIMP
ClearCache()2223 nsUrlClassifierDBService::ClearCache() {
2224 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2225
2226 return mWorkerProxy->ClearCache();
2227 }
2228
2229 NS_IMETHODIMP
GetCacheInfo(const nsACString & aTable,nsIUrlClassifierGetCacheCallback * aCallback)2230 nsUrlClassifierDBService::GetCacheInfo(
2231 const nsACString& aTable, nsIUrlClassifierGetCacheCallback* aCallback) {
2232 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2233
2234 return mWorkerProxy->GetCacheInfo(aTable, aCallback);
2235 }
2236
CacheCompletions(const ConstCacheResultArray & results)2237 nsresult nsUrlClassifierDBService::CacheCompletions(
2238 const ConstCacheResultArray& results) {
2239 NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
2240
2241 return mWorkerProxy->CacheCompletions(results);
2242 }
2243
CanComplete(const nsACString & aTableName)2244 bool nsUrlClassifierDBService::CanComplete(const nsACString& aTableName) {
2245 return !mDisallowCompletionsTables.Contains(aTableName);
2246 }
2247
GetCompleter(const nsACString & tableName,nsIUrlClassifierHashCompleter ** completer)2248 bool nsUrlClassifierDBService::GetCompleter(
2249 const nsACString& tableName, nsIUrlClassifierHashCompleter** completer) {
2250 // If we have specified a completer, go ahead and query it. This is only
2251 // used by tests.
2252 if (mCompleters.Get(tableName, completer)) {
2253 return true;
2254 }
2255
2256 if (!CanComplete(tableName)) {
2257 return false;
2258 }
2259
2260 // Otherwise, call gethash to find the hash completions.
2261 return NS_SUCCEEDED(
2262 CallGetService(NS_URLCLASSIFIERHASHCOMPLETER_CONTRACTID, completer));
2263 }
2264
2265 NS_IMETHODIMP
Observe(nsISupports * aSubject,const char * aTopic,const char16_t * aData)2266 nsUrlClassifierDBService::Observe(nsISupports* aSubject, const char* aTopic,
2267 const char16_t* aData) {
2268 if (!strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) {
2269 ReadDisallowCompletionsTablesFromPrefs();
2270 } else if (!strcmp(aTopic, "quit-application")) {
2271 // Tell the update thread to finish as soon as possible.
2272 gShuttingDownThread = true;
2273
2274 // The code in ::Shutdown() is run on a 'profile-before-change' event and
2275 // ensures that objects are freed by blocking on this freeing.
2276 // We can however speed up the shutdown time by using the worker thread to
2277 // release, in an earlier event, any objects that cannot affect an ongoing
2278 // update on the update thread.
2279 PreShutdown();
2280 } else if (!strcmp(aTopic, "profile-before-change")) {
2281 gShuttingDownThread = true;
2282 Shutdown();
2283 } else {
2284 return NS_ERROR_UNEXPECTED;
2285 }
2286
2287 return NS_OK;
2288 }
2289
2290 // Post a PreShutdown task to worker thread to release objects without blocking
2291 // main-thread. Notice that shutdown process may still be blocked by PreShutdown
2292 // task when ::Shutdown() is executed and synchronously waits for worker thread
2293 // to finish PreShutdown event.
PreShutdown()2294 nsresult nsUrlClassifierDBService::PreShutdown() {
2295 MOZ_ASSERT(XRE_IsParentProcess());
2296
2297 if (mWorkerProxy) {
2298 mWorkerProxy->PreShutdown();
2299 }
2300
2301 return NS_OK;
2302 }
2303
2304 // Join the background thread if it exists.
Shutdown()2305 nsresult nsUrlClassifierDBService::Shutdown() {
2306 LOG(("shutting down db service\n"));
2307 MOZ_ASSERT(XRE_IsParentProcess());
2308
2309 if (!gDbBackgroundThread) {
2310 return NS_OK;
2311 }
2312
2313 Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_SHUTDOWN_TIME> timer;
2314
2315 mCompleters.Clear();
2316
2317 nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
2318 if (prefs) {
2319 prefs->RemoveObserver(DISALLOW_COMPLETION_TABLE_PREF, this);
2320 }
2321
2322 // 1. Synchronize with worker thread and update thread by
2323 // *synchronously* dispatching an event to worker thread
2324 // for shutting down the update thread. The reason not
2325 // shutting down update thread directly from main thread
2326 // is to avoid racing for Classifier::mUpdateThread
2327 // between main thread and the worker thread. (Both threads
2328 // would access Classifier::mUpdateThread.)
2329 // This event is dispatched unconditionally to avoid
2330 // accessing mWorker->mClassifier on the main thread, which
2331 // would be a race.
2332 using Worker = nsUrlClassifierDBServiceWorker;
2333 RefPtr<nsIRunnable> r = NewRunnableMethod(
2334 "nsUrlClassifierDBServiceWorker::FlushAndDisableAsyncUpdate", mWorker,
2335 &Worker::FlushAndDisableAsyncUpdate);
2336 SyncRunnable::DispatchToThread(gDbBackgroundThread, r);
2337 // At this point the update thread has been shut down and
2338 // the worker thread should only have at most one event,
2339 // which is the callback event.
2340
2341 // 2. Send CancelUpdate() event to notify the dangling update.
2342 // (i.e. BeginUpdate is called but FinishUpdate is not.)
2343 // and CloseDb() to clear mClassifier. They will be the last two
2344 // events on the worker thread in the shutdown process.
2345 DebugOnly<nsresult> rv;
2346 rv = mWorkerProxy->CancelUpdate();
2347 MOZ_ASSERT(NS_SUCCEEDED(rv), "failed to post 'cancel update' event");
2348 rv = mWorkerProxy->CloseDb();
2349 MOZ_ASSERT(NS_SUCCEEDED(rv), "failed to post 'close db' event");
2350 mWorkerProxy = nullptr;
2351
2352 // 3. Invalidate XPCOM APIs by nulling out gDbBackgroundThread
2353 // since every API checks gDbBackgroundThread first. This has
2354 // to be done before calling nsIThread.shutdown because it
2355 // will cause the pending events on the joining thread to
2356 // be processed.
2357 nsIThread* backgroundThread = nullptr;
2358 std::swap(backgroundThread, gDbBackgroundThread);
2359
2360 // 4. Wait until the worker thread is down.
2361 if (backgroundThread) {
2362 backgroundThread->Shutdown();
2363 NS_RELEASE(backgroundThread);
2364 }
2365
2366 mWorker = nullptr;
2367 return NS_OK;
2368 }
2369
BackgroundThread()2370 nsIThread* nsUrlClassifierDBService::BackgroundThread() {
2371 return gDbBackgroundThread;
2372 }
2373
2374 // static
ShutdownHasStarted()2375 bool nsUrlClassifierDBService::ShutdownHasStarted() {
2376 return gShuttingDownThread;
2377 }
2378
2379 // static
GetWorker()2380 nsUrlClassifierDBServiceWorker* nsUrlClassifierDBService::GetWorker() {
2381 nsresult rv;
2382 RefPtr<nsUrlClassifierDBService> service =
2383 nsUrlClassifierDBService::GetInstance(&rv);
2384 if (!service) {
2385 return nullptr;
2386 }
2387
2388 return service->mWorker;
2389 }
2390
2391 NS_IMETHODIMP
AsyncClassifyLocalWithFeatures(nsIURI * aURI,const nsTArray<RefPtr<nsIUrlClassifierFeature>> & aFeatures,nsIUrlClassifierFeature::listType aListType,nsIUrlClassifierFeatureCallback * aCallback)2392 nsUrlClassifierDBService::AsyncClassifyLocalWithFeatures(
2393 nsIURI* aURI, const nsTArray<RefPtr<nsIUrlClassifierFeature>>& aFeatures,
2394 nsIUrlClassifierFeature::listType aListType,
2395 nsIUrlClassifierFeatureCallback* aCallback) {
2396 MOZ_ASSERT(NS_IsMainThread());
2397
2398 if (gShuttingDownThread) {
2399 return NS_ERROR_ABORT;
2400 }
2401
2402 nsCOMPtr<nsIURI> uri = NS_GetInnermostURI(aURI);
2403 NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE);
2404
2405 // Let's try to use the preferences.
2406 if (AsyncClassifyLocalWithFeaturesUsingPreferences(uri, aFeatures, aListType,
2407 aCallback)) {
2408 return NS_OK;
2409 }
2410
2411 nsUrlClassifierUtils* utilsService = nsUrlClassifierUtils::GetInstance();
2412 if (NS_WARN_IF(!utilsService)) {
2413 return NS_ERROR_FAILURE;
2414 }
2415
2416 nsAutoCString key;
2417 // Canonicalize the url
2418 nsresult rv = utilsService->GetKeyForURI(uri, key);
2419 NS_ENSURE_SUCCESS(rv, rv);
2420
2421 if (XRE_IsContentProcess()) {
2422 using namespace mozilla::dom;
2423 using namespace mozilla::ipc;
2424
2425 ContentChild* content = ContentChild::GetSingleton();
2426 if (NS_WARN_IF(!content || content->IsShuttingDown())) {
2427 return NS_ERROR_FAILURE;
2428 }
2429
2430 auto actor = new URLClassifierLocalChild();
2431
2432 nsTArray<IPCURLClassifierFeature> ipcFeatures;
2433 for (nsIUrlClassifierFeature* feature : aFeatures) {
2434 nsAutoCString name;
2435 rv = feature->GetName(name);
2436 if (NS_WARN_IF(NS_FAILED(rv))) {
2437 continue;
2438 }
2439
2440 nsTArray<nsCString> tables;
2441 rv = feature->GetTables(aListType, tables);
2442 if (NS_WARN_IF(NS_FAILED(rv))) {
2443 continue;
2444 }
2445
2446 nsAutoCString exceptionHostList;
2447 if (aListType == nsIUrlClassifierFeature::blocklist) {
2448 rv = feature->GetExceptionHostList(exceptionHostList);
2449 if (NS_WARN_IF(NS_FAILED(rv))) {
2450 continue;
2451 }
2452 }
2453
2454 ipcFeatures.AppendElement(
2455 IPCURLClassifierFeature(name, tables, exceptionHostList));
2456 }
2457
2458 if (!content->SendPURLClassifierLocalConstructor(actor, aURI,
2459 ipcFeatures)) {
2460 return NS_ERROR_FAILURE;
2461 }
2462
2463 actor->SetFeaturesAndCallback(aFeatures, aCallback);
2464 return NS_OK;
2465 }
2466
2467 using namespace mozilla::Telemetry;
2468 auto startTime = TimeStamp::Now(); // For telemetry.
2469
2470 // Let's keep the features alive and release them on the correct thread.
2471 RefPtr<FeatureHolder> holder =
2472 FeatureHolder::Create(aURI, aFeatures, aListType);
2473 if (NS_WARN_IF(!holder)) {
2474 return NS_ERROR_FAILURE;
2475 }
2476
2477 auto worker = mWorker;
2478
2479 // Since aCallback will be passed around threads...
2480 nsMainThreadPtrHandle<nsIUrlClassifierFeatureCallback> callback(
2481 new nsMainThreadPtrHolder<nsIUrlClassifierFeatureCallback>(
2482 "nsIURIClassifierFeatureCallback", aCallback));
2483
2484 nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction(
2485 "nsUrlClassifierDBService::AsyncClassifyLocalWithFeatures",
2486 [worker, key, holder, callback, startTime]() -> void {
2487 holder->DoLocalLookup(key, worker);
2488
2489 nsCOMPtr<nsIRunnable> cbRunnable = NS_NewRunnableFunction(
2490 "nsUrlClassifierDBService::AsyncClassifyLocalWithFeatures",
2491 [callback, holder, startTime]() -> void {
2492 // Measure the time diff between calling and callback.
2493 AccumulateTimeDelta(
2494 Telemetry::URLCLASSIFIER_ASYNC_CLASSIFYLOCAL_TIME, startTime);
2495
2496 nsTArray<RefPtr<nsIUrlClassifierFeatureResult>> results;
2497 holder->GetResults(results);
2498
2499 // |callback| is captured as const value so ...
2500 auto cb =
2501 const_cast<nsIUrlClassifierFeatureCallback*>(callback.get());
2502 cb->OnClassifyComplete(results);
2503 });
2504
2505 NS_DispatchToMainThread(cbRunnable);
2506 });
2507
2508 return gDbBackgroundThread->Dispatch(r, NS_DISPATCH_NORMAL);
2509 }
2510
AsyncClassifyLocalWithFeaturesUsingPreferences(nsIURI * aURI,const nsTArray<RefPtr<nsIUrlClassifierFeature>> & aFeatures,nsIUrlClassifierFeature::listType aListType,nsIUrlClassifierFeatureCallback * aCallback)2511 bool nsUrlClassifierDBService::AsyncClassifyLocalWithFeaturesUsingPreferences(
2512 nsIURI* aURI, const nsTArray<RefPtr<nsIUrlClassifierFeature>>& aFeatures,
2513 nsIUrlClassifierFeature::listType aListType,
2514 nsIUrlClassifierFeatureCallback* aCallback) {
2515 MOZ_ASSERT(NS_IsMainThread());
2516
2517 nsAutoCString host;
2518 nsresult rv = aURI->GetHost(host);
2519 if (NS_WARN_IF(NS_FAILED(rv))) {
2520 return false;
2521 }
2522
2523 nsTArray<RefPtr<nsIUrlClassifierFeatureResult>> results;
2524
2525 // Let's see if we have special entries set by prefs.
2526 for (nsIUrlClassifierFeature* feature : aFeatures) {
2527 bool found = false;
2528
2529 nsAutoCString tableName;
2530 rv = feature->HasHostInPreferences(host, aListType, tableName, &found);
2531 NS_ENSURE_SUCCESS(rv, false);
2532
2533 if (found) {
2534 MOZ_ASSERT(!tableName.IsEmpty());
2535 LOG(("URI found in preferences. Table: %s", tableName.get()));
2536
2537 RefPtr<mozilla::net::UrlClassifierFeatureResult> result =
2538 new mozilla::net::UrlClassifierFeatureResult(aURI, feature,
2539 tableName);
2540 results.AppendElement(result);
2541 }
2542 }
2543
2544 if (results.IsEmpty()) {
2545 return false;
2546 }
2547
2548 // If we have some match using the preferences, we don't need to continue.
2549 nsCOMPtr<nsIUrlClassifierFeatureCallback> callback(aCallback);
2550 nsCOMPtr<nsIRunnable> cbRunnable = NS_NewRunnableFunction(
2551 "nsUrlClassifierDBService::AsyncClassifyLocalWithFeatures",
2552 [callback, results = std::move(results)]() {
2553 callback->OnClassifyComplete(results);
2554 });
2555
2556 NS_DispatchToMainThread(cbRunnable);
2557 return true;
2558 }
2559
2560 NS_IMETHODIMP
GetFeatureByName(const nsACString & aFeatureName,nsIUrlClassifierFeature ** aFeature)2561 nsUrlClassifierDBService::GetFeatureByName(const nsACString& aFeatureName,
2562 nsIUrlClassifierFeature** aFeature) {
2563 NS_ENSURE_ARG_POINTER(aFeature);
2564 nsCOMPtr<nsIUrlClassifierFeature> feature =
2565 mozilla::net::UrlClassifierFeatureFactory::GetFeatureByName(aFeatureName);
2566 if (NS_WARN_IF(!feature)) {
2567 return NS_ERROR_FAILURE;
2568 }
2569
2570 feature.forget(aFeature);
2571 return NS_OK;
2572 }
2573
2574 NS_IMETHODIMP
GetFeatureNames(nsTArray<nsCString> & aArray)2575 nsUrlClassifierDBService::GetFeatureNames(nsTArray<nsCString>& aArray) {
2576 mozilla::net::UrlClassifierFeatureFactory::GetFeatureNames(aArray);
2577 return NS_OK;
2578 }
2579
2580 NS_IMETHODIMP
CreateFeatureWithTables(const nsACString & aName,const nsTArray<nsCString> & aBlocklistTables,const nsTArray<nsCString> & aEntitylistTables,nsIUrlClassifierFeature ** aFeature)2581 nsUrlClassifierDBService::CreateFeatureWithTables(
2582 const nsACString& aName, const nsTArray<nsCString>& aBlocklistTables,
2583 const nsTArray<nsCString>& aEntitylistTables,
2584 nsIUrlClassifierFeature** aFeature) {
2585 NS_ENSURE_ARG_POINTER(aFeature);
2586 nsCOMPtr<nsIUrlClassifierFeature> feature =
2587 mozilla::net::UrlClassifierFeatureFactory::CreateFeatureWithTables(
2588 aName, aBlocklistTables, aEntitylistTables);
2589 if (NS_WARN_IF(!feature)) {
2590 return NS_ERROR_FAILURE;
2591 }
2592
2593 feature.forget(aFeature);
2594 return NS_OK;
2595 }
2596