1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "components/safe_browsing/core/db/v4_get_hash_protocol_manager.h"
6 
7 #include <memory>
8 #include <vector>
9 
10 #include "base/base64.h"
11 #include "base/bind.h"
12 #include "base/run_loop.h"
13 #include "base/stl_util.h"
14 #include "base/strings/stringprintf.h"
15 #include "base/test/metrics/histogram_tester.h"
16 #include "base/test/simple_test_clock.h"
17 #include "base/time/time.h"
18 #include "build/build_config.h"
19 #include "components/safe_browsing/core/common/test_task_environment.h"
20 #include "components/safe_browsing/core/db/safebrowsing.pb.h"
21 #include "components/safe_browsing/core/db/util.h"
22 #include "components/safe_browsing/core/db/v4_test_util.h"
23 #include "net/base/escape.h"
24 #include "net/base/load_flags.h"
25 #include "net/base/net_errors.h"
26 #include "services/network/public/cpp/shared_url_loader_factory.h"
27 #include "services/network/public/cpp/weak_wrapper_shared_url_loader_factory.h"
28 #include "services/network/test/test_url_loader_factory.h"
29 #include "testing/platform_test.h"
30 
31 using base::Time;
32 using base::TimeDelta;
33 
34 namespace safe_browsing {
35 
36 namespace {
37 
38 struct KeyValue {
39   std::string key;
40   std::string value;
41 
KeyValuesafe_browsing::__anon967436c90111::KeyValue42   explicit KeyValue(const std::string key, const std::string value)
43       : key(key), value(value) {}
44   explicit KeyValue(const KeyValue& other) = default;
45 
46  private:
47   KeyValue();
48 };
49 
50 struct ResponseInfo {
51   FullHash full_hash;
52   ListIdentifier list_id;
53   std::vector<KeyValue> key_values;
54 
ResponseInfosafe_browsing::__anon967436c90111::ResponseInfo55   explicit ResponseInfo(FullHash full_hash, ListIdentifier list_id)
56       : full_hash(full_hash), list_id(list_id) {}
ResponseInfosafe_browsing::__anon967436c90111::ResponseInfo57   explicit ResponseInfo(const ResponseInfo& other)
58       : full_hash(other.full_hash),
59         list_id(other.list_id),
60         key_values(other.key_values) {}
61 
62  private:
63   ResponseInfo();
64 };
65 
66 }  // namespace
67 
68 class V4GetHashProtocolManagerTest : public PlatformTest {
69  public:
SetUp()70   void SetUp() override {
71     PlatformTest::SetUp();
72     task_environment_ = CreateTestTaskEnvironment();
73     callback_called_ = false;
74     test_shared_loader_factory_ =
75         base::MakeRefCounted<network::WeakWrapperSharedURLLoaderFactory>(
76             &test_url_loader_factory_);
77   }
78 
CreateProtocolManager()79   std::unique_ptr<V4GetHashProtocolManager> CreateProtocolManager() {
80     StoresToCheck stores_to_check(
81         {GetUrlMalwareId(), GetChromeUrlApiId(),
82          ListIdentifier(CHROME_PLATFORM, URL, SOCIAL_ENGINEERING),
83          ListIdentifier(CHROME_PLATFORM, URL, POTENTIALLY_HARMFUL_APPLICATION),
84          ListIdentifier(CHROME_PLATFORM, URL, SUBRESOURCE_FILTER)});
85     return V4GetHashProtocolManager::Create(test_shared_loader_factory_,
86                                             stores_to_check,
87                                             GetTestV4ProtocolConfig());
88   }
89 
SetupFetcherToReturnResponse(V4GetHashProtocolManager * pm,int net_error,int response_code,const std::string & data)90   static void SetupFetcherToReturnResponse(V4GetHashProtocolManager* pm,
91                                            int net_error,
92                                            int response_code,
93                                            const std::string& data) {
94     CHECK_EQ(pm->pending_hash_requests_.size(), 1u);
95     pm->OnURLLoaderCompleteInternal(
96         pm->pending_hash_requests_.begin()->second->loader.get(), net_error,
97         response_code, data);
98   }
99 
SetupFullHashFetcherToReturnResponse(V4GetHashProtocolManager * pm,const FullHash & full_hash,int net_error,int response_code,const std::string & data)100   static void SetupFullHashFetcherToReturnResponse(V4GetHashProtocolManager* pm,
101                                                    const FullHash& full_hash,
102                                                    int net_error,
103                                                    int response_code,
104                                                    const std::string& data) {
105     network::SimpleURLLoader* url_loader = nullptr;
106     for (const auto& pending_request : pm->pending_hash_requests_) {
107       if (pending_request.second->full_hash_to_store_and_hash_prefixes.count(
108               full_hash)) {
109         EXPECT_EQ(nullptr, url_loader);
110         url_loader = pending_request.second->loader.get();
111       }
112     }
113     ASSERT_TRUE(url_loader);
114 
115     pm->OnURLLoaderCompleteInternal(url_loader, net_error, response_code, data);
116   }
117 
SetupFetcherToReturnOKResponse(V4GetHashProtocolManager * pm,const std::vector<ResponseInfo> & infos)118   static void SetupFetcherToReturnOKResponse(
119       V4GetHashProtocolManager* pm,
120       const std::vector<ResponseInfo>& infos) {
121     SetupFetcherToReturnResponse(pm, net::OK, 200, GetV4HashResponse(infos));
122   }
123 
GetStockV4HashResponseInfos()124   static std::vector<ResponseInfo> GetStockV4HashResponseInfos() {
125     ResponseInfo info(FullHash("Everything's shiny, Cap'n."),
126                       GetChromeUrlApiId());
127     info.key_values.emplace_back("permission", "NOTIFICATIONS");
128     std::vector<ResponseInfo> infos;
129     infos.push_back(info);
130     return infos;
131   }
132 
GetStockV4HashResponse()133   static std::string GetStockV4HashResponse() {
134     return GetV4HashResponse(GetStockV4HashResponseInfos());
135   }
136 
SetTestClock(base::Time now,V4GetHashProtocolManager * pm)137   void SetTestClock(base::Time now, V4GetHashProtocolManager* pm) {
138     clock_.SetNow(now);
139     pm->SetClockForTests(&clock_);
140   }
141 
ValidateGetV4ApiResults(const ThreatMetadata & expected_md,const ThreatMetadata & actual_md)142   void ValidateGetV4ApiResults(const ThreatMetadata& expected_md,
143                                const ThreatMetadata& actual_md) {
144     EXPECT_EQ(expected_md, actual_md);
145     callback_called_ = true;
146   }
147 
ValidateGetV4HashResults(const std::vector<FullHashInfo> & expected_results,const std::vector<FullHashInfo> & actual_results)148   void ValidateGetV4HashResults(
149       const std::vector<FullHashInfo>& expected_results,
150       const std::vector<FullHashInfo>& actual_results) {
151     EXPECT_EQ(expected_results.size(), actual_results.size());
152     for (size_t i = 0; i < actual_results.size(); i++) {
153       EXPECT_TRUE(expected_results[i] == actual_results[i]);
154     }
155     callback_called_ = true;
156   }
157 
callback_called() const158   bool callback_called() const { return callback_called_; }
reset_callback_called()159   void reset_callback_called() { callback_called_ = false; }
160 
161  private:
GetV4HashResponse(std::vector<ResponseInfo> response_infos)162   static std::string GetV4HashResponse(
163       std::vector<ResponseInfo> response_infos) {
164     FindFullHashesResponse res;
165     res.mutable_negative_cache_duration()->set_seconds(600);
166     for (const ResponseInfo& info : response_infos) {
167       ThreatMatch* m = res.add_matches();
168       m->set_platform_type(info.list_id.platform_type());
169       m->set_threat_entry_type(info.list_id.threat_entry_type());
170       m->set_threat_type(info.list_id.threat_type());
171       m->mutable_cache_duration()->set_seconds(300);
172       m->mutable_threat()->set_hash(info.full_hash);
173 
174       for (const KeyValue& key_value : info.key_values) {
175         ThreatEntryMetadata::MetadataEntry* e =
176             m->mutable_threat_entry_metadata()->add_entries();
177         e->set_key(key_value.key);
178         e->set_value(key_value.value);
179       }
180     }
181 
182     // Serialize.
183     std::string res_data;
184     res.SerializeToString(&res_data);
185 
186     return res_data;
187   }
188 
189   bool callback_called_;
190   base::SimpleTestClock clock_;
191   network::TestURLLoaderFactory test_url_loader_factory_;
192   scoped_refptr<network::SharedURLLoaderFactory> test_shared_loader_factory_;
193   std::unique_ptr<base::test::TaskEnvironment> task_environment_;
194 };
195 
TEST_F(V4GetHashProtocolManagerTest,TestGetHashErrorHandlingNetwork)196 TEST_F(V4GetHashProtocolManagerTest, TestGetHashErrorHandlingNetwork) {
197   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
198 
199   FullHashToStoreAndHashPrefixesMap matched_locally;
200   matched_locally[FullHash("AHashFull")].emplace_back(GetUrlSocEngId(),
201                                                       HashPrefix("AHash"));
202   std::vector<FullHashInfo> expected_results;
203   pm->GetFullHashes(
204       matched_locally, {},
205       base::BindOnce(&V4GetHashProtocolManagerTest::ValidateGetV4HashResults,
206                      base::Unretained(this), expected_results));
207 
208   // Failed request status should result in error.
209   SetupFetcherToReturnResponse(pm.get(), net::ERR_CONNECTION_RESET, 200,
210                                GetStockV4HashResponse());
211 
212   // Should have recorded one error, but back off multiplier is unchanged.
213   EXPECT_EQ(1ul, pm->gethash_error_count_);
214   EXPECT_EQ(1ul, pm->gethash_back_off_mult_);
215   EXPECT_TRUE(callback_called());
216 }
217 
TEST_F(V4GetHashProtocolManagerTest,TestGetHashErrorHandlingResponseCode)218 TEST_F(V4GetHashProtocolManagerTest, TestGetHashErrorHandlingResponseCode) {
219   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
220 
221   FullHashToStoreAndHashPrefixesMap matched_locally;
222   matched_locally[FullHash("AHashFull")].emplace_back(GetUrlSocEngId(),
223                                                       HashPrefix("AHash"));
224   std::vector<FullHashInfo> expected_results;
225   pm->GetFullHashes(
226       matched_locally, {},
227       base::BindOnce(&V4GetHashProtocolManagerTest::ValidateGetV4HashResults,
228                      base::Unretained(this), expected_results));
229 
230   // Response code of anything other than 200 should result in error.
231   SetupFetcherToReturnResponse(pm.get(), net::OK, 204,
232                                GetStockV4HashResponse());
233 
234   // Should have recorded one error, but back off multiplier is unchanged.
235   EXPECT_EQ(1ul, pm->gethash_error_count_);
236   EXPECT_EQ(1ul, pm->gethash_back_off_mult_);
237   EXPECT_TRUE(callback_called());
238 }
239 
TEST_F(V4GetHashProtocolManagerTest,TestGetHashErrorHandlingParallelRequests)240 TEST_F(V4GetHashProtocolManagerTest, TestGetHashErrorHandlingParallelRequests) {
241   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
242   std::vector<FullHashInfo> empty_results;
243   base::HistogramTester histogram_tester;
244 
245   FullHashToStoreAndHashPrefixesMap matched_locally1;
246   matched_locally1[FullHash("AHash1Full")].emplace_back(GetUrlSocEngId(),
247                                                         HashPrefix("AHash1"));
248   pm->GetFullHashes(matched_locally1, {},
249                     base::BindRepeating(
250                         &V4GetHashProtocolManagerTest::ValidateGetV4HashResults,
251                         base::Unretained(this), empty_results));
252 
253   FullHashToStoreAndHashPrefixesMap matched_locally2;
254   matched_locally2[FullHash("AHash2Full")].emplace_back(GetUrlSocEngId(),
255                                                         HashPrefix("AHash2"));
256   pm->GetFullHashes(matched_locally2, {},
257                     base::BindRepeating(
258                         &V4GetHashProtocolManagerTest::ValidateGetV4HashResults,
259                         base::Unretained(this), empty_results));
260 
261   // Fail the first request.
262   SetupFullHashFetcherToReturnResponse(pm.get(), FullHash("AHash1Full"),
263                                        net::ERR_CONNECTION_RESET, 200,
264                                        GetStockV4HashResponse());
265 
266   // Should have recorded one error, but back off multiplier is unchanged.
267   EXPECT_EQ(1ul, pm->gethash_error_count_);
268   EXPECT_EQ(1ul, pm->gethash_back_off_mult_);
269   EXPECT_TRUE(callback_called());
270   histogram_tester.ExpectUniqueSample("SafeBrowsing.V4GetHash.Result",
271                                       V4OperationResult::NETWORK_ERROR, 1);
272 
273   reset_callback_called();
274 
275   // Comple the second request successfully.
276   SetupFullHashFetcherToReturnResponse(pm.get(), FullHash("AHash2Full"),
277                                        net::OK, 200, GetStockV4HashResponse());
278 
279   // Error counters are reset.
280   EXPECT_EQ(0ul, pm->gethash_error_count_);
281   EXPECT_EQ(1ul, pm->gethash_back_off_mult_);
282   EXPECT_TRUE(callback_called());
283   histogram_tester.ExpectBucketCount("SafeBrowsing.V4GetHash.Result",
284                                      V4OperationResult::STATUS_200, 1);
285 
286   reset_callback_called();
287 
288   // Start the third request.
289   FullHashToStoreAndHashPrefixesMap matched_locally3;
290   matched_locally3[FullHash("AHash3Full")].emplace_back(GetUrlSocEngId(),
291                                                         HashPrefix("AHash3"));
292   pm->GetFullHashes(matched_locally3, {},
293                     base::BindRepeating(
294                         &V4GetHashProtocolManagerTest::ValidateGetV4HashResults,
295                         base::Unretained(this), empty_results));
296 
297   // The request is not failed right away.
298   EXPECT_FALSE(callback_called());
299   // The request is not reported as MIN_WAIT_DURATION_ERROR.
300   histogram_tester.ExpectBucketCount("SafeBrowsing.V4GetHash.Result",
301                                      V4OperationResult::MIN_WAIT_DURATION_ERROR,
302                                      0);
303 }
304 
TEST_F(V4GetHashProtocolManagerTest,TestGetHashErrorHandlingOK)305 TEST_F(V4GetHashProtocolManagerTest, TestGetHashErrorHandlingOK) {
306   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
307 
308   base::Time now = base::Time::UnixEpoch();
309   SetTestClock(now, pm.get());
310 
311   HashPrefix prefix("Everything");
312   FullHash full_hash("Everything's shiny, Cap'n.");
313   FullHashToStoreAndHashPrefixesMap matched_locally;
314   matched_locally[full_hash].push_back(
315       StoreAndHashPrefix(GetChromeUrlApiId(), prefix));
316   std::vector<FullHashInfo> expected_results;
317   FullHashInfo fhi(full_hash, GetChromeUrlApiId(),
318                    now + base::TimeDelta::FromSeconds(300));
319   fhi.metadata.api_permissions.insert("NOTIFICATIONS");
320   expected_results.push_back(fhi);
321 
322   pm->GetFullHashes(
323       matched_locally, {},
324       base::BindOnce(&V4GetHashProtocolManagerTest::ValidateGetV4HashResults,
325                      base::Unretained(this), expected_results));
326 
327   SetupFetcherToReturnOKResponse(pm.get(), GetStockV4HashResponseInfos());
328 
329   // No error, back off multiplier is unchanged.
330   EXPECT_EQ(0ul, pm->gethash_error_count_);
331   EXPECT_EQ(1ul, pm->gethash_back_off_mult_);
332 
333   // Verify the state of the cache.
334   const FullHashCache* cache = pm->full_hash_cache_for_tests();
335   // Check the cache.
336   ASSERT_EQ(1u, cache->size());
337   EXPECT_EQ(1u, cache->count(prefix));
338   const CachedHashPrefixInfo& cached_result = cache->at(prefix);
339   EXPECT_EQ(cached_result.negative_expiry,
340             now + base::TimeDelta::FromSeconds(600));
341   ASSERT_EQ(1u, cached_result.full_hash_infos.size());
342   EXPECT_EQ(FullHash("Everything's shiny, Cap'n."),
343             cached_result.full_hash_infos[0].full_hash);
344   EXPECT_TRUE(callback_called());
345 }
346 
TEST_F(V4GetHashProtocolManagerTest,TestResultsNotCachedForNegativeCacheDuration)347 TEST_F(V4GetHashProtocolManagerTest,
348        TestResultsNotCachedForNegativeCacheDuration) {
349   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
350 
351   HashPrefix prefix("Everything");
352   std::vector<HashPrefix> prefixes_requested({prefix});
353   base::Time negative_cache_expire;
354   FullHash full_hash("Everything's shiny, Cap'n.");
355   std::vector<FullHashInfo> fhis;
356   fhis.emplace_back(full_hash, GetChromeUrlApiId(), base::Time::UnixEpoch());
357 
358   pm->UpdateCache(prefixes_requested, fhis, negative_cache_expire);
359 
360   // Verify the state of the cache.
361   const FullHashCache* cache = pm->full_hash_cache_for_tests();
362   // Check the cache.
363   EXPECT_EQ(0u, cache->size());
364 }
365 
TEST_F(V4GetHashProtocolManagerTest,TestGetHashRequest)366 TEST_F(V4GetHashProtocolManagerTest, TestGetHashRequest) {
367   FindFullHashesRequest req;
368   ThreatInfo* info = req.mutable_threat_info();
369 
370   const std::set<PlatformType> platform_types = {
371     GetCurrentPlatformType(),
372     CHROME_PLATFORM,
373   // TODO(crbug.com/1030487): This special case for Android will no longer be
374   // needed once GetCurrentPlatformType() returns ANDROID_PLATFORM on Android.
375 #if defined(OS_ANDROID)
376     ANDROID_PLATFORM,
377 #endif
378   };
379 
380   for (const PlatformType& p : platform_types) {
381     info->add_platform_types(p);
382   }
383 
384   info->add_threat_entry_types(URL);
385 
386   for (const ThreatType& tt : std::set<ThreatType>{
387            MALWARE_THREAT, SOCIAL_ENGINEERING, POTENTIALLY_HARMFUL_APPLICATION,
388            API_ABUSE, SUBRESOURCE_FILTER}) {
389     info->add_threat_types(tt);
390   }
391 
392   HashPrefix one = "hashone";
393   HashPrefix two = "hashtwo";
394   info->add_threat_entries()->set_hash(one);
395   info->add_threat_entries()->set_hash(two);
396 
397   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
398   req.mutable_client()->set_client_id(pm->config_.client_name);
399   req.mutable_client()->set_client_version(pm->config_.version);
400 
401   std::vector<std::string> client_states = {"client_state_1", "client_state_2"};
402   for (const auto& client_state : client_states) {
403     req.add_client_states(client_state);
404   }
405 
406   // Serialize and Base64 encode.
407   std::string req_data, req_base64;
408   req.SerializeToString(&req_data);
409   base::Base64Encode(req_data, &req_base64);
410 
411   std::vector<HashPrefix> prefixes_to_request = {one, two};
412   EXPECT_EQ(req_base64, pm->GetHashRequest(prefixes_to_request, client_states));
413 }
414 
TEST_F(V4GetHashProtocolManagerTest,TestParseHashResponse)415 TEST_F(V4GetHashProtocolManagerTest, TestParseHashResponse) {
416   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
417 
418   base::Time now = base::Time::UnixEpoch();
419   SetTestClock(now, pm.get());
420 
421   FullHash full_hash("Everything's shiny, Cap'n.");
422   FindFullHashesResponse res;
423   res.mutable_negative_cache_duration()->set_seconds(600);
424   res.mutable_minimum_wait_duration()->set_seconds(400);
425   ThreatMatch* m = res.add_matches();
426   m->set_threat_type(API_ABUSE);
427   // TODO(crbug.com/1030487): This special case for Android will no longer be
428   // needed once GetCurrentPlatformType() returns ANDROID_PLATFORM on Android.
429 #if defined(OS_ANDROID)
430   m->set_platform_type(ANDROID_PLATFORM);
431 #else
432   m->set_platform_type(GetCurrentPlatformType());
433 #endif
434   m->set_threat_entry_type(URL);
435   m->mutable_cache_duration()->set_seconds(300);
436   m->mutable_threat()->set_hash(full_hash);
437   ThreatEntryMetadata::MetadataEntry* e =
438       m->mutable_threat_entry_metadata()->add_entries();
439   e->set_key("permission");
440   e->set_value("NOTIFICATIONS");
441   // Add another ThreatMatch for a list we don't track. This response should
442   // get dropped.
443   m = res.add_matches();
444   m->set_threat_type(THREAT_TYPE_UNSPECIFIED);
445   m->set_platform_type(CHROME_PLATFORM);
446   m->set_threat_entry_type(URL);
447   m->mutable_cache_duration()->set_seconds(300);
448   m->mutable_threat()->set_hash(full_hash);
449 
450   // Serialize.
451   std::string res_data;
452   res.SerializeToString(&res_data);
453 
454   std::vector<FullHashInfo> full_hash_infos;
455   base::Time cache_expire;
456   EXPECT_TRUE(pm->ParseHashResponse(res_data, &full_hash_infos, &cache_expire));
457 
458   EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
459   // Even though the server responded with two ThreatMatch responses, one
460   // should have been dropped.
461   ASSERT_EQ(1ul, full_hash_infos.size());
462   const FullHashInfo& fhi = full_hash_infos[0];
463   EXPECT_EQ(full_hash, fhi.full_hash);
464   EXPECT_EQ(GetChromeUrlApiId(), fhi.list_id);
465   EXPECT_EQ(1ul, fhi.metadata.api_permissions.size());
466   EXPECT_EQ(1ul, fhi.metadata.api_permissions.count("NOTIFICATIONS"));
467   EXPECT_EQ(now + base::TimeDelta::FromSeconds(300), fhi.positive_expiry);
468   EXPECT_EQ(now + base::TimeDelta::FromSeconds(400), pm->next_gethash_time_);
469 }
470 
471 // Adds an entry with an ignored ThreatEntryType.
TEST_F(V4GetHashProtocolManagerTest,TestParseHashResponseWrongThreatEntryType)472 TEST_F(V4GetHashProtocolManagerTest,
473        TestParseHashResponseWrongThreatEntryType) {
474   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
475 
476   base::Time now = base::Time::UnixEpoch();
477   SetTestClock(now, pm.get());
478 
479   FindFullHashesResponse res;
480   res.mutable_negative_cache_duration()->set_seconds(600);
481   res.add_matches()->set_threat_entry_type(EXECUTABLE);
482 
483   // Serialize.
484   std::string res_data;
485   res.SerializeToString(&res_data);
486 
487   std::vector<FullHashInfo> full_hash_infos;
488   base::Time cache_expire;
489   EXPECT_FALSE(
490       pm->ParseHashResponse(res_data, &full_hash_infos, &cache_expire));
491 
492   EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
493   // There should be no hash results.
494   EXPECT_EQ(0ul, full_hash_infos.size());
495 }
496 
497 // Adds entries with a ThreatPatternType metadata.
TEST_F(V4GetHashProtocolManagerTest,TestParseHashThreatPatternType)498 TEST_F(V4GetHashProtocolManagerTest, TestParseHashThreatPatternType) {
499   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
500 
501   base::Time now = base::Time::UnixEpoch();
502   SetTestClock(now, pm.get());
503 
504   {
505     // Test social engineering pattern type.
506     FindFullHashesResponse se_res;
507     se_res.mutable_negative_cache_duration()->set_seconds(600);
508     ThreatMatch* se = se_res.add_matches();
509     se->set_threat_type(SOCIAL_ENGINEERING);
510     se->set_platform_type(CHROME_PLATFORM);
511     se->set_threat_entry_type(URL);
512     FullHash full_hash("Everything's shiny, Cap'n.");
513     se->mutable_threat()->set_hash(full_hash);
514     ThreatEntryMetadata::MetadataEntry* se_meta =
515         se->mutable_threat_entry_metadata()->add_entries();
516     se_meta->set_key("se_pattern_type");
517     se_meta->set_value("SOCIAL_ENGINEERING_LANDING");
518 
519     std::string se_data;
520     se_res.SerializeToString(&se_data);
521 
522     std::vector<FullHashInfo> full_hash_infos;
523     base::Time cache_expire;
524     EXPECT_TRUE(
525         pm->ParseHashResponse(se_data, &full_hash_infos, &cache_expire));
526     EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
527 
528     // Ensure that the threat remains valid since we found a full hash match,
529     // even though the metadata information could not be parsed correctly.
530     ASSERT_EQ(1ul, full_hash_infos.size());
531     const FullHashInfo& fhi = full_hash_infos[0];
532     EXPECT_EQ(full_hash, fhi.full_hash);
533     const ListIdentifier list_id(CHROME_PLATFORM, URL, SOCIAL_ENGINEERING);
534     EXPECT_EQ(list_id, fhi.list_id);
535     EXPECT_EQ(ThreatPatternType::SOCIAL_ENGINEERING_LANDING,
536               fhi.metadata.threat_pattern_type);
537   }
538 
539   {
540     // Test potentially harmful application pattern type.
541     FindFullHashesResponse pha_res;
542     pha_res.mutable_negative_cache_duration()->set_seconds(600);
543     ThreatMatch* pha = pha_res.add_matches();
544     pha->set_threat_type(POTENTIALLY_HARMFUL_APPLICATION);
545     pha->set_threat_entry_type(URL);
546     pha->set_platform_type(CHROME_PLATFORM);
547     FullHash full_hash("Not to fret.");
548     pha->mutable_threat()->set_hash(full_hash);
549     ThreatEntryMetadata::MetadataEntry* pha_meta =
550         pha->mutable_threat_entry_metadata()->add_entries();
551     pha_meta->set_key("pha_pattern_type");
552     pha_meta->set_value("LANDING");
553 
554     std::string pha_data;
555     pha_res.SerializeToString(&pha_data);
556     std::vector<FullHashInfo> full_hash_infos;
557     base::Time cache_expire;
558     EXPECT_TRUE(
559         pm->ParseHashResponse(pha_data, &full_hash_infos, &cache_expire));
560     EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
561 
562     ASSERT_EQ(1ul, full_hash_infos.size());
563     const FullHashInfo& fhi = full_hash_infos[0];
564     EXPECT_EQ(full_hash, fhi.full_hash);
565     const ListIdentifier list_id(CHROME_PLATFORM, URL,
566                                  POTENTIALLY_HARMFUL_APPLICATION);
567     EXPECT_EQ(list_id, fhi.list_id);
568     EXPECT_EQ(ThreatPatternType::MALWARE_LANDING,
569               fhi.metadata.threat_pattern_type);
570   }
571 
572   {
573     // Test invalid pattern type.
574     FullHash full_hash("Not to fret.");
575     FindFullHashesResponse invalid_res;
576     invalid_res.mutable_negative_cache_duration()->set_seconds(600);
577     ThreatMatch* invalid = invalid_res.add_matches();
578     invalid->set_threat_type(POTENTIALLY_HARMFUL_APPLICATION);
579     invalid->set_threat_entry_type(URL);
580     invalid->set_platform_type(CHROME_PLATFORM);
581     invalid->mutable_threat()->set_hash(full_hash);
582     ThreatEntryMetadata::MetadataEntry* invalid_meta =
583         invalid->mutable_threat_entry_metadata()->add_entries();
584     invalid_meta->set_key("pha_pattern_type");
585     invalid_meta->set_value("INVALIDE_VALUE");
586 
587     std::string invalid_data;
588     invalid_res.SerializeToString(&invalid_data);
589     std::vector<FullHashInfo> full_hash_infos;
590     base::Time cache_expire;
591     EXPECT_TRUE(
592         pm->ParseHashResponse(invalid_data, &full_hash_infos, &cache_expire));
593 
594     // Ensure that the threat remains valid since we found a full hash match,
595     // even though the metadata information could not be parsed correctly.
596     ASSERT_EQ(1ul, full_hash_infos.size());
597     const auto& fhi = full_hash_infos[0];
598     EXPECT_EQ(full_hash, fhi.full_hash);
599     EXPECT_EQ(
600         ListIdentifier(CHROME_PLATFORM, URL, POTENTIALLY_HARMFUL_APPLICATION),
601         fhi.list_id);
602     EXPECT_EQ(ThreatPatternType::NONE, fhi.metadata.threat_pattern_type);
603   }
604 }
605 
TEST_F(V4GetHashProtocolManagerTest,TestParseSubresourceFilterMetadata)606 TEST_F(V4GetHashProtocolManagerTest, TestParseSubresourceFilterMetadata) {
607   typedef SubresourceFilterLevel Level;
608   typedef SubresourceFilterType Type;
609   const struct {
610     const char* abusive_type;
611     const char* bas_type;
612     SubresourceFilterMatch expected_match;
613   } test_cases[] = {
614       {"warn",
615        "enforce",
616        {{Type::ABUSIVE, Level::WARN}, {Type::BETTER_ADS, Level::ENFORCE}}},
617       {nullptr, "warn", {{Type::BETTER_ADS, Level::WARN}}},
618       {"asdf",
619        "",
620        {{Type::ABUSIVE, Level::ENFORCE}, {Type::BETTER_ADS, Level::ENFORCE}}},
621       {"warn", nullptr, {{Type::ABUSIVE, Level::WARN}}},
622       {nullptr, nullptr, {}},
623       {"",
624        "",
625        {{Type::ABUSIVE, Level::ENFORCE}, {Type::BETTER_ADS, Level::ENFORCE}}},
626   };
627 
628   for (const auto& test_case : test_cases) {
629     SCOPED_TRACE(testing::Message() << "abusive: " << test_case.abusive_type
630                                     << " better ads: " << test_case.bas_type);
631     std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
632 
633     base::Time now = base::Time::UnixEpoch();
634     SetTestClock(now, pm.get());
635     FindFullHashesResponse sf_res;
636     sf_res.mutable_negative_cache_duration()->set_seconds(600);
637     ThreatMatch* sf = sf_res.add_matches();
638     sf->set_threat_type(SUBRESOURCE_FILTER);
639     sf->set_platform_type(CHROME_PLATFORM);
640     sf->set_threat_entry_type(URL);
641     FullHash full_hash("Everything's shiny, Cap'n.");
642     sf->mutable_threat()->set_hash(full_hash);
643 
644     // sf_absv.
645     if (test_case.abusive_type != nullptr) {
646       ThreatEntryMetadata::MetadataEntry* sf_absv =
647           sf->mutable_threat_entry_metadata()->add_entries();
648       sf_absv->set_key("sf_absv");
649       sf_absv->set_value(test_case.abusive_type);
650     }
651 
652     // sf_bas
653     if (test_case.bas_type != nullptr) {
654       ThreatEntryMetadata::MetadataEntry* sf_bas =
655           sf->mutable_threat_entry_metadata()->add_entries();
656       sf_bas->set_key("sf_bas");
657       sf_bas->set_value(test_case.bas_type);
658     }
659 
660     std::string sf_data;
661     sf_res.SerializeToString(&sf_data);
662 
663     std::vector<FullHashInfo> full_hash_infos;
664     base::Time cache_expire;
665     EXPECT_TRUE(
666         pm->ParseHashResponse(sf_data, &full_hash_infos, &cache_expire));
667     EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
668 
669     ASSERT_EQ(1ul, full_hash_infos.size());
670     const FullHashInfo& fhi = full_hash_infos[0];
671     EXPECT_EQ(full_hash, fhi.full_hash);
672     const ListIdentifier list_id(CHROME_PLATFORM, URL, SUBRESOURCE_FILTER);
673     EXPECT_EQ(list_id, fhi.list_id);
674     EXPECT_EQ(test_case.expected_match, fhi.metadata.subresource_filter_match);
675   }
676 }
677 
678 // Adds metadata with a key value that is not "permission".
TEST_F(V4GetHashProtocolManagerTest,TestParseHashResponseNonPermissionMetadata)679 TEST_F(V4GetHashProtocolManagerTest,
680        TestParseHashResponseNonPermissionMetadata) {
681   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
682 
683   base::Time now = base::Time::UnixEpoch();
684   SetTestClock(now, pm.get());
685 
686   FullHash full_hash("Not to fret.");
687   FindFullHashesResponse res;
688   res.mutable_negative_cache_duration()->set_seconds(600);
689   ThreatMatch* m = res.add_matches();
690   m->set_threat_type(API_ABUSE);
691   // TODO(crbug.com/1030487): This special case for Android will no longer be
692   // needed once GetCurrentPlatformType() returns ANDROID_PLATFORM on Android.
693 #if defined(OS_ANDROID)
694   m->set_platform_type(ANDROID_PLATFORM);
695 #else
696   m->set_platform_type(GetCurrentPlatformType());
697 #endif
698   m->set_threat_entry_type(URL);
699   m->mutable_threat()->set_hash(full_hash);
700   ThreatEntryMetadata::MetadataEntry* e =
701       m->mutable_threat_entry_metadata()->add_entries();
702   e->set_key("notpermission");
703   e->set_value("NOTGEOLOCATION");
704 
705   // Serialize.
706   std::string res_data;
707   res.SerializeToString(&res_data);
708 
709   std::vector<FullHashInfo> full_hash_infos;
710   base::Time cache_expire;
711   EXPECT_TRUE(pm->ParseHashResponse(res_data, &full_hash_infos, &cache_expire));
712 
713   EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
714   ASSERT_EQ(1ul, full_hash_infos.size());
715   const auto& fhi = full_hash_infos[0];
716   EXPECT_EQ(full_hash, fhi.full_hash);
717   EXPECT_EQ(GetChromeUrlApiId(), fhi.list_id);
718   EXPECT_TRUE(fhi.metadata.api_permissions.empty());
719 }
720 
TEST_F(V4GetHashProtocolManagerTest,TestParseHashResponseInconsistentThreatTypes)721 TEST_F(V4GetHashProtocolManagerTest,
722        TestParseHashResponseInconsistentThreatTypes) {
723   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
724 
725   FindFullHashesResponse res;
726   res.mutable_negative_cache_duration()->set_seconds(600);
727   ThreatMatch* m1 = res.add_matches();
728   m1->set_threat_type(API_ABUSE);
729   m1->set_platform_type(CHROME_PLATFORM);
730   m1->set_threat_entry_type(URL);
731   m1->mutable_threat()->set_hash(FullHash("Everything's shiny, Cap'n."));
732   m1->mutable_threat_entry_metadata()->add_entries();
733   ThreatMatch* m2 = res.add_matches();
734   m2->set_threat_type(MALWARE_THREAT);
735   m2->set_threat_entry_type(URL);
736   m2->mutable_threat()->set_hash(FullHash("Not to fret."));
737 
738   // Serialize.
739   std::string res_data;
740   res.SerializeToString(&res_data);
741 
742   std::vector<FullHashInfo> full_hash_infos;
743   base::Time cache_expire;
744   EXPECT_FALSE(
745       pm->ParseHashResponse(res_data, &full_hash_infos, &cache_expire));
746 }
747 
748 // Checks that results are looked up correctly in the cache.
TEST_F(V4GetHashProtocolManagerTest,GetCachedResults)749 TEST_F(V4GetHashProtocolManagerTest, GetCachedResults) {
750   base::Time now = base::Time::UnixEpoch();
751   FullHash full_hash("example");
752   HashPrefix prefix("exam");
753   FullHashToStoreAndHashPrefixesMap matched_locally;
754   matched_locally[full_hash].emplace_back(GetUrlMalwareId(), prefix);
755   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
756   FullHashCache* cache = pm->full_hash_cache_for_tests();
757 
758   {
759     std::vector<HashPrefix> prefixes_to_request;
760     std::vector<FullHashInfo> cached_full_hash_infos;
761     cache->clear();
762 
763     // Test with an empty cache. (Case: 2)
764     pm->GetFullHashCachedResults(matched_locally, now, &prefixes_to_request,
765                                  &cached_full_hash_infos);
766     EXPECT_TRUE(cache->empty());
767     ASSERT_EQ(1ul, prefixes_to_request.size());
768     EXPECT_EQ(prefix, prefixes_to_request[0]);
769     EXPECT_TRUE(cached_full_hash_infos.empty());
770   }
771 
772   {
773     std::vector<HashPrefix> prefixes_to_request;
774     std::vector<FullHashInfo> cached_full_hash_infos;
775     cache->clear();
776 
777     // Prefix has a cache entry but full hash is not there. (Case: 1-b-i)
778     CachedHashPrefixInfo* entry = &(*cache)[prefix];
779     entry->negative_expiry = now + base::TimeDelta::FromMinutes(5);
780     pm->GetFullHashCachedResults(matched_locally, now, &prefixes_to_request,
781                                  &cached_full_hash_infos);
782     EXPECT_TRUE(prefixes_to_request.empty());
783     EXPECT_TRUE(cached_full_hash_infos.empty());
784   }
785 
786   {
787     std::vector<HashPrefix> prefixes_to_request;
788     std::vector<FullHashInfo> cached_full_hash_infos;
789     cache->clear();
790 
791     // Expired negative cache entry. (Case: 1-b-ii)
792     CachedHashPrefixInfo* entry = &(*cache)[prefix];
793     entry->negative_expiry = now - base::TimeDelta::FromMinutes(5);
794     pm->GetFullHashCachedResults(matched_locally, now, &prefixes_to_request,
795                                  &cached_full_hash_infos);
796     ASSERT_EQ(1ul, prefixes_to_request.size());
797     EXPECT_EQ(prefix, prefixes_to_request[0]);
798     EXPECT_TRUE(cached_full_hash_infos.empty());
799   }
800 
801   {
802     std::vector<HashPrefix> prefixes_to_request;
803     std::vector<FullHashInfo> cached_full_hash_infos;
804     cache->clear();
805 
806     // Now put unexpired full hash in the cache. (Case: 1-a-i)
807     CachedHashPrefixInfo* entry = &(*cache)[prefix];
808     entry->negative_expiry = now + base::TimeDelta::FromMinutes(5);
809     entry->full_hash_infos.emplace_back(full_hash, GetUrlMalwareId(),
810                                         now + base::TimeDelta::FromMinutes(3));
811     pm->GetFullHashCachedResults(matched_locally, now, &prefixes_to_request,
812                                  &cached_full_hash_infos);
813     EXPECT_TRUE(prefixes_to_request.empty());
814     ASSERT_EQ(1ul, cached_full_hash_infos.size());
815     EXPECT_EQ(full_hash, cached_full_hash_infos[0].full_hash);
816   }
817 
818   {
819     std::vector<HashPrefix> prefixes_to_request;
820     std::vector<FullHashInfo> cached_full_hash_infos;
821     cache->clear();
822 
823     // Expire the full hash in the cache. (Case: 1-a-ii)
824     CachedHashPrefixInfo* entry = &(*cache)[prefix];
825     entry->negative_expiry = now + base::TimeDelta::FromMinutes(5);
826     entry->full_hash_infos.emplace_back(full_hash, GetUrlMalwareId(),
827                                         now - base::TimeDelta::FromMinutes(3));
828     pm->GetFullHashCachedResults(matched_locally, now, &prefixes_to_request,
829                                  &cached_full_hash_infos);
830     ASSERT_EQ(1ul, prefixes_to_request.size());
831     EXPECT_EQ(prefix, prefixes_to_request[0]);
832     EXPECT_TRUE(cached_full_hash_infos.empty());
833   }
834 }
835 
TEST_F(V4GetHashProtocolManagerTest,TestUpdatesAreMerged)836 TEST_F(V4GetHashProtocolManagerTest, TestUpdatesAreMerged) {
837   // We'll add one of the requested FullHashInfo objects into the cache, and
838   // inject the other one as a response from the server. The result should
839   // include both FullHashInfo objects.
840 
841   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
842   HashPrefix prefix_1("exam");
843   FullHash full_hash_1("example");
844   HashPrefix prefix_2("Everything");
845   FullHash full_hash_2("Everything's shiny, Cap'n.");
846 
847   base::Time now = Time::Now();
848   SetTestClock(now, pm.get());
849 
850   FullHashCache* cache = pm->full_hash_cache_for_tests();
851   CachedHashPrefixInfo* entry = &(*cache)[prefix_1];
852   entry->negative_expiry = now + base::TimeDelta::FromMinutes(100);
853   // Put one unexpired full hash in the cache for a store we'll look in.
854   entry->full_hash_infos.emplace_back(full_hash_1, GetUrlMalwareId(),
855                                       now + base::TimeDelta::FromSeconds(200));
856   // Put one unexpired full hash in the cache for a store we'll not look in.
857   entry->full_hash_infos.emplace_back(full_hash_1, GetUrlSocEngId(),
858                                       now + base::TimeDelta::FromSeconds(200));
859 
860   // Request full hash information from two stores.
861   FullHashToStoreAndHashPrefixesMap matched_locally;
862   matched_locally[full_hash_1].push_back(
863       StoreAndHashPrefix(GetUrlMalwareId(), prefix_1));
864   matched_locally[full_hash_2].push_back(
865       StoreAndHashPrefix(GetChromeUrlApiId(), prefix_2));
866 
867   // Expect full hash information from both stores.
868   std::vector<FullHashInfo> expected_results;
869   expected_results.emplace_back(full_hash_1, GetUrlMalwareId(),
870                                 now + base::TimeDelta::FromSeconds(200));
871   expected_results.emplace_back(full_hash_2, GetChromeUrlApiId(),
872                                 now + base::TimeDelta::FromSeconds(300));
873   expected_results[1].metadata.api_permissions.insert("NOTIFICATIONS");
874 
875   pm->GetFullHashes(
876       matched_locally, {},
877       base::BindOnce(&V4GetHashProtocolManagerTest::ValidateGetV4HashResults,
878                      base::Unretained(this), expected_results));
879 
880   SetupFetcherToReturnOKResponse(pm.get(), GetStockV4HashResponseInfos());
881 
882   // No error, back off multiplier is unchanged.
883   EXPECT_EQ(0ul, pm->gethash_error_count_);
884   EXPECT_EQ(1ul, pm->gethash_back_off_mult_);
885 
886   // Verify the state of the cache.
887   ASSERT_EQ(2u, cache->size());
888   const CachedHashPrefixInfo& cached_result_1 = cache->at(prefix_1);
889   EXPECT_EQ(cached_result_1.negative_expiry,
890             now + base::TimeDelta::FromMinutes(100));
891   ASSERT_EQ(2u, cached_result_1.full_hash_infos.size());
892   EXPECT_EQ(full_hash_1, cached_result_1.full_hash_infos[0].full_hash);
893   EXPECT_EQ(GetUrlMalwareId(), cached_result_1.full_hash_infos[0].list_id);
894 
895   const CachedHashPrefixInfo& cached_result_2 = cache->at(prefix_2);
896   EXPECT_EQ(cached_result_2.negative_expiry,
897             now + base::TimeDelta::FromSeconds(600));
898   ASSERT_EQ(1u, cached_result_2.full_hash_infos.size());
899   EXPECT_EQ(full_hash_2, cached_result_2.full_hash_infos[0].full_hash);
900   EXPECT_EQ(GetChromeUrlApiId(), cached_result_2.full_hash_infos[0].list_id);
901   EXPECT_TRUE(callback_called());
902 }
903 
904 // The server responds back with full hash information containing metadata
905 // information for one of the full hashes for the URL in test.
TEST_F(V4GetHashProtocolManagerTest,TestGetFullHashesWithApisMergesMetadata)906 TEST_F(V4GetHashProtocolManagerTest, TestGetFullHashesWithApisMergesMetadata) {
907   const GURL url("https://www.example.com/more");
908   ThreatMetadata expected_md;
909   expected_md.api_permissions.insert("NOTIFICATIONS");
910   expected_md.api_permissions.insert("AUDIO_CAPTURE");
911   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
912   pm->GetFullHashesWithApis(
913       url, {} /* list_client_states */,
914       base::BindOnce(&V4GetHashProtocolManagerTest::ValidateGetV4ApiResults,
915                      base::Unretained(this), expected_md));
916 
917   // The following two random looking strings value are two of the full hashes
918   // produced by UrlToFullHashes in v4_protocol_manager_util.h for the URL:
919   // "https://www.example.com"
920   std::vector<ResponseInfo> infos;
921   FullHash full_hash;
922   base::Base64Decode("1ZzJ0/7NjPkg6t0DAS8L5Jf7jA48Pn7opQcP4UXYeXc=",
923                      &full_hash);
924   ResponseInfo info(full_hash, GetChromeUrlApiId());
925   info.key_values.emplace_back("permission", "NOTIFICATIONS");
926   infos.push_back(info);
927 
928   base::Base64Decode("c9mG4AkGXxgsELy2pF2z1u2pSY-JMGVK8mU_ipOM2AE=",
929                      &full_hash);
930   info = ResponseInfo(full_hash, GetChromeUrlApiId());
931   info.key_values.emplace_back("permission", "AUDIO_CAPTURE");
932   infos.push_back(info);
933 
934   full_hash = FullHash("Everything's shiny, Cap'n.");
935   info = ResponseInfo(full_hash, GetChromeUrlApiId());
936   info.key_values.emplace_back("permission", "GEOLOCATION");
937   infos.push_back(info);
938   SetupFetcherToReturnOKResponse(pm.get(), infos);
939 
940   EXPECT_TRUE(callback_called());
941 }
942 
943 }  // namespace safe_browsing
944