core/db/v4_get_hash_protocol_manager.h

// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef COMPONENTS_SAFE_BROWSING_CORE_DB_V4_GET_HASH_PROTOCOL_MANAGER_H_
#define COMPONENTS_SAFE_BROWSING_CORE_DB_V4_GET_HASH_PROTOCOL_MANAGER_H_

// A class that implements Chrome's interface with the SafeBrowsing V4 protocol.
//
// The V4GetHashProtocolManager handles formatting and making requests of, and
// handling responses from, Google's SafeBrowsing servers. The purpose of this
// class is to get full hash matches from the SB server for the given set of
// hash prefixes.
//
// Design doc: go/design-doc-v4-full-hash-manager

#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>

#include "base/gtest_prod_util.h"
#include "base/macros.h"
#include "base/sequence_checker.h"
#include "base/time/default_clock.h"
#include "base/time/time.h"
#include "base/timer/timer.h"
#include "components/safe_browsing/core/db/safebrowsing.pb.h"
#include "components/safe_browsing/core/db/util.h"
#include "components/safe_browsing/core/db/v4_protocol_manager_util.h"
#include "components/safe_browsing/core/proto/webui.pb.h"

class GURL;

namespace network {
class SimpleURLLoader;
class SharedURLLoaderFactory;
}  // namespace network

namespace safe_browsing {

class V4GetHashProtocolManagerFuzzer;

// The matching hash prefixes and corresponding stores, for each full hash
// generated for a given URL.
typedef std::unordered_map<FullHash, StoreAndHashPrefixes>
    FullHashToStoreAndHashPrefixesMap;

// ----------------------------------------------------------------

// All information about a particular full hash i.e. negative TTL, store for
// which it is valid, and metadata associated with that store.
struct FullHashInfo {
 public:
  FullHash full_hash;

  // The list for which this full hash is applicable.
  ListIdentifier list_id;

  // The expiration time of the full hash for a particular store.
  base::Time positive_expiry;

  // Any metadata for this full hash for a particular store.
  ThreatMetadata metadata;

  FullHashInfo(const FullHash& full_hash,
               const ListIdentifier& list_id,
               const base::Time& positive_expiry);
  FullHashInfo(const FullHashInfo& other);
  ~FullHashInfo();

  bool operator==(const FullHashInfo& other) const;
  bool operator!=(const FullHashInfo& other) const;

 private:
  FullHashInfo();
};

// Caches individual response from GETHASH response.
struct CachedHashPrefixInfo {
  // The negative TTL for the hash prefix that leads to this
  // CachedHashPrefixInfo. The client should not send any more requests for that
  // hash prefix until this time.
  base::Time negative_expiry;

  // The list of all full hashes (and related info) that start with a
  // particular hash prefix and are known to be unsafe.
  std::vector<FullHashInfo> full_hash_infos;

  CachedHashPrefixInfo();
  CachedHashPrefixInfo(const CachedHashPrefixInfo& other);
  ~CachedHashPrefixInfo();
};

// Cached full hashes received from the server for the corresponding hash
// prefixes.
typedef std::unordered_map<HashPrefix, CachedHashPrefixInfo> FullHashCache;

// FullHashCallback is invoked when GetFullHashes completes. The parameter is
// the vector of full hash results. If empty, indicates that there were no
// matches, and that the resource is safe.
using FullHashCallback =
    base::OnceCallback<void(const std::vector<FullHashInfo>&)>;

// Information needed to update the cache and call the callback to post the
// results.
struct FullHashCallbackInfo {
  FullHashCallbackInfo();
  FullHashCallbackInfo(const std::vector<FullHashInfo>& cached_full_hash_infos,
                       const std::vector<HashPrefix>& prefixes_requested,
                       std::unique_ptr<network::SimpleURLLoader> loader,
                       const FullHashToStoreAndHashPrefixesMap&
                           full_hash_to_store_and_hash_prefixes,
                       FullHashCallback callback,
                       const base::Time& network_start_time);
  ~FullHashCallbackInfo();

  // The FullHashInfo objects retrieved from cache. These are merged with the
  // results received from the server before invoking the callback.
  std::vector<FullHashInfo> cached_full_hash_infos;

  // The callback method to call after collecting the full hashes for given
  // hash prefixes.
  FullHashCallback callback;

  // The loader that will return the response from the server. This is stored
  // here as a unique pointer to be able to reason about its lifetime easily.
  std::unique_ptr<network::SimpleURLLoader> loader;

  // The generated full hashes and the corresponding prefixes and the stores in
  // which to look for a full hash match.
  FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes;

  // Used to measure how long did it take to fetch the full hash response from
  // the server.
  base::Time network_start_time;

  // The prefixes that were requested from the server.
  std::vector<HashPrefix> prefixes_requested;
};

// ----------------------------------------------------------------

class V4GetHashProtocolManagerFactory;

class V4GetHashProtocolManager {
 public:
  // Invoked when GetFullHashesWithApis completes.
  // Parameters:
  //   - The API threat metadata for the given URL.
  using ThreatMetadataForApiCallback =
      base::OnceCallback<void(const ThreatMetadata& md)>;

  virtual ~V4GetHashProtocolManager();

  // Create an instance of the safe browsing v4 protocol manager.
  static std::unique_ptr<V4GetHashProtocolManager> Create(
      scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
      const StoresToCheck& stores_to_check,
      const V4ProtocolConfig& config);

  // Makes the passed |factory| the factory used to instantiate
  // a V4GetHashProtocolManager. Useful for tests.
  static void RegisterFactory(
      std::unique_ptr<V4GetHashProtocolManagerFactory> factory);

  // Empties the cache.
  void ClearCache();

  // Retrieve the full hash for a set of prefixes, and invoke the callback
  // argument when the results are retrieved. The callback may be invoked
  // synchronously. |list_client_states| is needed for reporting the current
  // state of the lists on the client; it does not affect the response from the
  // server.
  virtual void GetFullHashes(const FullHashToStoreAndHashPrefixesMap
                                 full_hash_to_matching_hash_prefixes,
                             const std::vector<std::string>& list_client_states,
                             FullHashCallback callback);

  // Retrieve the full hash and API metadata for the origin of |url|, and invoke
  // the callback argument when the results are retrieved. The callback may be
  // invoked synchronously.
  virtual void GetFullHashesWithApis(
      const GURL& url,
      const std::vector<std::string>& list_client_states,
      ThreatMetadataForApiCallback api_callback);

  // Callback when the request completes
  void OnURLLoaderComplete(network::SimpleURLLoader* url_loader,
                           std::unique_ptr<std::string> response_body);

  // Populates the protobuf with the FullHashCache data.
  void CollectFullHashCacheInfo(FullHashCacheInfo* full_hash_cache_info);

 protected:
  // Constructs a V4GetHashProtocolManager that issues network requests using
  // |url_loader_factory|.
  V4GetHashProtocolManager(
      scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
      const StoresToCheck& stores_to_check,
      const V4ProtocolConfig& config);

 private:
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest, TestGetHashRequest);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest, TestParseHashResponse);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest,
                           TestParseHashResponseWrongThreatEntryType);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest,
                           TestParseHashThreatPatternType);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest,
                           TestParseSubresourceFilterMetadata);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest,
                           TestParseHashResponseNonPermissionMetadata);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest,
                           TestParseHashResponseInconsistentThreatTypes);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest,
                           TestGetHashErrorHandlingOK);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest,
                           TestResultsNotCachedForNegativeCacheDuration);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest,
                           TestGetHashErrorHandlingNetwork);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest,
                           TestGetHashErrorHandlingResponseCode);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest,
                           TestGetHashErrorHandlingParallelRequests);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest, GetCachedResults);
  FRIEND_TEST_ALL_PREFIXES(V4GetHashProtocolManagerTest, TestUpdatesAreMerged);
  friend class V4GetHashProtocolManagerTest;
  friend class V4GetHashProtocolManagerFuzzer;
  friend class V4GetHashProtocolManagerFactoryImpl;

  FullHashCache* full_hash_cache_for_tests() { return &full_hash_cache_; }

  void OnURLLoaderCompleteInternal(network::SimpleURLLoader* url_loader,
                                   int net_error,
                                   int response_code,
                                   const std::string& data);

  // Looks up the cached results for full hashes in
  // |full_hash_to_store_and_hash_prefixes|. Fills |prefixes_to_request| with
  // the prefixes that need to be requested. Fills |cached_full_hash_infos|
  // with the cached results.
  // Note: It is valid for both |prefixes_to_request| and
  // |cached_full_hash_infos| to be empty after this function finishes.
  void GetFullHashCachedResults(
      const FullHashToStoreAndHashPrefixesMap&
          full_hash_to_store_and_hash_prefixes,
      const base::Time& now,
      std::vector<HashPrefix>* prefixes_to_request,
      std::vector<FullHashInfo>* cached_full_hash_infos);

  // Fills a FindFullHashesRequest protocol buffer for a request.
  // Returns the serialized and base 64 encoded request as a string.
  // |prefixes_to_request| is the list of hash prefixes to get full hashes for.
  // |list_client_states| is the client_state of each of the lists being synced.
  std::string GetHashRequest(
      const std::vector<HashPrefix>& prefixes_to_request,
      const std::vector<std::string>& list_client_states);

  void GetHashUrlAndHeaders(const std::string& request_base64,
                            GURL* gurl,
                            net::HttpRequestHeaders* headers) const;

  // Updates internal state for each GetHash response error, assuming that
  // the current time is |now|.
  void HandleGetHashError(const base::Time& now);

  // Merges the results from the cache and the results from the server. The
  // response from the server may include information for full hashes from
  // stores other than those required by this client so it filters out those
  // results that the client did not ask for.
  void MergeResults(const FullHashToStoreAndHashPrefixesMap&
                        full_hash_to_store_and_hash_prefixes,
                    const std::vector<FullHashInfo>& full_hash_infos,
                    std::vector<FullHashInfo>* merged_full_hash_infos);

  // Calls |api_callback| with an object of ThreatMetadata that contains
  // permission API metadata for full hashes in those |full_hash_infos| that
  // have a full hash in |full_hashes|.
  void OnFullHashForApi(ThreatMetadataForApiCallback api_callback,
                        const std::vector<FullHash>& full_hashes,
                        const std::vector<FullHashInfo>& full_hash_infos);

  // Parses a FindFullHashesResponse protocol buffer and fills the results in
  // |full_hash_infos| and |negative_cache_expire|. |response_data| is a
  // serialized FindFullHashes protocol buffer. |negative_cache_expire| is the
  // cache expiry time of the hash prefixes that were requested. Returns true if
  // parsing is successful; false otherwise.
  bool ParseHashResponse(const std::string& response_data,
                         std::vector<FullHashInfo>* full_hash_infos,
                         base::Time* negative_cache_expire);

  // Parses the store specific |metadata| information from |match|. Logs errors
  // to UMA if the metadata information was not parsed correctly or was
  // inconsistent with what's expected from that corresponding store.
  static void ParseMetadata(const ThreatMatch& match, ThreatMetadata* metadata);

  // Resets the gethash error counter and multiplier.
  void ResetGetHashErrors();

  // Overrides the clock used to check the time.
  void SetClockForTests(base::Clock* clock);

  // Updates the state of the full hash cache upon receiving a valid response
  // from the server.
  void UpdateCache(const std::vector<HashPrefix>& prefixes_requested,
                   const std::vector<FullHashInfo>& full_hash_infos,
                   const base::Time& negative_cache_expire);

 protected:
  // A cache of full hash results.
  FullHashCache full_hash_cache_;

 private:
  // Map of GetHash requests to parameters which created it.
  using PendingHashRequests =
      std::unordered_map<const network::SimpleURLLoader*,
                         std::unique_ptr<FullHashCallbackInfo>>;

  // The factory that controls the creation of V4GetHashProtocolManager.
  // This is used by tests.
  static V4GetHashProtocolManagerFactory* factory_;

  // The number of HTTP response errors since the the last successful HTTP
  // response, used for request backoff timing.
  size_t gethash_error_count_;

  // Multiplier for the backoff error after the second.
  size_t gethash_back_off_mult_;

  PendingHashRequests pending_hash_requests_;

  // For v4, the next gethash time is set to the backoff time is the last
  // response was an error, or the minimum wait time if the last response was
  // successful.
  base::Time next_gethash_time_;

  // The config of the client making Pver4 requests.
  const V4ProtocolConfig config_;

  // The URLLoaderFactory we use to issue network requests.
  scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_;

  // Records number of cache hits since the beginning of this session.
  int number_of_hits_ = 0;

  // The clock used to vend times.
  base::Clock* clock_;

  // The following sets represent the combination of lists that we would always
  // request from the server, irrespective of which list we found the hash
  // prefix match in.
  std::vector<PlatformType> platform_types_;
  std::vector<ThreatEntryType> threat_entry_types_;
  std::vector<ThreatType> threat_types_;

  SEQUENCE_CHECKER(sequence_checker_);

  DISALLOW_COPY_AND_ASSIGN(V4GetHashProtocolManager);
};

// Interface of a factory to create V4GetHashProtocolManager.  Useful for tests.
class V4GetHashProtocolManagerFactory {
 public:
  V4GetHashProtocolManagerFactory() {}
  virtual ~V4GetHashProtocolManagerFactory() {}
  virtual std::unique_ptr<V4GetHashProtocolManager> CreateProtocolManager(
      scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
      const StoresToCheck& stores_to_check,
      const V4ProtocolConfig& config) = 0;

 private:
  DISALLOW_COPY_AND_ASSIGN(V4GetHashProtocolManagerFactory);
};

#ifndef NDEBUG
std::ostream& operator<<(std::ostream& os, const FullHashInfo& id);
#endif

}  // namespace safe_browsing

#endif  // COMPONENTS_SAFE_BROWSING_CORE_DB_V4_GET_HASH_PROTOCOL_MANAGER_H_