1 // Copyright 2019 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef CHROME_BROWSER_OPTIMIZATION_GUIDE_OPTIMIZATION_GUIDE_HINTS_MANAGER_H_ 6 #define CHROME_BROWSER_OPTIMIZATION_GUIDE_OPTIMIZATION_GUIDE_HINTS_MANAGER_H_ 7 8 #include <memory> 9 #include <string> 10 #include <vector> 11 12 #include "base/callback_forward.h" 13 #include "base/containers/flat_map.h" 14 #include "base/containers/flat_set.h" 15 #include "base/containers/mru_cache.h" 16 #include "base/memory/scoped_refptr.h" 17 #include "base/memory/weak_ptr.h" 18 #include "base/optional.h" 19 #include "base/sequenced_task_runner.h" 20 #include "base/synchronization/lock.h" 21 #include "base/time/clock.h" 22 #include "base/timer/timer.h" 23 #include "chrome/browser/navigation_predictor/navigation_predictor_keyed_service.h" 24 #include "components/optimization_guide/hints_component_info.h" 25 #include "components/optimization_guide/hints_fetcher.h" 26 #include "components/optimization_guide/optimization_guide_decider.h" 27 #include "components/optimization_guide/optimization_guide_service_observer.h" 28 #include "components/optimization_guide/proto/hints.pb.h" 29 #include "components/optimization_guide/proto/models.pb.h" 30 #include "net/nqe/effective_connection_type.h" 31 #include "services/network/public/cpp/network_quality_tracker.h" 32 33 namespace base { 34 class FilePath; 35 } // namespace base 36 37 namespace content { 38 class NavigationHandle; 39 } // namespace content 40 41 namespace leveldb_proto { 42 class ProtoDatabaseProvider; 43 } // namespace leveldb_proto 44 45 namespace network { 46 class SharedURLLoaderFactory; 47 } // namespace network 48 49 namespace optimization_guide { 50 class HintCache; 51 class HintsFetcherFactory; 52 class OptimizationFilter; 53 class OptimizationMetadata; 54 class OptimizationGuideService; 55 enum class OptimizationTargetDecision; 56 enum class OptimizationTypeDecision; 57 class StoreUpdateData; 58 class TopHostProvider; 59 } // namespace optimization_guide 60 61 class OptimizationGuideNavigationData; 62 class PrefService; 63 class Profile; 64 65 class OptimizationGuideHintsManager 66 : public optimization_guide::OptimizationGuideServiceObserver, 67 public network::NetworkQualityTracker::EffectiveConnectionTypeObserver, 68 public NavigationPredictorKeyedService::Observer { 69 public: 70 OptimizationGuideHintsManager( 71 const std::vector<optimization_guide::proto::OptimizationType>& 72 optimization_types_at_initialization, 73 optimization_guide::OptimizationGuideService* optimization_guide_service, 74 Profile* profile, 75 const base::FilePath& profile_path, 76 PrefService* pref_service, 77 leveldb_proto::ProtoDatabaseProvider* database_provider, 78 optimization_guide::TopHostProvider* top_host_provider, 79 scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory); 80 81 ~OptimizationGuideHintsManager() override; 82 83 // Unhooks the observer to |optimization_guide_service_|. 84 void Shutdown(); 85 86 // optimization_guide::OptimizationGuideServiceObserver implementation: 87 void OnHintsComponentAvailable( 88 const optimization_guide::HintsComponentInfo& info) override; 89 90 // |next_update_closure| is called the next time OnHintsComponentAvailable() 91 // is called and the corresponding hints have been updated. 92 void ListenForNextUpdateForTesting(base::OnceClosure next_update_closure); 93 94 // Registers the optimization types that have the potential for hints to be 95 // called by consumers of the Optimization Guide. 96 void RegisterOptimizationTypes( 97 const std::vector<optimization_guide::proto::OptimizationType>& 98 optimization_types); 99 100 // Returns the optimization types that are registered. 101 base::flat_set<optimization_guide::proto::OptimizationType> registered_optimization_types()102 registered_optimization_types() const { 103 return registered_optimization_types_; 104 } 105 106 // Returns whether there is an optimization allowlist loaded for 107 // |optimization_type|. 108 bool HasLoadedOptimizationAllowlist( 109 optimization_guide::proto::OptimizationType optimization_type); 110 // Returns whether there is an optimization blocklist loaded for 111 // |optimization_type|. 112 bool HasLoadedOptimizationBlocklist( 113 optimization_guide::proto::OptimizationType optimization_type); 114 115 // Returns the OptimizationTypeDecision based on the given parameters. 116 // |optimization_metadata| will be populated, if applicable. 117 optimization_guide::OptimizationTypeDecision CanApplyOptimization( 118 const GURL& navigation_url, 119 const base::Optional<int64_t>& navigation_id, 120 optimization_guide::proto::OptimizationType optimization_type, 121 optimization_guide::OptimizationMetadata* optimization_metadata); 122 123 // Invokes |callback| with the decision for |navigation_url| and 124 // |optimization_type|, when sufficient information has been collected by 125 // |this| to make the decision. Virtual for testing. 126 virtual void CanApplyOptimizationAsync( 127 const GURL& navigation_url, 128 const base::Optional<int64_t>& navigation_id, 129 optimization_guide::proto::OptimizationType optimization_type, 130 optimization_guide::OptimizationGuideDecisionCallback callback); 131 132 // Clears all fetched hints from |hint_cache_|. 133 void ClearFetchedHints(); 134 135 // Clears the host-keyed fetched hints from |hint_cache_|, both the persisted 136 // and in memory ones. 137 void ClearHostKeyedHints(); 138 139 // Returns the current batch update hints fetcher. batch_update_hints_fetcher()140 optimization_guide::HintsFetcher* batch_update_hints_fetcher() const { 141 return batch_update_hints_fetcher_.get(); 142 } 143 144 // Overrides |hints_fetcher_factory| for testing. 145 void SetHintsFetcherFactoryForTesting( 146 std::unique_ptr<optimization_guide::HintsFetcherFactory> 147 hints_fetcher_factory); 148 149 // Overrides |clock_| for testing. 150 void SetClockForTesting(const base::Clock* clock); 151 152 // network::NetworkQualityTracker::EffectiveConnectionTypeObserver 153 // implementation: 154 void OnEffectiveConnectionTypeChanged( 155 net::EffectiveConnectionType type) override; 156 157 // Notifies |this| that a navigation with |navigation_handle| has started. 158 // |callback| is run when the request has finished regardless of whether there 159 // was actually a hint for that load or not. The callback can be used as a 160 // signal for tests. 161 void OnNavigationStartOrRedirect(content::NavigationHandle* navigation_handle, 162 base::OnceClosure callback); 163 164 // Notifies |this| that a navigation with redirect chain 165 // |navigation_redirect_chain| has finished. 166 void OnNavigationFinish(const std::vector<GURL>& navigation_redirect_chain); 167 168 // Add hints to the cache with the provided metadata. For testing only. 169 void AddHintForTesting( 170 const GURL& url, 171 optimization_guide::proto::OptimizationType optimization_type, 172 const base::Optional<optimization_guide::OptimizationMetadata>& metadata); 173 174 // Override the decision returned by |ShouldTargetNavigation| 175 // for |optimization_target|. For testing purposes only. 176 void OverrideTargetDecisionForTesting( 177 optimization_guide::proto::OptimizationTarget optimization_target, 178 optimization_guide::OptimizationGuideDecision 179 optimization_guide_decision); 180 181 private: 182 FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerTest, IsGoogleURL); 183 FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerFetchingTest, 184 HintsFetched_AtSRP_NoRegisteredOptimizationTypes); 185 FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerFetchingTest, 186 HintsFetched_AtSRP_ECT_SLOW_2G); 187 FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerFetchingTest, 188 HintsFetched_AtSRP_ECT_4G); 189 FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerFetchingTest, 190 HintsFetched_AtNonSRP_ECT_SLOW_2G); 191 FRIEND_TEST_ALL_PREFIXES(OptimizationGuideHintsManagerFetchingTest, 192 HintsFetched_AtSRP_ECT_SLOW_2G_DuplicatesRemoved); 193 FRIEND_TEST_ALL_PREFIXES( 194 OptimizationGuideHintsManagerFetchingTest, 195 HintsFetched_AtSRP_ECT_SLOW_2G_NonHTTPOrHTTPSHostsRemoved); 196 FRIEND_TEST_ALL_PREFIXES( 197 OptimizationGuideHintsManagerFetchingTest, 198 HintsFetched_ExternalAndroidApp_ECT_SLOW_2G_NonHTTPOrHTTPSHostsRemovedAppWhitelisted); 199 FRIEND_TEST_ALL_PREFIXES( 200 OptimizationGuideHintsManagerFetchingTest, 201 HintsFetched_ExternalAndroidApp_ECT_SLOW_2G_NonHTTPOrHTTPSHostsRemovedNotAllAppsWhitelisted); 202 FRIEND_TEST_ALL_PREFIXES( 203 OptimizationGuideHintsManagerFetchingTest, 204 HintsFetched_ExternalAndroidApp_ECT_SLOW_2G_NonHTTPOrHTTPSHostsRemovedAppNotWhitelisted); 205 206 // Processes the hints component. 207 // 208 // Should always be called on the thread that belongs to 209 // |background_task_runner_|. 210 std::unique_ptr<optimization_guide::StoreUpdateData> ProcessHintsComponent( 211 const optimization_guide::HintsComponentInfo& info, 212 const base::flat_set<optimization_guide::proto::OptimizationType>& 213 registered_optimization_types, 214 std::unique_ptr<optimization_guide::StoreUpdateData> update_data); 215 216 // Processes the optimization filters contained in the hints component. 217 // 218 // Should always be called on the thread that belongs to 219 // |background_task_runner_|. 220 void ProcessOptimizationFilters( 221 const google::protobuf::RepeatedPtrField< 222 optimization_guide::proto::OptimizationFilter>& 223 allowlist_optimization_filters, 224 const google::protobuf::RepeatedPtrField< 225 optimization_guide::proto::OptimizationFilter>& 226 blocklist_optimization_filters, 227 const base::flat_set<optimization_guide::proto::OptimizationType>& 228 registered_optimization_types); 229 230 // Process a set of optimization filters. 231 // 232 // |is_allowlist| will be used to ensure that the filters are either uses as 233 // allowlists or blocklists. |optimization_filters_lock_| should be held 234 // before calling this function. 235 void ProcessOptimizationFilterSet( 236 const google::protobuf::RepeatedPtrField< 237 optimization_guide::proto::OptimizationFilter>& filters, 238 bool is_allowlist, 239 const base::flat_set<optimization_guide::proto::OptimizationType>& 240 registered_optimization_types) 241 EXCLUSIVE_LOCKS_REQUIRED(optimization_filters_lock_); 242 243 // Callback run after the hint cache is fully initialized. At this point, 244 // the OptimizationGuideHintsManager is ready to process hints. 245 void OnHintCacheInitialized(); 246 247 // Updates the cache with the latest hints sent by the Component Updater. 248 void UpdateComponentHints( 249 base::OnceClosure update_closure, 250 std::unique_ptr<optimization_guide::StoreUpdateData> update_data); 251 252 // Called when the hints have been fully updated with the latest hints from 253 // the Component Updater. This is used as a signal during tests. 254 void OnComponentHintsUpdated(base::OnceClosure update_closure, 255 bool hints_updated); 256 257 // Method to decide whether to fetch new hints for user's top sites and 258 // proceeds to schedule the fetch. 259 void MaybeScheduleTopHostsHintsFetch(); 260 261 // Schedules |hints_fetch_timer_| to fire based on: 262 // 1. The update time for the fetched hints in the store and 263 // 2. The last time a fetch attempt was made. 264 void ScheduleTopHostsHintsFetch(); 265 266 // Called to make a request to fetch hints from the remote Optimization Guide 267 // Service. Used to fetch hints for origins frequently visited by the user. 268 void FetchTopHostsHints(); 269 270 // Called when the hints for the top hosts have been fetched from the remote 271 // Optimization Guide Service and are ready for parsing. This is used when 272 // fetching hints in batch mode. 273 void OnTopHostsHintsFetched( 274 const base::flat_set<std::string>& hosts_fetched, 275 base::Optional< 276 std::unique_ptr<optimization_guide::proto::GetHintsResponse>> 277 get_hints_response); 278 279 // Called when the hints for a navigation have been fetched from the remote 280 // Optimization Guide Service and are ready for parsing. This is used when 281 // fetching hints in real-time. |navigation_url| is the URL associated with 282 // the navigation handle that initiated the fetch. 283 // |page_navigation_urls_requested| contains the URLs that were requested by 284 // |this| to be fetched. |page_navigation_hosts_requested| contains the hosts 285 // that were requested by |this| to be fetched. 286 void OnPageNavigationHintsFetched( 287 base::WeakPtr<OptimizationGuideNavigationData> navigation_data_weak_ptr, 288 const base::Optional<GURL>& navigation_url, 289 const base::flat_set<GURL>& page_navigation_urls_requested, 290 const base::flat_set<std::string>& page_navigation_hosts_requested, 291 base::Optional< 292 std::unique_ptr<optimization_guide::proto::GetHintsResponse>> 293 get_hints_response); 294 295 // Called when the fetched hints have been stored in |hint_cache| and are 296 // ready to be used. This is used when hints were fetched in batch mode. 297 void OnFetchedTopHostsHintsStored(); 298 299 // Called when the fetched hints have been stored in |hint_cache| and are 300 // ready to be used. This is used when hints were fetched in real-time. 301 // |navigation_url| is the URL associated with the navigation handle that 302 // initiated the fetch. |page_navigation_hosts_requested| contains the hosts 303 // whose hints should be loaded into memory when invoked. 304 void OnFetchedPageNavigationHintsStored( 305 base::WeakPtr<OptimizationGuideNavigationData> navigation_data_weak_ptr, 306 const base::Optional<GURL>& navigation_url, 307 const base::flat_set<std::string>& page_navigation_hosts_requested); 308 309 // Returns true if there is a fetch currently in-flight for |navigation_url|. 310 bool IsHintBeingFetchedForNavigation(const GURL& navigation_url); 311 312 // Cleans up the hints fetcher for |navigation_url|, if applicable. 313 void CleanUpFetcherForNavigation(const GURL& navigation_url); 314 315 // Returns the time when a hints fetch request was last attempted. 316 base::Time GetLastHintsFetchAttemptTime() const; 317 318 // Sets the time when a hints fetch was last attempted to |last_attempt_time|. 319 void SetLastHintsFetchAttemptTime(base::Time last_attempt_time); 320 321 // Called when the request to load a hint has completed. 322 void OnHintLoaded(base::OnceClosure callback, 323 const optimization_guide::proto::Hint* loaded_hint) const; 324 325 // Returns true if |this| is allowed to fetch hints at the navigation time for 326 // |url|. 327 bool IsAllowedToFetchNavigationHints(const GURL& url); 328 329 // Loads the hint if available. 330 // |callback| is run when the request has finished regardless of whether there 331 // was actually a hint for that load or not. The callback can be used as a 332 // signal for tests. 333 void LoadHintForNavigation(content::NavigationHandle* navigation_handle, 334 base::OnceClosure callback); 335 336 // Loads the hint for |host| if available. 337 // |callback| is run when the request has finished regardless of whether there 338 // was actually a hint for that |host| or not. The callback can be used as a 339 // signal for tests. 340 void LoadHintForHost(const std::string& host, base::OnceClosure callback); 341 342 // Returns true if the hostname for |url| matches the host of google web 343 // search results page (www.google.*). 344 bool IsGoogleURL(const GURL& url) const; 345 346 // Returns true if we can make a request for hints for |prediction|. 347 bool IsAllowedToFetchForNavigationPrediction( 348 const base::Optional<NavigationPredictorKeyedService::Prediction> 349 prediction) const; 350 351 // NavigationPredictorKeyedService::Observer: 352 void OnPredictionUpdated( 353 const base::Optional<NavigationPredictorKeyedService::Prediction> 354 prediction) override; 355 356 // Returns whether there is an optimization type to fetch for. Will return 357 // false if no optimization types are registered or if all registered 358 // optimization types are covered by optimization filters. 359 bool HasOptimizationTypeToFetchFor(); 360 361 // Creates a hints fetch for |navigation_handle| if it is allowed. The 362 // fetch will include the host and URL of the |navigation_handle| if the 363 // associated hints for each are not already in the cache. 364 void MaybeFetchHintsForNavigation( 365 content::NavigationHandle* navigation_handle); 366 367 // If an entry for |navigation_url| is contained in |registered_callbacks_|, 368 // it will load the hint for |navigation_url|'s host and upon completion, will 369 // invoke the registered callbacks for |navigation_url|. 370 void PrepareToInvokeRegisteredCallbacks(const GURL& navigation_url); 371 372 // Invokes the registered callbacks for |navigation_url|, if applicable. 373 void OnReadyToInvokeRegisteredCallbacks(const GURL& navigation_url); 374 375 // Whether all information was available to make a decision for 376 // |navigation_url| and |optimization type}. 377 bool HasAllInformationForDecisionAvailable( 378 const GURL& navigation_url, 379 optimization_guide::proto::OptimizationType optimization_type); 380 381 // The OptimizationGuideService that this guide is listening to. Not owned. 382 optimization_guide::OptimizationGuideService* const 383 optimization_guide_service_; 384 385 // The information of the latest component delivered by 386 // |optimization_guide_service_|. 387 base::Optional<optimization_guide::HintsComponentInfo> hints_component_info_; 388 389 // The set of optimization types that have been registered with the hints 390 // manager. 391 // 392 // Should only be read and modified on the UI thread. 393 base::flat_set<optimization_guide::proto::OptimizationType> 394 registered_optimization_types_; 395 396 // Synchronizes access to member variables related to optimization filters. 397 base::Lock optimization_filters_lock_; 398 399 // The set of optimization types that the component specified by 400 // |component_info_| has optimization filters for. 401 base::flat_set<optimization_guide::proto::OptimizationType> 402 optimization_types_with_filter_ GUARDED_BY(optimization_filters_lock_); 403 404 // A map from optimization type to the host filter that holds the allowlist 405 // for that type. 406 base::flat_map<optimization_guide::proto::OptimizationType, 407 std::unique_ptr<optimization_guide::OptimizationFilter>> 408 allowlist_optimization_filters_ GUARDED_BY(optimization_filters_lock_); 409 410 // A map from optimization type to the host filter that holds the blocklist 411 // for that type. 412 base::flat_map<optimization_guide::proto::OptimizationType, 413 std::unique_ptr<optimization_guide::OptimizationFilter>> 414 blocklist_optimization_filters_ GUARDED_BY(optimization_filters_lock_); 415 416 // A map from URL to a map of callbacks (along with the navigation IDs that 417 // they were called for) keyed by their optimization type. 418 base::flat_map< 419 GURL, 420 base::flat_map< 421 optimization_guide::proto::OptimizationType, 422 std::vector<std::pair< 423 base::Optional<int64_t>, 424 optimization_guide::OptimizationGuideDecisionCallback>>>> 425 registered_callbacks_; 426 427 // Background thread where hints processing should be performed. 428 scoped_refptr<base::SequencedTaskRunner> background_task_runner_; 429 430 // A reference to the profile. Not owned. 431 Profile* profile_ = nullptr; 432 433 // A reference to the PrefService for this profile. Not owned. 434 PrefService* pref_service_ = nullptr; 435 436 // The hint cache that holds both hints received from the component and 437 // fetched from the remote Optimization Guide Service. 438 std::unique_ptr<optimization_guide::HintCache> hint_cache_; 439 440 // The fetcher that handles making requests for hints for multiple hosts from 441 // the remote Optimization Guide Service. 442 std::unique_ptr<optimization_guide::HintsFetcher> batch_update_hints_fetcher_; 443 444 // A cache keyed by navigation URL to the fetcher making a request for a hint 445 // for that URL and/or host to the remote Optimization Guide Service that 446 // keeps track of when an entry has been placed in the cache. 447 base::MRUCache<GURL, std::unique_ptr<optimization_guide::HintsFetcher>> 448 page_navigation_hints_fetchers_; 449 450 // The factory used to create hints fetchers. It is mostly used to create 451 // new fetchers for use under the page navigation context, but will also be 452 // used to create the initial fetcher for the batch update context. 453 std::unique_ptr<optimization_guide::HintsFetcherFactory> 454 hints_fetcher_factory_; 455 456 // The external app packages that have been approved for fetching from the 457 // remote Optimization Guide Service. 458 base::flat_set<std::string> external_app_packages_approved_for_fetch_; 459 460 // The top host provider that can be queried. Not owned. 461 optimization_guide::TopHostProvider* top_host_provider_ = nullptr; 462 463 // The timer used to schedule fetching hints from the remote Optimization 464 // Guide Service. 465 base::OneShotTimer top_hosts_hints_fetch_timer_; 466 467 // The clock used to schedule fetching from the remote Optimization Guide 468 // Service. 469 const base::Clock* clock_; 470 471 // Whether fetched hints should be cleared when the store is initialized 472 // because a new optimization type was registered. 473 bool should_clear_hints_for_new_type_ = false; 474 475 // The current estimate of the EffectiveConnectionType. 476 net::EffectiveConnectionType current_effective_connection_type_ = 477 net::EffectiveConnectionType::EFFECTIVE_CONNECTION_TYPE_UNKNOWN; 478 479 // Used in testing to subscribe to an update event in this class. 480 base::OnceClosure next_update_closure_; 481 482 // Used to get |weak_ptr_| to self on the UI thread. 483 base::WeakPtrFactory<OptimizationGuideHintsManager> ui_weak_ptr_factory_{ 484 this}; 485 486 DISALLOW_COPY_AND_ASSIGN(OptimizationGuideHintsManager); 487 }; 488 489 #endif // CHROME_BROWSER_OPTIMIZATION_GUIDE_OPTIMIZATION_GUIDE_HINTS_MANAGER_H_ 490