1 // Copyright 2019 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include <memory>
6
7 #include "base/base64.h"
8 #include "base/callback_helpers.h"
9 #include "base/run_loop.h"
10 #include "base/task/thread_pool/thread_pool_instance.h"
11 #include "base/test/metrics/histogram_tester.h"
12 #include "base/test/scoped_feature_list.h"
13 #include "build/build_config.h"
14 #include "chrome/browser/browser_process.h"
15 #include "chrome/browser/optimization_guide/optimization_guide_keyed_service.h"
16 #include "chrome/browser/optimization_guide/optimization_guide_keyed_service_factory.h"
17 #include "chrome/browser/optimization_guide/optimization_guide_session_statistic.h"
18 #include "chrome/browser/optimization_guide/prediction/prediction_manager.h"
19 #include "chrome/browser/profiles/profile.h"
20 #include "chrome/browser/ui/browser.h"
21 #include "chrome/services/machine_learning/public/cpp/service_connection.h"
22 #include "chrome/services/machine_learning/public/mojom/machine_learning_service.mojom.h"
23 #include "chrome/test/base/in_process_browser_test.h"
24 #include "chrome/test/base/ui_test_utils.h"
25 #include "components/data_reduction_proxy/core/common/data_reduction_proxy_switches.h"
26 #include "components/metrics/content/subprocess_metrics_provider.h"
27 #include "components/optimization_guide/optimization_guide_constants.h"
28 #include "components/optimization_guide/optimization_guide_features.h"
29 #include "components/optimization_guide/optimization_guide_store.h"
30 #include "components/optimization_guide/optimization_guide_switches.h"
31 #include "components/optimization_guide/optimization_guide_test_util.h"
32 #include "components/optimization_guide/proto/models.pb.h"
33 #include "components/optimization_guide/store_update_data.h"
34 #include "components/page_load_metrics/browser/page_load_metrics_test_waiter.h"
35 #include "components/previews/core/previews_switches.h"
36 #include "components/variations/hashing.h"
37 #include "content/public/browser/service_process_host.h"
38 #include "content/public/test/browser_test.h"
39 #include "content/public/test/browser_test_utils.h"
40 #include "content/public/test/network_connection_change_simulator.h"
41 #include "net/test/embedded_test_server/http_request.h"
42 #include "net/test/embedded_test_server/http_response.h"
43
44 namespace {
45
46 // Fetch and calculate the total number of samples from all the bins for
47 // |histogram_name|. Note: from some browertests run (such as chromeos) there
48 // might be two profiles created, and this will return the total sample count
49 // across profiles.
GetTotalHistogramSamples(const base::HistogramTester * histogram_tester,const std::string & histogram_name)50 int GetTotalHistogramSamples(const base::HistogramTester* histogram_tester,
51 const std::string& histogram_name) {
52 std::vector<base::Bucket> buckets =
53 histogram_tester->GetAllSamples(histogram_name);
54 int total = 0;
55 for (const auto& bucket : buckets)
56 total += bucket.count;
57
58 return total;
59 }
60
61 // Retries fetching |histogram_name| until it contains at least |count| samples.
RetryForHistogramUntilCountReached(const base::HistogramTester * histogram_tester,const std::string & histogram_name,int count)62 void RetryForHistogramUntilCountReached(
63 const base::HistogramTester* histogram_tester,
64 const std::string& histogram_name,
65 int count) {
66 while (true) {
67 base::ThreadPoolInstance::Get()->FlushForTesting();
68 base::RunLoop().RunUntilIdle();
69
70 content::FetchHistogramsFromChildProcesses();
71 metrics::SubprocessMetricsProvider::MergeHistogramDeltasForTesting();
72
73 int total = GetTotalHistogramSamples(histogram_tester, histogram_name);
74 if (total >= count)
75 return;
76 }
77 }
78
79 std::unique_ptr<optimization_guide::proto::PredictionModel>
GetValidDecisionTreePredictionModel()80 GetValidDecisionTreePredictionModel() {
81 std::unique_ptr<optimization_guide::proto::PredictionModel> prediction_model =
82 GetMinimalDecisionTreePredictionModel(/* threshold= */ 5.0,
83 /* weight= */ 2.0);
84
85 optimization_guide::proto::DecisionTree* decision_tree_model =
86 prediction_model->mutable_model()->mutable_decision_tree();
87
88 optimization_guide::proto::TreeNode* tree_node =
89 decision_tree_model->mutable_nodes(0);
90 tree_node->mutable_binary_node()->mutable_left_child_id()->set_value(1);
91 tree_node->mutable_binary_node()->mutable_right_child_id()->set_value(2);
92 tree_node->mutable_binary_node()
93 ->mutable_inequality_left_child_test()
94 ->mutable_feature_id()
95 ->mutable_id()
96 ->set_value("agg1");
97 tree_node->mutable_binary_node()
98 ->mutable_inequality_left_child_test()
99 ->set_type(optimization_guide::proto::InequalityTest::LESS_OR_EQUAL);
100 tree_node->mutable_binary_node()
101 ->mutable_inequality_left_child_test()
102 ->mutable_threshold()
103 ->set_float_value(1.0);
104
105 tree_node = decision_tree_model->add_nodes();
106 tree_node->mutable_node_id()->set_value(1);
107 tree_node->mutable_leaf()->mutable_vector()->add_value()->set_double_value(
108 2.);
109
110 tree_node = decision_tree_model->add_nodes();
111 tree_node->mutable_node_id()->set_value(2);
112 tree_node->mutable_leaf()->mutable_vector()->add_value()->set_double_value(
113 4.);
114
115 return prediction_model;
116 }
117
118 std::unique_ptr<optimization_guide::proto::PredictionModel>
GetValidEnsemblePredictionModel()119 GetValidEnsemblePredictionModel() {
120 std::unique_ptr<optimization_guide::proto::PredictionModel> prediction_model =
121 std::make_unique<optimization_guide::proto::PredictionModel>();
122 prediction_model->mutable_model()->mutable_threshold()->set_value(5.0);
123
124 optimization_guide::proto::Model valid_decision_tree_model =
125 GetValidDecisionTreePredictionModel()->model();
126 optimization_guide::proto::Ensemble* ensemble =
127 prediction_model->mutable_model()->mutable_ensemble();
128 *ensemble->add_members()->mutable_submodel() = valid_decision_tree_model;
129 *ensemble->add_members()->mutable_submodel() = valid_decision_tree_model;
130 return prediction_model;
131 }
132
133 std::unique_ptr<optimization_guide::proto::PredictionModel>
CreatePredictionModel()134 CreatePredictionModel() {
135 std::unique_ptr<optimization_guide::proto::PredictionModel> prediction_model =
136 GetValidEnsemblePredictionModel();
137
138 optimization_guide::proto::ModelInfo* model_info =
139 prediction_model->mutable_model_info();
140 model_info->set_version(1);
141 model_info->add_supported_model_features(
142 optimization_guide::proto::
143 CLIENT_MODEL_FEATURE_EFFECTIVE_CONNECTION_TYPE);
144 model_info->add_supported_host_model_features("agg1");
145 model_info->set_optimization_target(
146 optimization_guide::proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD);
147 model_info->add_supported_model_types(
148 optimization_guide::proto::ModelType::MODEL_TYPE_DECISION_TREE);
149 return prediction_model;
150 }
151
152 std::unique_ptr<optimization_guide::proto::GetModelsResponse>
BuildGetModelsResponse(const std::vector<std::string> & hosts,const std::vector<optimization_guide::proto::ClientModelFeature> & client_model_features)153 BuildGetModelsResponse(
154 const std::vector<std::string>& hosts,
155 const std::vector<optimization_guide::proto::ClientModelFeature>&
156 client_model_features) {
157 std::unique_ptr<optimization_guide::proto::GetModelsResponse>
158 get_models_response =
159 std::make_unique<optimization_guide::proto::GetModelsResponse>();
160
161 for (const auto& host : hosts) {
162 optimization_guide::proto::HostModelFeatures* host_model_features =
163 get_models_response->add_host_model_features();
164 host_model_features->set_host(host);
165 optimization_guide::proto::ModelFeature* model_feature =
166 host_model_features->add_model_features();
167 model_feature->set_feature_name("agg1");
168 model_feature->set_double_value(2.0);
169 }
170
171 std::unique_ptr<optimization_guide::proto::PredictionModel> prediction_model =
172 CreatePredictionModel();
173 for (const auto& client_model_feature : client_model_features) {
174 prediction_model->mutable_model_info()->add_supported_model_features(
175 client_model_feature);
176 }
177 prediction_model->mutable_model_info()->set_version(2);
178 *get_models_response->add_models() = *prediction_model.get();
179
180 return get_models_response;
181 }
182
183 enum class PredictionModelsFetcherRemoteResponseType {
184 kSuccessfulWithModelsAndFeatures = 0,
185 kSuccessfulWithFeaturesAndNoModels = 1,
186 kSuccessfulWithModelsAndNoFeatures = 2,
187 kUnsuccessful = 3,
188 };
189
190 // A WebContentsObserver that asks whether an optimization target can be
191 // applied.
192 class OptimizationGuideConsumerWebContentsObserver
193 : public content::WebContentsObserver {
194 public:
OptimizationGuideConsumerWebContentsObserver(content::WebContents * web_contents)195 explicit OptimizationGuideConsumerWebContentsObserver(
196 content::WebContents* web_contents)
197 : content::WebContentsObserver(web_contents) {}
198 ~OptimizationGuideConsumerWebContentsObserver() override = default;
199
200 // contents::WebContentsObserver implementation:
DidFinishNavigation(content::NavigationHandle * navigation_handle)201 void DidFinishNavigation(
202 content::NavigationHandle* navigation_handle) override {
203 OptimizationGuideKeyedService* service =
204 OptimizationGuideKeyedServiceFactory::GetForProfile(
205 Profile::FromBrowserContext(web_contents()->GetBrowserContext()));
206 if (callback_) {
207 // Intentionally do not set client model feature values to override to
208 // make sure decisions are the same in both sync and async variants.
209 service->ShouldTargetNavigationAsync(
210 navigation_handle,
211 optimization_guide::proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD, {},
212 std::move(callback_));
213 }
214 }
215
set_callback(optimization_guide::OptimizationGuideTargetDecisionCallback callback)216 void set_callback(
217 optimization_guide::OptimizationGuideTargetDecisionCallback callback) {
218 callback_ = std::move(callback);
219 }
220
221 private:
222 optimization_guide::OptimizationGuideTargetDecisionCallback callback_;
223 };
224
225 // A ServiceProcessHost::Observer that monitors ML Service launch events.
226 class MLServiceProcessObserver : public content::ServiceProcessHost::Observer {
227 public:
MLServiceProcessObserver()228 MLServiceProcessObserver() { content::ServiceProcessHost::AddObserver(this); }
229
~MLServiceProcessObserver()230 ~MLServiceProcessObserver() override {
231 content::ServiceProcessHost::RemoveObserver(this);
232 }
233
234 MLServiceProcessObserver(const MLServiceProcessObserver&) = delete;
235 MLServiceProcessObserver& operator=(const MLServiceProcessObserver&) = delete;
236
237 // Whether the service is launched.
IsLaunched() const238 int IsLaunched() const { return is_launched_; }
239
240 // Launch |launch_wait_loop_| to wait until a service launch is detected.
WaitForLaunch()241 void WaitForLaunch() {
242 if (!is_launched_)
243 launch_wait_loop_.Run();
244 }
245
OnServiceProcessLaunched(const content::ServiceProcessInfo & info)246 void OnServiceProcessLaunched(
247 const content::ServiceProcessInfo& info) override {
248 if (info.IsService<machine_learning::mojom::MachineLearningService>()) {
249 is_launched_ = true;
250 if (launch_wait_loop_.running())
251 launch_wait_loop_.Quit();
252 }
253 }
254
255 private:
256 base::RunLoop launch_wait_loop_;
257 bool is_launched_ = false;
258 };
259
260 } // namespace
261
262 namespace optimization_guide {
263
264 // Abstract base class for browser testing Prediction Manager.
265 // Actual class fixtures should implement InitializeFeatureList to set up
266 // features used in tests.
267 class PredictionManagerBrowserTestBase : public InProcessBrowserTest {
268 public:
269 PredictionManagerBrowserTestBase() = default;
270 ~PredictionManagerBrowserTestBase() override = default;
271
272 PredictionManagerBrowserTestBase(const PredictionManagerBrowserTestBase&) =
273 delete;
274 PredictionManagerBrowserTestBase& operator=(
275 const PredictionManagerBrowserTestBase&) = delete;
276
SetUp()277 void SetUp() override {
278 InitializeFeatureList();
279
280 models_server_ = std::make_unique<net::EmbeddedTestServer>(
281 net::EmbeddedTestServer::TYPE_HTTPS);
282 models_server_->ServeFilesFromSourceDirectory("chrome/test/data/previews");
283 models_server_->RegisterRequestHandler(base::BindRepeating(
284 &PredictionManagerBrowserTestBase::HandleGetModelsRequest,
285 base::Unretained(this)));
286
287 ASSERT_TRUE(models_server_->Start());
288 InProcessBrowserTest::SetUp();
289 }
290
SetUpOnMainThread()291 void SetUpOnMainThread() override {
292 content::NetworkConnectionChangeSimulator().SetConnectionType(
293 network::mojom::ConnectionType::CONNECTION_2G);
294 https_server_ = std::make_unique<net::EmbeddedTestServer>(
295 net::EmbeddedTestServer::TYPE_HTTPS);
296 https_server_->ServeFilesFromSourceDirectory(GetChromeTestDataDir());
297 ASSERT_TRUE(https_server_->Start());
298 https_url_with_content_ = https_server_->GetURL("/english_page.html");
299 https_url_without_content_ = https_server_->GetURL("/empty.html");
300
301 // Set up an OptimizationGuideKeyedService consumer.
302 consumer_ = std::make_unique<OptimizationGuideConsumerWebContentsObserver>(
303 browser()->tab_strip_model()->GetActiveWebContents());
304
305 InProcessBrowserTest::SetUpOnMainThread();
306 }
307
TearDownOnMainThread()308 void TearDownOnMainThread() override {
309 EXPECT_TRUE(https_server_->ShutdownAndWaitUntilComplete());
310 EXPECT_TRUE(models_server_->ShutdownAndWaitUntilComplete());
311 InProcessBrowserTest::TearDownOnMainThread();
312 }
313
SetUpCommandLine(base::CommandLine * cmd)314 void SetUpCommandLine(base::CommandLine* cmd) override {
315 cmd->AppendSwitch("enable-spdy-proxy-auth");
316 cmd->AppendSwitch(optimization_guide::switches::
317 kFetchModelsAndHostModelFeaturesOverrideTimer);
318
319 cmd->AppendSwitch(optimization_guide::switches::
320 kDisableCheckingUserPermissionsForTesting);
321 cmd->AppendSwitchASCII(optimization_guide::switches::kFetchHintsOverride,
322 "whatever.com,somehost.com");
323 cmd->AppendSwitchASCII(
324 optimization_guide::switches::kOptimizationGuideServiceGetModelsURL,
325 models_server_
326 ->GetURL(GURL(optimization_guide::
327 kOptimizationGuideServiceGetModelsDefaultURL)
328 .host(),
329 "/")
330 .spec());
331 cmd->AppendSwitchASCII("host-rules", "MAP * 127.0.0.1");
332 cmd->AppendSwitchASCII("force-variation-ids", "4");
333 }
334
SetResponseType(PredictionModelsFetcherRemoteResponseType response_type)335 void SetResponseType(
336 PredictionModelsFetcherRemoteResponseType response_type) {
337 response_type_ = response_type;
338 }
339
RegisterWithKeyedService()340 void RegisterWithKeyedService() {
341 OptimizationGuideKeyedServiceFactory::GetForProfile(browser()->profile())
342 ->RegisterOptimizationTargets(
343 {optimization_guide::proto::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD});
344 }
345
346 // Sets the callback on the consumer of the OptimizationGuideKeyedService. If
347 // set, this will call the async version of ShouldTargetNavigation.
SetCallbackOnConsumer(optimization_guide::OptimizationGuideTargetDecisionCallback callback)348 void SetCallbackOnConsumer(
349 optimization_guide::OptimizationGuideTargetDecisionCallback callback) {
350 ASSERT_TRUE(consumer_);
351
352 consumer_->set_callback(std::move(callback));
353 }
354
consumer()355 OptimizationGuideConsumerWebContentsObserver* consumer() {
356 return consumer_.get();
357 }
358
GetPredictionManager()359 PredictionManager* GetPredictionManager() {
360 OptimizationGuideKeyedService* optimization_guide_keyed_service =
361 OptimizationGuideKeyedServiceFactory::GetForProfile(
362 browser()->profile());
363 return optimization_guide_keyed_service->GetPredictionManager();
364 }
365
366 std::unique_ptr<page_load_metrics::PageLoadMetricsTestWaiter>
CreatePageLoadMetricsTestWaiter()367 CreatePageLoadMetricsTestWaiter() {
368 content::WebContents* web_contents =
369 browser()->tab_strip_model()->GetActiveWebContents();
370 return std::make_unique<page_load_metrics::PageLoadMetricsTestWaiter>(
371 web_contents);
372 }
373
SetExpectedFieldTrialNames(const base::flat_set<uint32_t> & expected_field_trial_name_hashes)374 void SetExpectedFieldTrialNames(
375 const base::flat_set<uint32_t>& expected_field_trial_name_hashes) {
376 expected_field_trial_name_hashes_ = expected_field_trial_name_hashes;
377 }
378
using_ml_service() const379 bool using_ml_service() const { return using_ml_service_; }
https_url_with_content()380 GURL https_url_with_content() { return https_url_with_content_; }
https_url_without_content()381 GURL https_url_without_content() { return https_url_without_content_; }
382
383 protected:
384 // Virtualize for testing different feature configurations.
385 virtual void InitializeFeatureList() = 0;
386
387 base::test::ScopedFeatureList scoped_feature_list_;
388
389 // Feature that the model server should return in response to
390 // GetModelsRequest.
391 proto::ClientModelFeature client_model_feature_ =
392 optimization_guide::proto::CLIENT_MODEL_FEATURE_SITE_ENGAGEMENT_SCORE;
393
394 private:
HandleGetModelsRequest(const net::test_server::HttpRequest & request)395 std::unique_ptr<net::test_server::HttpResponse> HandleGetModelsRequest(
396 const net::test_server::HttpRequest& request) {
397 std::unique_ptr<net::test_server::BasicHttpResponse> response;
398
399 response = std::make_unique<net::test_server::BasicHttpResponse>();
400 // The request to the remote Optimization Guide Service should always be a
401 // POST.
402 EXPECT_EQ(request.method, net::test_server::METHOD_POST);
403 EXPECT_NE(request.headers.end(), request.headers.find("X-Client-Data"));
404 optimization_guide::proto::GetModelsRequest models_request;
405 EXPECT_TRUE(models_request.ParseFromString(request.content));
406 // Make sure we actually filter field trials appropriately.
407 EXPECT_EQ(expected_field_trial_name_hashes_.size(),
408 static_cast<size_t>(models_request.active_field_trials_size()));
409 base::flat_set<uint32_t> seen_field_trial_name_hashes;
410 for (const auto& field_trial : models_request.active_field_trials()) {
411 EXPECT_TRUE(
412 expected_field_trial_name_hashes_.find(field_trial.name_hash()) !=
413 expected_field_trial_name_hashes_.end());
414 seen_field_trial_name_hashes.insert(field_trial.name_hash());
415 }
416 EXPECT_EQ(seen_field_trial_name_hashes.size(),
417 expected_field_trial_name_hashes_.size());
418
419 response->set_code(net::HTTP_OK);
420 std::unique_ptr<optimization_guide::proto::GetModelsResponse>
421 get_models_response = BuildGetModelsResponse(
422 {"example1.com", https_server_->GetURL("/").host()},
423 {client_model_feature_});
424 if (response_type_ == PredictionModelsFetcherRemoteResponseType::
425 kSuccessfulWithFeaturesAndNoModels) {
426 get_models_response->clear_models();
427 } else if (response_type_ == PredictionModelsFetcherRemoteResponseType::
428 kSuccessfulWithModelsAndNoFeatures) {
429 get_models_response->clear_host_model_features();
430 } else if (response_type_ ==
431 PredictionModelsFetcherRemoteResponseType::kUnsuccessful) {
432 response->set_code(net::HTTP_NOT_FOUND);
433 }
434
435 std::string serialized_response;
436 get_models_response->SerializeToString(&serialized_response);
437 response->set_content(serialized_response);
438 return std::move(response);
439 }
440
441 bool using_ml_service_ = false;
442 GURL https_url_with_content_, https_url_without_content_;
443 std::unique_ptr<net::EmbeddedTestServer> https_server_;
444 std::unique_ptr<net::EmbeddedTestServer> models_server_;
445 PredictionModelsFetcherRemoteResponseType response_type_ =
446 PredictionModelsFetcherRemoteResponseType::
447 kSuccessfulWithModelsAndFeatures;
448 std::unique_ptr<OptimizationGuideConsumerWebContentsObserver> consumer_;
449 base::flat_set<uint32_t> expected_field_trial_name_hashes_;
450 };
451
452 // Parametrized on whether the ML Service path is enabled.
453 class PredictionManagerBrowserTest
454 : public PredictionManagerBrowserTestBase,
455 public ::testing::WithParamInterface<bool> {
456 public:
PredictionManagerBrowserTest()457 PredictionManagerBrowserTest() : using_ml_service_(GetParam()) {}
458 ~PredictionManagerBrowserTest() override = default;
459
460 PredictionManagerBrowserTest(const PredictionManagerBrowserTest&) = delete;
461 PredictionManagerBrowserTest& operator=(const PredictionManagerBrowserTest&) =
462 delete;
463
using_ml_service() const464 bool using_ml_service() const { return using_ml_service_; }
465
466 private:
InitializeFeatureList()467 void InitializeFeatureList() override {
468 if (using_ml_service_) {
469 scoped_feature_list_.InitWithFeatures(
470 {optimization_guide::features::kOptimizationHints,
471 optimization_guide::features::kRemoteOptimizationGuideFetching,
472 optimization_guide::features::kOptimizationTargetPrediction,
473 optimization_guide::features::
474 kOptimizationTargetPredictionUsingMLService},
475 {});
476 } else {
477 scoped_feature_list_.InitWithFeatures(
478 {optimization_guide::features::kOptimizationHints,
479 optimization_guide::features::kRemoteOptimizationGuideFetching,
480 optimization_guide::features::kOptimizationTargetPrediction},
481 {});
482 }
483 }
484
485 bool using_ml_service_ = false;
486 };
487
488 #if defined(OS_WIN) || defined(OS_MAC) || defined(OS_CHROMEOS)
489 #define DISABLE_ON_WIN_MAC_CHROMEOS(x) DISABLED_##x
490 #else
491 #define DISABLE_ON_WIN_MAC_CHROMEOS(x) x
492 #endif
493
494 INSTANTIATE_TEST_SUITE_P(UsingMLService,
495 PredictionManagerBrowserTest,
496 ::testing::Bool(),
497 ::testing::PrintToStringParamName());
498
IN_PROC_BROWSER_TEST_P(PredictionManagerBrowserTest,DISABLE_ON_WIN_MAC_CHROMEOS (FCPReachedSessionStatisticsUpdated))499 IN_PROC_BROWSER_TEST_P(
500 PredictionManagerBrowserTest,
501 DISABLE_ON_WIN_MAC_CHROMEOS(FCPReachedSessionStatisticsUpdated)) {
502 RegisterWithKeyedService();
503 auto waiter = CreatePageLoadMetricsTestWaiter();
504 waiter->AddPageExpectation(
505 page_load_metrics::PageLoadMetricsTestWaiter::TimingField::kFirstPaint);
506 ui_test_utils::NavigateToURL(browser(), https_url_with_content());
507 waiter->Wait();
508
509 const OptimizationGuideSessionStatistic* session_fcp =
510 GetPredictionManager()->GetFCPSessionStatisticsForTesting();
511 EXPECT_TRUE(session_fcp);
512 EXPECT_EQ(1u, session_fcp->GetNumberOfSamples());
513 }
514
IN_PROC_BROWSER_TEST_P(PredictionManagerBrowserTest,DISABLE_ON_WIN_MAC_CHROMEOS (NoFCPSessionStatisticsUnchanged))515 IN_PROC_BROWSER_TEST_P(
516 PredictionManagerBrowserTest,
517 DISABLE_ON_WIN_MAC_CHROMEOS(NoFCPSessionStatisticsUnchanged)) {
518 RegisterWithKeyedService();
519 auto waiter = CreatePageLoadMetricsTestWaiter();
520 waiter->AddPageExpectation(
521 page_load_metrics::PageLoadMetricsTestWaiter::TimingField::kFirstPaint);
522 ui_test_utils::NavigateToURL(browser(), https_url_with_content());
523 waiter->Wait();
524
525 const OptimizationGuideSessionStatistic* session_fcp =
526 GetPredictionManager()->GetFCPSessionStatisticsForTesting();
527 float current_mean = session_fcp->GetMean();
528
529 waiter = CreatePageLoadMetricsTestWaiter();
530 waiter->AddPageExpectation(
531 page_load_metrics::PageLoadMetricsTestWaiter::TimingField::kLoadEvent);
532 ui_test_utils::NavigateToURL(browser(), https_url_without_content());
533 waiter->Wait();
534 EXPECT_EQ(1u, session_fcp->GetNumberOfSamples());
535 EXPECT_EQ(current_mean, session_fcp->GetMean());
536 }
537
IN_PROC_BROWSER_TEST_P(PredictionManagerBrowserTest,DISABLE_ON_WIN_MAC_CHROMEOS (ModelsAndFeaturesStoreInitialized))538 IN_PROC_BROWSER_TEST_P(
539 PredictionManagerBrowserTest,
540 DISABLE_ON_WIN_MAC_CHROMEOS(ModelsAndFeaturesStoreInitialized)) {
541 base::HistogramTester histogram_tester;
542 MLServiceProcessObserver ml_service_observer;
543 content::NetworkConnectionChangeSimulator().SetConnectionType(
544 network::mojom::ConnectionType::CONNECTION_2G);
545
546 RegisterWithKeyedService();
547 RetryForHistogramUntilCountReached(
548 &histogram_tester,
549 "OptimizationGuide.PredictionManager.PredictionModelsStored", 1);
550 RetryForHistogramUntilCountReached(
551 &histogram_tester,
552 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 1);
553
554 histogram_tester.ExpectUniqueSample(
555 "OptimizationGuide.PredictionManager.HostModelFeaturesStored", true, 1);
556 histogram_tester.ExpectUniqueSample(
557 "OptimizationGuide.PredictionManager.PredictionModelsStored", true, 1);
558 histogram_tester.ExpectUniqueSample(
559 "OptimizationGuide.PredictionModelUpdateVersion.PainfulPageLoad", 2, 1);
560 histogram_tester.ExpectUniqueSample(
561 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 2, 1);
562 EXPECT_EQ(ml_service_observer.IsLaunched(), using_ml_service());
563 }
564
IN_PROC_BROWSER_TEST_P(PredictionManagerBrowserTest,DISABLE_ON_WIN_MAC_CHROMEOS (OnlyHostModelFeaturesInGetModelsResponse))565 IN_PROC_BROWSER_TEST_P(
566 PredictionManagerBrowserTest,
567 DISABLE_ON_WIN_MAC_CHROMEOS(OnlyHostModelFeaturesInGetModelsResponse)) {
568 base::HistogramTester histogram_tester;
569 MLServiceProcessObserver ml_service_observer;
570
571 SetResponseType(PredictionModelsFetcherRemoteResponseType::
572 kSuccessfulWithFeaturesAndNoModels);
573 RegisterWithKeyedService();
574 RetryForHistogramUntilCountReached(
575 &histogram_tester,
576 "OptimizationGuide.PredictionManager.HostModelFeaturesStored", 1);
577 histogram_tester.ExpectUniqueSample(
578 "OptimizationGuide.PredictionManager.HostModelFeaturesStored", true, 1);
579 histogram_tester.ExpectTotalCount(
580 "OptimizationGuide.PredictionManager.PredictionModelsStored", 0);
581 histogram_tester.ExpectTotalCount(
582 "OptimizationGuide.PredictionModelUpdateVersion.PainfulPageLoad", 0);
583 histogram_tester.ExpectTotalCount(
584 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 0);
585 EXPECT_FALSE(ml_service_observer.IsLaunched());
586 }
587
IN_PROC_BROWSER_TEST_P(PredictionManagerBrowserTest,DISABLE_ON_WIN_MAC_CHROMEOS (OnlyPredictionModelsInGetModelsResponse))588 IN_PROC_BROWSER_TEST_P(
589 PredictionManagerBrowserTest,
590 DISABLE_ON_WIN_MAC_CHROMEOS(OnlyPredictionModelsInGetModelsResponse)) {
591 base::HistogramTester histogram_tester;
592 MLServiceProcessObserver ml_service_observer;
593
594 SetResponseType(PredictionModelsFetcherRemoteResponseType::
595 kSuccessfulWithModelsAndNoFeatures);
596 RegisterWithKeyedService();
597 RetryForHistogramUntilCountReached(
598 &histogram_tester,
599 "OptimizationGuide.PredictionManager.PredictionModelsStored", 1);
600 RetryForHistogramUntilCountReached(
601 &histogram_tester,
602 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 1);
603
604 // A metadata entry will always be stored for host model features, regardless
605 // of whether any host model features were actually returned.
606 histogram_tester.ExpectUniqueSample(
607 "OptimizationGuide.PredictionManager.HostModelFeaturesStored", true, 1);
608 histogram_tester.ExpectUniqueSample(
609 "OptimizationGuide.PredictionManager.PredictionModelsStored", true, 1);
610 histogram_tester.ExpectUniqueSample(
611 "OptimizationGuide.PredictionModelUpdateVersion.PainfulPageLoad", 2, 1);
612 histogram_tester.ExpectUniqueSample(
613 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 2, 1);
614 EXPECT_EQ(ml_service_observer.IsLaunched(), using_ml_service());
615 }
616
IN_PROC_BROWSER_TEST_P(PredictionManagerBrowserTest,DISABLE_ON_WIN_MAC_CHROMEOS (PredictionModelFetchFailed))617 IN_PROC_BROWSER_TEST_P(
618 PredictionManagerBrowserTest,
619 DISABLE_ON_WIN_MAC_CHROMEOS(PredictionModelFetchFailed)) {
620 SetResponseType(PredictionModelsFetcherRemoteResponseType::kUnsuccessful);
621 base::HistogramTester histogram_tester;
622 MLServiceProcessObserver ml_service_observer;
623
624 RegisterWithKeyedService();
625
626 // Wait until histograms have been updated before performing checks for
627 // correct behavior based on the response.
628 RetryForHistogramUntilCountReached(
629 &histogram_tester,
630 "OptimizationGuide.PredictionModelFetcher.GetModelsResponse.Status", 1);
631
632 histogram_tester.ExpectBucketCount(
633 "OptimizationGuide.PredictionModelFetcher.GetModelsResponse.Status",
634 net::HTTP_NOT_FOUND, 1);
635
636 histogram_tester.ExpectTotalCount(
637 "OptimizationGuide.PredictionManager.HostModelFeaturesStored", 0);
638 histogram_tester.ExpectTotalCount(
639 "OptimizationGuide.PredictionManager.PredictionModelsStored", 0);
640 histogram_tester.ExpectTotalCount(
641 "OptimizationGuide.PredictionModelUpdateVersion.PainfulPageLoad", 0);
642 histogram_tester.ExpectTotalCount(
643 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 0);
644 EXPECT_FALSE(ml_service_observer.IsLaunched());
645 }
646
IN_PROC_BROWSER_TEST_P(PredictionManagerBrowserTest,DISABLE_ON_WIN_MAC_CHROMEOS (HostModelFeaturesClearedOnHistoryClear))647 IN_PROC_BROWSER_TEST_P(
648 PredictionManagerBrowserTest,
649 DISABLE_ON_WIN_MAC_CHROMEOS(HostModelFeaturesClearedOnHistoryClear)) {
650 base::HistogramTester histogram_tester;
651 MLServiceProcessObserver ml_service_observer;
652
653 RegisterWithKeyedService();
654
655 // Wait until histograms have been updated before performing checks for
656 // correct behavior based on the response.
657 RetryForHistogramUntilCountReached(
658 &histogram_tester,
659 "OptimizationGuide.PredictionModelFetcher.GetModelsResponse.Status", 1);
660
661 RetryForHistogramUntilCountReached(
662 &histogram_tester,
663 "OptimizationGuide.PredictionManager.HostModelFeaturesStored", 1);
664
665 RetryForHistogramUntilCountReached(
666 &histogram_tester,
667 "OptimizationGuide.PredictionManager.PredictionModelsStored", 1);
668
669 RetryForHistogramUntilCountReached(
670 &histogram_tester,
671 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 1);
672
673 EXPECT_EQ(ml_service_observer.IsLaunched(), using_ml_service());
674
675 SetCallbackOnConsumer(base::DoNothing());
676 ui_test_utils::NavigateToURL(browser(), https_url_with_content());
677
678 histogram_tester.ExpectUniqueSample(
679 "OptimizationGuide.PredictionManager.HasHostModelFeaturesForHost", true,
680 1);
681
682 // Wipe the browser history - clears all the host model features.
683 browser()->profile()->Wipe();
684 histogram_tester.ExpectBucketCount(
685 "OptimizationGuide.ClearHostModelFeatures.StoreAvailable", true, 1);
686
687 SetCallbackOnConsumer(base::DoNothing());
688 ui_test_utils::NavigateToURL(browser(), https_url_with_content());
689 histogram_tester.ExpectBucketCount(
690 "OptimizationGuide.PredictionManager.HasHostModelFeaturesForHost", false,
691 1);
692 }
693
694 class PredictionManagerBrowserSameOriginTest
695 : public PredictionManagerBrowserTest {
696 public:
697 PredictionManagerBrowserSameOriginTest() = default;
698 ~PredictionManagerBrowserSameOriginTest() override = default;
699
SetUp()700 void SetUp() override {
701 client_model_feature_ =
702 optimization_guide::proto::CLIENT_MODEL_FEATURE_SAME_ORIGIN_NAVIGATION;
703 PredictionManagerBrowserTest::SetUp();
704 }
705 };
706
707 INSTANTIATE_TEST_SUITE_P(UsingMLService,
708 PredictionManagerBrowserSameOriginTest,
709 ::testing::Bool(),
710 ::testing::PrintToStringParamName());
711
712 // Regression test for https://crbug.com/1037945. Tests that the origin of the
713 // previous navigation is computed correctly.
IN_PROC_BROWSER_TEST_P(PredictionManagerBrowserSameOriginTest,DISABLE_ON_WIN_MAC_CHROMEOS (IsSameOriginNavigation))714 IN_PROC_BROWSER_TEST_P(PredictionManagerBrowserSameOriginTest,
715 DISABLE_ON_WIN_MAC_CHROMEOS(IsSameOriginNavigation)) {
716 base::HistogramTester histogram_tester;
717 MLServiceProcessObserver ml_service_observer;
718
719 RegisterWithKeyedService();
720
721 // Wait until histograms have been updated before performing checks for
722 // correct behavior based on the response.
723 RetryForHistogramUntilCountReached(
724 &histogram_tester,
725 "OptimizationGuide.PredictionModelFetcher.GetModelsResponse.Status", 1);
726
727 RetryForHistogramUntilCountReached(
728 &histogram_tester,
729 "OptimizationGuide.PredictionManager.HostModelFeaturesStored", 1);
730
731 RetryForHistogramUntilCountReached(
732 &histogram_tester,
733 "OptimizationGuide.PredictionManager.PredictionModelsStored", 1);
734
735 RetryForHistogramUntilCountReached(
736 &histogram_tester,
737 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 1);
738
739 EXPECT_EQ(ml_service_observer.IsLaunched(), using_ml_service());
740
741 SetCallbackOnConsumer(base::DoNothing());
742 ui_test_utils::NavigateToURL(browser(), https_url_with_content());
743 RetryForHistogramUntilCountReached(
744 &histogram_tester, "OptimizationGuide.PredictionManager.IsSameOrigin", 1);
745 histogram_tester.ExpectUniqueSample(
746 "OptimizationGuide.PredictionManager.IsSameOrigin", false, 1);
747
748 // Navigate to the same URL in the same tab. This should count as a
749 // same-origin navigation.
750 SetCallbackOnConsumer(base::DoNothing());
751 ui_test_utils::NavigateToURL(browser(), https_url_with_content());
752 RetryForHistogramUntilCountReached(
753 &histogram_tester, "OptimizationGuide.PredictionManager.IsSameOrigin", 2);
754 histogram_tester.ExpectBucketCount(
755 "OptimizationGuide.PredictionManager.IsSameOrigin", false, 1);
756 histogram_tester.ExpectBucketCount(
757 "OptimizationGuide.PredictionManager.IsSameOrigin", true, 1);
758
759 // Navigate to a cross-origin URL. This should count as a cross-origin
760 // navigation.
761 SetCallbackOnConsumer(base::DoNothing());
762 ui_test_utils::NavigateToURL(browser(), GURL("https://www.google.com/"));
763 RetryForHistogramUntilCountReached(
764 &histogram_tester, "OptimizationGuide.PredictionManager.IsSameOrigin", 3);
765 histogram_tester.ExpectBucketCount(
766 "OptimizationGuide.PredictionManager.IsSameOrigin", false, 2);
767 histogram_tester.ExpectBucketCount(
768 "OptimizationGuide.PredictionManager.IsSameOrigin", true, 1);
769 }
770
IN_PROC_BROWSER_TEST_P(PredictionManagerBrowserSameOriginTest,DISABLE_ON_WIN_MAC_CHROMEOS (ShouldTargetNavigationAsync))771 IN_PROC_BROWSER_TEST_P(
772 PredictionManagerBrowserSameOriginTest,
773 DISABLE_ON_WIN_MAC_CHROMEOS(ShouldTargetNavigationAsync)) {
774 base::HistogramTester histogram_tester;
775 MLServiceProcessObserver ml_service_observer;
776
777 RegisterWithKeyedService();
778
779 // Wait until histograms have been updated before performing checks for
780 // correct behavior based on the response.
781 RetryForHistogramUntilCountReached(
782 &histogram_tester,
783 "OptimizationGuide.PredictionModelFetcher.GetModelsResponse.Status", 1);
784
785 RetryForHistogramUntilCountReached(
786 &histogram_tester,
787 "OptimizationGuide.PredictionManager.HostModelFeaturesStored", 1);
788
789 RetryForHistogramUntilCountReached(
790 &histogram_tester,
791 "OptimizationGuide.PredictionManager.PredictionModelsStored", 1);
792
793 RetryForHistogramUntilCountReached(
794 &histogram_tester,
795 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 1);
796
797 EXPECT_EQ(ml_service_observer.IsLaunched(), using_ml_service());
798
799 std::unique_ptr<base::RunLoop> run_loop = std::make_unique<base::RunLoop>();
800 SetCallbackOnConsumer(base::BindOnce(
801 [](base::RunLoop* run_loop,
802 OptimizationGuideConsumerWebContentsObserver* consumer,
803 optimization_guide::OptimizationGuideDecision decision) {
804 // The model should be evaluated with an actual decision since the model
805 // and all features provided are valid.
806 EXPECT_NE(decision,
807 optimization_guide::OptimizationGuideDecision::kUnknown);
808 run_loop->Quit();
809 },
810 run_loop.get(), consumer()));
811
812 ui_test_utils::NavigateToURL(browser(), https_url_with_content());
813 run_loop->Run();
814 }
815
816 class PredictionManagerUsingMLServiceBrowserSameOriginTest
817 : public PredictionManagerBrowserSameOriginTest {};
818
819 // Only instantiate with ML Service enabled.
820 INSTANTIATE_TEST_SUITE_P(UsingMLService,
821 PredictionManagerUsingMLServiceBrowserSameOriginTest,
822 ::testing::Values(true),
823 ::testing::PrintToStringParamName());
824
IN_PROC_BROWSER_TEST_P(PredictionManagerUsingMLServiceBrowserSameOriginTest,DISABLE_ON_WIN_MAC_CHROMEOS (ShouldTargetNavigationAsyncWithServiceDisconnection))825 IN_PROC_BROWSER_TEST_P(
826 PredictionManagerUsingMLServiceBrowserSameOriginTest,
827 DISABLE_ON_WIN_MAC_CHROMEOS(
828 ShouldTargetNavigationAsyncWithServiceDisconnection)) {
829 base::HistogramTester histogram_tester;
830 MLServiceProcessObserver ml_service_observer;
831
832 RegisterWithKeyedService();
833
834 // Wait until histograms have been updated before performing checks for
835 // correct behavior based on the response.
836 RetryForHistogramUntilCountReached(
837 &histogram_tester,
838 "OptimizationGuide.PredictionModelFetcher.GetModelsResponse.Status", 1);
839
840 RetryForHistogramUntilCountReached(
841 &histogram_tester,
842 "OptimizationGuide.PredictionManager.HostModelFeaturesStored", 1);
843
844 RetryForHistogramUntilCountReached(
845 &histogram_tester,
846 "OptimizationGuide.PredictionManager.PredictionModelsStored", 1);
847
848 RetryForHistogramUntilCountReached(
849 &histogram_tester,
850 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 1);
851
852 EXPECT_TRUE(ml_service_observer.IsLaunched());
853
854 // Force termination of the service: model predictors will become invalid.
855 machine_learning::ServiceConnection::GetInstance()->ResetServiceForTesting();
856
857 SetCallbackOnConsumer(base::BindOnce(
858 [](OptimizationGuideConsumerWebContentsObserver* consumer,
859 optimization_guide::OptimizationGuideDecision decision) {
860 EXPECT_EQ(decision,
861 optimization_guide::OptimizationGuideDecision::kUnknown);
862 },
863 consumer()));
864
865 ui_test_utils::NavigateToURL(browser(), https_url_with_content());
866 }
867
868 class PredictionManagerUsingMLServiceMetricsOnlyBrowserTest
869 : public PredictionManagerBrowserTestBase {
870 public:
871 PredictionManagerUsingMLServiceMetricsOnlyBrowserTest() = default;
872 ~PredictionManagerUsingMLServiceMetricsOnlyBrowserTest() override = default;
873
874 private:
InitializeFeatureList()875 void InitializeFeatureList() override {
876 scoped_feature_list_.InitWithFeaturesAndParameters(
877 {
878 {optimization_guide::features::kOptimizationHints, {}},
879 {optimization_guide::features::kRemoteOptimizationGuideFetching,
880 {}},
881 {optimization_guide::features::kOptimizationTargetPrediction,
882 {{"painful_page_load_metrics_only", "true"}}},
883 {optimization_guide::features::
884 kOptimizationTargetPredictionUsingMLService,
885 {}},
886 {optimization_guide::features::kOptimizationHintsFieldTrials,
887 {{"allowed_field_trial_names",
888 "scoped_feature_list_trial_for_OptimizationHints,scoped_feature_"
889 "list_trial_for_OptimizationHintsFetching"}}},
890 },
891 {});
892 SetExpectedFieldTrialNames(base::flat_set<uint32_t>(
893 {variations::HashName(
894 "scoped_feature_list_trial_for_OptimizationHints"),
895 variations::HashName(
896 "scoped_feature_list_trial_for_OptimizationHintsFetching")}));
897 }
898 };
899
IN_PROC_BROWSER_TEST_F(PredictionManagerUsingMLServiceMetricsOnlyBrowserTest,DISABLE_ON_WIN_MAC_CHROMEOS (ShouldTargetNavigationAsync))900 IN_PROC_BROWSER_TEST_F(
901 PredictionManagerUsingMLServiceMetricsOnlyBrowserTest,
902 DISABLE_ON_WIN_MAC_CHROMEOS(ShouldTargetNavigationAsync)) {
903 base::HistogramTester histogram_tester;
904 MLServiceProcessObserver ml_service_observer;
905
906 EXPECT_TRUE(
907 features::ShouldOverrideOptimizationTargetDecisionForMetricsPurposes(
908 proto::OptimizationTarget::OPTIMIZATION_TARGET_PAINFUL_PAGE_LOAD));
909
910 RegisterWithKeyedService();
911
912 // Wait until histograms have been updated before performing checks for
913 // correct behavior based on the response.
914 RetryForHistogramUntilCountReached(
915 &histogram_tester,
916 "OptimizationGuide.PredictionModelFetcher.GetModelsResponse.Status", 1);
917
918 RetryForHistogramUntilCountReached(
919 &histogram_tester,
920 "OptimizationGuide.PredictionManager.HostModelFeaturesStored", 1);
921
922 RetryForHistogramUntilCountReached(
923 &histogram_tester,
924 "OptimizationGuide.PredictionManager.PredictionModelsStored", 1);
925
926 RetryForHistogramUntilCountReached(
927 &histogram_tester,
928 "OptimizationGuide.PredictionModelLoadedVersion.PainfulPageLoad", 1);
929
930 EXPECT_TRUE(ml_service_observer.IsLaunched());
931
932 SetCallbackOnConsumer(base::BindOnce(
933 [](OptimizationGuideConsumerWebContentsObserver* consumer,
934 optimization_guide::OptimizationGuideDecision decision) {
935 EXPECT_EQ(decision,
936 optimization_guide::OptimizationGuideDecision::kFalse);
937 },
938 consumer()));
939
940 ui_test_utils::NavigateToURL(browser(), https_url_with_content());
941 }
942
943 } // namespace optimization_guide
944