1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "components/ntp_snippets/remote/json_request.h"
6 
7 #include <algorithm>
8 #include <utility>
9 #include <vector>
10 
11 #include "base/bind.h"
12 #include "base/command_line.h"
13 #include "base/feature_list.h"
14 #include "base/json/json_writer.h"
15 #include "base/metrics/histogram_functions.h"
16 #include "base/metrics/histogram_macros.h"
17 #include "base/strings/stringprintf.h"
18 #include "base/time/clock.h"
19 #include "base/values.h"
20 #include "components/ntp_snippets/category_info.h"
21 #include "components/ntp_snippets/features.h"
22 #include "components/ntp_snippets/remote/request_params.h"
23 #include "components/ntp_snippets/user_classifier.h"
24 #include "components/strings/grit/components_strings.h"
25 #include "components/variations/net/variations_http_headers.h"
26 #include "components/variations/variations_associated_data.h"
27 #include "net/base/load_flags.h"
28 #include "net/http/http_response_headers.h"
29 #include "net/http/http_status_code.h"
30 #include "net/traffic_annotation/network_traffic_annotation.h"
31 #include "services/network/public/cpp/resource_request.h"
32 #include "services/network/public/cpp/shared_url_loader_factory.h"
33 #include "services/network/public/cpp/simple_url_loader.h"
34 #include "third_party/icu/source/common/unicode/uloc.h"
35 #include "third_party/icu/source/common/unicode/utypes.h"
36 #include "ui/base/l10n/l10n_util.h"
37 
38 using language::UrlLanguageHistogram;
39 
40 namespace ntp_snippets {
41 
42 namespace internal {
43 
44 namespace {
45 
46 // Variation parameter for disabling the retry.
47 const char kBackground5xxRetriesName[] = "background_5xx_retries_count";
48 
49 // Variation parameter for sending UrlLanguageHistogram info to the server.
50 const char kSendTopLanguagesName[] = "send_top_languages";
51 
52 // Variation parameter for sending UserClassifier info to the server.
53 const char kSendUserClassName[] = "send_user_class";
54 
IsSendingTopLanguagesEnabled()55 bool IsSendingTopLanguagesEnabled() {
56   return variations::GetVariationParamByFeatureAsBool(
57       ntp_snippets::kArticleSuggestionsFeature, kSendTopLanguagesName,
58       /*default_value=*/true);
59 }
60 
IsSendingUserClassEnabled()61 bool IsSendingUserClassEnabled() {
62   return variations::GetVariationParamByFeatureAsBool(
63       ntp_snippets::kArticleSuggestionsFeature, kSendUserClassName,
64       /*default_value=*/true);
65 }
66 
IsSendingOptionalImagesCapabilityEnabled()67 bool IsSendingOptionalImagesCapabilityEnabled() {
68   return base::FeatureList::IsEnabled(
69       ntp_snippets::kOptionalImagesEnabledFeature);
70 }
71 
72 // Translate the BCP 47 |language_code| into a posix locale string.
PosixLocaleFromBCP47Language(const std::string & language_code)73 std::string PosixLocaleFromBCP47Language(const std::string& language_code) {
74   char locale[ULOC_FULLNAME_CAPACITY];
75   UErrorCode error = U_ZERO_ERROR;
76   // Translate the input to a posix locale.
77   uloc_forLanguageTag(language_code.c_str(), locale, ULOC_FULLNAME_CAPACITY,
78                       nullptr, &error);
79   if (error != U_ZERO_ERROR) {
80     DLOG(WARNING) << "Error in translating language code to a locale string: "
81                   << error;
82     return std::string();
83   }
84   return locale;
85 }
86 
ISO639FromPosixLocale(const std::string & locale)87 std::string ISO639FromPosixLocale(const std::string& locale) {
88   char language[ULOC_LANG_CAPACITY];
89   UErrorCode error = U_ZERO_ERROR;
90   uloc_getLanguage(locale.c_str(), language, ULOC_LANG_CAPACITY, &error);
91   if (error != U_ZERO_ERROR) {
92     DLOG(WARNING)
93         << "Error in translating locale string to a ISO639 language code: "
94         << error;
95     return std::string();
96   }
97   return language;
98 }
99 
AppendLanguageInfoToList(base::ListValue * list,const UrlLanguageHistogram::LanguageInfo & info)100 void AppendLanguageInfoToList(base::ListValue* list,
101                               const UrlLanguageHistogram::LanguageInfo& info) {
102   auto lang = std::make_unique<base::DictionaryValue>();
103   lang->SetString("language", info.language_code);
104   lang->SetDouble("frequency", info.frequency);
105   list->Append(std::move(lang));
106 }
107 
GetUserClassString(UserClassifier::UserClass user_class)108 std::string GetUserClassString(UserClassifier::UserClass user_class) {
109   switch (user_class) {
110     case UserClassifier::UserClass::RARE_NTP_USER:
111       return "RARE_NTP_USER";
112     case UserClassifier::UserClass::ACTIVE_NTP_USER:
113       return "ACTIVE_NTP_USER";
114     case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER:
115       return "ACTIVE_SUGGESTIONS_CONSUMER";
116   }
117   NOTREACHED();
118   return std::string();
119 }
120 
121 }  // namespace
122 
JsonRequest(base::Optional<Category> exclusive_category,const base::Clock * clock,const ParseJSONCallback & callback)123 JsonRequest::JsonRequest(
124     base::Optional<Category> exclusive_category,
125     const base::Clock* clock,  // Needed until destruction of the request.
126     const ParseJSONCallback& callback)
127     : exclusive_category_(exclusive_category),
128       clock_(clock),
129       parse_json_callback_(callback) {
130   creation_time_ = clock_->Now();
131 }
132 
~JsonRequest()133 JsonRequest::~JsonRequest() {
134   LOG_IF(DFATAL, !request_completed_callback_.is_null())
135       << "The CompletionCallback was never called!";
136 }
137 
Start(CompletedCallback callback)138 void JsonRequest::Start(CompletedCallback callback) {
139   DCHECK(simple_url_loader_);
140   DCHECK(url_loader_factory_);
141   request_completed_callback_ = std::move(callback);
142   last_response_string_.clear();
143   simple_url_loader_->SetAllowHttpErrorResults(true);
144   simple_url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie(
145       url_loader_factory_.get(),
146       base::BindOnce(&JsonRequest::OnSimpleLoaderComplete,
147                      base::Unretained(this)));
148 }
149 
150 // static
Get5xxRetryCount(bool interactive_request)151 int JsonRequest::Get5xxRetryCount(bool interactive_request) {
152   if (interactive_request) {
153     return 2;
154   }
155   return std::max(0, variations::GetVariationParamByFeatureAsInt(
156                          ntp_snippets::kArticleSuggestionsFeature,
157                          kBackground5xxRetriesName, 0));
158 }
159 
GetFetchDuration() const160 base::TimeDelta JsonRequest::GetFetchDuration() const {
161   return clock_->Now() - creation_time_;
162 }
163 
GetResponseString() const164 std::string JsonRequest::GetResponseString() const {
165   return last_response_string_;
166 }
167 
OnSimpleLoaderComplete(std::unique_ptr<std::string> response_body)168 void JsonRequest::OnSimpleLoaderComplete(
169     std::unique_ptr<std::string> response_body) {
170   int net_error = simple_url_loader_->NetError();
171   int response_code = -1;
172   if (simple_url_loader_->ResponseInfo() &&
173       simple_url_loader_->ResponseInfo()->headers) {
174     response_code =
175         simple_url_loader_->ResponseInfo()->headers->response_code();
176   }
177   simple_url_loader_.reset();
178   base::UmaHistogramSparse("NewTabPage.Snippets.FetchHttpResponseOrErrorCode",
179                            net_error == net::OK ? response_code : net_error);
180   if (net_error != net::OK) {
181     std::move(request_completed_callback_)
182         .Run(/*result=*/base::Value(), FetchResult::URL_REQUEST_STATUS_ERROR,
183              /*error_details=*/base::StringPrintf(" %d", net_error));
184   } else if (response_code / 100 != 2) {
185     FetchResult result = response_code == net::HTTP_UNAUTHORIZED
186                              ? FetchResult::HTTP_ERROR_UNAUTHORIZED
187                              : FetchResult::HTTP_ERROR;
188     std::move(request_completed_callback_)
189         .Run(/*result=*/base::Value(), result,
190              /*error_details=*/base::StringPrintf(" %d", response_code));
191   } else {
192     last_response_string_ = std::move(*response_body);
193     parse_json_callback_.Run(last_response_string_,
194                              base::BindOnce(&JsonRequest::OnJsonParsed,
195                                             weak_ptr_factory_.GetWeakPtr()),
196                              base::BindOnce(&JsonRequest::OnJsonError,
197                                             weak_ptr_factory_.GetWeakPtr()));
198   }
199 }
200 
OnJsonParsed(base::Value result)201 void JsonRequest::OnJsonParsed(base::Value result) {
202   std::move(request_completed_callback_)
203       .Run(std::move(result), FetchResult::SUCCESS,
204            /*error_details=*/std::string());
205 }
206 
OnJsonError(const std::string & error)207 void JsonRequest::OnJsonError(const std::string& error) {
208   LOG(WARNING) << "Received invalid JSON (" << error
209                << "): " << last_response_string_;
210   std::move(request_completed_callback_)
211       .Run(/*result=*/base::Value(), FetchResult::JSON_PARSE_ERROR,
212            /*error_details=*/base::StringPrintf(" (error %s)", error.c_str()));
213 }
214 
Builder()215 JsonRequest::Builder::Builder() : language_histogram_(nullptr) {}
216 JsonRequest::Builder::Builder(JsonRequest::Builder&&) = default;
217 JsonRequest::Builder::~Builder() = default;
218 
Build() const219 std::unique_ptr<JsonRequest> JsonRequest::Builder::Build() const {
220   DCHECK(!url_.is_empty());
221   DCHECK(url_loader_factory_);
222   DCHECK(clock_);
223   auto request = std::make_unique<JsonRequest>(params_.exclusive_category,
224                                                clock_, parse_json_callback_);
225   std::string body = BuildBody();
226   request->simple_url_loader_ = BuildURLLoader(body);
227   request->url_loader_factory_ = std::move(url_loader_factory_);
228 
229   return request;
230 }
231 
SetAuthentication(const std::string & auth_header)232 JsonRequest::Builder& JsonRequest::Builder::SetAuthentication(
233     const std::string& auth_header) {
234   auth_header_ = auth_header;
235   return *this;
236 }
237 
SetLanguageHistogram(const language::UrlLanguageHistogram * language_histogram)238 JsonRequest::Builder& JsonRequest::Builder::SetLanguageHistogram(
239     const language::UrlLanguageHistogram* language_histogram) {
240   language_histogram_ = language_histogram;
241   return *this;
242 }
243 
SetParams(const RequestParams & params)244 JsonRequest::Builder& JsonRequest::Builder::SetParams(
245     const RequestParams& params) {
246   params_ = params;
247   return *this;
248 }
249 
SetParseJsonCallback(ParseJSONCallback callback)250 JsonRequest::Builder& JsonRequest::Builder::SetParseJsonCallback(
251     ParseJSONCallback callback) {
252   parse_json_callback_ = callback;
253   return *this;
254 }
255 
SetClock(const base::Clock * clock)256 JsonRequest::Builder& JsonRequest::Builder::SetClock(const base::Clock* clock) {
257   clock_ = clock;
258   return *this;
259 }
260 
SetUrl(const GURL & url)261 JsonRequest::Builder& JsonRequest::Builder::SetUrl(const GURL& url) {
262   url_ = url;
263   return *this;
264 }
265 
SetUrlLoaderFactory(scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory)266 JsonRequest::Builder& JsonRequest::Builder::SetUrlLoaderFactory(
267     scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory) {
268   url_loader_factory_ = std::move(url_loader_factory);
269   return *this;
270 }
271 
SetUserClassifier(const UserClassifier & user_classifier)272 JsonRequest::Builder& JsonRequest::Builder::SetUserClassifier(
273     const UserClassifier& user_classifier) {
274   if (IsSendingUserClassEnabled()) {
275     user_class_ = GetUserClassString(user_classifier.GetUserClass());
276   }
277   return *this;
278 }
279 
SetOptionalImagesCapability(bool supports_optional_images)280 JsonRequest::Builder& JsonRequest::Builder::SetOptionalImagesCapability(
281     bool supports_optional_images) {
282   if (supports_optional_images && IsSendingOptionalImagesCapabilityEnabled()) {
283     display_capability_ = "CAPABILITY_OPTIONAL_IMAGES";
284   }
285   return *this;
286 }
287 
288 std::unique_ptr<network::ResourceRequest>
BuildResourceRequest() const289 JsonRequest::Builder::BuildResourceRequest() const {
290   auto resource_request = std::make_unique<network::ResourceRequest>();
291   resource_request->url = url_;
292   resource_request->credentials_mode = network::mojom::CredentialsMode::kOmit;
293   resource_request->method = "POST";
294   resource_request->headers.SetHeader("Content-Type",
295                                       "application/json; charset=UTF-8");
296   if (!auth_header_.empty()) {
297     resource_request->headers.SetHeader("Authorization", auth_header_);
298   }
299   // Add X-Client-Data header with experiment IDs from field trials.
300   // TODO: We should call AppendVariationHeaders with explicit
301   // variations::SignedIn::kNo If the auth_header_ is empty
302   variations::AppendVariationsHeaderUnknownSignedIn(
303       url_, variations::InIncognito::kNo, resource_request.get());
304   return resource_request;
305 }
306 
BuildBody() const307 std::string JsonRequest::Builder::BuildBody() const {
308   auto request = std::make_unique<base::DictionaryValue>();
309   std::string user_locale = PosixLocaleFromBCP47Language(params_.language_code);
310   if (!user_locale.empty()) {
311     request->SetString("uiLanguage", user_locale);
312   }
313 
314   request->SetString("priority", params_.interactive_request
315                                      ? "USER_ACTION"
316                                      : "BACKGROUND_PREFETCH");
317 
318   auto excluded = std::make_unique<base::ListValue>();
319   for (const auto& id : params_.excluded_ids) {
320     excluded->AppendString(id);
321   }
322   request->Set("excludedSuggestionIds", std::move(excluded));
323 
324   if (!user_class_.empty()) {
325     request->SetString("userActivenessClass", user_class_);
326   }
327 
328   if (!display_capability_.empty()) {
329     request->SetString("displayCapability", display_capability_);
330   }
331 
332   language::UrlLanguageHistogram::LanguageInfo ui_language;
333   language::UrlLanguageHistogram::LanguageInfo other_top_language;
334   PrepareLanguages(&ui_language, &other_top_language);
335   if (ui_language.frequency != 0 || other_top_language.frequency != 0) {
336     auto language_list = std::make_unique<base::ListValue>();
337     if (ui_language.frequency > 0) {
338       AppendLanguageInfoToList(language_list.get(), ui_language);
339     }
340     if (other_top_language.frequency > 0) {
341       AppendLanguageInfoToList(language_list.get(), other_top_language);
342     }
343     request->Set("topLanguages", std::move(language_list));
344   }
345 
346   // TODO(vitaliii): Support count_to_fetch without requiring
347   // |exclusive_category|.
348   if (params_.exclusive_category.has_value()) {
349     base::DictionaryValue exclusive_category_parameters;
350     exclusive_category_parameters.SetInteger(
351         "id", params_.exclusive_category->remote_id());
352     exclusive_category_parameters.SetInteger("numSuggestions",
353                                              params_.count_to_fetch);
354     base::ListValue category_parameters;
355     category_parameters.Append(std::move(exclusive_category_parameters));
356     request->SetKey("categoryParameters", std::move(category_parameters));
357   }
358 
359   std::string request_json;
360   bool success = base::JSONWriter::WriteWithOptions(
361       *request, base::JSONWriter::OPTIONS_PRETTY_PRINT, &request_json);
362   DCHECK(success);
363   return request_json;
364 }
365 
BuildURLLoader(const std::string & body) const366 std::unique_ptr<network::SimpleURLLoader> JsonRequest::Builder::BuildURLLoader(
367     const std::string& body) const {
368   net::NetworkTrafficAnnotationTag traffic_annotation =
369       net::DefineNetworkTrafficAnnotation("ntp_snippets_fetch", R"(
370         semantics {
371           sender: "New Tab Page Content Suggestions Fetch"
372           description:
373             "Chromium can show content suggestions (e.g. news articles) on the "
374             "New Tab page. For signed-in users, these may be personalized "
375             "based on the user's synced browsing history."
376           trigger:
377             "Triggered periodically in the background, or upon explicit user "
378             "request."
379           data:
380             "The Chromium UI language, as well as a second language the user "
381             "understands, based on language::UrlLanguageHistogram. For "
382             "signed-in users, the requests is authenticated."
383           destination: GOOGLE_OWNED_SERVICE
384         }
385         policy {
386           cookies_allowed: NO
387           setting:
388             "This feature cannot be disabled by settings now (but is requested "
389             "to be implemented in crbug.com/695129)."
390           chrome_policy {
391             NTPContentSuggestionsEnabled {
392               policy_options {mode: MANDATORY}
393               NTPContentSuggestionsEnabled: false
394             }
395           }
396         })");
397   auto resource_request = BuildResourceRequest();
398 
399   // Log the request for debugging network issues.
400   VLOG(1) << "Sending a NTP snippets request to " << url_ << ":\n"
401           << resource_request->headers.ToString() << "\n"
402           << body;
403 
404   auto loader = network::SimpleURLLoader::Create(std::move(resource_request),
405                                                  traffic_annotation);
406   loader->AttachStringForUpload(body, "application/json");
407   int max_retries = JsonRequest::Get5xxRetryCount(params_.interactive_request);
408   if (max_retries > 0) {
409     loader->SetRetryOptions(
410         max_retries, network::SimpleURLLoader::RETRY_ON_5XX |
411                          network::SimpleURLLoader::RETRY_ON_NETWORK_CHANGE);
412   }
413   return loader;
414 }
415 
416 void JsonRequest::Builder::PrepareLanguages(
417     language::UrlLanguageHistogram::LanguageInfo* ui_language,
418     language::UrlLanguageHistogram::LanguageInfo* other_top_language) const {
419   // TODO(jkrcal): Add language model factory for iOS and add fakes to tests so
420   // that |language_histogram| is never nullptr. Remove this check and add a
421   // DCHECK into the constructor.
422   if (!language_histogram_ || !IsSendingTopLanguagesEnabled()) {
423     return;
424   }
425 
426   // TODO(jkrcal): Is this back-and-forth converting necessary?
427   ui_language->language_code = ISO639FromPosixLocale(
428       PosixLocaleFromBCP47Language(params_.language_code));
429   ui_language->frequency =
430       language_histogram_->GetLanguageFrequency(ui_language->language_code);
431 
432   std::vector<UrlLanguageHistogram::LanguageInfo> top_languages =
433       language_histogram_->GetTopLanguages();
434   for (const UrlLanguageHistogram::LanguageInfo& info : top_languages) {
435     if (info.language_code != ui_language->language_code) {
436       *other_top_language = info;
437 
438       // Report to UMA how important the UI language is.
439       DCHECK_GT(other_top_language->frequency, 0)
440           << "GetTopLanguages() should not return languages with 0 frequency";
441       float ratio_ui_in_both_languages =
442           ui_language->frequency /
443           (ui_language->frequency + other_top_language->frequency);
444       UMA_HISTOGRAM_PERCENTAGE(
445           "NewTabPage.Languages.UILanguageRatioInTwoTopLanguages",
446           ratio_ui_in_both_languages * 100);
447       break;
448     }
449   }
450 }
451 
452 }  // namespace internal
453 
454 }  // namespace ntp_snippets
455