1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "components/ntp_tiles/popular_sites_impl.h"
6 
7 #include <map>
8 #include <memory>
9 #include <string>
10 #include <utility>
11 #include <vector>
12 
13 #include "base/bind.h"
14 #include "base/command_line.h"
15 #include "base/json/json_writer.h"
16 #include "base/optional.h"
17 #include "base/run_loop.h"
18 #include "base/strings/string_number_conversions.h"
19 #include "base/strings/utf_string_conversions.h"
20 #include "base/test/scoped_feature_list.h"
21 #include "base/test/task_environment.h"
22 #include "base/threading/thread_task_runner_handle.h"
23 #include "base/values.h"
24 #include "build/branding_buildflags.h"
25 #include "build/build_config.h"
26 #include "components/ntp_tiles/features.h"
27 #include "components/ntp_tiles/pref_names.h"
28 #include "components/ntp_tiles/tile_source.h"
29 #include "components/pref_registry/pref_registry_syncable.h"
30 #include "components/sync_preferences/testing_pref_service_syncable.h"
31 #include "net/http/http_status_code.h"
32 #include "services/data_decoder/public/cpp/test_support/in_process_data_decoder.h"
33 #include "services/network/public/cpp/shared_url_loader_factory.h"
34 #include "services/network/public/cpp/weak_wrapper_shared_url_loader_factory.h"
35 #include "services/network/test/test_url_loader_factory.h"
36 #include "testing/gmock/include/gmock/gmock.h"
37 #include "testing/gtest/include/gtest/gtest.h"
38 
39 using testing::_;
40 using testing::Contains;
41 using testing::ElementsAre;
42 using testing::Eq;
43 using testing::Gt;
44 using testing::IsEmpty;
45 using testing::Not;
46 using testing::Pair;
47 using testing::SizeIs;
48 
49 namespace ntp_tiles {
50 namespace {
51 
52 const char kTitle[] = "title";
53 const char kUrl[] = "url";
54 const char kLargeIconUrl[] = "large_icon_url";
55 const char kFaviconUrl[] = "favicon_url";
56 const char kSection[] = "section";
57 const char kSites[] = "sites";
58 const char kTitleSource[] = "title_source";
59 
60 using TestPopularSite = std::map<std::string, std::string>;
61 using TestPopularSiteVector = std::vector<TestPopularSite>;
62 using TestPopularSection = std::pair<SectionType, TestPopularSiteVector>;
63 using TestPopularSectionVector = std::vector<TestPopularSection>;
64 
Str16Eq(const std::string & s)65 ::testing::Matcher<const base::string16&> Str16Eq(const std::string& s) {
66   return ::testing::Eq(base::UTF8ToUTF16(s));
67 }
68 
URLEq(const std::string & s)69 ::testing::Matcher<const GURL&> URLEq(const std::string& s) {
70   return ::testing::Eq(GURL(s));
71 }
72 
GetNumberOfDefaultPopularSitesForPlatform()73 size_t GetNumberOfDefaultPopularSitesForPlatform() {
74 #if defined(OS_ANDROID) || defined(OS_IOS)
75   return 8ul;
76 #else
77   return 0ul;
78 #endif
79 }
80 
81 class PopularSitesTest : public ::testing::Test {
82  protected:
PopularSitesTest()83   PopularSitesTest()
84       : kWikipedia{
85             {kTitle, "Wikipedia, fhta Ph'nglui mglw'nafh"},
86             {kUrl, "https://zz.m.wikipedia.org/"},
87             {kLargeIconUrl, "https://zz.m.wikipedia.org/wikipedia.png"},
88             {kTitleSource, "3"},  // Title extracted from title tag.
89         },
90         kYouTube{
91             {kTitle, "YouTube"},
92             {kUrl, "https://m.youtube.com/"},
93             {kLargeIconUrl, "https://s.ytimg.com/apple-touch-icon.png"},
94             {kTitleSource, "1"},  // Title extracted from manifest.
95         },
96         kChromium{
97             {kTitle, "The Chromium Project"},
98             {kUrl, "https://www.chromium.org/"},
99             {kFaviconUrl, "https://www.chromium.org/favicon.ico"},
100             // No "title_source" (like in v5 or earlier). Defaults to TITLE_TAG.
101         },
102         prefs_(new sync_preferences::TestingPrefServiceSyncable()),
103         test_shared_loader_factory_(
104             base::MakeRefCounted<network::WeakWrapperSharedURLLoaderFactory>(
105                 &test_url_loader_factory_)) {
106     PopularSitesImpl::RegisterProfilePrefs(prefs_->registry());
107   }
108 
SetCountryAndVersion(const std::string & country,const std::string & version)109   void SetCountryAndVersion(const std::string& country,
110                             const std::string& version) {
111     prefs_->SetString(prefs::kPopularSitesOverrideCountry, country);
112     prefs_->SetString(prefs::kPopularSitesOverrideVersion, version);
113   }
114 
CreateListFromTestSites(const TestPopularSiteVector & sites)115   std::unique_ptr<base::ListValue> CreateListFromTestSites(
116       const TestPopularSiteVector& sites) {
117     auto sites_value = std::make_unique<base::ListValue>();
118     for (const TestPopularSite& site : sites) {
119       auto site_value = std::make_unique<base::DictionaryValue>();
120       for (const std::pair<const std::string, std::string>& kv : site) {
121         if (kv.first == kTitleSource) {
122           int source;
123           bool convert_success = base::StringToInt(kv.second, &source);
124           DCHECK(convert_success);
125           site_value->SetInteger(kv.first, source);
126           continue;
127         }
128         site_value->SetString(kv.first, kv.second);
129       }
130       sites_value->Append(std::move(site_value));
131     }
132     return sites_value;
133   }
134 
RespondWithV5JSON(const std::string & url,const TestPopularSiteVector & sites)135   void RespondWithV5JSON(const std::string& url,
136                          const TestPopularSiteVector& sites) {
137     std::string sites_string;
138     base::JSONWriter::Write(*CreateListFromTestSites(sites), &sites_string);
139     test_url_loader_factory_.AddResponse(url, sites_string);
140   }
141 
RespondWithV6JSON(const std::string & url,const TestPopularSectionVector & sections)142   void RespondWithV6JSON(const std::string& url,
143                          const TestPopularSectionVector& sections) {
144     base::ListValue sections_value;
145     for (const TestPopularSection& section : sections) {
146       auto section_value = std::make_unique<base::DictionaryValue>();
147       section_value->SetInteger(kSection, static_cast<int>(section.first));
148       section_value->SetList(kSites, CreateListFromTestSites(section.second));
149       sections_value.Append(std::move(section_value));
150     }
151     std::string sites_string;
152     base::JSONWriter::Write(sections_value, &sites_string);
153     test_url_loader_factory_.AddResponse(url, sites_string);
154   }
155 
RespondWithData(const std::string & url,const std::string & data)156   void RespondWithData(const std::string& url, const std::string& data) {
157     test_url_loader_factory_.AddResponse(url, data);
158   }
159 
RespondWith404(const std::string & url)160   void RespondWith404(const std::string& url) {
161     test_url_loader_factory_.AddResponse(url, "", net::HTTP_NOT_FOUND);
162   }
163 
ReregisterProfilePrefs()164   void ReregisterProfilePrefs() {
165     prefs_ = std::make_unique<sync_preferences::TestingPrefServiceSyncable>();
166     PopularSitesImpl::RegisterProfilePrefs(prefs_->registry());
167   }
168 
169   // Returns an optional bool representing whether the completion callback was
170   // called at all, and if yes which was the returned bool value.
FetchPopularSites(bool force_download,PopularSites::SitesVector * sites)171   base::Optional<bool> FetchPopularSites(bool force_download,
172                                          PopularSites::SitesVector* sites) {
173     std::map<SectionType, PopularSites::SitesVector> sections;
174     base::Optional<bool> save_success =
175         FetchAllSections(force_download, &sections);
176     *sites = sections.at(SectionType::PERSONALIZED);
177     return save_success;
178   }
179 
180   // Returns an optional bool representing whether the completion callback was
181   // called at all, and if yes which was the returned bool value.
FetchAllSections(bool force_download,std::map<SectionType,PopularSites::SitesVector> * sections)182   base::Optional<bool> FetchAllSections(
183       bool force_download,
184       std::map<SectionType, PopularSites::SitesVector>* sections) {
185     std::unique_ptr<PopularSites> popular_sites = CreatePopularSites();
186 
187     base::RunLoop loop;
188     base::Optional<bool> save_success;
189     if (popular_sites->MaybeStartFetch(
190             force_download, base::BindOnce(
191                                 [](base::Optional<bool>* save_success,
192                                    base::RunLoop* loop, bool success) {
193                                   save_success->emplace(success);
194                                   loop->Quit();
195                                 },
196                                 &save_success, &loop))) {
197       loop.Run();
198     }
199     *sections = popular_sites->sections();
200     return save_success;
201   }
202 
CreatePopularSites()203   std::unique_ptr<PopularSites> CreatePopularSites() {
204     return std::make_unique<PopularSitesImpl>(prefs_.get(),
205                                               /*template_url_service=*/nullptr,
206                                               /*variations_service=*/nullptr,
207                                               test_shared_loader_factory_);
208   }
209 
210   const TestPopularSite kWikipedia;
211   const TestPopularSite kYouTube;
212   const TestPopularSite kChromium;
213 
214   base::test::SingleThreadTaskEnvironment task_environment_{
215       base::test::SingleThreadTaskEnvironment::MainThreadType::UI};
216   data_decoder::test::InProcessDataDecoder in_process_data_decoder_;
217   std::unique_ptr<sync_preferences::TestingPrefServiceSyncable> prefs_;
218   network::TestURLLoaderFactory test_url_loader_factory_;
219   scoped_refptr<network::SharedURLLoaderFactory> test_shared_loader_factory_;
220 };
221 
TEST_F(PopularSitesTest,ContainsDefaultTilesRightAfterConstruction)222 TEST_F(PopularSitesTest, ContainsDefaultTilesRightAfterConstruction) {
223   auto popular_sites = CreatePopularSites();
224   EXPECT_THAT(
225       popular_sites->sections(),
226       ElementsAre(Pair(SectionType::PERSONALIZED,
227                        SizeIs(GetNumberOfDefaultPopularSitesForPlatform()))));
228 }
229 
TEST_F(PopularSitesTest,IsEmptyOnConstructionIfDisabledByTrial)230 TEST_F(PopularSitesTest, IsEmptyOnConstructionIfDisabledByTrial) {
231   base::test::ScopedFeatureList override_features;
232   override_features.InitAndDisableFeature(kPopularSitesBakedInContentFeature);
233   ReregisterProfilePrefs();
234 
235   auto popular_sites = CreatePopularSites();
236 
237   EXPECT_THAT(popular_sites->sections(),
238               ElementsAre(Pair(SectionType::PERSONALIZED, IsEmpty())));
239 }
240 
TEST_F(PopularSitesTest,ShouldSucceedFetching)241 TEST_F(PopularSitesTest, ShouldSucceedFetching) {
242   SetCountryAndVersion("ZZ", "5");
243   RespondWithV5JSON(
244       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
245       {kWikipedia});
246 
247   PopularSites::SitesVector sites;
248   EXPECT_THAT(FetchPopularSites(/*force_download=*/true, &sites),
249               Eq(base::Optional<bool>(true)));
250 
251   ASSERT_THAT(sites.size(), Eq(1u));
252   EXPECT_THAT(sites[0].title, Str16Eq("Wikipedia, fhta Ph'nglui mglw'nafh"));
253   EXPECT_THAT(sites[0].url, URLEq("https://zz.m.wikipedia.org/"));
254   EXPECT_THAT(sites[0].large_icon_url,
255               URLEq("https://zz.m.wikipedia.org/wikipedia.png"));
256   EXPECT_THAT(sites[0].favicon_url, URLEq(""));
257   EXPECT_THAT(sites[0].title_source, Eq(TileTitleSource::TITLE_TAG));
258 }
259 
TEST_F(PopularSitesTest,Fallback)260 TEST_F(PopularSitesTest, Fallback) {
261   SetCountryAndVersion("ZZ", "5");
262   RespondWith404(
263       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json");
264   RespondWithV5JSON(
265       "https://www.gstatic.com/chrome/ntp/suggested_sites_DEFAULT_5.json",
266       {kYouTube, kChromium});
267 
268   PopularSites::SitesVector sites;
269   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
270               Eq(base::Optional<bool>(true)));
271 
272   ASSERT_THAT(sites.size(), Eq(2u));
273   EXPECT_THAT(sites[0].title, Str16Eq("YouTube"));
274   EXPECT_THAT(sites[0].url, URLEq("https://m.youtube.com/"));
275   EXPECT_THAT(sites[0].large_icon_url,
276               URLEq("https://s.ytimg.com/apple-touch-icon.png"));
277   EXPECT_THAT(sites[0].favicon_url, URLEq(""));
278   EXPECT_THAT(sites[0].title_source, Eq(TileTitleSource::MANIFEST));
279   EXPECT_THAT(sites[1].title, Str16Eq("The Chromium Project"));
280   EXPECT_THAT(sites[1].url, URLEq("https://www.chromium.org/"));
281   EXPECT_THAT(sites[1].large_icon_url, URLEq(""));
282   EXPECT_THAT(sites[1].favicon_url,
283               URLEq("https://www.chromium.org/favicon.ico"));
284   // Fall back to TITLE_TAG if there is no "title_source". Version 5 or before
285   // haven't had this property and get titles from <title> tags exclusively.
286   EXPECT_THAT(sites[1].title_source, Eq(TileTitleSource::TITLE_TAG));
287 }
288 
TEST_F(PopularSitesTest,PopulatesWithDefaultResoucesOnFailure)289 TEST_F(PopularSitesTest, PopulatesWithDefaultResoucesOnFailure) {
290   SetCountryAndVersion("ZZ", "5");
291   RespondWith404(
292       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json");
293   RespondWith404(
294       "https://www.gstatic.com/chrome/ntp/suggested_sites_DEFAULT_5.json");
295 
296   PopularSites::SitesVector sites;
297   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
298               Eq(base::Optional<bool>(false)));
299   EXPECT_THAT(sites.size(), Eq(GetNumberOfDefaultPopularSitesForPlatform()));
300 }
301 
302 #if defined(OS_ANDROID) || defined(OS_IOS)
TEST_F(PopularSitesTest,AddsIconResourcesToDefaultPages)303 TEST_F(PopularSitesTest, AddsIconResourcesToDefaultPages) {
304   std::unique_ptr<PopularSites> popular_sites = CreatePopularSites();
305 
306   const PopularSites::SitesVector& sites =
307       popular_sites->sections().at(SectionType::PERSONALIZED);
308   ASSERT_FALSE(sites.empty());
309   for (const auto& site : sites) {
310     EXPECT_TRUE(site.baked_in);
311 #if BUILDFLAG(GOOGLE_CHROME_BRANDING)
312     EXPECT_THAT(site.default_icon_resource, Gt(0));
313 #endif
314   }
315 }
316 #endif
317 
TEST_F(PopularSitesTest,ProvidesDefaultSitesUntilCallbackReturns)318 TEST_F(PopularSitesTest, ProvidesDefaultSitesUntilCallbackReturns) {
319   SetCountryAndVersion("ZZ", "5");
320   RespondWithV5JSON(
321       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
322       {kWikipedia});
323   std::unique_ptr<PopularSites> popular_sites = CreatePopularSites();
324 
325   base::RunLoop loop;
326   base::Optional<bool> save_success = false;
327 
328   bool callback_was_scheduled = popular_sites->MaybeStartFetch(
329       /*force_download=*/true, base::BindOnce(
330                                    [](base::Optional<bool>* save_success,
331                                       base::RunLoop* loop, bool success) {
332                                      save_success->emplace(success);
333                                      loop->Quit();
334                                    },
335                                    &save_success, &loop));
336 
337   // Assert that callback was scheduled so we can wait for its completion.
338   ASSERT_TRUE(callback_was_scheduled);
339   // There should be 8 default sites as nothing was fetched yet.
340   EXPECT_THAT(popular_sites->sections().at(SectionType::PERSONALIZED).size(),
341               Eq(GetNumberOfDefaultPopularSitesForPlatform()));
342 
343   loop.Run();  // Wait for the fetch to finish and the callback to return.
344 
345   EXPECT_TRUE(save_success.value());
346   // The 1 fetched site should replace the default sites.
347   EXPECT_THAT(popular_sites->sections(),
348               ElementsAre(Pair(SectionType::PERSONALIZED, SizeIs(1ul))));
349 }
350 
TEST_F(PopularSitesTest,UsesCachedJson)351 TEST_F(PopularSitesTest, UsesCachedJson) {
352   SetCountryAndVersion("ZZ", "5");
353   RespondWithV5JSON(
354       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
355       {kWikipedia});
356 
357   // First request succeeds and gets cached.
358   PopularSites::SitesVector sites;
359   ASSERT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
360               Eq(base::Optional<bool>(true)));
361 
362   // File disappears from server, but we don't need it because it's cached.
363   RespondWith404(
364       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json");
365   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
366               Eq(base::nullopt));
367   EXPECT_THAT(sites[0].url, URLEq("https://zz.m.wikipedia.org/"));
368 }
369 
TEST_F(PopularSitesTest,CachesEmptyFile)370 TEST_F(PopularSitesTest, CachesEmptyFile) {
371   SetCountryAndVersion("ZZ", "5");
372   RespondWithData(
373       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json", "[]");
374   RespondWithV5JSON(
375       "https://www.gstatic.com/chrome/ntp/suggested_sites_DEFAULT_5.json",
376       {kWikipedia});
377 
378   // First request succeeds and caches empty suggestions list (no fallback).
379   PopularSites::SitesVector sites;
380   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
381               Eq(base::Optional<bool>(true)));
382   EXPECT_THAT(sites, IsEmpty());
383 
384   // File appears on server, but we continue to use our cached empty file.
385   RespondWithV5JSON(
386       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
387       {kWikipedia});
388   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
389               Eq(base::nullopt));
390   EXPECT_THAT(sites, IsEmpty());
391 }
392 
TEST_F(PopularSitesTest,DoesntUseCachedFileIfDownloadForced)393 TEST_F(PopularSitesTest, DoesntUseCachedFileIfDownloadForced) {
394   SetCountryAndVersion("ZZ", "5");
395   RespondWithV5JSON(
396       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
397       {kWikipedia});
398 
399   // First request succeeds and gets cached.
400   PopularSites::SitesVector sites;
401   EXPECT_THAT(FetchPopularSites(/*force_download=*/true, &sites),
402               Eq(base::Optional<bool>(true)));
403   EXPECT_THAT(sites[0].url, URLEq("https://zz.m.wikipedia.org/"));
404 
405   // File disappears from server. Download is forced, so we get the new file.
406   RespondWithV5JSON(
407       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
408       {kChromium});
409   EXPECT_THAT(FetchPopularSites(/*force_download=*/true, &sites),
410               Eq(base::Optional<bool>(true)));
411   EXPECT_THAT(sites[0].url, URLEq("https://www.chromium.org/"));
412 }
413 
TEST_F(PopularSitesTest,DoesntUseCacheWithDeprecatedVersion)414 TEST_F(PopularSitesTest, DoesntUseCacheWithDeprecatedVersion) {
415   SetCountryAndVersion("ZZ", "5");
416   RespondWithV5JSON(
417       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
418       {kWikipedia});
419 
420   // First request succeeds and gets cached.
421   PopularSites::SitesVector sites;
422   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
423               Eq(base::Optional<bool>(true)));
424   EXPECT_THAT(sites[0].url, URLEq("https://zz.m.wikipedia.org/"));
425   EXPECT_THAT(prefs_->GetInteger(prefs::kPopularSitesVersionPref), Eq(5));
426 
427   // The client is updated to use V6. Drop old data and refetch.
428   SetCountryAndVersion("ZZ", "6");
429   RespondWithV6JSON(
430       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_6.json",
431       {{SectionType::PERSONALIZED, {kChromium}}});
432   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
433               Eq(base::Optional<bool>(true)));
434   EXPECT_THAT(sites[0].url, URLEq("https://www.chromium.org/"));
435   EXPECT_THAT(prefs_->GetInteger(prefs::kPopularSitesVersionPref), Eq(6));
436 }
437 
TEST_F(PopularSitesTest,FallsBackToDefaultParserIfVersionContainsNoNumber)438 TEST_F(PopularSitesTest, FallsBackToDefaultParserIfVersionContainsNoNumber) {
439   SetCountryAndVersion("ZZ", "staging");
440   // The version is used in the URL, as planned when setting it.
441   RespondWithV5JSON(
442       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_staging.json",
443       {kChromium});
444   PopularSites::SitesVector sites;
445   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
446               Eq(base::Optional<bool>(true)));
447   ASSERT_THAT(sites.size(), Eq(1u));
448   EXPECT_THAT(sites[0].url, URLEq("https://www.chromium.org/"));
449 }
450 
TEST_F(PopularSitesTest,RefetchesAfterCountryMoved)451 TEST_F(PopularSitesTest, RefetchesAfterCountryMoved) {
452   RespondWithV5JSON(
453       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
454       {kWikipedia});
455   RespondWithV5JSON(
456       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZX_5.json",
457       {kChromium});
458 
459   PopularSites::SitesVector sites;
460 
461   // First request (in ZZ) saves Wikipedia.
462   SetCountryAndVersion("ZZ", "5");
463   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
464               Eq(base::Optional<bool>(true)));
465   EXPECT_THAT(sites[0].url, URLEq("https://zz.m.wikipedia.org/"));
466 
467   // Second request (now in ZX) saves Chromium.
468   SetCountryAndVersion("ZX", "5");
469   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
470               base::Optional<bool>(true));
471   EXPECT_THAT(sites[0].url, URLEq("https://www.chromium.org/"));
472 }
473 
TEST_F(PopularSitesTest,DoesntCacheInvalidFile)474 TEST_F(PopularSitesTest, DoesntCacheInvalidFile) {
475   SetCountryAndVersion("ZZ", "5");
476   RespondWithData(
477       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
478       "ceci n'est pas un json");
479   RespondWith404(
480       "https://www.gstatic.com/chrome/ntp/suggested_sites_DEFAULT_5.json");
481 
482   // First request falls back and gets nothing there either.
483   PopularSites::SitesVector sites;
484   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
485               Eq(base::Optional<bool>(false)));
486 
487   // Second request refetches ZZ_9, which now has data.
488   RespondWithV5JSON(
489       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
490       {kChromium});
491   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
492               Eq(base::Optional<bool>(true)));
493   ASSERT_THAT(sites.size(), Eq(1u));
494   EXPECT_THAT(sites[0].url, URLEq("https://www.chromium.org/"));
495 }
496 
TEST_F(PopularSitesTest,RefetchesAfterFallback)497 TEST_F(PopularSitesTest, RefetchesAfterFallback) {
498   SetCountryAndVersion("ZZ", "5");
499   RespondWith404(
500       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json");
501   RespondWithV5JSON(
502       "https://www.gstatic.com/chrome/ntp/suggested_sites_DEFAULT_5.json",
503       {kWikipedia});
504 
505   // First request falls back.
506   PopularSites::SitesVector sites;
507   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
508               Eq(base::Optional<bool>(true)));
509   ASSERT_THAT(sites.size(), Eq(1u));
510   EXPECT_THAT(sites[0].url, URLEq("https://zz.m.wikipedia.org/"));
511 
512   // Second request refetches ZZ_9, which now has data.
513   RespondWithV5JSON(
514       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_5.json",
515       {kChromium});
516   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
517               Eq(base::Optional<bool>(true)));
518   ASSERT_THAT(sites.size(), Eq(1u));
519   EXPECT_THAT(sites[0].url, URLEq("https://www.chromium.org/"));
520 }
521 
TEST_F(PopularSitesTest,ShouldOverrideDirectory)522 TEST_F(PopularSitesTest, ShouldOverrideDirectory) {
523   SetCountryAndVersion("ZZ", "5");
524   prefs_->SetString(prefs::kPopularSitesOverrideDirectory, "foo/bar/");
525   RespondWithV5JSON("https://www.gstatic.com/foo/bar/suggested_sites_ZZ_5.json",
526                     {kWikipedia});
527 
528   PopularSites::SitesVector sites;
529   EXPECT_THAT(FetchPopularSites(/*force_download=*/false, &sites),
530               Eq(base::Optional<bool>(true)));
531 
532   EXPECT_THAT(sites.size(), Eq(1u));
533 }
534 
TEST_F(PopularSitesTest,DoesNotFetchExplorationSites)535 TEST_F(PopularSitesTest, DoesNotFetchExplorationSites) {
536   SetCountryAndVersion("ZZ", "6");
537   RespondWithV6JSON(
538       "https://www.gstatic.com/chrome/ntp/suggested_sites_ZZ_6.json",
539       {{SectionType::PERSONALIZED, {kChromium}},
540        {SectionType::NEWS, {kYouTube}}});
541 
542   std::map<SectionType, PopularSites::SitesVector> sections;
543   EXPECT_THAT(FetchAllSections(/*force_download=*/false, &sections),
544               Eq(base::Optional<bool>(true)));
545 
546   // The fetched news section should not be propagated without enabled feature.
547   EXPECT_THAT(sections, Not(Contains(Pair(SectionType::NEWS, _))));
548 }
549 
550 }  // namespace
551 }  // namespace ntp_tiles
552