1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import os
6import sys
7
8from telemetry import benchmark
9from telemetry import story
10from telemetry.core import util
11from telemetry import decorators
12from telemetry.page import page as page_module
13from telemetry.page import legacy_page_test
14from telemetry import record_wpr
15from telemetry.testing import options_for_unittests
16from telemetry.testing import tab_test_case
17from telemetry.util import wpr_modes
18
19
20class MockPage(page_module.Page):
21  def __init__(self, story_set, url):
22    super(MockPage, self).__init__(url=url,
23                                   page_set=story_set,
24                                   base_dir=util.GetUnittestDataDir(),
25                                   name=url)
26    self.func_calls = []
27
28  def RunPageInteractions(self, _):
29    self.func_calls.append('RunPageInteractions')
30
31  def RunSmoothness(self, _):
32    self.func_calls.append('RunSmoothness')
33
34
35class MockStorySet(story.StorySet):
36  def __init__(self, url=''):
37    super(MockStorySet, self).__init__(
38        archive_data_file='data/archive_files/test.json')
39    self.AddStory(MockPage(self, url))
40
41
42class MockPageTest(legacy_page_test.LegacyPageTest):
43  def __init__(self):
44    super(MockPageTest, self).__init__()
45    self._action_name_to_run = "RunPageInteractions"
46    self.func_calls = []
47
48  def CustomizeBrowserOptions(self, options):
49    self.func_calls.append('CustomizeBrowserOptions')
50
51  def WillNavigateToPage(self, page, tab):
52    self.func_calls.append('WillNavigateToPage')
53
54  def DidNavigateToPage(self, page, tab):
55    self.func_calls.append('DidNavigateToPage')
56
57  def ValidateAndMeasurePage(self, page, tab, results):
58    self.func_calls.append('ValidateAndMeasurePage')
59
60  def WillStartBrowser(self, platform):
61    self.func_calls.append('WillStartBrowser')
62
63  def DidStartBrowser(self, browser):
64    self.func_calls.append('DidStartBrowser')
65
66
67class MockBenchmark(benchmark.Benchmark):
68  test = MockPageTest
69
70  def __init__(self):
71    super(MockBenchmark, self).__init__()
72    self.mock_story_set = None
73
74  @classmethod
75  def AddBenchmarkCommandLineArgs(cls, group):
76    group.add_option('', '--mock-benchmark-url', action='store', type='string')
77
78  def CreateStorySet(self, options):
79    kwargs = {}
80    if options.mock_benchmark_url:
81      kwargs['url'] = options.mock_benchmark_url
82    self.mock_story_set = MockStorySet(**kwargs)
83    return self.mock_story_set
84
85
86class MockTimelineBasedMeasurementBenchmark(benchmark.Benchmark):
87
88  def __init__(self):
89    super(MockTimelineBasedMeasurementBenchmark, self).__init__()
90    self.mock_story_set = None
91
92  @classmethod
93  def AddBenchmarkCommandLineArgs(cls, group):
94    group.add_option('', '--mock-benchmark-url', action='store', type='string')
95
96  def CreateStorySet(self, options):
97    kwargs = {}
98    if options.mock_benchmark_url:
99      kwargs['url'] = options.mock_benchmark_url
100    self.mock_story_set = MockStorySet(**kwargs)
101    return self.mock_story_set
102
103
104def _SuccessfulStories(results):
105  return set(run.story for run in results.IterStoryRuns() if run.ok)
106
107
108class RecordWprUnitTests(tab_test_case.TabTestCase):
109
110  _base_dir = util.GetUnittestDataDir()
111  _test_data_dir = os.path.join(util.GetUnittestDataDir(), 'page_tests')
112
113  @classmethod
114  def setUpClass(cls):
115    sys.path.extend([cls._base_dir, cls._test_data_dir])
116    super(RecordWprUnitTests, cls).setUpClass()
117    cls._url = cls.UrlOfUnittestFile('blank.html')
118    cls._test_options = options_for_unittests.GetCopy()
119
120  # When the RecorderPageTest is created from a Benchmark, the benchmark will
121  # have a PageTest, specified by its test attribute.
122  def testRunPage_OnlyRunBenchmarkAction(self):
123    page_test = MockBenchmark().test()
124    record_page_test = record_wpr.RecorderPageTest(page_test)
125    page = MockPage(story_set=MockStorySet(url=self._url), url=self._url)
126    record_page_test.ValidateAndMeasurePage(page, self._tab, results=None)
127
128  def testRunPage_CallBenchmarksPageTestsFunctions(self):
129    page_test = MockBenchmark().test()
130    record_page_test = record_wpr.RecorderPageTest(page_test)
131    page = MockPage(story_set=MockStorySet(url=self._url), url=self._url)
132    record_page_test.ValidateAndMeasurePage(page, self._tab, results=None)
133    self.assertEqual(1, len(page_test.func_calls))
134    self.assertEqual('ValidateAndMeasurePage', page_test.func_calls[0])
135
136  def GetBrowserDeviceFlags(self):
137    flags = ['--browser', self._browser.browser_type,
138             '--remote', self._test_options.cros_remote,
139             '--device', self._device]
140    if self._browser.browser_type == 'exact':
141      flags += ['--browser-executable', self._test_options.browser_executable]
142    if self._test_options.chrome_root:
143      flags += ['--chrome-root', self._test_options.chrome_root]
144    return flags
145
146  @decorators.Disabled('chromeos') # crbug.com/404868.
147  def testWprRecorderWithPageSet(self):
148    flags = self.GetBrowserDeviceFlags()
149    mock_story_set = MockStorySet(url=self._url)
150    with record_wpr.WprRecorder(self._test_data_dir,
151                                mock_story_set, flags) as wpr_recorder:
152      with wpr_recorder.CreateResults() as results:
153        wpr_recorder.Record(results)
154        self.assertItemsEqual(mock_story_set.stories,
155                              _SuccessfulStories(results))
156
157  def testWprRecorderWithBenchmark(self):
158    flags = self.GetBrowserDeviceFlags()
159    flags.extend(['--mock-benchmark-url', self._url])
160    mock_benchmark = MockBenchmark()
161    with record_wpr.WprRecorder(self._test_data_dir, mock_benchmark,
162                                flags) as wpr_recorder:
163      with wpr_recorder.CreateResults() as results:
164        wpr_recorder.Record(results)
165        self.assertItemsEqual(mock_benchmark.mock_story_set.stories,
166                              _SuccessfulStories(results))
167
168  def testWprRecorderWithTimelineBasedMeasurementBenchmark(self):
169    flags = self.GetBrowserDeviceFlags()
170    flags.extend(['--mock-benchmark-url', self._url])
171    mock_benchmark = MockTimelineBasedMeasurementBenchmark()
172    with record_wpr.WprRecorder(self._test_data_dir, mock_benchmark,
173                                flags) as wpr_recorder:
174      with wpr_recorder.CreateResults() as results:
175        wpr_recorder.Record(results)
176        self.assertItemsEqual(mock_benchmark.mock_story_set.stories,
177                              _SuccessfulStories(results))
178
179  def testPageSetBaseDirFlag(self):
180    flags = self.GetBrowserDeviceFlags()
181    flags.extend(['--page-set-base-dir', self._test_data_dir,
182                  '--mock-benchmark-url', self._url])
183    mock_benchmark = MockBenchmark()
184    with record_wpr.WprRecorder(
185        'non-existent-dummy-dir', mock_benchmark, flags) as wpr_recorder:
186      with wpr_recorder.CreateResults() as results:
187        wpr_recorder.Record(results)
188        self.assertItemsEqual(mock_benchmark.mock_story_set.stories,
189                              _SuccessfulStories(results))
190
191  def testCommandLineFlags(self):
192    flags = [
193        '--pageset-repeat', '2',
194        '--mock-benchmark-url', self._url,
195        '--upload',
196    ]
197    with record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(),
198                                flags) as wpr_recorder:
199      # page_runner command-line args
200      self.assertEquals(2, wpr_recorder.options.pageset_repeat)
201      # benchmark command-line args
202      self.assertEquals(self._url, wpr_recorder.options.mock_benchmark_url)
203      # record_wpr command-line arg to upload to cloud-storage.
204      self.assertTrue(wpr_recorder.options.upload)
205      # invalid command-line args
206      self.assertFalse(hasattr(wpr_recorder.options, 'not_a_real_option'))
207
208  def testRecordingEnabled(self):
209    flags = ['--mock-benchmark-url', self._url]
210    with record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(),
211                                flags) as wpr_recorder:
212      self.assertEqual(wpr_modes.WPR_RECORD,
213                       wpr_recorder.options.browser_options.wpr_mode)
214
215  # When the RecorderPageTest CustomizeBrowserOptions/WillStartBrowser/
216  # DidStartBrowser function is called, it forwards the call to the PageTest
217  def testRecorderPageTest_BrowserMethods(self):
218    flags = ['--mock-benchmark-url', self._url]
219    page_test = MockBenchmark().test()
220    record_page_test = record_wpr.RecorderPageTest(page_test)
221    with record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(),
222                                flags) as wpr_recorder:
223      record_page_test.CustomizeBrowserOptions(wpr_recorder.options)
224      record_page_test.WillStartBrowser(self._tab.browser.platform)
225      record_page_test.DidStartBrowser(self._tab.browser)
226      self.assertTrue('CustomizeBrowserOptions' in page_test.func_calls)
227      self.assertTrue('WillStartBrowser' in page_test.func_calls)
228      self.assertTrue('DidStartBrowser' in page_test.func_calls)
229
230  def testUseLiveSitesUnsupported(self):
231    flags = ['--use-live-sites']
232    with self.assertRaises(SystemExit):
233      record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(), flags)
234