1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import os 6import sys 7 8from telemetry import benchmark 9from telemetry import story 10from telemetry.core import util 11from telemetry import decorators 12from telemetry.page import page as page_module 13from telemetry.page import legacy_page_test 14from telemetry import record_wpr 15from telemetry.testing import options_for_unittests 16from telemetry.testing import tab_test_case 17from telemetry.util import wpr_modes 18 19 20class MockPage(page_module.Page): 21 def __init__(self, story_set, url): 22 super(MockPage, self).__init__(url=url, 23 page_set=story_set, 24 base_dir=util.GetUnittestDataDir(), 25 name=url) 26 self.func_calls = [] 27 28 def RunPageInteractions(self, _): 29 self.func_calls.append('RunPageInteractions') 30 31 def RunSmoothness(self, _): 32 self.func_calls.append('RunSmoothness') 33 34 35class MockStorySet(story.StorySet): 36 def __init__(self, url=''): 37 super(MockStorySet, self).__init__( 38 archive_data_file='data/archive_files/test.json') 39 self.AddStory(MockPage(self, url)) 40 41 42class MockPageTest(legacy_page_test.LegacyPageTest): 43 def __init__(self): 44 super(MockPageTest, self).__init__() 45 self._action_name_to_run = "RunPageInteractions" 46 self.func_calls = [] 47 48 def CustomizeBrowserOptions(self, options): 49 self.func_calls.append('CustomizeBrowserOptions') 50 51 def WillNavigateToPage(self, page, tab): 52 self.func_calls.append('WillNavigateToPage') 53 54 def DidNavigateToPage(self, page, tab): 55 self.func_calls.append('DidNavigateToPage') 56 57 def ValidateAndMeasurePage(self, page, tab, results): 58 self.func_calls.append('ValidateAndMeasurePage') 59 60 def WillStartBrowser(self, platform): 61 self.func_calls.append('WillStartBrowser') 62 63 def DidStartBrowser(self, browser): 64 self.func_calls.append('DidStartBrowser') 65 66 67class MockBenchmark(benchmark.Benchmark): 68 test = MockPageTest 69 70 def __init__(self): 71 super(MockBenchmark, self).__init__() 72 self.mock_story_set = None 73 74 @classmethod 75 def AddBenchmarkCommandLineArgs(cls, group): 76 group.add_option('', '--mock-benchmark-url', action='store', type='string') 77 78 def CreateStorySet(self, options): 79 kwargs = {} 80 if options.mock_benchmark_url: 81 kwargs['url'] = options.mock_benchmark_url 82 self.mock_story_set = MockStorySet(**kwargs) 83 return self.mock_story_set 84 85 86class MockTimelineBasedMeasurementBenchmark(benchmark.Benchmark): 87 88 def __init__(self): 89 super(MockTimelineBasedMeasurementBenchmark, self).__init__() 90 self.mock_story_set = None 91 92 @classmethod 93 def AddBenchmarkCommandLineArgs(cls, group): 94 group.add_option('', '--mock-benchmark-url', action='store', type='string') 95 96 def CreateStorySet(self, options): 97 kwargs = {} 98 if options.mock_benchmark_url: 99 kwargs['url'] = options.mock_benchmark_url 100 self.mock_story_set = MockStorySet(**kwargs) 101 return self.mock_story_set 102 103 104def _SuccessfulStories(results): 105 return set(run.story for run in results.IterStoryRuns() if run.ok) 106 107 108class RecordWprUnitTests(tab_test_case.TabTestCase): 109 110 _base_dir = util.GetUnittestDataDir() 111 _test_data_dir = os.path.join(util.GetUnittestDataDir(), 'page_tests') 112 113 @classmethod 114 def setUpClass(cls): 115 sys.path.extend([cls._base_dir, cls._test_data_dir]) 116 super(RecordWprUnitTests, cls).setUpClass() 117 cls._url = cls.UrlOfUnittestFile('blank.html') 118 cls._test_options = options_for_unittests.GetCopy() 119 120 # When the RecorderPageTest is created from a Benchmark, the benchmark will 121 # have a PageTest, specified by its test attribute. 122 def testRunPage_OnlyRunBenchmarkAction(self): 123 page_test = MockBenchmark().test() 124 record_page_test = record_wpr.RecorderPageTest(page_test) 125 page = MockPage(story_set=MockStorySet(url=self._url), url=self._url) 126 record_page_test.ValidateAndMeasurePage(page, self._tab, results=None) 127 128 def testRunPage_CallBenchmarksPageTestsFunctions(self): 129 page_test = MockBenchmark().test() 130 record_page_test = record_wpr.RecorderPageTest(page_test) 131 page = MockPage(story_set=MockStorySet(url=self._url), url=self._url) 132 record_page_test.ValidateAndMeasurePage(page, self._tab, results=None) 133 self.assertEqual(1, len(page_test.func_calls)) 134 self.assertEqual('ValidateAndMeasurePage', page_test.func_calls[0]) 135 136 def GetBrowserDeviceFlags(self): 137 flags = ['--browser', self._browser.browser_type, 138 '--remote', self._test_options.cros_remote, 139 '--device', self._device] 140 if self._browser.browser_type == 'exact': 141 flags += ['--browser-executable', self._test_options.browser_executable] 142 if self._test_options.chrome_root: 143 flags += ['--chrome-root', self._test_options.chrome_root] 144 return flags 145 146 @decorators.Disabled('chromeos') # crbug.com/404868. 147 def testWprRecorderWithPageSet(self): 148 flags = self.GetBrowserDeviceFlags() 149 mock_story_set = MockStorySet(url=self._url) 150 with record_wpr.WprRecorder(self._test_data_dir, 151 mock_story_set, flags) as wpr_recorder: 152 with wpr_recorder.CreateResults() as results: 153 wpr_recorder.Record(results) 154 self.assertItemsEqual(mock_story_set.stories, 155 _SuccessfulStories(results)) 156 157 def testWprRecorderWithBenchmark(self): 158 flags = self.GetBrowserDeviceFlags() 159 flags.extend(['--mock-benchmark-url', self._url]) 160 mock_benchmark = MockBenchmark() 161 with record_wpr.WprRecorder(self._test_data_dir, mock_benchmark, 162 flags) as wpr_recorder: 163 with wpr_recorder.CreateResults() as results: 164 wpr_recorder.Record(results) 165 self.assertItemsEqual(mock_benchmark.mock_story_set.stories, 166 _SuccessfulStories(results)) 167 168 def testWprRecorderWithTimelineBasedMeasurementBenchmark(self): 169 flags = self.GetBrowserDeviceFlags() 170 flags.extend(['--mock-benchmark-url', self._url]) 171 mock_benchmark = MockTimelineBasedMeasurementBenchmark() 172 with record_wpr.WprRecorder(self._test_data_dir, mock_benchmark, 173 flags) as wpr_recorder: 174 with wpr_recorder.CreateResults() as results: 175 wpr_recorder.Record(results) 176 self.assertItemsEqual(mock_benchmark.mock_story_set.stories, 177 _SuccessfulStories(results)) 178 179 def testPageSetBaseDirFlag(self): 180 flags = self.GetBrowserDeviceFlags() 181 flags.extend(['--page-set-base-dir', self._test_data_dir, 182 '--mock-benchmark-url', self._url]) 183 mock_benchmark = MockBenchmark() 184 with record_wpr.WprRecorder( 185 'non-existent-dummy-dir', mock_benchmark, flags) as wpr_recorder: 186 with wpr_recorder.CreateResults() as results: 187 wpr_recorder.Record(results) 188 self.assertItemsEqual(mock_benchmark.mock_story_set.stories, 189 _SuccessfulStories(results)) 190 191 def testCommandLineFlags(self): 192 flags = [ 193 '--pageset-repeat', '2', 194 '--mock-benchmark-url', self._url, 195 '--upload', 196 ] 197 with record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(), 198 flags) as wpr_recorder: 199 # page_runner command-line args 200 self.assertEquals(2, wpr_recorder.options.pageset_repeat) 201 # benchmark command-line args 202 self.assertEquals(self._url, wpr_recorder.options.mock_benchmark_url) 203 # record_wpr command-line arg to upload to cloud-storage. 204 self.assertTrue(wpr_recorder.options.upload) 205 # invalid command-line args 206 self.assertFalse(hasattr(wpr_recorder.options, 'not_a_real_option')) 207 208 def testRecordingEnabled(self): 209 flags = ['--mock-benchmark-url', self._url] 210 with record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(), 211 flags) as wpr_recorder: 212 self.assertEqual(wpr_modes.WPR_RECORD, 213 wpr_recorder.options.browser_options.wpr_mode) 214 215 # When the RecorderPageTest CustomizeBrowserOptions/WillStartBrowser/ 216 # DidStartBrowser function is called, it forwards the call to the PageTest 217 def testRecorderPageTest_BrowserMethods(self): 218 flags = ['--mock-benchmark-url', self._url] 219 page_test = MockBenchmark().test() 220 record_page_test = record_wpr.RecorderPageTest(page_test) 221 with record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(), 222 flags) as wpr_recorder: 223 record_page_test.CustomizeBrowserOptions(wpr_recorder.options) 224 record_page_test.WillStartBrowser(self._tab.browser.platform) 225 record_page_test.DidStartBrowser(self._tab.browser) 226 self.assertTrue('CustomizeBrowserOptions' in page_test.func_calls) 227 self.assertTrue('WillStartBrowser' in page_test.func_calls) 228 self.assertTrue('DidStartBrowser' in page_test.func_calls) 229 230 def testUseLiveSitesUnsupported(self): 231 flags = ['--use-live-sites'] 232 with self.assertRaises(SystemExit): 233 record_wpr.WprRecorder(self._test_data_dir, MockBenchmark(), flags) 234