1#!/usr/bin/env python 2import os 3import mozunit 4from unittest import mock 5import shutil 6import string 7import random 8import pathlib 9 10import pytest 11 12from mozperftest.tests.support import get_running_env, EXAMPLE_TEST 13from mozperftest.environment import TEST, SYSTEM 14from mozperftest.test.browsertime import add_options 15from mozperftest.test.browsertime.runner import ( 16 NodeException, 17 matches, 18 extract_browser_name, 19) 20from mozperftest.utils import silence, temporary_env 21 22 23HERE = os.path.dirname(__file__) 24 25 26def fetch(self, url): 27 return os.path.join(HERE, "fetched_artifact.zip") 28 29 30def mocked_jsonload(val): 31 return val.__iter__.return_value 32 33 34def build_mock_open(files_data): 35 mocked_opens = [] 36 37 for data in files_data: 38 mocked_file = mock.MagicMock() 39 mocked_file.__enter__.return_value.__iter__.return_value = data 40 mocked_opens.append(mocked_file) 41 42 m = mock.mock_open() 43 m.side_effect = mocked_opens 44 return m 45 46 47@mock.patch("mozperftest.test.browsertime.runner.install_package") 48@mock.patch( 49 "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True 50) 51@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) 52@mock.patch( 53 "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages", 54 new=lambda x, y: None, 55) 56def test_browser(*mocked): 57 mach_cmd, metadata, env = get_running_env( 58 android=True, 59 android_app_name="something", 60 browsertime_geckodriver="GECKODRIVER", 61 browsertime_iterations=1, 62 browsertime_extra_options="one=1,two=2", 63 tests=[EXAMPLE_TEST], 64 browsertime_no_window_recorder=False, 65 browsertime_viewport_size="1234x567", 66 ) 67 68 sys = env.layers[SYSTEM] 69 browser = env.layers[TEST] 70 try: 71 with sys as s, browser as b, silence(): 72 b(s(metadata)) 73 finally: 74 shutil.rmtree(mach_cmd._mach_context.state_dir) 75 assert mach_cmd.run_process.call_count == 1 76 77 # Make sure all arguments are of type str 78 for option in mach_cmd.run_process.call_args[0][0]: 79 assert isinstance(option, str) 80 81 cmd = " ".join(mach_cmd.run_process.call_args[0][0]) 82 assert EXAMPLE_TEST in cmd 83 assert "--firefox.geckodriverPath GECKODRIVER" in cmd 84 assert "--one 1" in cmd 85 assert "--two 2" in cmd 86 87 results = metadata.get_results() 88 assert len(results) == 1 89 assert set(list(results[0].keys())) - set(["name", "results"]) == set() 90 assert results[0]["name"] == "Example" 91 92 93@mock.patch( 94 "mozperftest.test.browsertime.runner.BrowsertimeRunner.browsertime_js", 95 new=pathlib.Path("doesn't-exist"), 96) 97@mock.patch( 98 "mozperftest.test.browsertime.runner.BrowsertimeRunner.visualmetrics_py", 99 new=pathlib.Path("doesn't-exist-either"), 100) 101def test_browsertime_not_existing(): 102 _, _, env = get_running_env( 103 android=True, 104 android_app_name="something", 105 browsertime_geckodriver="GECKODRIVER", 106 browsertime_iterations=1, 107 browsertime_extra_options="one=1,two=2", 108 tests=[EXAMPLE_TEST], 109 ) 110 browser = env.layers[TEST] 111 btime_layer = browser.layers[0] 112 assert btime_layer._should_install() 113 114 115@mock.patch( 116 "mozperftest.test.browsertime.runner.pathlib.Path.exists", new=lambda x: True 117) 118def test_browsertime_no_reinstall(): 119 _, _, env = get_running_env( 120 android=True, 121 android_app_name="something", 122 browsertime_geckodriver="GECKODRIVER", 123 browsertime_iterations=1, 124 browsertime_extra_options="one=1,two=2", 125 tests=[EXAMPLE_TEST], 126 ) 127 128 with mock.patch( 129 "mozperftest.test.browsertime.runner.pathlib.Path.open", 130 build_mock_open( 131 [ 132 { 133 "devDependencies": { 134 "browsertime": "89771a1d6be54114db190427dbc281582cba3d47" 135 } 136 }, 137 { 138 "_from": ( 139 "browsertime@https://github.com/sitespeedio/browsertime" 140 "/tarball/89771a1d6be54114db190427dbc281582cba3d47" 141 ) 142 }, 143 ] 144 ), 145 ), mock.patch("mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload): 146 browser = env.layers[TEST] 147 btime_layer = browser.layers[0] 148 assert not btime_layer._should_install() 149 150 151@mock.patch( 152 "mozperftest.test.browsertime.runner.pathlib.Path.exists", new=lambda x: True 153) 154def test_browsertime_should_reinstall(): 155 _, _, env = get_running_env( 156 android=True, 157 android_app_name="something", 158 browsertime_geckodriver="GECKODRIVER", 159 browsertime_iterations=1, 160 browsertime_extra_options="one=1,two=2", 161 tests=[EXAMPLE_TEST], 162 ) 163 164 with mock.patch( 165 "mozperftest.test.browsertime.runner.pathlib.Path.open", 166 build_mock_open( 167 [ 168 { 169 "devDependencies": { 170 "browsertime": "89771a1d6be54114db190427dbc281582cba3d47" 171 } 172 }, 173 { 174 "_from": ( 175 "browsertime@https://github.com/sitespeedio/browsertime" 176 "/tarball/98747854be54114db190427dbc281582cba3d47" 177 ) 178 }, 179 ] 180 ), 181 ), mock.patch("mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload): 182 browser = env.layers[TEST] 183 btime_layer = browser.layers[0] 184 assert btime_layer._should_install() 185 186 187@mock.patch("mozperftest.test.browsertime.runner.install_package") 188@mock.patch( 189 "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True 190) 191@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) 192@mock.patch( 193 "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages", 194 new=lambda x, y: None, 195) 196def test_browser_failed(*mocked): 197 mach_cmd, metadata, env = get_running_env( 198 android=True, 199 android_app_name="something", 200 browsertime_geckodriver="GECKODRIVER", 201 browsertime_iterations=1, 202 browsertime_extra_options="one=1,two=2", 203 tests=[EXAMPLE_TEST], 204 browsertime_no_window_recorder=False, 205 browsertime_viewport_size="1234x567", 206 ) 207 # set the return value to 1 to simulate a node failure 208 mach_cmd.run_process.return_value = 1 209 browser = env.layers[TEST] 210 sys = env.layers[SYSTEM] 211 212 with sys as s, browser as b, silence(), pytest.raises(NodeException): 213 b(s(metadata)) 214 215 216@mock.patch("mozperftest.test.browsertime.runner.install_package") 217@mock.patch( 218 "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True 219) 220@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) 221@mock.patch( 222 "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages", 223 new=lambda x, y: None, 224) 225def test_browser_desktop(*mocked): 226 mach_cmd, metadata, env = get_running_env( 227 browsertime_iterations=1, 228 browsertime_extra_options="one=1,two=2", 229 tests=[EXAMPLE_TEST], 230 browsertime_no_window_recorder=False, 231 browsertime_viewport_size="1234x567", 232 ) 233 browser = env.layers[TEST] 234 sys = env.layers[SYSTEM] 235 236 try: 237 with sys as s, browser as b, silence(): 238 # just checking that the setup_helper property gets 239 # correctly initialized 240 browsertime = browser.layers[-1] 241 assert browsertime.setup_helper is not None 242 helper = browsertime.setup_helper 243 assert browsertime.setup_helper is helper 244 245 b(s(metadata)) 246 finally: 247 shutil.rmtree(mach_cmd._mach_context.state_dir) 248 249 assert mach_cmd.run_process.call_count == 1 250 cmd = " ".join(mach_cmd.run_process.call_args[0][0]) 251 # check that --firefox.binaryPath is set automatically 252 assert "--firefox.binaryPath" in cmd 253 254 255def test_add_options(): 256 mach_cmd, metadata, env = get_running_env() 257 options = [("one", 1), ("two", 2)] 258 add_options(env, options) 259 extra = env.get_arg("browsertime-extra-options") 260 assert "one=1" in extra 261 assert "two=2" in extra 262 263 264@mock.patch("mozperftest.test.browsertime.runner.install_package") 265@mock.patch( 266 "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True 267) 268@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) 269@mock.patch("mozperftest.test.browsertime.runner.BrowsertimeRunner.setup_helper") 270def test_install_url(*mocked): 271 url = "https://here/tarball/" + "".join( 272 [random.choice(string.hexdigits[:-6]) for c in range(40)] 273 ) 274 mach, metadata, env = get_running_env( 275 browsertime_install_url=url, 276 tests=[EXAMPLE_TEST], 277 browsertime_no_window_recorder=False, 278 browsertime_viewport_size="1234x567", 279 ) 280 browser = env.layers[TEST] 281 sys = env.layers[SYSTEM] 282 283 try: 284 with sys as s, temporary_env(MOZ_AUTOMATION="1"), browser as b, silence(): 285 b(s(metadata)) 286 finally: 287 shutil.rmtree(mach._mach_context.state_dir) 288 289 assert mach.run_process.call_count == 1 290 291 292@mock.patch("mozperftest.test.browsertime.runner.install_package") 293@mock.patch( 294 "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True 295) 296@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch) 297@mock.patch( 298 "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages", 299 new=lambda x, y: None, 300) 301def test_install_url_bad(*mocked): 302 mach, metadata, env = get_running_env( 303 browsertime_install_url="meh", 304 tests=[EXAMPLE_TEST], 305 ) 306 browser = env.layers[TEST] 307 sys = env.layers[SYSTEM] 308 309 with pytest.raises(ValueError): 310 try: 311 with sys as s, browser as b, silence(): 312 b(s(metadata)) 313 finally: 314 shutil.rmtree(mach._mach_context.state_dir) 315 316 317def test_matches(): 318 args = ["arg1=1", "--arg2=value2"] 319 320 assert matches(args, "arg1") 321 assert not matches(args, "arg3") 322 323 324def test_extract_browser_name(): 325 args = ["arg1=1", "--arg2=value2", "--browser=me", "--zome"] 326 assert extract_browser_name(args) == "me" 327 328 329if __name__ == "__main__": 330 mozunit.main() 331