1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4import collections 5import json 6import os 7import pathlib 8import sys 9import re 10import shutil 11from pathlib import Path 12 13from mozperftest.utils import install_package, get_output_dir 14from mozperftest.test.noderunner import NodeRunner 15from mozperftest.test.browsertime.visualtools import get_dependencies, xvfb 16 17 18BROWSERTIME_SRC_ROOT = Path(__file__).parent 19 20 21def matches(args, *flags): 22 """Returns True if any argument matches any of the given flags 23 24 Maybe with an argument. 25 """ 26 27 for flag in flags: 28 if flag in args or any(arg.startswith(flag + "=") for arg in args): 29 return True 30 return False 31 32 33def extract_browser_name(args): 34 "Extracts the browser name if any" 35 # These are BT arguments, it's BT job to check them 36 # here we just want to extract the browser name 37 res = re.findall(r"(--browser|-b)[= ]([\w]+)", " ".join(args)) 38 if res == []: 39 return None 40 return res[0][-1] 41 42 43class NodeException(Exception): 44 pass 45 46 47class BrowsertimeRunner(NodeRunner): 48 """Runs a browsertime test.""" 49 50 name = "browsertime" 51 activated = True 52 user_exception = True 53 54 arguments = { 55 "cycles": {"type": int, "default": 1, "help": "Number of full cycles"}, 56 "iterations": {"type": int, "default": 1, "help": "Number of iterations"}, 57 "node": {"type": str, "default": None, "help": "Path to Node.js"}, 58 "geckodriver": {"type": str, "default": None, "help": "Path to geckodriver"}, 59 "binary": { 60 "type": str, 61 "default": None, 62 "help": "Path to the desktop browser, or Android app name.", 63 }, 64 "clobber": { 65 "action": "store_true", 66 "default": False, 67 "help": "Force-update the installation.", 68 }, 69 "install-url": { 70 "type": str, 71 "default": None, 72 "help": "Use this URL as the install url.", 73 }, 74 "extra-options": { 75 "type": str, 76 "default": "", 77 "help": "Extra options passed to browsertime.js", 78 }, 79 "xvfb": {"action": "store_true", "default": False, "help": "Use xvfb"}, 80 "no-window-recorder": { 81 "action": "store_true", 82 "default": False, 83 "help": "Use the window recorder", 84 }, 85 "viewport-size": {"type": str, "default": "1366x695", "help": "Viewport size"}, 86 } 87 88 def __init__(self, env, mach_cmd): 89 super(BrowsertimeRunner, self).__init__(env, mach_cmd) 90 self.topsrcdir = mach_cmd.topsrcdir 91 self._mach_context = mach_cmd._mach_context 92 self.virtualenv_manager = mach_cmd.virtualenv_manager 93 self._created_dirs = [] 94 self._test_script = None 95 self._setup_helper = None 96 self.get_binary_path = mach_cmd.get_binary_path 97 98 @property 99 def setup_helper(self): 100 if self._setup_helper is not None: 101 return self._setup_helper 102 sys.path.append(str(Path(self.topsrcdir, "tools", "lint", "eslint"))) 103 import setup_helper 104 105 self._setup_helper = setup_helper 106 return self._setup_helper 107 108 @property 109 def artifact_cache_path(self): 110 """Downloaded artifacts will be kept here.""" 111 # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE. 112 return Path(self._mach_context.state_dir, "cache", "browsertime") 113 114 @property 115 def state_path(self): 116 """Unpacked artifacts will be kept here.""" 117 # The convention is $MOZBUILD_STATE_PATH/$FEATURE. 118 res = Path(self._mach_context.state_dir, "browsertime") 119 os.makedirs(str(res), exist_ok=True) 120 return res 121 122 @property 123 def browsertime_js(self): 124 root = os.environ.get("BROWSERTIME", self.state_path) 125 path = Path(root, "node_modules", "browsertime", "bin", "browsertime.js") 126 if path.exists(): 127 os.environ["BROWSERTIME_JS"] = str(path) 128 return path 129 130 @property 131 def visualmetrics_py(self): 132 root = os.environ.get("BROWSERTIME", self.state_path) 133 path = Path( 134 root, "node_modules", "browsertime", "browsertime", "visualmetrics.py" 135 ) 136 if path.exists(): 137 os.environ["VISUALMETRICS_PY"] = str(path) 138 return path 139 140 def _should_install(self): 141 # If browsertime doesn't exist, install it 142 if not self.visualmetrics_py.exists() or not self.browsertime_js.exists(): 143 return True 144 145 # Browsertime exists, check if it's outdated 146 with Path(BROWSERTIME_SRC_ROOT, "package.json").open() as new, Path( 147 os.environ.get("BROWSERTIME", self.state_path), 148 "node_modules", 149 "browsertime", 150 "package.json", 151 ).open() as old: 152 old_pkg = json.load(old) 153 new_pkg = json.load(new) 154 155 return not old_pkg["_from"].endswith(new_pkg["devDependencies"]["browsertime"]) 156 157 def setup(self): 158 """Install browsertime and visualmetrics.py prerequisites and the Node.js package.""" 159 160 node = self.get_arg("node") 161 if node is not None: 162 os.environ["NODEJS"] = node 163 164 super(BrowsertimeRunner, self).setup() 165 install_url = self.get_arg("install-url") 166 167 # installing Python deps on the fly 168 visualmetrics = self.get_arg("visualmetrics", False) 169 170 if visualmetrics: 171 # installing Python deps on the fly 172 for dep in get_dependencies(): 173 install_package(self.virtualenv_manager, dep, ignore_failure=True) 174 175 # check if the browsertime package has been deployed correctly 176 # for this we just check for the browsertime directory presence 177 # we also make sure the visual metrics module is there *if* 178 # we need it 179 if not self._should_install() and not self.get_arg("clobber"): 180 return 181 182 # preparing ~/.mozbuild/browsertime 183 for file in ("package.json", "package-lock.json"): 184 src = BROWSERTIME_SRC_ROOT / file 185 target = self.state_path / file 186 # Overwrite the existing files 187 shutil.copyfile(str(src), str(target)) 188 189 package_json_path = self.state_path / "package.json" 190 191 if install_url is not None: 192 self.info( 193 "Updating browsertime node module version in {package_json_path} " 194 "to {install_url}", 195 install_url=install_url, 196 package_json_path=str(package_json_path), 197 ) 198 199 expr = r"/tarball/[a-f0-9]{40}$" 200 if not re.search(expr, install_url): 201 raise ValueError( 202 "New upstream URL does not end with {}: '{}'".format( 203 expr[:-1], install_url 204 ) 205 ) 206 207 with package_json_path.open() as f: 208 existing_body = json.loads( 209 f.read(), object_pairs_hook=collections.OrderedDict 210 ) 211 212 existing_body["devDependencies"]["browsertime"] = install_url 213 updated_body = json.dumps(existing_body) 214 with package_json_path.open("w") as f: 215 f.write(updated_body) 216 217 self._setup_node_packages(package_json_path) 218 219 def _setup_node_packages(self, package_json_path): 220 # Install the browsertime Node.js requirements. 221 if not self.setup_helper.check_node_executables_valid(): 222 return 223 224 should_clobber = self.get_arg("clobber") 225 # To use a custom `geckodriver`, set 226 # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url) 227 # to an endpoint with binaries named like 228 # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31. 229 automation = "MOZ_AUTOMATION" in os.environ 230 231 if automation: 232 os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true" 233 os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true" 234 235 self.info( 236 "Installing browsertime node module from {package_json}", 237 package_json=str(package_json_path), 238 ) 239 install_url = self.get_arg("install-url") 240 241 self.setup_helper.package_setup( 242 str(self.state_path), 243 "browsertime", 244 should_update=install_url is not None, 245 should_clobber=should_clobber, 246 no_optional=install_url or automation, 247 ) 248 249 def extra_default_args(self, args=[]): 250 # Add Mozilla-specific default arguments. This is tricky because browsertime is quite 251 # loose about arguments; repeat arguments are generally accepted but then produce 252 # difficult to interpret type errors. 253 extra_args = [] 254 255 # Default to Firefox. Override with `-b ...` or `--browser=...`. 256 if not matches(args, "-b", "--browser"): 257 extra_args.extend(("-b", "firefox")) 258 259 # Default to not collect HAR. Override with `--skipHar=false`. 260 if not matches(args, "--har", "--skipHar", "--gzipHar"): 261 extra_args.append("--skipHar") 262 263 extra_args.extend(["--viewPort", self.get_arg("viewport-size")]) 264 265 if not matches(args, "--android"): 266 binary = self.get_arg("binary") 267 if binary is not None: 268 extra_args.extend(("--firefox.binaryPath", binary)) 269 else: 270 # If --firefox.binaryPath is not specified, default to the objdir binary 271 # Note: --firefox.release is not a real browsertime option, but it will 272 # silently ignore it instead and default to a release installation. 273 if ( 274 not matches( 275 args, 276 "--firefox.binaryPath", 277 "--firefox.release", 278 "--firefox.nightly", 279 "--firefox.beta", 280 "--firefox.developer", 281 ) 282 and extract_browser_name(args) != "chrome" 283 ): 284 extra_args.extend(("--firefox.binaryPath", self.get_binary_path())) 285 286 geckodriver = self.get_arg("geckodriver") 287 if geckodriver is not None: 288 extra_args.extend(("--firefox.geckodriverPath", geckodriver)) 289 290 if extra_args: 291 self.debug( 292 "Running browsertime with extra default arguments: {extra_args}", 293 extra_args=extra_args, 294 ) 295 296 return extra_args 297 298 def _android_args(self, metadata): 299 app_name = self.get_arg("android-app-name") 300 301 args_list = [ 302 "--android", 303 # Work around a `selenium-webdriver` issue where Browsertime 304 # fails to find a Firefox binary even though we're going to 305 # actually do things on an Android device. 306 "--firefox.binaryPath", 307 self.node_path, 308 "--firefox.android.package", 309 app_name, 310 ] 311 activity = self.get_arg("android-activity") 312 if activity is not None: 313 args_list += ["--firefox.android.activity", activity] 314 315 return args_list 316 317 def run(self, metadata): 318 self._test_script = metadata.script 319 self.setup() 320 cycles = self.get_arg("cycles", 1) 321 for cycle in range(1, cycles + 1): 322 323 # Build an output directory 324 output = self.get_arg("output") 325 if output is None: 326 output = pathlib.Path(self.topsrcdir, "artifacts") 327 result_dir = get_output_dir(output, f"browsertime-results-{cycle}") 328 329 # Run the test cycle 330 metadata.run_hook( 331 "before_cycle", metadata, self.env, cycle, self._test_script 332 ) 333 try: 334 metadata = self._one_cycle(metadata, result_dir) 335 finally: 336 metadata.run_hook( 337 "after_cycle", metadata, self.env, cycle, self._test_script 338 ) 339 return metadata 340 341 def _one_cycle(self, metadata, result_dir): 342 profile = self.get_arg("profile-directory") 343 344 args = [ 345 "--resultDir", 346 str(result_dir), 347 "--firefox.profileTemplate", 348 profile, 349 "--iterations", 350 str(self.get_arg("iterations")), 351 self._test_script["filename"], 352 ] 353 354 # Set *all* prefs found in browser_prefs because 355 # browsertime will override the ones found in firefox.profileTemplate 356 # with its own defaults at `firefoxPreferences.js` 357 # Using `--firefox.preference` ensures we override them. 358 # see https://github.com/sitespeedio/browsertime/issues/1427 359 browser_prefs = metadata.get_options("browser_prefs") 360 for key, value in browser_prefs.items(): 361 args += ["--firefox.preference", f"{key}:{value}"] 362 363 if self.get_arg("verbose"): 364 args += ["-vvv"] 365 366 # if the visualmetrics layer is activated, we want to feed it 367 visualmetrics = self.get_arg("visualmetrics", False) 368 if visualmetrics: 369 args += ["--video", "true"] 370 if not self.get_arg("no-window-recorder"): 371 args += ["--firefox.windowRecorder", "true"] 372 373 extra_options = self.get_arg("extra-options") 374 if extra_options: 375 for option in extra_options.split(","): 376 option = option.strip() 377 if not option: 378 continue 379 option = option.split("=", 1) 380 if len(option) != 2: 381 self.warning( 382 f"Skipping browsertime option {option} as it " 383 "is missing a name/value pairing. We expect options " 384 "to be formatted as: --browsertime-extra-options " 385 "'browserRestartTries=1,timeouts.browserStart=10'" 386 ) 387 continue 388 name, value = option 389 args += ["--" + name, value] 390 391 if self.get_arg("android"): 392 args.extend(self._android_args(metadata)) 393 394 extra = self.extra_default_args(args=args) 395 command = [str(self.browsertime_js)] + extra + args 396 self.info("Running browsertime with this command %s" % " ".join(command)) 397 398 if visualmetrics and self.get_arg("xvfb"): 399 with xvfb(): 400 exit_code = self.node(command) 401 else: 402 exit_code = self.node(command) 403 404 if exit_code != 0: 405 raise NodeException(exit_code) 406 407 metadata.add_result( 408 {"results": str(result_dir), "name": self._test_script["name"]} 409 ) 410 411 return metadata 412