1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4import collections
5import json
6import os
7import pathlib
8import sys
9import re
10import shutil
11from pathlib import Path
12
13from mozperftest.utils import install_package, get_output_dir
14from mozperftest.test.noderunner import NodeRunner
15from mozperftest.test.browsertime.visualtools import get_dependencies, xvfb
16
17
18BROWSERTIME_SRC_ROOT = Path(__file__).parent
19
20
21def matches(args, *flags):
22    """Returns True if any argument matches any of the given flags
23
24    Maybe with an argument.
25    """
26
27    for flag in flags:
28        if flag in args or any(arg.startswith(flag + "=") for arg in args):
29            return True
30    return False
31
32
33def extract_browser_name(args):
34    "Extracts the browser name if any"
35    # These are BT arguments, it's BT job to check them
36    # here we just want to extract the browser name
37    res = re.findall(r"(--browser|-b)[= ]([\w]+)", " ".join(args))
38    if res == []:
39        return None
40    return res[0][-1]
41
42
43class NodeException(Exception):
44    pass
45
46
47class BrowsertimeRunner(NodeRunner):
48    """Runs a browsertime test."""
49
50    name = "browsertime"
51    activated = True
52    user_exception = True
53
54    arguments = {
55        "cycles": {"type": int, "default": 1, "help": "Number of full cycles"},
56        "iterations": {"type": int, "default": 1, "help": "Number of iterations"},
57        "node": {"type": str, "default": None, "help": "Path to Node.js"},
58        "geckodriver": {"type": str, "default": None, "help": "Path to geckodriver"},
59        "binary": {
60            "type": str,
61            "default": None,
62            "help": "Path to the desktop browser, or Android app name.",
63        },
64        "clobber": {
65            "action": "store_true",
66            "default": False,
67            "help": "Force-update the installation.",
68        },
69        "install-url": {
70            "type": str,
71            "default": None,
72            "help": "Use this URL as the install url.",
73        },
74        "extra-options": {
75            "type": str,
76            "default": "",
77            "help": "Extra options passed to browsertime.js",
78        },
79        "xvfb": {"action": "store_true", "default": False, "help": "Use xvfb"},
80        "no-window-recorder": {
81            "action": "store_true",
82            "default": False,
83            "help": "Use the window recorder",
84        },
85        "viewport-size": {"type": str, "default": "1366x695", "help": "Viewport size"},
86    }
87
88    def __init__(self, env, mach_cmd):
89        super(BrowsertimeRunner, self).__init__(env, mach_cmd)
90        self.topsrcdir = mach_cmd.topsrcdir
91        self._mach_context = mach_cmd._mach_context
92        self.virtualenv_manager = mach_cmd.virtualenv_manager
93        self._created_dirs = []
94        self._test_script = None
95        self._setup_helper = None
96        self.get_binary_path = mach_cmd.get_binary_path
97
98    @property
99    def setup_helper(self):
100        if self._setup_helper is not None:
101            return self._setup_helper
102        sys.path.append(str(Path(self.topsrcdir, "tools", "lint", "eslint")))
103        import setup_helper
104
105        self._setup_helper = setup_helper
106        return self._setup_helper
107
108    @property
109    def artifact_cache_path(self):
110        """Downloaded artifacts will be kept here."""
111        # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
112        return Path(self._mach_context.state_dir, "cache", "browsertime")
113
114    @property
115    def state_path(self):
116        """Unpacked artifacts will be kept here."""
117        # The convention is $MOZBUILD_STATE_PATH/$FEATURE.
118        res = Path(self._mach_context.state_dir, "browsertime")
119        os.makedirs(str(res), exist_ok=True)
120        return res
121
122    @property
123    def browsertime_js(self):
124        root = os.environ.get("BROWSERTIME", self.state_path)
125        path = Path(root, "node_modules", "browsertime", "bin", "browsertime.js")
126        if path.exists():
127            os.environ["BROWSERTIME_JS"] = str(path)
128        return path
129
130    @property
131    def visualmetrics_py(self):
132        root = os.environ.get("BROWSERTIME", self.state_path)
133        path = Path(
134            root, "node_modules", "browsertime", "browsertime", "visualmetrics.py"
135        )
136        if path.exists():
137            os.environ["VISUALMETRICS_PY"] = str(path)
138        return path
139
140    def _should_install(self):
141        # If browsertime doesn't exist, install it
142        if not self.visualmetrics_py.exists() or not self.browsertime_js.exists():
143            return True
144
145        # Browsertime exists, check if it's outdated
146        with Path(BROWSERTIME_SRC_ROOT, "package.json").open() as new, Path(
147            os.environ.get("BROWSERTIME", self.state_path),
148            "node_modules",
149            "browsertime",
150            "package.json",
151        ).open() as old:
152            old_pkg = json.load(old)
153            new_pkg = json.load(new)
154
155        return not old_pkg["_from"].endswith(new_pkg["devDependencies"]["browsertime"])
156
157    def setup(self):
158        """Install browsertime and visualmetrics.py prerequisites and the Node.js package."""
159
160        node = self.get_arg("node")
161        if node is not None:
162            os.environ["NODEJS"] = node
163
164        super(BrowsertimeRunner, self).setup()
165        install_url = self.get_arg("install-url")
166
167        # installing Python deps on the fly
168        visualmetrics = self.get_arg("visualmetrics", False)
169
170        if visualmetrics:
171            # installing Python deps on the fly
172            for dep in get_dependencies():
173                install_package(self.virtualenv_manager, dep, ignore_failure=True)
174
175        # check if the browsertime package has been deployed correctly
176        # for this we just check for the browsertime directory presence
177        # we also make sure the visual metrics module is there *if*
178        # we need it
179        if not self._should_install() and not self.get_arg("clobber"):
180            return
181
182        # preparing ~/.mozbuild/browsertime
183        for file in ("package.json", "package-lock.json"):
184            src = BROWSERTIME_SRC_ROOT / file
185            target = self.state_path / file
186            # Overwrite the existing files
187            shutil.copyfile(str(src), str(target))
188
189        package_json_path = self.state_path / "package.json"
190
191        if install_url is not None:
192            self.info(
193                "Updating browsertime node module version in {package_json_path} "
194                "to {install_url}",
195                install_url=install_url,
196                package_json_path=str(package_json_path),
197            )
198
199            expr = r"/tarball/[a-f0-9]{40}$"
200            if not re.search(expr, install_url):
201                raise ValueError(
202                    "New upstream URL does not end with {}: '{}'".format(
203                        expr[:-1], install_url
204                    )
205                )
206
207            with package_json_path.open() as f:
208                existing_body = json.loads(
209                    f.read(), object_pairs_hook=collections.OrderedDict
210                )
211
212            existing_body["devDependencies"]["browsertime"] = install_url
213            updated_body = json.dumps(existing_body)
214            with package_json_path.open("w") as f:
215                f.write(updated_body)
216
217        self._setup_node_packages(package_json_path)
218
219    def _setup_node_packages(self, package_json_path):
220        # Install the browsertime Node.js requirements.
221        if not self.setup_helper.check_node_executables_valid():
222            return
223
224        should_clobber = self.get_arg("clobber")
225        # To use a custom `geckodriver`, set
226        # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url)
227        # to an endpoint with binaries named like
228        # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31.
229        automation = "MOZ_AUTOMATION" in os.environ
230
231        if automation:
232            os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true"
233            os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true"
234
235        self.info(
236            "Installing browsertime node module from {package_json}",
237            package_json=str(package_json_path),
238        )
239        install_url = self.get_arg("install-url")
240
241        self.setup_helper.package_setup(
242            str(self.state_path),
243            "browsertime",
244            should_update=install_url is not None,
245            should_clobber=should_clobber,
246            no_optional=install_url or automation,
247        )
248
249    def extra_default_args(self, args=[]):
250        # Add Mozilla-specific default arguments.  This is tricky because browsertime is quite
251        # loose about arguments; repeat arguments are generally accepted but then produce
252        # difficult to interpret type errors.
253        extra_args = []
254
255        # Default to Firefox.  Override with `-b ...` or `--browser=...`.
256        if not matches(args, "-b", "--browser"):
257            extra_args.extend(("-b", "firefox"))
258
259        # Default to not collect HAR.  Override with `--skipHar=false`.
260        if not matches(args, "--har", "--skipHar", "--gzipHar"):
261            extra_args.append("--skipHar")
262
263        extra_args.extend(["--viewPort", self.get_arg("viewport-size")])
264
265        if not matches(args, "--android"):
266            binary = self.get_arg("binary")
267            if binary is not None:
268                extra_args.extend(("--firefox.binaryPath", binary))
269            else:
270                # If --firefox.binaryPath is not specified, default to the objdir binary
271                # Note: --firefox.release is not a real browsertime option, but it will
272                #       silently ignore it instead and default to a release installation.
273                if (
274                    not matches(
275                        args,
276                        "--firefox.binaryPath",
277                        "--firefox.release",
278                        "--firefox.nightly",
279                        "--firefox.beta",
280                        "--firefox.developer",
281                    )
282                    and extract_browser_name(args) != "chrome"
283                ):
284                    extra_args.extend(("--firefox.binaryPath", self.get_binary_path()))
285
286        geckodriver = self.get_arg("geckodriver")
287        if geckodriver is not None:
288            extra_args.extend(("--firefox.geckodriverPath", geckodriver))
289
290        if extra_args:
291            self.debug(
292                "Running browsertime with extra default arguments: {extra_args}",
293                extra_args=extra_args,
294            )
295
296        return extra_args
297
298    def _android_args(self, metadata):
299        app_name = self.get_arg("android-app-name")
300
301        args_list = [
302            "--android",
303            # Work around a `selenium-webdriver` issue where Browsertime
304            # fails to find a Firefox binary even though we're going to
305            # actually do things on an Android device.
306            "--firefox.binaryPath",
307            self.node_path,
308            "--firefox.android.package",
309            app_name,
310        ]
311        activity = self.get_arg("android-activity")
312        if activity is not None:
313            args_list += ["--firefox.android.activity", activity]
314
315        return args_list
316
317    def run(self, metadata):
318        self._test_script = metadata.script
319        self.setup()
320        cycles = self.get_arg("cycles", 1)
321        for cycle in range(1, cycles + 1):
322
323            # Build an output directory
324            output = self.get_arg("output")
325            if output is None:
326                output = pathlib.Path(self.topsrcdir, "artifacts")
327            result_dir = get_output_dir(output, f"browsertime-results-{cycle}")
328
329            # Run the test cycle
330            metadata.run_hook(
331                "before_cycle", metadata, self.env, cycle, self._test_script
332            )
333            try:
334                metadata = self._one_cycle(metadata, result_dir)
335            finally:
336                metadata.run_hook(
337                    "after_cycle", metadata, self.env, cycle, self._test_script
338                )
339        return metadata
340
341    def _one_cycle(self, metadata, result_dir):
342        profile = self.get_arg("profile-directory")
343
344        args = [
345            "--resultDir",
346            str(result_dir),
347            "--firefox.profileTemplate",
348            profile,
349            "--iterations",
350            str(self.get_arg("iterations")),
351            self._test_script["filename"],
352        ]
353
354        # Set *all* prefs found in browser_prefs because
355        # browsertime will override the ones found in firefox.profileTemplate
356        # with its own defaults at `firefoxPreferences.js`
357        # Using `--firefox.preference` ensures we override them.
358        # see https://github.com/sitespeedio/browsertime/issues/1427
359        browser_prefs = metadata.get_options("browser_prefs")
360        for key, value in browser_prefs.items():
361            args += ["--firefox.preference", f"{key}:{value}"]
362
363        if self.get_arg("verbose"):
364            args += ["-vvv"]
365
366        # if the visualmetrics layer is activated, we want to feed it
367        visualmetrics = self.get_arg("visualmetrics", False)
368        if visualmetrics:
369            args += ["--video", "true"]
370            if not self.get_arg("no-window-recorder"):
371                args += ["--firefox.windowRecorder", "true"]
372
373        extra_options = self.get_arg("extra-options")
374        if extra_options:
375            for option in extra_options.split(","):
376                option = option.strip()
377                if not option:
378                    continue
379                option = option.split("=", 1)
380                if len(option) != 2:
381                    self.warning(
382                        f"Skipping browsertime option {option} as it "
383                        "is missing a name/value pairing. We expect options "
384                        "to be formatted as: --browsertime-extra-options "
385                        "'browserRestartTries=1,timeouts.browserStart=10'"
386                    )
387                    continue
388                name, value = option
389                args += ["--" + name, value]
390
391        if self.get_arg("android"):
392            args.extend(self._android_args(metadata))
393
394        extra = self.extra_default_args(args=args)
395        command = [str(self.browsertime_js)] + extra + args
396        self.info("Running browsertime with this command %s" % " ".join(command))
397
398        if visualmetrics and self.get_arg("xvfb"):
399            with xvfb():
400                exit_code = self.node(command)
401        else:
402            exit_code = self.node(command)
403
404        if exit_code != 0:
405            raise NodeException(exit_code)
406
407        metadata.add_result(
408            {"results": str(result_dir), "name": self._test_script["name"]}
409        )
410
411        return metadata
412