1#!/usr/bin/env python
2import os
3import mozunit
4from unittest import mock
5import shutil
6import string
7import random
8import pathlib
9
10import pytest
11
12from mozperftest.tests.support import get_running_env, EXAMPLE_TEST
13from mozperftest.environment import TEST, SYSTEM
14from mozperftest.test.browsertime import add_options
15from mozperftest.test.browsertime.runner import (
16    NodeException,
17    matches,
18    extract_browser_name,
19)
20from mozperftest.utils import silence, temporary_env
21
22
23HERE = os.path.dirname(__file__)
24
25
26def fetch(self, url):
27    return os.path.join(HERE, "fetched_artifact.zip")
28
29
30def mocked_jsonload(val):
31    return val.__iter__.return_value
32
33
34def build_mock_open(files_data):
35    mocked_opens = []
36
37    for data in files_data:
38        mocked_file = mock.MagicMock()
39        mocked_file.__enter__.return_value.__iter__.return_value = data
40        mocked_opens.append(mocked_file)
41
42    m = mock.mock_open()
43    m.side_effect = mocked_opens
44    return m
45
46
47@mock.patch("mozperftest.test.browsertime.runner.install_package")
48@mock.patch(
49    "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True
50)
51@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
52@mock.patch(
53    "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages",
54    new=lambda x, y: None,
55)
56def test_browser(*mocked):
57    mach_cmd, metadata, env = get_running_env(
58        android=True,
59        android_app_name="something",
60        browsertime_geckodriver="GECKODRIVER",
61        browsertime_iterations=1,
62        browsertime_extra_options="one=1,two=2",
63        tests=[EXAMPLE_TEST],
64        browsertime_no_window_recorder=False,
65        browsertime_viewport_size="1234x567",
66    )
67
68    sys = env.layers[SYSTEM]
69    browser = env.layers[TEST]
70    try:
71        with sys as s, browser as b, silence():
72            b(s(metadata))
73    finally:
74        shutil.rmtree(mach_cmd._mach_context.state_dir)
75    assert mach_cmd.run_process.call_count == 1
76
77    # Make sure all arguments are of type str
78    for option in mach_cmd.run_process.call_args[0][0]:
79        assert isinstance(option, str)
80
81    cmd = " ".join(mach_cmd.run_process.call_args[0][0])
82    assert EXAMPLE_TEST in cmd
83    assert "--firefox.geckodriverPath GECKODRIVER" in cmd
84    assert "--one 1" in cmd
85    assert "--two 2" in cmd
86
87    results = metadata.get_results()
88    assert len(results) == 1
89    assert set(list(results[0].keys())) - set(["name", "results"]) == set()
90    assert results[0]["name"] == "Example"
91
92
93@mock.patch(
94    "mozperftest.test.browsertime.runner.BrowsertimeRunner.browsertime_js",
95    new=pathlib.Path("doesn't-exist"),
96)
97@mock.patch(
98    "mozperftest.test.browsertime.runner.BrowsertimeRunner.visualmetrics_py",
99    new=pathlib.Path("doesn't-exist-either"),
100)
101def test_browsertime_not_existing():
102    _, _, env = get_running_env(
103        android=True,
104        android_app_name="something",
105        browsertime_geckodriver="GECKODRIVER",
106        browsertime_iterations=1,
107        browsertime_extra_options="one=1,two=2",
108        tests=[EXAMPLE_TEST],
109    )
110    browser = env.layers[TEST]
111    btime_layer = browser.layers[0]
112    assert btime_layer._should_install()
113
114
115@mock.patch(
116    "mozperftest.test.browsertime.runner.pathlib.Path.exists", new=lambda x: True
117)
118def test_browsertime_no_reinstall():
119    _, _, env = get_running_env(
120        android=True,
121        android_app_name="something",
122        browsertime_geckodriver="GECKODRIVER",
123        browsertime_iterations=1,
124        browsertime_extra_options="one=1,two=2",
125        tests=[EXAMPLE_TEST],
126    )
127
128    with mock.patch(
129        "mozperftest.test.browsertime.runner.pathlib.Path.open",
130        build_mock_open(
131            [
132                {
133                    "devDependencies": {
134                        "browsertime": "89771a1d6be54114db190427dbc281582cba3d47"
135                    }
136                },
137                {
138                    "_from": (
139                        "browsertime@https://github.com/sitespeedio/browsertime"
140                        "/tarball/89771a1d6be54114db190427dbc281582cba3d47"
141                    )
142                },
143            ]
144        ),
145    ), mock.patch("mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload):
146        browser = env.layers[TEST]
147        btime_layer = browser.layers[0]
148        assert not btime_layer._should_install()
149
150
151@mock.patch(
152    "mozperftest.test.browsertime.runner.pathlib.Path.exists", new=lambda x: True
153)
154def test_browsertime_should_reinstall():
155    _, _, env = get_running_env(
156        android=True,
157        android_app_name="something",
158        browsertime_geckodriver="GECKODRIVER",
159        browsertime_iterations=1,
160        browsertime_extra_options="one=1,two=2",
161        tests=[EXAMPLE_TEST],
162    )
163
164    with mock.patch(
165        "mozperftest.test.browsertime.runner.pathlib.Path.open",
166        build_mock_open(
167            [
168                {
169                    "devDependencies": {
170                        "browsertime": "89771a1d6be54114db190427dbc281582cba3d47"
171                    }
172                },
173                {
174                    "_from": (
175                        "browsertime@https://github.com/sitespeedio/browsertime"
176                        "/tarball/98747854be54114db190427dbc281582cba3d47"
177                    )
178                },
179            ]
180        ),
181    ), mock.patch("mozperftest.test.browsertime.runner.json.load", new=mocked_jsonload):
182        browser = env.layers[TEST]
183        btime_layer = browser.layers[0]
184        assert btime_layer._should_install()
185
186
187@mock.patch("mozperftest.test.browsertime.runner.install_package")
188@mock.patch(
189    "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True
190)
191@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
192@mock.patch(
193    "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages",
194    new=lambda x, y: None,
195)
196def test_browser_failed(*mocked):
197    mach_cmd, metadata, env = get_running_env(
198        android=True,
199        android_app_name="something",
200        browsertime_geckodriver="GECKODRIVER",
201        browsertime_iterations=1,
202        browsertime_extra_options="one=1,two=2",
203        tests=[EXAMPLE_TEST],
204        browsertime_no_window_recorder=False,
205        browsertime_viewport_size="1234x567",
206    )
207    # set the return value to 1 to simulate a node failure
208    mach_cmd.run_process.return_value = 1
209    browser = env.layers[TEST]
210    sys = env.layers[SYSTEM]
211
212    with sys as s, browser as b, silence(), pytest.raises(NodeException):
213        b(s(metadata))
214
215
216@mock.patch("mozperftest.test.browsertime.runner.install_package")
217@mock.patch(
218    "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True
219)
220@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
221@mock.patch(
222    "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages",
223    new=lambda x, y: None,
224)
225def test_browser_desktop(*mocked):
226    mach_cmd, metadata, env = get_running_env(
227        browsertime_iterations=1,
228        browsertime_extra_options="one=1,two=2",
229        tests=[EXAMPLE_TEST],
230        browsertime_no_window_recorder=False,
231        browsertime_viewport_size="1234x567",
232    )
233    browser = env.layers[TEST]
234    sys = env.layers[SYSTEM]
235
236    try:
237        with sys as s, browser as b, silence():
238            # just checking that the setup_helper property gets
239            # correctly initialized
240            browsertime = browser.layers[-1]
241            assert browsertime.setup_helper is not None
242            helper = browsertime.setup_helper
243            assert browsertime.setup_helper is helper
244
245            b(s(metadata))
246    finally:
247        shutil.rmtree(mach_cmd._mach_context.state_dir)
248
249    assert mach_cmd.run_process.call_count == 1
250    cmd = " ".join(mach_cmd.run_process.call_args[0][0])
251    # check that --firefox.binaryPath is set automatically
252    assert "--firefox.binaryPath" in cmd
253
254
255def test_add_options():
256    mach_cmd, metadata, env = get_running_env()
257    options = [("one", 1), ("two", 2)]
258    add_options(env, options)
259    extra = env.get_arg("browsertime-extra-options")
260    assert "one=1" in extra
261    assert "two=2" in extra
262
263
264@mock.patch("mozperftest.test.browsertime.runner.install_package")
265@mock.patch(
266    "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True
267)
268@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
269@mock.patch("mozperftest.test.browsertime.runner.BrowsertimeRunner.setup_helper")
270def test_install_url(*mocked):
271    url = "https://here/tarball/" + "".join(
272        [random.choice(string.hexdigits[:-6]) for c in range(40)]
273    )
274    mach, metadata, env = get_running_env(
275        browsertime_install_url=url,
276        tests=[EXAMPLE_TEST],
277        browsertime_no_window_recorder=False,
278        browsertime_viewport_size="1234x567",
279    )
280    browser = env.layers[TEST]
281    sys = env.layers[SYSTEM]
282
283    try:
284        with sys as s, temporary_env(MOZ_AUTOMATION="1"), browser as b, silence():
285            b(s(metadata))
286    finally:
287        shutil.rmtree(mach._mach_context.state_dir)
288
289    assert mach.run_process.call_count == 1
290
291
292@mock.patch("mozperftest.test.browsertime.runner.install_package")
293@mock.patch(
294    "mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True
295)
296@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
297@mock.patch(
298    "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages",
299    new=lambda x, y: None,
300)
301def test_install_url_bad(*mocked):
302    mach, metadata, env = get_running_env(
303        browsertime_install_url="meh",
304        tests=[EXAMPLE_TEST],
305    )
306    browser = env.layers[TEST]
307    sys = env.layers[SYSTEM]
308
309    with pytest.raises(ValueError):
310        try:
311            with sys as s, browser as b, silence():
312                b(s(metadata))
313        finally:
314            shutil.rmtree(mach._mach_context.state_dir)
315
316
317def test_matches():
318    args = ["arg1=1", "--arg2=value2"]
319
320    assert matches(args, "arg1")
321    assert not matches(args, "arg3")
322
323
324def test_extract_browser_name():
325    args = ["arg1=1", "--arg2=value2", "--browser=me", "--zome"]
326    assert extract_browser_name(args) == "me"
327
328
329if __name__ == "__main__":
330    mozunit.main()
331