1from __future__ import absolute_import, unicode_literals
2
3import os
4import sys
5import threading
6import time
7import traceback
8from unittest import mock
9import pytest
10from unittest.mock import Mock
11from six import reraise
12
13import mozunit
14from mozprofile import BaseProfile
15from mozrunner.errors import RunnerNotStartedError
16
17# need this so the raptor unit tests can find output & filter classes
18here = os.path.abspath(os.path.dirname(__file__))
19raptor_dir = os.path.join(os.path.dirname(here), "raptor")
20sys.path.insert(0, raptor_dir)
21
22
23from browsertime import BrowsertimeDesktop, BrowsertimeAndroid
24from webextension import (
25    WebExtensionFirefox,
26    WebExtensionDesktopChrome,
27    WebExtensionAndroid,
28)
29
30
31DEFAULT_TIMEOUT = 125
32
33
34class TestBrowserThread(threading.Thread):
35    def __init__(self, raptor_instance, tests, names):
36        super(TestBrowserThread, self).__init__()
37        self.raptor_instance = raptor_instance
38        self.tests = tests
39        self.names = names
40        self.exc = None
41
42    def print_error(self):
43        if self.exc is None:
44            return
45        type, value, tb = self.exc
46        traceback.print_exception(type, value, tb, None, sys.stderr)
47
48    def run(self):
49        try:
50            self.raptor_instance.run_tests(self.tests, self.names)
51        except BaseException:
52            self.exc = sys.exc_info()
53
54
55# Perftest tests
56@pytest.mark.parametrize(
57    "perftest_class, app_name",
58    [
59        [WebExtensionFirefox, "firefox"],
60        [WebExtensionDesktopChrome, "chrome"],
61        [WebExtensionDesktopChrome, "chromium"],
62        [WebExtensionAndroid, "geckoview"],
63        [BrowsertimeDesktop, "firefox"],
64        [BrowsertimeDesktop, "chrome"],
65        [BrowsertimeDesktop, "chromium"],
66        [BrowsertimeAndroid, "geckoview"],
67    ],
68)
69def test_build_profile(options, perftest_class, app_name, get_prefs):
70    options["app"] = app_name
71    perftest_instance = perftest_class(**options)
72    assert isinstance(perftest_instance.profile, BaseProfile)
73    if app_name != "firefox":
74        return
75
76    # These prefs are set in mozprofile
77    firefox_prefs = [
78        'user_pref("app.update.checkInstallTime", false);',
79        'user_pref("app.update.disabledForTesting", true);',
80        'user_pref("'
81        'security.turn_off_all_security_so_that_viruses_can_take_over_this_computer", true);',
82    ]
83    # This pref is set in raptor
84    raptor_pref = 'user_pref("security.enable_java", false);'
85
86    prefs_file = os.path.join(perftest_instance.profile.profile, "user.js")
87    with open(prefs_file, "r") as fh:
88        prefs = fh.read()
89        for firefox_pref in firefox_prefs:
90            assert firefox_pref in prefs
91        assert raptor_pref in prefs
92
93
94def test_perftest_host_ip(ConcretePerftest, options, get_prefs):
95    os.environ["HOST_IP"] = "some_dummy_host_ip"
96    options["host"] = "HOST_IP"
97
98    perftest = ConcretePerftest(**options)
99
100    assert perftest.config["host"] == os.environ["HOST_IP"]
101
102
103@pytest.mark.parametrize(
104    "app_name, expected_e10s_flag",
105    [["firefox", True], ["geckoview", True]],
106)
107def test_e10s_enabling(ConcretePerftest, options, app_name, expected_e10s_flag):
108    options["app"] = app_name
109    perftest = ConcretePerftest(profile_class="firefox", **options)
110    assert perftest.config["e10s"] == expected_e10s_flag
111
112
113def test_profile_was_provided_locally(ConcretePerftest, options):
114    perftest = ConcretePerftest(**options)
115    assert os.path.isdir(perftest.config["local_profile_dir"])
116
117
118@pytest.mark.parametrize(
119    "profile_class, app, expected_profile",
120    [
121        ["firefox", "firefox", "firefox"],
122        [None, "firefox", "firefox"],
123        ["firefox", None, "firefox"],
124    ],
125)
126def test_profile_class_assignation(
127    ConcretePerftest, options, profile_class, app, expected_profile
128):
129    options["app"] = app
130    perftest = ConcretePerftest(profile_class=profile_class, **options)
131    assert perftest.profile_class == expected_profile
132
133
134def test_raptor_venv(ConcretePerftest, options):
135    perftest = ConcretePerftest(**options)
136    assert perftest.raptor_venv.endswith("raptor-venv")
137
138
139@pytest.mark.parametrize(
140    "run_local,"
141    "debug_mode,"
142    "post_startup_delay,"
143    "expected_post_startup_delay,"
144    "expected_debug_mode",
145    [
146        [True, True, 1234, 1234, True],
147        [True, True, 12345, 3000, True],
148        [False, False, 1234, 1234, False],
149        [False, False, 12345, 12345, False],
150        [True, False, 1234, 1234, False],
151        [True, False, 12345, 12345, False],
152        [False, True, 1234, 1234, False],
153        [False, True, 12345, 12345, False],
154    ],
155)
156def test_post_startup_delay(
157    ConcretePerftest,
158    options,
159    run_local,
160    debug_mode,
161    post_startup_delay,
162    expected_post_startup_delay,
163    expected_debug_mode,
164):
165    perftest = ConcretePerftest(
166        run_local=run_local,
167        debug_mode=debug_mode,
168        post_startup_delay=post_startup_delay,
169        **options
170    )
171    assert perftest.post_startup_delay == expected_post_startup_delay
172    assert perftest.debug_mode == expected_debug_mode
173
174
175@pytest.mark.parametrize(
176    "alert, expected_alert", [["test_to_alert_on", "test_to_alert_on"], [None, None]]
177)
178def test_perftest_run_test_setup(
179    ConcretePerftest, options, mock_test, alert, expected_alert
180):
181    perftest = ConcretePerftest(**options)
182    mock_test["alert_on"] = alert
183
184    perftest.run_test_setup(mock_test)
185
186    assert perftest.config["subtest_alert_on"] == expected_alert
187
188
189# WebExtension tests
190@pytest.mark.parametrize(
191    "app", ["firefox", pytest.mark.xfail("chrome"), pytest.mark.xfail("chromium")]
192)
193def test_start_browser(get_binary, app):
194    binary = get_binary(app)
195    assert binary
196
197    raptor = WebExtensionFirefox(app, binary, post_startup_delay=0)
198
199    tests = [{"name": "raptor-{}-tp6".format(app), "page_timeout": 1000}]
200    test_names = [test["name"] for test in tests]
201
202    thread = TestBrowserThread(raptor, tests, test_names)
203    thread.start()
204
205    timeout = time.time() + 5  # seconds
206    while time.time() < timeout:
207        try:
208            is_running = raptor.runner.is_running()
209            assert is_running
210            break
211        except RunnerNotStartedError:
212            time.sleep(0.1)
213    else:
214        # browser didn't start
215        # if the thread had an error, display it here
216        thread.print_error()
217        assert False
218
219    raptor.clean_up()
220    thread.join(5)
221
222    if thread.exc is not None:
223        exc, value, tb = thread.exc
224        reraise(exc, value, tb)
225
226    assert not raptor.runner.is_running()
227    assert raptor.runner.returncode is not None
228
229
230# Browsertime tests
231def test_cmd_arguments(ConcreteBrowsertime, browsertime_options, mock_test):
232    expected_cmd = {
233        browsertime_options["browsertime_node"],
234        browsertime_options["browsertime_browsertimejs"],
235        "--firefox.geckodriverPath",
236        browsertime_options["browsertime_geckodriver"],
237        "--browsertime.page_cycles",
238        "1",
239        "--browsertime.url",
240        mock_test["test_url"],
241        "--browsertime.page_cycle_delay",
242        "1000",
243        "--browsertime.post_startup_delay",
244        str(DEFAULT_TIMEOUT),
245        "--firefox.profileTemplate",
246        "--skipHar",
247        "--video",
248        "true",
249        "--visualMetrics",
250        "false",
251        "--timeouts.pageLoad",
252        str(DEFAULT_TIMEOUT),
253        "--timeouts.script",
254        str(DEFAULT_TIMEOUT),
255        "--resultDir",
256        "-n",
257        "1",
258    }
259    if browsertime_options.get("app") in ["chrome", "chrome-m"]:
260        expected_cmd.add(
261            "--chrome.chromedriverPath", browsertime_options["browsertime_chromedriver"]
262        )
263
264    browsertime = ConcreteBrowsertime(
265        post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
266    )
267    browsertime.run_test_setup(mock_test)
268    cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT)
269
270    assert expected_cmd.issubset(set(cmd))
271
272
273def extract_arg_value(cmd, arg):
274    param_index = cmd.index(arg) + 1
275    return cmd[param_index]
276
277
278@pytest.mark.parametrize(
279    "arg_to_test, expected, test_patch, options_patch",
280    [
281        ["-n", "1", {}, {"browser_cycles": None}],
282        ["-n", "123", {"browser_cycles": 123}, {}],
283        ["--video", "false", {}, {"browsertime_video": None}],
284        ["--video", "true", {}, {"browsertime_video": "dummy_value"}],
285        ["--timeouts.script", str(DEFAULT_TIMEOUT), {}, {"page_cycles": None}],
286        ["--timeouts.script", str(123 * DEFAULT_TIMEOUT), {"page_cycles": 123}, {}],
287        ["--browsertime.page_cycles", "1", {}, {"page_cycles": None}],
288        ["--browsertime.page_cycles", "123", {"page_cycles": 123}, {}],
289    ],
290)
291def test_browsertime_arguments(
292    ConcreteBrowsertime,
293    browsertime_options,
294    mock_test,
295    arg_to_test,
296    expected,
297    test_patch,
298    options_patch,
299):
300    mock_test.update(test_patch)
301    browsertime_options.update(options_patch)
302    browsertime = ConcreteBrowsertime(
303        post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
304    )
305    browsertime.run_test_setup(mock_test)
306    cmd = browsertime._compose_cmd(mock_test, DEFAULT_TIMEOUT)
307
308    param_value = extract_arg_value(cmd, arg_to_test)
309    assert expected == param_value
310
311
312@pytest.mark.parametrize(
313    "timeout, expected_timeout, test_patch, options_patch",
314    [
315        [0, 20, {}, {}],
316        [0, 20, {}, {"gecko_profile": False}],
317        [1000, 321, {}, {"gecko_profile": True}],
318    ],
319)
320def test_compute_process_timeout(
321    ConcreteBrowsertime,
322    browsertime_options,
323    mock_test,
324    timeout,
325    expected_timeout,
326    test_patch,
327    options_patch,
328):
329    mock_test.update(test_patch)
330    browsertime_options.update(options_patch)
331    browsertime = ConcreteBrowsertime(
332        post_startup_delay=DEFAULT_TIMEOUT, **browsertime_options
333    )
334    bt_timeout = browsertime._compute_process_timeout(mock_test, timeout)
335    assert bt_timeout == expected_timeout
336
337
338@pytest.mark.parametrize(
339    "host, playback, benchmark",
340    [["127.0.0.1", True, False], ["localhost", False, True]],
341)
342def test_android_reverse_ports(host, playback, benchmark):
343    raptor = WebExtensionAndroid(
344        "geckoview", "org.mozilla.geckoview_example", host=host
345    )
346    if benchmark:
347        benchmark_mock = mock.patch("raptor.raptor.benchmark.Benchmark")
348        raptor.benchmark = benchmark_mock
349        raptor.benchmark.port = 1234
350
351    if playback:
352        playback_mock = mock.patch(
353            "mozbase.mozproxy.mozproxy.backends.mitm.mitm.MitmproxyAndroid"
354        )
355        playback_mock.port = 4321
356        raptor.playback = playback_mock
357
358    raptor.set_reverse_port = Mock()
359    raptor.set_reverse_ports()
360
361    raptor.set_reverse_port.assert_any_call(raptor.control_server.port)
362    if benchmark:
363        raptor.set_reverse_port.assert_any_call(1234)
364
365    if playback:
366        raptor.set_reverse_port.assert_any_call(4321)
367
368
369def test_android_reverse_ports_non_local_host():
370    raptor = WebExtensionAndroid(
371        "geckoview", "org.mozilla.geckoview_example", host="192.168.100.10"
372    )
373
374    raptor.set_reverse_port = Mock()
375    raptor.set_reverse_ports()
376
377    raptor.set_reverse_port.assert_not_called()
378
379
380if __name__ == "__main__":
381    mozunit.main()
382