1# Copyright (C) 2012 Google Inc. All rights reserved.
2# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30import sys
31import unittest
32
33from blinkpy.common.host_mock import MockHost
34from blinkpy.common.system.system_host_mock import MockSystemHost
35from blinkpy.web_tests import run_web_tests
36from blinkpy.web_tests.controllers.web_test_runner import WebTestRunner, Worker, Sharder, TestRunInterruptedException
37from blinkpy.web_tests.models import test_expectations
38from blinkpy.web_tests.models import test_failures
39from blinkpy.web_tests.models.test_run_results import TestRunResults
40from blinkpy.web_tests.models.test_input import TestInput
41from blinkpy.web_tests.models.test_results import TestResult
42from blinkpy.web_tests.port.test import TestPort
43from blinkpy.web_tests.port.driver import DriverOutput
44
45
46TestExpectations = test_expectations.TestExpectations
47
48
49class FakePrinter(object):
50    num_completed = 0
51    num_tests = 0
52
53    def print_expected(self, run_results, get_tests_with_result_type):
54        pass
55
56    def print_workers_and_shards(self, port, num_workers, num_shards, num_locked_shards):
57        pass
58
59    def print_started_test(self, test_name):
60        pass
61
62    def print_finished_test(self, port, result, expected, exp_str, got_str):
63        pass
64
65    def write(self, msg):
66        pass
67
68    def write_update(self, msg):
69        pass
70
71    def flush(self):
72        pass
73
74
75class LockCheckingRunner(WebTestRunner):
76
77    def __init__(self, port, options, printer, tester, http_lock):
78        super(LockCheckingRunner, self).__init__(options, port, printer, port.results_directory(), lambda test_name: False)
79        self._finished_list_called = False
80        self._tester = tester
81        self._should_have_http_lock = http_lock
82
83
84# TODO(crbug.com/926841): Debug running this test on Swarming on Windows.
85# Ensure that all child processes are always cleaned up.
86@unittest.skipIf(sys.platform == 'win32', 'may not clean up child processes')
87class WebTestRunnerTests(unittest.TestCase):
88
89    def setUp(self):
90        self._actual_output = DriverOutput(
91            text='', image=None, image_hash=None, audio=None)
92        self._expected_output = DriverOutput(
93            text='', image=None, image_hash=None, audio=None)
94
95    # pylint: disable=protected-access
96    def _runner(self, port=None):
97        # FIXME: we shouldn't have to use run_web_tests.py to get the options we need.
98        options = run_web_tests.parse_args(['--platform', 'test-mac-mac10.11'])[0]
99        options.child_processes = '1'
100
101        host = MockHost()
102        port = port or host.port_factory.get(options.platform, options=options)
103        return LockCheckingRunner(port, options, FakePrinter(), self, True)
104
105    def _run_tests(self, runner, tests):
106        test_inputs = [TestInput(test, timeout_ms=6000) for test in tests]
107        expectations = TestExpectations(runner._port, tests)
108        runner.run_tests(expectations, test_inputs, set(), num_workers=1)
109
110    def test_interrupt_if_at_failure_limits(self):
111        runner = self._runner()
112        runner._options.exit_after_n_failures = None
113        runner._options.exit_after_n_crashes_or_times = None
114        test_names = ['passes/text.html', 'passes/image.html']
115        runner._test_inputs = [TestInput(test_name, timeout_ms=6000) for test_name in test_names]
116
117        run_results = TestRunResults(TestExpectations(runner._port), len(test_names))
118        run_results.unexpected_failures = 100
119        run_results.unexpected_crashes = 50
120        run_results.unexpected_timeouts = 50
121        # No exception when the exit_after* options are None.
122        runner._interrupt_if_at_failure_limits(run_results)
123
124        # No exception when we haven't hit the limit yet.
125        runner._options.exit_after_n_failures = 101
126        runner._options.exit_after_n_crashes_or_timeouts = 101
127        runner._interrupt_if_at_failure_limits(run_results)
128
129        # Interrupt if we've exceeded either limit:
130        runner._options.exit_after_n_crashes_or_timeouts = 10
131        with self.assertRaises(TestRunInterruptedException):
132            runner._interrupt_if_at_failure_limits(run_results)
133        self.assertEqual(run_results.results_by_name['passes/text.html'].type, 'SKIP')
134        self.assertEqual(run_results.results_by_name['passes/image.html'].type, 'SKIP')
135
136        runner._options.exit_after_n_crashes_or_timeouts = None
137        runner._options.exit_after_n_failures = 10
138        with self.assertRaises(TestRunInterruptedException):
139            runner._interrupt_if_at_failure_limits(run_results)
140
141    def test_update_summary_with_result(self):
142        runner = self._runner()
143        test = 'failures/expected/reftest.html'
144        expectations = TestExpectations(runner._port)
145        runner._expectations = expectations
146
147        run_results = TestRunResults(expectations, 1)
148        result = TestResult(
149            test_name=test, failures=[
150                test_failures.FailureReftestMismatchDidNotOccur(
151                    self._actual_output, self._expected_output)],
152            reftest_type=['!='])
153        runner._update_summary_with_result(run_results, result)
154        self.assertEqual(1, run_results.expected)
155        self.assertEqual(0, run_results.unexpected)
156
157        run_results = TestRunResults(expectations, 1)
158        result = TestResult(test_name=test, failures=[], reftest_type=['=='])
159        runner._update_summary_with_result(run_results, result)
160        self.assertEqual(0, run_results.expected)
161        self.assertEqual(1, run_results.unexpected)
162
163
164class SharderTests(unittest.TestCase):
165
166    test_list = [
167        "http/tests/websocket/tests/unicode.htm",
168        "animations/keyframes.html",
169        "http/tests/security/view-source-no-refresh.html",
170        "http/tests/websocket/tests/websocket-protocol-ignored.html",
171        "fast/css/display-none-inline-style-change-crash.html",
172        "http/tests/xmlhttprequest/supported-xml-content-types.html",
173        "dom/html/level2/html/HTMLAnchorElement03.html",
174        "dom/html/level2/html/HTMLAnchorElement06.html",
175        "perf/object-keys.html",
176        "virtual/threaded/dir/test.html",
177        "virtual/threaded/fast/foo/test.html",
178    ]
179
180    def get_test_input(self, test_file):
181        return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf')))
182
183    def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None, max_locked_shards=1):
184        port = TestPort(MockSystemHost())
185        self.sharder = Sharder(port.split_test, max_locked_shards)
186        test_list = test_list or self.test_list
187        return self.sharder.shard_tests([self.get_test_input(test) for test in test_list],
188                                        num_workers, fully_parallel, False, run_singly)
189
190    def assert_shards(self, actual_shards, expected_shard_names):
191        self.assertEqual(len(actual_shards), len(expected_shard_names))
192        for i, shard in enumerate(actual_shards):
193            expected_shard_name, expected_test_names = expected_shard_names[i]
194            self.assertEqual(shard.name, expected_shard_name)
195            self.assertEqual([test_input.test_name for test_input in shard.test_inputs],
196                             expected_test_names)
197
198    def test_shard_by_dir(self):
199        locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, run_singly=False)
200
201        # Note that although there are tests in multiple dirs that need locks,
202        # they are crammed into a single shard in order to reduce the # of
203        # workers hitting the server at once.
204        self.assert_shards(locked,
205                           [('locked_shard_1',
206                             ['http/tests/security/view-source-no-refresh.html',
207                              'http/tests/websocket/tests/unicode.htm',
208                              'http/tests/websocket/tests/websocket-protocol-ignored.html',
209                              'http/tests/xmlhttprequest/supported-xml-content-types.html',
210                              'perf/object-keys.html'])])
211        self.assert_shards(unlocked,
212                           [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
213                            ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']),
214                            ('animations', ['animations/keyframes.html']),
215                            ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
216                                                      'dom/html/level2/html/HTMLAnchorElement06.html']),
217                            ('fast/css', ['fast/css/display-none-inline-style-change-crash.html'])])
218
219    def test_shard_every_file(self):
220        locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False)
221        self.assert_shards(locked,
222                           [('locked_shard_1',
223                             ['http/tests/websocket/tests/unicode.htm',
224                              'http/tests/security/view-source-no-refresh.html',
225                              'http/tests/websocket/tests/websocket-protocol-ignored.html']),
226                            ('locked_shard_2',
227                             ['http/tests/xmlhttprequest/supported-xml-content-types.html',
228                              'perf/object-keys.html'])])
229        self.assert_shards(unlocked,
230                           [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
231                            ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']),
232                            ('.', ['animations/keyframes.html']),
233                            ('.', ['fast/css/display-none-inline-style-change-crash.html']),
234                            ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
235                            ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
236
237    def test_shard_in_two(self):
238        locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False)
239        self.assert_shards(locked,
240                           [('locked_tests',
241                             ['http/tests/websocket/tests/unicode.htm',
242                              'http/tests/security/view-source-no-refresh.html',
243                              'http/tests/websocket/tests/websocket-protocol-ignored.html',
244                              'http/tests/xmlhttprequest/supported-xml-content-types.html',
245                              'perf/object-keys.html'])])
246        self.assert_shards(unlocked,
247                           [('unlocked_tests',
248                             ['animations/keyframes.html',
249                              'fast/css/display-none-inline-style-change-crash.html',
250                              'dom/html/level2/html/HTMLAnchorElement03.html',
251                              'dom/html/level2/html/HTMLAnchorElement06.html',
252                              'virtual/threaded/dir/test.html',
253                              'virtual/threaded/fast/foo/test.html'])])
254
255    def test_shard_in_two_has_no_locked_shards(self):
256        locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False,
257                                           test_list=['animations/keyframe.html'])
258        self.assertEqual(len(locked), 0)
259        self.assertEqual(len(unlocked), 1)
260
261    def test_shard_in_two_has_no_unlocked_shards(self):
262        locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False,
263                                           test_list=['http/tests/websocket/tests/unicode.htm'])
264        self.assertEqual(len(locked), 1)
265        self.assertEqual(len(unlocked), 0)
266
267    def test_multiple_locked_shards(self):
268        locked, _ = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2, run_singly=False)
269        self.assert_shards(locked,
270                           [('locked_shard_1',
271                             ['http/tests/security/view-source-no-refresh.html',
272                              'http/tests/websocket/tests/unicode.htm',
273                              'http/tests/websocket/tests/websocket-protocol-ignored.html']),
274                            ('locked_shard_2',
275                             ['http/tests/xmlhttprequest/supported-xml-content-types.html',
276                              'perf/object-keys.html'])])
277
278        locked, _ = self.get_shards(num_workers=4, fully_parallel=False, run_singly=False)
279        self.assert_shards(locked,
280                           [('locked_shard_1',
281                             ['http/tests/security/view-source-no-refresh.html',
282                              'http/tests/websocket/tests/unicode.htm',
283                              'http/tests/websocket/tests/websocket-protocol-ignored.html',
284                              'http/tests/xmlhttprequest/supported-xml-content-types.html',
285                              'perf/object-keys.html'])])
286
287    def test_virtual_shards(self):
288        # With run_singly=False, we try to keep all of the tests in a virtual suite together even
289        # when fully_parallel=True, so that we don't restart every time the command line args change.
290        _, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False,
291                                      test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
292        self.assert_shards(unlocked,
293                           [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])])
294
295        # But, with run_singly=True, we have to restart every time anyway, so we want full parallelism.
296        _, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=True,
297                                      test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
298        self.assert_shards(unlocked,
299                           [('.', ['virtual/foo/bar1.html']),
300                            ('.', ['virtual/foo/bar2.html'])])
301
302
303class WorkerTests(unittest.TestCase):
304
305    class DummyCaller(object):
306        worker_number = 1
307        name = 'dummy_caller'
308
309    def test_worker_no_manifest_update(self):
310        # pylint: disable=protected-access
311        options = run_web_tests.parse_args(['--platform', 'test-mac-mac10.11'])[0]
312        worker = Worker(self.DummyCaller(), '/results', options)
313        self.assertTrue(options.manifest_update)
314        self.assertFalse(worker._options.manifest_update)
315