1# Copyright (C) 2012 Google Inc. All rights reserved. 2# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30import sys 31import unittest 32 33from blinkpy.common.host_mock import MockHost 34from blinkpy.common.system.system_host_mock import MockSystemHost 35from blinkpy.web_tests import run_web_tests 36from blinkpy.web_tests.controllers.web_test_runner import WebTestRunner, Worker, Sharder, TestRunInterruptedException 37from blinkpy.web_tests.models import test_expectations 38from blinkpy.web_tests.models import test_failures 39from blinkpy.web_tests.models.test_run_results import TestRunResults 40from blinkpy.web_tests.models.test_input import TestInput 41from blinkpy.web_tests.models.test_results import TestResult 42from blinkpy.web_tests.port.test import TestPort 43from blinkpy.web_tests.port.driver import DriverOutput 44 45 46TestExpectations = test_expectations.TestExpectations 47 48 49class FakePrinter(object): 50 num_completed = 0 51 num_tests = 0 52 53 def print_expected(self, run_results, get_tests_with_result_type): 54 pass 55 56 def print_workers_and_shards(self, port, num_workers, num_shards, num_locked_shards): 57 pass 58 59 def print_started_test(self, test_name): 60 pass 61 62 def print_finished_test(self, port, result, expected, exp_str, got_str): 63 pass 64 65 def write(self, msg): 66 pass 67 68 def write_update(self, msg): 69 pass 70 71 def flush(self): 72 pass 73 74 75class LockCheckingRunner(WebTestRunner): 76 77 def __init__(self, port, options, printer, tester, http_lock): 78 super(LockCheckingRunner, self).__init__(options, port, printer, port.results_directory(), lambda test_name: False) 79 self._finished_list_called = False 80 self._tester = tester 81 self._should_have_http_lock = http_lock 82 83 84# TODO(crbug.com/926841): Debug running this test on Swarming on Windows. 85# Ensure that all child processes are always cleaned up. 86@unittest.skipIf(sys.platform == 'win32', 'may not clean up child processes') 87class WebTestRunnerTests(unittest.TestCase): 88 89 def setUp(self): 90 self._actual_output = DriverOutput( 91 text='', image=None, image_hash=None, audio=None) 92 self._expected_output = DriverOutput( 93 text='', image=None, image_hash=None, audio=None) 94 95 # pylint: disable=protected-access 96 def _runner(self, port=None): 97 # FIXME: we shouldn't have to use run_web_tests.py to get the options we need. 98 options = run_web_tests.parse_args(['--platform', 'test-mac-mac10.11'])[0] 99 options.child_processes = '1' 100 101 host = MockHost() 102 port = port or host.port_factory.get(options.platform, options=options) 103 return LockCheckingRunner(port, options, FakePrinter(), self, True) 104 105 def _run_tests(self, runner, tests): 106 test_inputs = [TestInput(test, timeout_ms=6000) for test in tests] 107 expectations = TestExpectations(runner._port, tests) 108 runner.run_tests(expectations, test_inputs, set(), num_workers=1) 109 110 def test_interrupt_if_at_failure_limits(self): 111 runner = self._runner() 112 runner._options.exit_after_n_failures = None 113 runner._options.exit_after_n_crashes_or_times = None 114 test_names = ['passes/text.html', 'passes/image.html'] 115 runner._test_inputs = [TestInput(test_name, timeout_ms=6000) for test_name in test_names] 116 117 run_results = TestRunResults(TestExpectations(runner._port), len(test_names)) 118 run_results.unexpected_failures = 100 119 run_results.unexpected_crashes = 50 120 run_results.unexpected_timeouts = 50 121 # No exception when the exit_after* options are None. 122 runner._interrupt_if_at_failure_limits(run_results) 123 124 # No exception when we haven't hit the limit yet. 125 runner._options.exit_after_n_failures = 101 126 runner._options.exit_after_n_crashes_or_timeouts = 101 127 runner._interrupt_if_at_failure_limits(run_results) 128 129 # Interrupt if we've exceeded either limit: 130 runner._options.exit_after_n_crashes_or_timeouts = 10 131 with self.assertRaises(TestRunInterruptedException): 132 runner._interrupt_if_at_failure_limits(run_results) 133 self.assertEqual(run_results.results_by_name['passes/text.html'].type, 'SKIP') 134 self.assertEqual(run_results.results_by_name['passes/image.html'].type, 'SKIP') 135 136 runner._options.exit_after_n_crashes_or_timeouts = None 137 runner._options.exit_after_n_failures = 10 138 with self.assertRaises(TestRunInterruptedException): 139 runner._interrupt_if_at_failure_limits(run_results) 140 141 def test_update_summary_with_result(self): 142 runner = self._runner() 143 test = 'failures/expected/reftest.html' 144 expectations = TestExpectations(runner._port) 145 runner._expectations = expectations 146 147 run_results = TestRunResults(expectations, 1) 148 result = TestResult( 149 test_name=test, failures=[ 150 test_failures.FailureReftestMismatchDidNotOccur( 151 self._actual_output, self._expected_output)], 152 reftest_type=['!=']) 153 runner._update_summary_with_result(run_results, result) 154 self.assertEqual(1, run_results.expected) 155 self.assertEqual(0, run_results.unexpected) 156 157 run_results = TestRunResults(expectations, 1) 158 result = TestResult(test_name=test, failures=[], reftest_type=['==']) 159 runner._update_summary_with_result(run_results, result) 160 self.assertEqual(0, run_results.expected) 161 self.assertEqual(1, run_results.unexpected) 162 163 164class SharderTests(unittest.TestCase): 165 166 test_list = [ 167 "http/tests/websocket/tests/unicode.htm", 168 "animations/keyframes.html", 169 "http/tests/security/view-source-no-refresh.html", 170 "http/tests/websocket/tests/websocket-protocol-ignored.html", 171 "fast/css/display-none-inline-style-change-crash.html", 172 "http/tests/xmlhttprequest/supported-xml-content-types.html", 173 "dom/html/level2/html/HTMLAnchorElement03.html", 174 "dom/html/level2/html/HTMLAnchorElement06.html", 175 "perf/object-keys.html", 176 "virtual/threaded/dir/test.html", 177 "virtual/threaded/fast/foo/test.html", 178 ] 179 180 def get_test_input(self, test_file): 181 return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf'))) 182 183 def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None, max_locked_shards=1): 184 port = TestPort(MockSystemHost()) 185 self.sharder = Sharder(port.split_test, max_locked_shards) 186 test_list = test_list or self.test_list 187 return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], 188 num_workers, fully_parallel, False, run_singly) 189 190 def assert_shards(self, actual_shards, expected_shard_names): 191 self.assertEqual(len(actual_shards), len(expected_shard_names)) 192 for i, shard in enumerate(actual_shards): 193 expected_shard_name, expected_test_names = expected_shard_names[i] 194 self.assertEqual(shard.name, expected_shard_name) 195 self.assertEqual([test_input.test_name for test_input in shard.test_inputs], 196 expected_test_names) 197 198 def test_shard_by_dir(self): 199 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, run_singly=False) 200 201 # Note that although there are tests in multiple dirs that need locks, 202 # they are crammed into a single shard in order to reduce the # of 203 # workers hitting the server at once. 204 self.assert_shards(locked, 205 [('locked_shard_1', 206 ['http/tests/security/view-source-no-refresh.html', 207 'http/tests/websocket/tests/unicode.htm', 208 'http/tests/websocket/tests/websocket-protocol-ignored.html', 209 'http/tests/xmlhttprequest/supported-xml-content-types.html', 210 'perf/object-keys.html'])]) 211 self.assert_shards(unlocked, 212 [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), 213 ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']), 214 ('animations', ['animations/keyframes.html']), 215 ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html', 216 'dom/html/level2/html/HTMLAnchorElement06.html']), 217 ('fast/css', ['fast/css/display-none-inline-style-change-crash.html'])]) 218 219 def test_shard_every_file(self): 220 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False) 221 self.assert_shards(locked, 222 [('locked_shard_1', 223 ['http/tests/websocket/tests/unicode.htm', 224 'http/tests/security/view-source-no-refresh.html', 225 'http/tests/websocket/tests/websocket-protocol-ignored.html']), 226 ('locked_shard_2', 227 ['http/tests/xmlhttprequest/supported-xml-content-types.html', 228 'perf/object-keys.html'])]) 229 self.assert_shards(unlocked, 230 [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), 231 ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']), 232 ('.', ['animations/keyframes.html']), 233 ('.', ['fast/css/display-none-inline-style-change-crash.html']), 234 ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), 235 ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])]) 236 237 def test_shard_in_two(self): 238 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False) 239 self.assert_shards(locked, 240 [('locked_tests', 241 ['http/tests/websocket/tests/unicode.htm', 242 'http/tests/security/view-source-no-refresh.html', 243 'http/tests/websocket/tests/websocket-protocol-ignored.html', 244 'http/tests/xmlhttprequest/supported-xml-content-types.html', 245 'perf/object-keys.html'])]) 246 self.assert_shards(unlocked, 247 [('unlocked_tests', 248 ['animations/keyframes.html', 249 'fast/css/display-none-inline-style-change-crash.html', 250 'dom/html/level2/html/HTMLAnchorElement03.html', 251 'dom/html/level2/html/HTMLAnchorElement06.html', 252 'virtual/threaded/dir/test.html', 253 'virtual/threaded/fast/foo/test.html'])]) 254 255 def test_shard_in_two_has_no_locked_shards(self): 256 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False, 257 test_list=['animations/keyframe.html']) 258 self.assertEqual(len(locked), 0) 259 self.assertEqual(len(unlocked), 1) 260 261 def test_shard_in_two_has_no_unlocked_shards(self): 262 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False, 263 test_list=['http/tests/websocket/tests/unicode.htm']) 264 self.assertEqual(len(locked), 1) 265 self.assertEqual(len(unlocked), 0) 266 267 def test_multiple_locked_shards(self): 268 locked, _ = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2, run_singly=False) 269 self.assert_shards(locked, 270 [('locked_shard_1', 271 ['http/tests/security/view-source-no-refresh.html', 272 'http/tests/websocket/tests/unicode.htm', 273 'http/tests/websocket/tests/websocket-protocol-ignored.html']), 274 ('locked_shard_2', 275 ['http/tests/xmlhttprequest/supported-xml-content-types.html', 276 'perf/object-keys.html'])]) 277 278 locked, _ = self.get_shards(num_workers=4, fully_parallel=False, run_singly=False) 279 self.assert_shards(locked, 280 [('locked_shard_1', 281 ['http/tests/security/view-source-no-refresh.html', 282 'http/tests/websocket/tests/unicode.htm', 283 'http/tests/websocket/tests/websocket-protocol-ignored.html', 284 'http/tests/xmlhttprequest/supported-xml-content-types.html', 285 'perf/object-keys.html'])]) 286 287 def test_virtual_shards(self): 288 # With run_singly=False, we try to keep all of the tests in a virtual suite together even 289 # when fully_parallel=True, so that we don't restart every time the command line args change. 290 _, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False, 291 test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) 292 self.assert_shards(unlocked, 293 [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])]) 294 295 # But, with run_singly=True, we have to restart every time anyway, so we want full parallelism. 296 _, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=True, 297 test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) 298 self.assert_shards(unlocked, 299 [('.', ['virtual/foo/bar1.html']), 300 ('.', ['virtual/foo/bar2.html'])]) 301 302 303class WorkerTests(unittest.TestCase): 304 305 class DummyCaller(object): 306 worker_number = 1 307 name = 'dummy_caller' 308 309 def test_worker_no_manifest_update(self): 310 # pylint: disable=protected-access 311 options = run_web_tests.parse_args(['--platform', 'test-mac-mac10.11'])[0] 312 worker = Worker(self.DummyCaller(), '/results', options) 313 self.assertTrue(options.manifest_update) 314 self.assertFalse(worker._options.manifest_update) 315