1# Copyright (C) 2010 Google Inc. All rights reserved. 2# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged 3# Copyright (C) 2011 Apple Inc. All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: 8# 9# * Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above 12# copyright notice, this list of conditions and the following disclaimer 13# in the documentation and/or other materials provided with the 14# distribution. 15# * Neither the name of Google Inc. nor the names of its 16# contributors may be used to endorse or promote products derived from 17# this software without specific prior written permission. 18# 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31import logging 32import optparse 33import sys 34import traceback 35 36from blinkpy.common import exit_codes 37from blinkpy.common.host import Host 38from blinkpy.web_tests.controllers.manager import Manager 39from blinkpy.web_tests.models import test_run_results 40from blinkpy.web_tests.port.factory import configuration_options 41from blinkpy.web_tests.port.factory import platform_options 42from blinkpy.web_tests.port.factory import wpt_options 43from blinkpy.web_tests.port.factory import python_server_options 44from blinkpy.web_tests.views import printing 45 46_log = logging.getLogger(__name__) 47 48 49def main(argv, stderr): 50 options, args = parse_args(argv) 51 52 if options.platform and 'test' in options.platform and not 'browser_test' in options.platform: 53 # It's a bit lame to import mocks into real code, but this allows the user 54 # to run tests against the test platform interactively, which is useful for 55 # debugging test failures. 56 from blinkpy.common.host_mock import MockHost 57 host = MockHost() 58 else: 59 host = Host() 60 61 printer = printing.Printer(host, options, stderr) 62 63 try: 64 port = host.port_factory.get(options.platform, options) 65 except (NotImplementedError, ValueError) as error: 66 _log.error(error) 67 printer.cleanup() 68 return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS 69 70 try: 71 return run(port, options, args, printer).exit_code 72 73 # We need to still handle KeyboardInterrupt, at least for blinkpy unittest cases. 74 except KeyboardInterrupt: 75 return exit_codes.INTERRUPTED_EXIT_STATUS 76 except test_run_results.TestRunException as error: 77 _log.error(error.msg) 78 return error.code 79 except BaseException as error: 80 if isinstance(error, Exception): 81 _log.error('\n%s raised: %s', error.__class__.__name__, error) 82 traceback.print_exc(file=stderr) 83 return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS 84 finally: 85 printer.cleanup() 86 87 88def deprecate(option, opt_str, _, parser): 89 """ 90 Prints a error message for a deprecated option. 91 Usage: 92 optparse.make_option( 93 '--some-option', 94 action='callback', 95 callback=deprecate, 96 help='....') 97 """ 98 parser.error('%s: %s' % (opt_str, option.help)) 99 100 101def parse_args(args): 102 option_group_definitions = [] 103 104 option_group_definitions.append( 105 ('Platform options', platform_options())) 106 107 option_group_definitions.append( 108 ('Configuration options', configuration_options())) 109 110 option_group_definitions.append( 111 ('Printing Options', printing.print_options())) 112 113 option_group_definitions.append( 114 ('web-platform-tests (WPT) Options', wpt_options())) 115 116 option_group_definitions.append(('Python Server Options', python_server_options())) 117 118 option_group_definitions.append( 119 ('Android-specific Options', [ 120 optparse.make_option( 121 '--adb-device', 122 action='append', 123 default=[], 124 dest='adb_devices', 125 help='Run Android web tests on these devices'), 126 # FIXME: Flip this to be off by default once we can log the 127 # device setup more cleanly. 128 optparse.make_option( 129 '--no-android-logging', 130 dest='android_logging', 131 action='store_false', 132 default=True, 133 help=('Do not log android-specific debug messages (default ' 134 'is to log as part of --debug-rwt-logging)')), 135 ])) 136 137 option_group_definitions.append( 138 ('Fuchsia-specific Options', [ 139 optparse.make_option( 140 '--zircon-logging', 141 dest='zircon_logging', 142 action='store_true', 143 default=True, 144 help=('Log Zircon debug messages (enabled by default).')), 145 optparse.make_option( 146 '--no-zircon-logging', 147 dest='zircon_logging', 148 action='store_false', 149 default=True, 150 help=('Do not log Zircon debug messages.')), 151 optparse.make_option( 152 '--device', 153 choices=['aemu','qemu'], 154 default='qemu', 155 help=('Choose device to launch Fuchsia with.')), 156 ])) 157 158 option_group_definitions.append( 159 ('Results Options', [ 160 optparse.make_option( 161 '--additional-driver-flag', 162 '--additional-drt-flag', 163 dest='additional_driver_flag', 164 action='append', 165 default=[], 166 help=('Additional command line flag to pass to the driver. Specify multiple ' 167 'times to add multiple flags.')), 168 optparse.make_option( 169 '--flag-specific', 170 dest='flag_specific', 171 action='store', 172 default=None, 173 help=('Name of a flag-specific configuration defined in FlagSpecificConfig, ' 174 ' as a shortcut of --additional-driver-flag options.')), 175 optparse.make_option( 176 '--additional-expectations', 177 action='append', 178 default=[], 179 help=('Path to a test_expectations file that will override previous ' 180 'expectations. Specify multiple times for multiple sets of overrides.')), 181 optparse.make_option( 182 '--ignore-default-expectations', 183 action='store_true', 184 help=('Do not use the default set of TestExpectations files.')), 185 optparse.make_option( 186 '--no-expectations', 187 action='store_true', 188 help=('Do not use TestExpectations, only run the tests without ' 189 'reporting any results. Useful for generating code ' 190 'coverage reports.')), 191 optparse.make_option( 192 '--additional-platform-directory', 193 action='append', 194 default=[], 195 help=('Additional directory where to look for test baselines (will take ' 196 'precedence over platform baselines). Specify multiple times to add ' 197 'multiple search path entries.')), 198 optparse.make_option( 199 '--build-directory', 200 default='out', 201 help=('Path to the directory where build files are kept, not including ' 202 'configuration. In general this will be "out".')), 203 optparse.make_option( 204 '--clobber-old-results', 205 action='store_true', 206 default=False, 207 help='Clobbers test results from previous runs.'), 208 optparse.make_option( 209 '--compare-port', 210 action='store', 211 default=None, 212 help="Use the specified port's baselines first"), 213 optparse.make_option( 214 '--copy-baselines', 215 action='store_true', 216 default=False, 217 help=('If the actual result is different from the current baseline, ' 218 'copy the current baseline into the *most-specific-platform* ' 219 'directory, or the flag-specific generic-platform directory if ' 220 '--additional-driver-flag is specified. See --reset-results.')), 221 optparse.make_option( 222 '--driver-name', 223 type='string', 224 help='Alternative driver binary to use'), 225 optparse.make_option( 226 '--json-test-results', # New name from json_results_generator 227 '--write-full-results-to', # Old argument name 228 '--isolated-script-test-output', # Isolated API 229 help='Path to write the JSON test results for *all* tests.'), 230 # FIXME(tansell): Remove this option if nobody is found who needs it. 231 optparse.make_option( 232 '--json-failing-test-results', 233 help='Path to write the JSON test results for only *failing* tests.'), 234 optparse.make_option( 235 '--no-show-results', 236 dest='show_results', 237 action='store_false', 238 default=True, 239 help="Don't launch a browser with results after the tests are done"), 240 optparse.make_option( 241 '--reset-results', 242 action='store_true', 243 default=False, 244 help=('Reset baselines to the generated results in their existing location or the default ' 245 'location if no baseline exists. For virtual tests, reset the virtual baselines. ' 246 'If --additional-driver-flag is specified, reset the flag-specific baselines. ' 247 'If --copy-baselines is specified, the copied baselines will be reset.')), 248 optparse.make_option( 249 '--results-directory', 250 help='Location of test results'), 251 optparse.make_option( 252 '--smoke', 253 action='store_true', 254 help='Run just the SmokeTests'), 255 optparse.make_option( 256 '--no-smoke', 257 dest='smoke', 258 action='store_false', 259 help='Do not run just the SmokeTests'), 260 ])) 261 262 option_group_definitions.append( 263 ('Testing Options', [ 264 optparse.make_option( 265 '--additional-env-var', 266 type='string', 267 action='append', 268 default=[], 269 help=('Passes that environment variable to the tests ' 270 '(--additional-env-var=NAME=VALUE)')), 271 optparse.make_option( 272 '--build', 273 dest='build', 274 action='store_true', 275 default=True, 276 help=('Check to ensure the build is up to date (default).')), 277 optparse.make_option( 278 '--no-build', 279 dest='build', 280 action='store_false', 281 help="Don't check to see if the build is up to date."), 282 optparse.make_option( 283 '--child-processes', '--jobs', '-j', 284 help='Number of drivers to run in parallel.'), 285 optparse.make_option( 286 '--disable-breakpad', 287 action='store_true', 288 help="Don't use breakpad to symbolize unexpected crashes."), 289 optparse.make_option( 290 '--driver-logging', 291 action='store_true', 292 help='Print detailed logging of the driver/content_shell'), 293 optparse.make_option( 294 '--enable-leak-detection', 295 action='store_true', 296 help='Enable the leak detection of DOM objects.'), 297 optparse.make_option( 298 '--enable-sanitizer', 299 action='store_true', 300 help='Only alert on sanitizer-related errors and crashes'), 301 optparse.make_option( 302 '--exit-after-n-crashes-or-timeouts', 303 type='int', 304 default=None, 305 help='Exit after the first N crashes instead of running all tests'), 306 optparse.make_option( 307 '--exit-after-n-failures', 308 type='int', 309 default=None, 310 help='Exit after the first N failures instead of running all tests'), 311 optparse.make_option( 312 '--fuzzy-diff', 313 action='store_true', 314 default=False, 315 help=('When running tests on an actual GPU, variance in pixel ' 316 'output can leads image differences causing failed expectations. ' 317 'Instead a fuzzy diff is used to account for this variance. ' 318 'See tools/imagediff/image_diff.cc')), 319 optparse.make_option( 320 '--ignore-builder-category', 321 action='store', 322 help=('The category of builders to use with the --ignore-flaky-tests option ' 323 "('layout' or 'deps').")), 324 optparse.make_option( 325 '--ignore-flaky-tests', 326 action='store', 327 help=('Control whether tests that are flaky on the bots get ignored. ' 328 "'very-flaky' == Ignore any tests that flaked more than once on the bot. " 329 "'maybe-flaky' == Ignore any tests that flaked once on the bot. " 330 "'unexpected' == Ignore any tests that had unexpected results on the bot.")), 331 optparse.make_option( 332 '--iterations', 333 '--isolated-script-test-repeat', 334 # TODO(crbug.com/893235): Remove the gtest alias when FindIt no longer uses it. 335 '--gtest_repeat', 336 type='int', 337 default=1, 338 help='Number of times to run the set of tests (e.g. ABCABCABC)'), 339 optparse.make_option( 340 '--layout-tests-directory', 341 help=('Path to a custom web tests directory')), 342 optparse.make_option( 343 '--max-locked-shards', 344 type='int', 345 default=0, 346 help='Set the maximum number of locked shards'), 347 optparse.make_option( 348 '--nocheck-sys-deps', 349 action='store_true', 350 default=False, 351 help="Don't check the system dependencies (themes)"), 352 optparse.make_option( 353 '--order', 354 action='store', 355 default='random', 356 help=('Determine the order in which the test cases will be run. ' 357 "'none' == use the order in which the tests were listed " 358 'either in arguments or test list, ' 359 "'random' == pseudo-random order (default). Seed can be specified " 360 'via --seed, otherwise it will default to the current unix timestamp. ' 361 "'natural' == use the natural order")), 362 optparse.make_option( 363 '--profile', 364 action='store_true', 365 help='Output per-test profile information.'), 366 optparse.make_option( 367 '--profiler', 368 action='store', 369 help='Output per-test profile information, using the specified profiler.'), 370 optparse.make_option( 371 '--restart-shell-between-tests', 372 type='choice', 373 action='store', 374 choices=['always', 'never', 'on_retry',], 375 default='on_retry', 376 help=( 377 'Restarting the shell between tests produces more ' 378 'consistent results, as it prevents state from carrying over ' 379 'from previous tests. It also increases test run time by at ' 380 'least 2X. By default, the shell is restarted when tests get ' 381 'retried, since leaking state between retries can sometimes ' 382 'mask underlying flakiness, and the whole point of retries is ' 383 'to look for flakiness.')), 384 optparse.make_option( 385 '--repeat-each', 386 type='int', 387 default=1, 388 help='Number of times to run each test (e.g. AAABBBCCC)'), 389 optparse.make_option( 390 '--num-retries', 391 '--test-launcher-retry-limit', 392 '--isolated-script-test-launcher-retry-limit', 393 type='int', 394 default=None, 395 help=('Number of times to retry failures. Default (when this ' 396 'flag is not specified) is to retry 3 times, unless an ' 397 'explicit list of tests is passed to run_web_tests.py. ' 398 'If a non-zero value is given explicitly, failures are ' 399 'retried regardless.')), 400 optparse.make_option( 401 '--no-retry-failures', 402 dest='num_retries', 403 action='store_const', 404 const=0, 405 help="Don't retry any failures (equivalent to --num-retries=0)."), 406 optparse.make_option( 407 '--total-shards', 408 type=int, 409 help=('Total number of shards being used for this test run. ' 410 'Must be used with --shard-index. ' 411 '(The user of this script is responsible for spawning ' 412 'all of the shards.)')), 413 optparse.make_option( 414 '--shard-index', 415 type=int, 416 help=('Shard index [0..total_shards) of this test run. ' 417 'Must be used with --total-shards.')), 418 optparse.make_option( 419 '--seed', 420 type='int', 421 help=('Seed to use for random test order (default: %default). ' 422 'Only applicable in combination with --order=random.')), 423 optparse.make_option( 424 '--skipped', 425 action='store', 426 default=None, 427 help=('Control how tests marked SKIP are run. ' 428 '"default" == Skip tests unless explicitly listed on the command line, ' 429 '"ignore" == Run them anyway, ' 430 '"only" == only run the SKIP tests, ' 431 '"always" == always skip, even if listed on the command line.')), 432 optparse.make_option( 433 '--isolated-script-test-also-run-disabled-tests', 434 # TODO(crbug.com/893235): Remove the gtest alias when FindIt no longer uses it. 435 '--gtest_also_run_disabled_tests', 436 action='store_const', 437 const='ignore', 438 dest='skipped', 439 help=('Equivalent to --skipped=ignore.')), 440 optparse.make_option( 441 '--skip-failing-tests', 442 action='store_true', 443 default=False, 444 help=('Skip tests that are expected to fail. Note: When using this option, ' 445 'you might miss new crashes in these tests.')), 446 optparse.make_option( 447 '--skip-timeouts', 448 action='store_true', 449 default=False, 450 help=('Skip tests marked TIMEOUT. Use it to speed up running the entire ' 451 'test suite.')), 452 optparse.make_option( 453 '--fastest', 454 action='store', 455 type='float', 456 help='Run the N% fastest tests as well as any tests listed on the command line'), 457 optparse.make_option( 458 '--test-list', 459 action='append', 460 metavar='FILE', 461 help='read list of tests to run from file, as if they were specified on the command line'), 462 optparse.make_option( 463 '--isolated-script-test-filter', 464 action='append', 465 type='string', 466 help='A list of test globs to run or skip, separated by TWO colons, e.g. fast::css/test.html; ' 467 'prefix the glob with "-" to skip it'), 468 # TODO(crbug.com/893235): Remove gtest_filter when FindIt no longer uses it. 469 optparse.make_option( 470 '--gtest_filter', 471 type='string', 472 help='A colon-separated list of tests to run. Wildcards are ' 473 'NOT supported. It is the same as listing the tests as ' 474 'positional arguments.'), 475 optparse.make_option( 476 '--time-out-ms', 477 help='Set the timeout for each test'), 478 optparse.make_option( 479 '--wrapper', 480 help=('wrapper command to insert before invocations of the driver; option ' 481 'is split on whitespace before running. (Example: --wrapper="valgrind ' 482 '--smc-check=all")')), 483 # FIXME: Display the default number of child processes that will run. 484 optparse.make_option( 485 '-f', '--fully-parallel', 486 action='store_true', 487 help='run all tests in parallel'), 488 optparse.make_option( 489 '--virtual-parallel', 490 action='store_true', 491 help='When running in parallel, include virtual tests. Useful for running a single ' 492 'virtual test suite, but will be slower in other cases.'), 493 optparse.make_option( 494 '-i', '--ignore-tests', 495 action='append', 496 default=[], 497 help='directories or test to ignore (may specify multiple times)'), 498 optparse.make_option( 499 '-n', '--dry-run', 500 action='store_true', 501 default=False, 502 help='Do everything but actually run the tests or upload results.'), 503 optparse.make_option( 504 '-w', '--watch', 505 action='store_true', 506 help='Re-run tests quickly (e.g. avoid restarting the server)'), 507 optparse.make_option( 508 '--zero-tests-executed-ok', 509 action='store_true', 510 help='If set, exit with a success code when no tests are run.' 511 ' Used on trybots when web tests are retried without patch.'), 512 optparse.make_option( 513 '--driver-kill-timeout-secs', 514 type=float, 515 default=1.0, 516 help=('Number of seconds to wait before killing a driver, and the main ' 517 'use case is to leave enough time to allow the process to ' 518 'finish post-run hooks, such as dumping code coverage data. ' 519 'Default is 1 second, can be overriden for specific use cases.')) 520 ])) 521 522 # FIXME: Move these into json_results_generator.py. 523 option_group_definitions.append( 524 ('Result JSON Options', [ 525 # TODO(qyearsley): --build-name is unused and should be removed. 526 optparse.make_option('--build-name', help=optparse.SUPPRESS_HELP), 527 optparse.make_option( 528 '--step-name', 529 default='blink_web_tests', 530 help='The name of the step in a build running this script.'), 531 optparse.make_option( 532 '--build-number', 533 default='DUMMY_BUILD_NUMBER', 534 help='The build number of the builder running this script.'), 535 optparse.make_option( 536 '--builder-name', 537 default='', 538 help='The name of the builder shown on the waterfall running ' 539 'this script, e.g. "Mac10.13 Tests".'), 540 # TODO(qyearsley): This is not actually a Buildbot master since 541 # Buildbot is gone; all instances of the term "master" in this 542 # code-base should be removed after test-results.appspot.com is 543 # removed. 544 optparse.make_option('--master-name'), 545 optparse.make_option( 546 '--test-results-server', 547 default='', 548 help='If specified, upload results JSON files to this ' 549 'App Engine server.'), 550 ])) 551 552 option_parser = optparse.OptionParser( 553 prog='run_web_tests.py', 554 usage='%prog [options] [tests]', 555 description='Runs Blink web tests as described in docs/testing/web_tests.md') 556 557 for group_name, group_options in option_group_definitions: 558 option_group = optparse.OptionGroup(option_parser, group_name) 559 option_group.add_options(group_options) 560 option_parser.add_option_group(option_group) 561 562 (options, args) = option_parser.parse_args(args) 563 564 return (options, args) 565 566 567def _set_up_derived_options(port, options, args): 568 """Sets the options values that depend on other options values.""" 569 # --restart-shell-between-tests is implemented by changing the batch size. 570 if options.restart_shell_between_tests == 'always': 571 options.derived_batch_size = 1 572 options.must_use_derived_batch_size = True 573 elif options.restart_shell_between_tests == 'never': 574 options.derived_batch_size = 0 575 options.must_use_derived_batch_size = True 576 else: 577 # If 'repeat_each' or 'iterations' has been set, then implicitly set the 578 # batch size to 1. If we're already repeating the tests more than once, 579 # then we're not particularly concerned with speed. Restarting content 580 # shell provides more consistent results. 581 if options.repeat_each > 1 or options.iterations > 1: 582 options.derived_batch_size = 1 583 options.must_use_derived_batch_size = True 584 else: 585 options.derived_batch_size = port.default_batch_size() 586 options.must_use_derived_batch_size = False 587 588 if not options.child_processes: 589 options.child_processes = port.host.environ.get( 590 'WEBKIT_TEST_CHILD_PROCESSES', str(port.default_child_processes())) 591 if not options.max_locked_shards: 592 options.max_locked_shards = int(port.host.environ.get( 593 'WEBKIT_TEST_MAX_LOCKED_SHARDS', str(port.default_max_locked_shards()))) 594 595 if not options.configuration: 596 options.configuration = port.default_configuration() 597 598 if not options.time_out_ms: 599 options.time_out_ms = str(port.default_timeout_ms()) 600 601 options.slow_time_out_ms = str(5 * int(options.time_out_ms)) 602 603 if options.additional_platform_directory: 604 additional_platform_directories = [] 605 for path in options.additional_platform_directory: 606 additional_platform_directories.append(port.host.filesystem.abspath(path)) 607 options.additional_platform_directory = additional_platform_directories 608 609 if not args and not options.test_list and options.smoke is None: 610 options.smoke = port.default_smoke_test_only() 611 if options.smoke: 612 if not args and not options.test_list and options.num_retries is None: 613 # Retry failures 3 times if we're running a smoke test without 614 # additional tests. SmokeTests is an explicit list of tests, so we 615 # wouldn't retry by default without this special case. 616 options.num_retries = 3 617 618 if not options.test_list: 619 options.test_list = [] 620 options.test_list.append(port.host.filesystem.join(port.web_tests_dir(), 'SmokeTests')) 621 if not options.skipped: 622 options.skipped = 'always' 623 624 if not options.skipped: 625 options.skipped = 'default' 626 627 if options.gtest_filter: 628 args.extend(options.gtest_filter.split(':')) 629 630 if not options.total_shards and 'GTEST_TOTAL_SHARDS' in port.host.environ: 631 options.total_shards = int(port.host.environ['GTEST_TOTAL_SHARDS']) 632 if not options.shard_index and 'GTEST_SHARD_INDEX' in port.host.environ: 633 options.shard_index = int(port.host.environ['GTEST_SHARD_INDEX']) 634 635 if not options.seed: 636 options.seed = port.host.time() 637 638 639def run(port, options, args, printer): 640 _set_up_derived_options(port, options, args) 641 manager = Manager(port, options, printer) 642 printer.print_config(port) 643 run_details = manager.run(args) 644 _log.debug('') 645 _log.debug('Testing completed. Exit status: %d', run_details.exit_code) 646 printer.flush() 647 return run_details 648 649 650if __name__ == '__main__': 651 sys.exit(main(sys.argv[1:], sys.stderr)) 652