1# Copyright (C) 2010 Google Inc. All rights reserved.
2# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
3# Copyright (C) 2011 Apple Inc. All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9#     * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11#     * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15#     * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31import logging
32import optparse
33import sys
34import traceback
35
36from blinkpy.common import exit_codes
37from blinkpy.common.host import Host
38from blinkpy.web_tests.controllers.manager import Manager
39from blinkpy.web_tests.models import test_run_results
40from blinkpy.web_tests.port.factory import configuration_options
41from blinkpy.web_tests.port.factory import platform_options
42from blinkpy.web_tests.port.factory import wpt_options
43from blinkpy.web_tests.port.factory import python_server_options
44from blinkpy.web_tests.views import printing
45
46_log = logging.getLogger(__name__)
47
48
49def main(argv, stderr):
50    options, args = parse_args(argv)
51
52    if options.platform and 'test' in options.platform and not 'browser_test' in options.platform:
53        # It's a bit lame to import mocks into real code, but this allows the user
54        # to run tests against the test platform interactively, which is useful for
55        # debugging test failures.
56        from blinkpy.common.host_mock import MockHost
57        host = MockHost()
58    else:
59        host = Host()
60
61    printer = printing.Printer(host, options, stderr)
62
63    try:
64        port = host.port_factory.get(options.platform, options)
65    except (NotImplementedError, ValueError) as error:
66        _log.error(error)
67        printer.cleanup()
68        return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS
69
70    try:
71        return run(port, options, args, printer).exit_code
72
73    # We need to still handle KeyboardInterrupt, at least for blinkpy unittest cases.
74    except KeyboardInterrupt:
75        return exit_codes.INTERRUPTED_EXIT_STATUS
76    except test_run_results.TestRunException as error:
77        _log.error(error.msg)
78        return error.code
79    except BaseException as error:
80        if isinstance(error, Exception):
81            _log.error('\n%s raised: %s', error.__class__.__name__, error)
82            traceback.print_exc(file=stderr)
83        return exit_codes.UNEXPECTED_ERROR_EXIT_STATUS
84    finally:
85        printer.cleanup()
86
87
88def deprecate(option, opt_str, _, parser):
89    """
90    Prints a error message for a deprecated option.
91    Usage:
92        optparse.make_option(
93            '--some-option',
94            action='callback',
95            callback=deprecate,
96            help='....')
97    """
98    parser.error('%s: %s' % (opt_str, option.help))
99
100
101def parse_args(args):
102    option_group_definitions = []
103
104    option_group_definitions.append(
105        ('Platform options', platform_options()))
106
107    option_group_definitions.append(
108        ('Configuration options', configuration_options()))
109
110    option_group_definitions.append(
111        ('Printing Options', printing.print_options()))
112
113    option_group_definitions.append(
114        ('web-platform-tests (WPT) Options', wpt_options()))
115
116    option_group_definitions.append(('Python Server Options', python_server_options()))
117
118    option_group_definitions.append(
119        ('Android-specific Options', [
120            optparse.make_option(
121                '--adb-device',
122                action='append',
123                default=[],
124                dest='adb_devices',
125                help='Run Android web tests on these devices'),
126            # FIXME: Flip this to be off by default once we can log the
127            # device setup more cleanly.
128            optparse.make_option(
129                '--no-android-logging',
130                dest='android_logging',
131                action='store_false',
132                default=True,
133                help=('Do not log android-specific debug messages (default '
134                      'is to log as part of --debug-rwt-logging)')),
135        ]))
136
137    option_group_definitions.append(
138        ('Fuchsia-specific Options', [
139            optparse.make_option(
140                '--zircon-logging',
141                dest='zircon_logging',
142                action='store_true',
143                default=True,
144                help=('Log Zircon debug messages (enabled by default).')),
145            optparse.make_option(
146                '--no-zircon-logging',
147                dest='zircon_logging',
148                action='store_false',
149                default=True,
150                help=('Do not log Zircon debug messages.')),
151            optparse.make_option(
152                '--device',
153                choices=['aemu','qemu'],
154                default='qemu',
155                help=('Choose device to launch Fuchsia with.')),
156        ]))
157
158    option_group_definitions.append(
159        ('Results Options', [
160            optparse.make_option(
161                '--additional-driver-flag',
162                '--additional-drt-flag',
163                dest='additional_driver_flag',
164                action='append',
165                default=[],
166                help=('Additional command line flag to pass to the driver. Specify multiple '
167                      'times to add multiple flags.')),
168            optparse.make_option(
169                '--flag-specific',
170                dest='flag_specific',
171                action='store',
172                default=None,
173                help=('Name of a flag-specific configuration defined in FlagSpecificConfig, '
174                      ' as a shortcut of --additional-driver-flag options.')),
175            optparse.make_option(
176                '--additional-expectations',
177                action='append',
178                default=[],
179                help=('Path to a test_expectations file that will override previous '
180                      'expectations. Specify multiple times for multiple sets of overrides.')),
181            optparse.make_option(
182                '--ignore-default-expectations',
183                action='store_true',
184                help=('Do not use the default set of TestExpectations files.')),
185            optparse.make_option(
186                '--no-expectations',
187                action='store_true',
188                help=('Do not use TestExpectations, only run the tests without '
189                      'reporting any results. Useful for generating code '
190                      'coverage reports.')),
191            optparse.make_option(
192                '--additional-platform-directory',
193                action='append',
194                default=[],
195                help=('Additional directory where to look for test baselines (will take '
196                      'precedence over platform baselines). Specify multiple times to add '
197                      'multiple search path entries.')),
198            optparse.make_option(
199                '--build-directory',
200                default='out',
201                help=('Path to the directory where build files are kept, not including '
202                      'configuration. In general this will be "out".')),
203            optparse.make_option(
204                '--clobber-old-results',
205                action='store_true',
206                default=False,
207                help='Clobbers test results from previous runs.'),
208            optparse.make_option(
209                '--compare-port',
210                action='store',
211                default=None,
212                help="Use the specified port's baselines first"),
213            optparse.make_option(
214                '--copy-baselines',
215                action='store_true',
216                default=False,
217                help=('If the actual result is different from the current baseline, '
218                      'copy the current baseline into the *most-specific-platform* '
219                      'directory, or the flag-specific generic-platform directory if '
220                      '--additional-driver-flag is specified. See --reset-results.')),
221            optparse.make_option(
222                '--driver-name',
223                type='string',
224                help='Alternative driver binary to use'),
225            optparse.make_option(
226                '--json-test-results',              # New name from json_results_generator
227                '--write-full-results-to',          # Old argument name
228                '--isolated-script-test-output',    # Isolated API
229                help='Path to write the JSON test results for *all* tests.'),
230            # FIXME(tansell): Remove this option if nobody is found who needs it.
231            optparse.make_option(
232                '--json-failing-test-results',
233                help='Path to write the JSON test results for only *failing* tests.'),
234            optparse.make_option(
235                '--no-show-results',
236                dest='show_results',
237                action='store_false',
238                default=True,
239                help="Don't launch a browser with results after the tests are done"),
240            optparse.make_option(
241                '--reset-results',
242                action='store_true',
243                default=False,
244                help=('Reset baselines to the generated results in their existing location or the default '
245                      'location if no baseline exists. For virtual tests, reset the virtual baselines. '
246                      'If --additional-driver-flag is specified, reset the flag-specific baselines. '
247                      'If --copy-baselines is specified, the copied baselines will be reset.')),
248            optparse.make_option(
249                '--results-directory',
250                help='Location of test results'),
251            optparse.make_option(
252                '--smoke',
253                action='store_true',
254                help='Run just the SmokeTests'),
255            optparse.make_option(
256                '--no-smoke',
257                dest='smoke',
258                action='store_false',
259                help='Do not run just the SmokeTests'),
260        ]))
261
262    option_group_definitions.append(
263        ('Testing Options', [
264            optparse.make_option(
265                '--additional-env-var',
266                type='string',
267                action='append',
268                default=[],
269                help=('Passes that environment variable to the tests '
270                      '(--additional-env-var=NAME=VALUE)')),
271            optparse.make_option(
272                '--build',
273                dest='build',
274                action='store_true',
275                default=True,
276                help=('Check to ensure the build is up to date (default).')),
277            optparse.make_option(
278                '--no-build',
279                dest='build',
280                action='store_false',
281                help="Don't check to see if the build is up to date."),
282            optparse.make_option(
283                '--child-processes', '--jobs', '-j',
284                help='Number of drivers to run in parallel.'),
285            optparse.make_option(
286                '--disable-breakpad',
287                action='store_true',
288                help="Don't use breakpad to symbolize unexpected crashes."),
289            optparse.make_option(
290                '--driver-logging',
291                action='store_true',
292                help='Print detailed logging of the driver/content_shell'),
293            optparse.make_option(
294                '--enable-leak-detection',
295                action='store_true',
296                help='Enable the leak detection of DOM objects.'),
297            optparse.make_option(
298                '--enable-sanitizer',
299                action='store_true',
300                help='Only alert on sanitizer-related errors and crashes'),
301            optparse.make_option(
302                '--exit-after-n-crashes-or-timeouts',
303                type='int',
304                default=None,
305                help='Exit after the first N crashes instead of running all tests'),
306            optparse.make_option(
307                '--exit-after-n-failures',
308                type='int',
309                default=None,
310                help='Exit after the first N failures instead of running all tests'),
311            optparse.make_option(
312                '--fuzzy-diff',
313                action='store_true',
314                default=False,
315                help=('When running tests on an actual GPU, variance in pixel '
316                      'output can leads image differences causing failed expectations. '
317                      'Instead a fuzzy diff is used to account for this variance. '
318                      'See tools/imagediff/image_diff.cc')),
319            optparse.make_option(
320                '--ignore-builder-category',
321                action='store',
322                help=('The category of builders to use with the --ignore-flaky-tests option '
323                      "('layout' or 'deps').")),
324            optparse.make_option(
325                '--ignore-flaky-tests',
326                action='store',
327                help=('Control whether tests that are flaky on the bots get ignored. '
328                      "'very-flaky' == Ignore any tests that flaked more than once on the bot. "
329                      "'maybe-flaky' == Ignore any tests that flaked once on the bot. "
330                      "'unexpected' == Ignore any tests that had unexpected results on the bot.")),
331            optparse.make_option(
332                '--iterations',
333                '--isolated-script-test-repeat',
334                # TODO(crbug.com/893235): Remove the gtest alias when FindIt no longer uses it.
335                '--gtest_repeat',
336                type='int',
337                default=1,
338                help='Number of times to run the set of tests (e.g. ABCABCABC)'),
339            optparse.make_option(
340                '--layout-tests-directory',
341                help=('Path to a custom web tests directory')),
342            optparse.make_option(
343                '--max-locked-shards',
344                type='int',
345                default=0,
346                help='Set the maximum number of locked shards'),
347            optparse.make_option(
348                '--nocheck-sys-deps',
349                action='store_true',
350                default=False,
351                help="Don't check the system dependencies (themes)"),
352            optparse.make_option(
353                '--order',
354                action='store',
355                default='random',
356                help=('Determine the order in which the test cases will be run. '
357                      "'none' == use the order in which the tests were listed "
358                      'either in arguments or test list, '
359                      "'random' == pseudo-random order (default). Seed can be specified "
360                      'via --seed, otherwise it will default to the current unix timestamp. '
361                      "'natural' == use the natural order")),
362            optparse.make_option(
363                '--profile',
364                action='store_true',
365                help='Output per-test profile information.'),
366            optparse.make_option(
367                '--profiler',
368                action='store',
369                help='Output per-test profile information, using the specified profiler.'),
370            optparse.make_option(
371                '--restart-shell-between-tests',
372                type='choice',
373                action='store',
374                choices=['always', 'never', 'on_retry',],
375                default='on_retry',
376                help=(
377                    'Restarting the shell between tests produces more '
378                    'consistent results, as it prevents state from carrying over '
379                    'from previous tests. It also increases test run time by at '
380                    'least 2X. By default, the shell is restarted when tests get '
381                    'retried, since leaking state between retries can sometimes '
382                    'mask underlying flakiness, and the whole point of retries is '
383                    'to look for flakiness.')),
384            optparse.make_option(
385                '--repeat-each',
386                type='int',
387                default=1,
388                help='Number of times to run each test (e.g. AAABBBCCC)'),
389            optparse.make_option(
390                '--num-retries',
391                '--test-launcher-retry-limit',
392                '--isolated-script-test-launcher-retry-limit',
393                type='int',
394                default=None,
395                help=('Number of times to retry failures. Default (when this '
396                      'flag is not specified) is to retry 3 times, unless an '
397                      'explicit list of tests is passed to run_web_tests.py. '
398                      'If a non-zero value is given explicitly, failures are '
399                      'retried regardless.')),
400            optparse.make_option(
401                '--no-retry-failures',
402                dest='num_retries',
403                action='store_const',
404                const=0,
405                help="Don't retry any failures (equivalent to --num-retries=0)."),
406            optparse.make_option(
407                '--total-shards',
408                type=int,
409                help=('Total number of shards being used for this test run. '
410                      'Must be used with --shard-index. '
411                      '(The user of this script is responsible for spawning '
412                      'all of the shards.)')),
413            optparse.make_option(
414                '--shard-index',
415                type=int,
416                help=('Shard index [0..total_shards) of this test run. '
417                      'Must be used with --total-shards.')),
418            optparse.make_option(
419                '--seed',
420                type='int',
421                help=('Seed to use for random test order (default: %default). '
422                      'Only applicable in combination with --order=random.')),
423            optparse.make_option(
424                '--skipped',
425                action='store',
426                default=None,
427                help=('Control how tests marked SKIP are run. '
428                      '"default" == Skip tests unless explicitly listed on the command line, '
429                      '"ignore" == Run them anyway, '
430                      '"only" == only run the SKIP tests, '
431                      '"always" == always skip, even if listed on the command line.')),
432            optparse.make_option(
433                '--isolated-script-test-also-run-disabled-tests',
434                # TODO(crbug.com/893235): Remove the gtest alias when FindIt no longer uses it.
435                '--gtest_also_run_disabled_tests',
436                action='store_const',
437                const='ignore',
438                dest='skipped',
439                help=('Equivalent to --skipped=ignore.')),
440            optparse.make_option(
441                '--skip-failing-tests',
442                action='store_true',
443                default=False,
444                help=('Skip tests that are expected to fail. Note: When using this option, '
445                      'you might miss new crashes in these tests.')),
446            optparse.make_option(
447                '--skip-timeouts',
448                action='store_true',
449                default=False,
450                help=('Skip tests marked TIMEOUT. Use it to speed up running the entire '
451                      'test suite.')),
452            optparse.make_option(
453                '--fastest',
454                action='store',
455                type='float',
456                help='Run the N% fastest tests as well as any tests listed on the command line'),
457            optparse.make_option(
458                '--test-list',
459                action='append',
460                metavar='FILE',
461                help='read list of tests to run from file, as if they were specified on the command line'),
462            optparse.make_option(
463                '--isolated-script-test-filter',
464                action='append',
465                type='string',
466                help='A list of test globs to run or skip, separated by TWO colons, e.g. fast::css/test.html; '
467                     'prefix the glob with "-" to skip it'),
468            # TODO(crbug.com/893235): Remove gtest_filter when FindIt no longer uses it.
469            optparse.make_option(
470                '--gtest_filter',
471                type='string',
472                help='A colon-separated list of tests to run. Wildcards are '
473                     'NOT supported. It is the same as listing the tests as '
474                     'positional arguments.'),
475            optparse.make_option(
476                '--time-out-ms',
477                help='Set the timeout for each test'),
478            optparse.make_option(
479                '--wrapper',
480                help=('wrapper command to insert before invocations of the driver; option '
481                      'is split on whitespace before running. (Example: --wrapper="valgrind '
482                      '--smc-check=all")')),
483            # FIXME: Display the default number of child processes that will run.
484            optparse.make_option(
485                '-f', '--fully-parallel',
486                action='store_true',
487                help='run all tests in parallel'),
488            optparse.make_option(
489                '--virtual-parallel',
490                action='store_true',
491                help='When running in parallel, include virtual tests. Useful for running a single '
492                     'virtual test suite, but will be slower in other cases.'),
493            optparse.make_option(
494                '-i', '--ignore-tests',
495                action='append',
496                default=[],
497                help='directories or test to ignore (may specify multiple times)'),
498            optparse.make_option(
499                '-n', '--dry-run',
500                action='store_true',
501                default=False,
502                help='Do everything but actually run the tests or upload results.'),
503            optparse.make_option(
504                '-w', '--watch',
505                action='store_true',
506                help='Re-run tests quickly (e.g. avoid restarting the server)'),
507            optparse.make_option(
508                '--zero-tests-executed-ok',
509                action='store_true',
510                help='If set, exit with a success code when no tests are run.'
511                ' Used on trybots when web tests are retried without patch.'),
512            optparse.make_option(
513                '--driver-kill-timeout-secs',
514                type=float,
515                default=1.0,
516                help=('Number of seconds to wait before killing a driver, and the main '
517                      'use case is to leave enough time to allow the process to '
518                      'finish post-run hooks, such as dumping code coverage data. '
519                      'Default is 1 second, can be overriden for specific use cases.'))
520        ]))
521
522    # FIXME: Move these into json_results_generator.py.
523    option_group_definitions.append(
524        ('Result JSON Options', [
525            # TODO(qyearsley): --build-name is unused and should be removed.
526            optparse.make_option('--build-name', help=optparse.SUPPRESS_HELP),
527            optparse.make_option(
528                '--step-name',
529                default='blink_web_tests',
530                help='The name of the step in a build running this script.'),
531            optparse.make_option(
532                '--build-number',
533                default='DUMMY_BUILD_NUMBER',
534                help='The build number of the builder running this script.'),
535            optparse.make_option(
536                '--builder-name',
537                default='',
538                help='The name of the builder shown on the waterfall running '
539                     'this script, e.g. "Mac10.13 Tests".'),
540            # TODO(qyearsley): This is not actually a Buildbot master since
541            # Buildbot is gone; all instances of the term "master" in this
542            # code-base should be removed after test-results.appspot.com is
543            # removed.
544            optparse.make_option('--master-name'),
545            optparse.make_option(
546                '--test-results-server',
547                default='',
548                help='If specified, upload results JSON files to this '
549                     'App Engine server.'),
550        ]))
551
552    option_parser = optparse.OptionParser(
553        prog='run_web_tests.py',
554        usage='%prog [options] [tests]',
555        description='Runs Blink web tests as described in docs/testing/web_tests.md')
556
557    for group_name, group_options in option_group_definitions:
558        option_group = optparse.OptionGroup(option_parser, group_name)
559        option_group.add_options(group_options)
560        option_parser.add_option_group(option_group)
561
562    (options, args) = option_parser.parse_args(args)
563
564    return (options, args)
565
566
567def _set_up_derived_options(port, options, args):
568    """Sets the options values that depend on other options values."""
569    # --restart-shell-between-tests is implemented by changing the batch size.
570    if options.restart_shell_between_tests == 'always':
571        options.derived_batch_size = 1
572        options.must_use_derived_batch_size = True
573    elif options.restart_shell_between_tests == 'never':
574        options.derived_batch_size = 0
575        options.must_use_derived_batch_size = True
576    else:
577        # If 'repeat_each' or 'iterations' has been set, then implicitly set the
578        # batch size to 1. If we're already repeating the tests more than once,
579        # then we're not particularly concerned with speed. Restarting content
580        # shell provides more consistent results.
581        if options.repeat_each > 1 or options.iterations > 1:
582            options.derived_batch_size = 1
583            options.must_use_derived_batch_size = True
584        else:
585            options.derived_batch_size = port.default_batch_size()
586            options.must_use_derived_batch_size = False
587
588    if not options.child_processes:
589        options.child_processes = port.host.environ.get(
590            'WEBKIT_TEST_CHILD_PROCESSES', str(port.default_child_processes()))
591    if not options.max_locked_shards:
592        options.max_locked_shards = int(port.host.environ.get(
593            'WEBKIT_TEST_MAX_LOCKED_SHARDS', str(port.default_max_locked_shards())))
594
595    if not options.configuration:
596        options.configuration = port.default_configuration()
597
598    if not options.time_out_ms:
599        options.time_out_ms = str(port.default_timeout_ms())
600
601    options.slow_time_out_ms = str(5 * int(options.time_out_ms))
602
603    if options.additional_platform_directory:
604        additional_platform_directories = []
605        for path in options.additional_platform_directory:
606            additional_platform_directories.append(port.host.filesystem.abspath(path))
607        options.additional_platform_directory = additional_platform_directories
608
609    if not args and not options.test_list and options.smoke is None:
610        options.smoke = port.default_smoke_test_only()
611    if options.smoke:
612        if not args and not options.test_list and options.num_retries is None:
613            # Retry failures 3 times if we're running a smoke test without
614            # additional tests. SmokeTests is an explicit list of tests, so we
615            # wouldn't retry by default without this special case.
616            options.num_retries = 3
617
618        if not options.test_list:
619            options.test_list = []
620        options.test_list.append(port.host.filesystem.join(port.web_tests_dir(), 'SmokeTests'))
621        if not options.skipped:
622            options.skipped = 'always'
623
624    if not options.skipped:
625        options.skipped = 'default'
626
627    if options.gtest_filter:
628        args.extend(options.gtest_filter.split(':'))
629
630    if not options.total_shards and 'GTEST_TOTAL_SHARDS' in port.host.environ:
631        options.total_shards = int(port.host.environ['GTEST_TOTAL_SHARDS'])
632    if not options.shard_index and 'GTEST_SHARD_INDEX' in port.host.environ:
633        options.shard_index = int(port.host.environ['GTEST_SHARD_INDEX'])
634
635    if not options.seed:
636        options.seed = port.host.time()
637
638
639def run(port, options, args, printer):
640    _set_up_derived_options(port, options, args)
641    manager = Manager(port, options, printer)
642    printer.print_config(port)
643    run_details = manager.run(args)
644    _log.debug('')
645    _log.debug('Testing completed. Exit status: %d', run_details.exit_code)
646    printer.flush()
647    return run_details
648
649
650if __name__ == '__main__':
651    sys.exit(main(sys.argv[1:], sys.stderr))
652