1#!/usr/bin/env python
2# Copyright 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Runs the WebDriver Java acceptance tests."""
7
8import optparse
9import os
10import re
11import stat
12import sys
13
14_THIS_DIR = os.path.abspath(os.path.dirname(__file__))
15sys.path.insert(1, os.path.join(_THIS_DIR, os.pardir))
16
17import chrome_paths
18import test_environment
19import util
20import glob
21
22if util.IsLinux():
23  sys.path.insert(0, os.path.join(chrome_paths.GetSrc(), 'build', 'android'))
24  from pylib import constants
25
26
27def _Run(java_tests_src_dir, test_filter, ready_to_run_tests,
28         chromedriver_path, chrome_path, log_path, android_package_key,
29         debug, tests_report_file):
30  """Run the WebDriver Java tests and return the test results.
31
32  Args:
33    java_tests_src_dir: the java test source code directory.
34    test_filter: the filter to use when choosing tests to run. Format is same
35        as Google C++ Test format.
36    readyToRunTests: tests that need to run regardless of
37        @NotYetImplemented annotation
38    chromedriver_path: path to ChromeDriver exe.
39    chrome_path: path to Chrome exe.
40    log_path: path to server log.
41    android_package_key: name of Chrome's Android package.
42    debug: whether the tests should wait until attached by a debugger.
43  """
44
45  sys_props = ['selenium.browser=chrome',
46               'webdriver.chrome.driver=' + os.path.abspath(chromedriver_path)]
47  if chrome_path:
48    if util.IsLinux() and android_package_key is None:
49      # Workaround for crbug.com/611886 and
50      # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1695
51      chrome_wrapper_path = os.path.join(java_tests_src_dir,
52                                         'chrome-wrapper-no-sandbox')
53      with open(chrome_wrapper_path, 'w') as f:
54        f.write('#!/bin/sh\n')
55        f.write('exec "%s" --no-sandbox --disable-gpu "$@"\n' %
56            os.path.abspath(chrome_path))
57      st = os.stat(chrome_wrapper_path)
58      os.chmod(chrome_wrapper_path, st.st_mode | stat.S_IEXEC)
59    elif util.IsMac():
60      # Use srgb color profile, otherwise the default color profile on Mac
61      # causes some color adjustments, so screenshots have unexpected colors.
62      chrome_wrapper_path = os.path.join(java_tests_src_dir, 'chrome-wrapper')
63      with open(chrome_wrapper_path, 'w') as f:
64        f.write('#!/bin/sh\n')
65        f.write('exec "%s" --force-color-profile=srgb "$@"\n' %
66            os.path.abspath(chrome_path))
67      st = os.stat(chrome_wrapper_path)
68      os.chmod(chrome_wrapper_path, st.st_mode | stat.S_IEXEC)
69    else:
70      chrome_wrapper_path = os.path.abspath(chrome_path)
71    sys_props += ['webdriver.chrome.binary=' + chrome_wrapper_path]
72  if log_path:
73    sys_props += ['webdriver.chrome.logfile=' + log_path]
74  if android_package_key:
75    android_package = constants.PACKAGE_INFO[android_package_key].package
76    sys_props += ['webdriver.chrome.android_package=' + android_package]
77    if android_package_key == 'chromedriver_webview_shell':
78      android_activity = constants.PACKAGE_INFO[android_package_key].activity
79      android_process = '%s:main' % android_package
80      sys_props += ['webdriver.chrome.android_activity=' + android_activity]
81      sys_props += ['webdriver.chrome.android_process=' + android_process]
82  if test_filter:
83    # Test jar actually takes a regex. Convert from glob.
84    test_filter = test_filter.replace('*', '.*')
85    sys_props += ['filter=' + test_filter]
86  if ready_to_run_tests:
87    sys_props += ['readyToRun=' + ready_to_run_tests]
88
89  jvm_args = []
90  if debug:
91    transport = 'dt_socket'
92    if util.IsWindows():
93      transport = 'dt_shmem'
94    jvm_args += ['-agentlib:jdwp=transport=%s,server=y,suspend=y,'
95                 'address=33081' % transport]
96
97  _RunTest(java_tests_src_dir, jvm_args, sys_props, tests_report_file)
98
99def _RunTest(java_tests_src_dir, jvm_args, sys_props, tests_report_file):
100  """Runs a single JUnit test suite.
101
102  Args:
103    java_tests_src_dir: the directory to run the tests in.
104    sys_props: Java system properties to set when running the tests.
105    jvm_args: Java VM command line args to use.
106  """
107
108  classpath = []
109  for name in glob.glob(java_tests_src_dir + "/jar/*.jar"):
110    classpath.append(name)
111
112  if util.IsWindows():
113    separator = ';'
114  else:
115    separator = ':'
116
117  code = util.RunCommand(
118                         ['java'] +
119                         ['-D%s' % sys_prop for sys_prop in sys_props] +
120                         ['-D%s' % jvm_arg for jvm_arg in jvm_args] +
121                         ['-cp', separator.join(classpath),
122                          'org.junit.runner.JUnitCore',
123                          'org.openqa.selenium.chrome.ChromeDriverTests'],
124                         java_tests_src_dir,
125                         tests_report_file)
126
127  if code != 0:
128    print 'FAILED to run java tests of ChromeDriverTests'
129
130def _PrintTestResults(results_path):
131  """Prints the given results in a format recognized by the buildbot."""
132  with open(results_path, "r") as myfile:
133    contents = myfile.read()
134
135  successJunitTestsCount = re.search(r'OK \((\d* tests)', contents)
136
137  if successJunitTestsCount:
138    testsCount = re.findall(r'INFO: <<< Finished (.*)\)', contents)
139    print("Ran %s tests " % len(testsCount))
140    myfile.close()
141    return 0
142
143  print("============================")
144  print("FAILURES DETAILS")
145  print("============================")
146  start = 'There w'
147  end = 'FAILURES!!!'
148  print contents[contents.find(start):contents.rfind(end)]
149
150  print("============================")
151  print("SUMMARY")
152  print("============================")
153  testsCount = re.findall(r'INFO: <<< Finished (.*)\)', contents)
154  print("Ran %s tests " % len(testsCount))
155
156  failuresCount = re.search(r'There w.* (.*) failure', contents)
157  if failuresCount:
158    print("Failed %s tests" % failuresCount.group(1))
159  failedTests = re.findall(r'^\d*\) (.*\(org.openqa.*\))',
160       contents, re.MULTILINE)
161  testsToReRun = []
162  for test in failedTests:
163    testName = test.split('(')[0]
164    testClass = test.split('(')[1].split('.')[-1]
165    testsToReRun.append(testClass[0:-1] + '.' + testName)
166  print 'Rerun failing tests with filter: ' + ':'.join(testsToReRun)
167
168  myfile.close()
169  return failuresCount.group(1)
170
171def main():
172  parser = optparse.OptionParser()
173  parser.add_option(
174      '', '--verbose', action='store_true', default=False,
175      help='Whether output should be verbose')
176  parser.add_option(
177      '', '--debug', action='store_true', default=False,
178      help='Whether to wait to be attached by a debugger')
179  parser.add_option(
180      '', '--chromedriver', type='string', default=None,
181      help='Path to a build of the chromedriver library(REQUIRED!)')
182  parser.add_option(
183      '', '--chrome', type='string', default=None,
184      help='Path to a build of the chrome binary')
185  parser.add_option(
186      '', '--log-path',
187      help='Output verbose server logs to this file')
188  parser.add_option(
189      '', '--android-package', help='Android package key')
190  parser.add_option(
191      '', '--filter', type='string', default=None,
192      help='Filter for specifying what tests to run, "*" will run all. E.g., '
193           '*testShouldReturnTitleOfPageIfSet')
194  parser.add_option(
195      '', '--also-run-disabled-tests', action='store_true', default=False,
196      help='Include disabled tests while running the tests')
197  parser.add_option(
198      '', '--isolate-tests', action='store_true', default=False,
199      help='Relaunch the jar test harness after each test')
200  options, _ = parser.parse_args()
201
202  options.chromedriver = util.GetAbsolutePathOfUserPath(options.chromedriver)
203  if options.chromedriver is None or not os.path.exists(options.chromedriver):
204    parser.error('chromedriver is required or the given path is invalid.' +
205                 'Please run "%s --help" for help' % __file__)
206
207  if options.android_package:
208    if options.android_package not in constants.PACKAGE_INFO:
209      parser.error('Invalid --android-package')
210    environment = test_environment.AndroidTestEnvironment(
211        options.android_package)
212  else:
213    environment = test_environment.DesktopTestEnvironment()
214
215  try:
216    environment.GlobalSetUp()
217    # Run passed tests when filter is not provided.
218    if options.isolate_tests:
219      test_filters = environment.GetPassedJavaTests()
220    else:
221      if options.filter:
222        test_filter = options.filter
223      else:
224        test_filter = '*'
225      if not options.also_run_disabled_tests:
226        if '-' in test_filter:
227          test_filter += ':'
228        else:
229          test_filter += '-'
230        test_filter += ':'.join(environment.GetDisabledJavaTestMatchers())
231      test_filters = [test_filter]
232    ready_to_run_tests = ':'.join(environment.GetReadyToRunJavaTestMatchers())
233
234    java_tests_src_dir = os.path.join(chrome_paths.GetSrc(), 'chrome', 'test',
235                                      'chromedriver', 'third_party',
236                                      'java_tests')
237    tests_report_file = os.path.join(java_tests_src_dir, 'results.txt')
238
239    if (not os.path.exists(java_tests_src_dir) or
240        not os.listdir(java_tests_src_dir)):
241      java_tests_url = ('https://chromium.googlesource.com/chromium/deps'
242                        '/webdriver')
243      print ('"%s" is empty or it doesn\'t exist. ' % java_tests_src_dir +
244             'Need to map ' + java_tests_url + ' to '
245             'chrome/test/chromedriver/third_party/java_tests in .gclient.\n'
246             'Alternatively, do:\n'
247             '  $ cd chrome/test/chromedriver/third_party\n'
248             '  $ git clone %s java_tests' % java_tests_url)
249      return 1
250
251    _Run(
252      java_tests_src_dir=java_tests_src_dir,
253      test_filter=test_filter,
254      ready_to_run_tests=ready_to_run_tests,
255      chromedriver_path=options.chromedriver,
256      chrome_path=util.GetAbsolutePathOfUserPath(options.chrome),
257      log_path=options.log_path,
258      android_package_key=options.android_package,
259      debug=options.debug,
260      tests_report_file=tests_report_file)
261    return _PrintTestResults(tests_report_file)
262  finally:
263    environment.GlobalTearDown()
264    if(os.path.exists(tests_report_file)):
265     os.remove(tests_report_file)
266    if(os.path.exists(os.path.join(java_tests_src_dir,
267                                   "chrome-wrapper-no-sandbox"))):
268      os.remove(os.path.join(java_tests_src_dir, "chrome-wrapper-no-sandbox"))
269    if(os.path.exists(os.path.join(java_tests_src_dir, "chrome-wrapper"))):
270      os.remove(os.path.join(java_tests_src_dir, "chrome-wrapper"))
271
272if __name__ == '__main__':
273  sys.exit(main())
274