1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import glob
6import logging
7import os
8import sys
9
10from base_test_runner import BaseTestRunner
11import debug_info
12import constants
13import perf_tests_helper
14import run_tests_helper
15from test_package_apk import TestPackageApk
16from test_package_executable import TestPackageExecutable
17from test_result import TestResults
18
19
20class SingleTestRunner(BaseTestRunner):
21  """Single test suite attached to a single device.
22
23  Args:
24    device: Device to run the tests.
25    test_suite: A specific test suite to run, empty to run all.
26    gtest_filter: A gtest_filter flag.
27    test_arguments: Additional arguments to pass to the test binary.
28    timeout: Timeout for each test.
29    rebaseline: Whether or not to run tests in isolation and update the filter.
30    performance_test: Whether or not performance test(s).
31    cleanup_test_files: Whether or not to cleanup test files on device.
32    tool: Name of the Valgrind tool.
33    shard_index: index number of the shard on which the test suite will run.
34    dump_debug_info: Whether or not to dump debug information.
35    build_type: 'Release' or 'Debug'.
36  """
37
38  def __init__(self, device, test_suite, gtest_filter, test_arguments, timeout,
39               rebaseline, performance_test, cleanup_test_files, tool_name,
40               shard_index, dump_debug_info, fast_and_loose, build_type):
41    BaseTestRunner.__init__(self, device, tool_name, shard_index, build_type)
42    self._running_on_emulator = self.device.startswith('emulator')
43    self._gtest_filter = gtest_filter
44    self._test_arguments = test_arguments
45    self.test_results = TestResults()
46    if dump_debug_info:
47      self.dump_debug_info = debug_info.GTestDebugInfo(self.adb, device,
48           os.path.basename(test_suite), gtest_filter)
49    else:
50      self.dump_debug_info = None
51    self.fast_and_loose = fast_and_loose
52
53    logging.warning('Test suite: ' + test_suite)
54    if os.path.splitext(test_suite)[1] == '.apk':
55      self.test_package = TestPackageApk(self.adb, device,
56          test_suite, timeout, rebaseline, performance_test, cleanup_test_files,
57          self.tool, self.dump_debug_info)
58    else:
59      self.test_package = TestPackageExecutable(
60          self.adb, device,
61          test_suite, timeout, rebaseline, performance_test, cleanup_test_files,
62          self.tool, self.dump_debug_info)
63    self._performance_test_setup = None
64    if performance_test:
65      self._performance_test_setup = perf_tests_helper.PerfTestSetup(self.adb)
66
67  def _TestSuiteRequiresMockTestServer(self):
68    """Returns True if the test suite requires mock test server."""
69    return False
70  # TODO(yfriedman): Disabled because of flakiness.
71  # (self.test_package.test_suite_basename == 'unit_tests' or
72  #          self.test_package.test_suite_basename == 'net_unittests' or
73  #          False)
74
75  def _GetFilterFileName(self):
76    """Returns the filename of gtest filter."""
77    return os.path.join(sys.path[0], 'gtest_filter',
78        self.test_package.test_suite_basename + '_disabled')
79
80  def _GetAdditionalEmulatorFilterName(self):
81    """Returns the filename of additional gtest filter for emulator."""
82    return os.path.join(sys.path[0], 'gtest_filter',
83        self.test_package.test_suite_basename +
84        '_emulator_additional_disabled')
85
86  def GetDisabledTests(self):
87    """Returns a list of disabled tests.
88
89    Returns:
90      A list of disabled tests obtained from gtest_filter/test_suite_disabled.
91    """
92    disabled_tests = run_tests_helper.GetExpectations(self._GetFilterFileName())
93    if self._running_on_emulator:
94      # Append emulator's filter file.
95      disabled_tests.extend(run_tests_helper.GetExpectations(
96          self._GetAdditionalEmulatorFilterName()))
97    return disabled_tests
98
99  def UpdateFilter(self, failed_tests):
100    """Updates test_suite_disabled file with the new filter (deletes if empty).
101
102    If running in Emulator, only the failed tests which are not in the normal
103    filter returned by _GetFilterFileName() are written to emulator's
104    additional filter file.
105
106    Args:
107      failed_tests: A sorted list of failed tests.
108    """
109    disabled_tests = []
110    if not self._running_on_emulator:
111      filter_file_name = self._GetFilterFileName()
112    else:
113      filter_file_name = self._GetAdditionalEmulatorFilterName()
114      disabled_tests.extend(
115          run_tests_helper.GetExpectations(self._GetFilterFileName()))
116      logging.info('About to update emulator\'s additional filter (%s).'
117          % filter_file_name)
118
119    new_failed_tests = []
120    if failed_tests:
121      for test in failed_tests:
122        if test.name not in disabled_tests:
123          new_failed_tests.append(test.name)
124
125    if not new_failed_tests:
126      if os.path.exists(filter_file_name):
127        os.unlink(filter_file_name)
128      return
129
130    filter_file = file(filter_file_name, 'w')
131    if self._running_on_emulator:
132      filter_file.write('# Addtional list of suppressions from emulator\n')
133    else:
134      filter_file.write('# List of suppressions\n')
135    filter_file.write('# This file was automatically generated by %s\n'
136        % sys.argv[0])
137    filter_file.write('\n'.join(sorted(new_failed_tests)))
138    filter_file.write('\n')
139    filter_file.close()
140
141  def GetDataFilesForTestSuite(self):
142    """Returns a list of data files/dirs needed by the test suite."""
143    # Ideally, we'd just push all test data. However, it has >100MB, and a lot
144    # of the files are not relevant (some are used for browser_tests, others for
145    # features not supported, etc..).
146    if self.test_package.test_suite_basename in ['base_unittests',
147                                                 'sql_unittests',
148                                                 'unit_tests']:
149      test_files = [
150          'base/data/file_util_unittest',
151          'base/data/json/bom_feff.json',
152          'chrome/test/data/download-test1.lib',
153          'chrome/test/data/extensions/bad_magic.crx',
154          'chrome/test/data/extensions/good.crx',
155          'chrome/test/data/extensions/icon1.png',
156          'chrome/test/data/extensions/icon2.png',
157          'chrome/test/data/extensions/icon3.png',
158          'chrome/test/data/extensions/allow_silent_upgrade/',
159          'chrome/test/data/extensions/app/',
160          'chrome/test/data/extensions/bad/',
161          'chrome/test/data/extensions/effective_host_permissions/',
162          'chrome/test/data/extensions/empty_manifest/',
163          'chrome/test/data/extensions/good/Extensions/',
164          'chrome/test/data/extensions/manifest_tests/',
165          'chrome/test/data/extensions/page_action/',
166          'chrome/test/data/extensions/permissions/',
167          'chrome/test/data/extensions/script_and_capture/',
168          'chrome/test/data/extensions/unpacker/',
169          'chrome/test/data/bookmarks/',
170          'chrome/test/data/components/',
171          'chrome/test/data/extensions/json_schema_test.js',
172          'chrome/test/data/History/',
173          'chrome/test/data/json_schema_validator/',
174          'chrome/test/data/pref_service/',
175          'chrome/test/data/serializer_nested_test.js',
176          'chrome/test/data/serializer_test.js',
177          'chrome/test/data/serializer_test_nowhitespace.js',
178          'chrome/test/data/top_sites/',
179          'chrome/test/data/web_app_info/',
180          'chrome/test/data/web_database',
181          'chrome/test/data/webui/',
182          'chrome/test/data/zip',
183          'chrome/third_party/mock4js/',
184          'content/browser/gpu/software_rendering_list.json',
185          'net/data/cache_tests/insert_load1',
186          'net/data/cache_tests/dirty_entry5',
187          'net/data/ssl/certificates/',
188          'ui/base/test/data/data_pack_unittest',
189        ]
190      if self.test_package.test_suite_basename == 'unit_tests':
191        test_files += ['chrome/test/data/simple_open_search.xml']
192        # The following are spell check data. Now only list the data under
193        # third_party/hunspell_dictionaries which are used by unit tests.
194        old_cwd = os.getcwd()
195        os.chdir(constants.CHROME_DIR)
196        test_files += glob.glob('third_party/hunspell_dictionaries/*.bdic')
197        os.chdir(old_cwd)
198      return test_files
199    elif self.test_package.test_suite_basename == 'net_unittests':
200      return [
201          'net/data/cache_tests',
202          'net/data/filter_unittests',
203          'net/data/ftp',
204          'net/data/proxy_resolver_v8_unittest',
205          'net/data/ssl/certificates',
206          'net/data/url_request_unittest/',
207          'net/data/proxy_script_fetcher_unittest'
208          ]
209    elif self.test_package.test_suite_basename == 'ui_tests':
210      return [
211          'chrome/test/data/dromaeo',
212          'chrome/test/data/json2.js',
213          'chrome/test/data/sunspider',
214          'chrome/test/data/v8_benchmark',
215          'chrome/test/perf/sunspider_uitest.js',
216          'chrome/test/perf/v8_benchmark_uitest.js',
217          ]
218    elif self.test_package.test_suite_basename == 'page_cycler_tests':
219      data = [
220          'tools/page_cycler',
221          'data/page_cycler',
222          ]
223      for d in data:
224        if not os.path.exists(d):
225          raise Exception('Page cycler data not found.')
226      return data
227    elif self.test_package.test_suite_basename == 'webkit_unit_tests':
228      return [
229          'third_party/WebKit/Source/WebKit/chromium/tests/data',
230          ]
231    elif self.test_package.test_suite_basename == 'content_unittests':
232      return [
233          'content/test/data/gpu/webgl_conformance_test_expectations.txt',
234          'net/data/ssl/certificates/',
235          'webkit/data/dom_storage/webcore_test_database.localstorage',
236          'third_party/hyphen/hyph_en_US.dic',
237          ]
238    elif self.test_package.test_suite_basename == 'media_unittests':
239      return [
240          'media/test/data',
241          ]
242    return []
243
244  def LaunchHelperToolsForTestSuite(self):
245    """Launches helper tools for the test suite.
246
247    Sometimes one test may need to run some helper tools first in order to
248    successfully complete the test.
249    """
250    if self._TestSuiteRequiresMockTestServer():
251      self.LaunchChromeTestServerSpawner()
252
253  def StripAndCopyFiles(self):
254    """Strips and copies the required data files for the test suite."""
255    self.test_package.StripAndCopyExecutable()
256    self.test_package.PushDataAndPakFiles()
257    self.tool.CopyFiles()
258    test_data = self.GetDataFilesForTestSuite()
259    if test_data and not self.fast_and_loose:
260      # Make sure SD card is ready.
261      self.adb.WaitForSdCardReady(20)
262      for data in test_data:
263        self.CopyTestData([data], self.adb.GetExternalStorage())
264
265  def RunTestsWithFilter(self):
266    """Runs a tests via a small, temporary shell script."""
267    self.test_package.CreateTestRunnerScript(self._gtest_filter,
268                                             self._test_arguments)
269    self.test_results = self.test_package.RunTestsAndListResults()
270
271  def RebaselineTests(self):
272    """Runs all available tests, restarting in case of failures."""
273    if self._gtest_filter:
274      all_tests = set(self._gtest_filter.split(':'))
275    else:
276      all_tests = set(self.test_package.GetAllTests())
277    failed_results = set()
278    executed_results = set()
279    while True:
280      executed_names = set([f.name for f in executed_results])
281      self._gtest_filter = ':'.join(all_tests - executed_names)
282      self.RunTestsWithFilter()
283      failed_results.update(self.test_results.crashed,
284          self.test_results.failed)
285      executed_results.update(self.test_results.crashed,
286                              self.test_results.failed,
287                              self.test_results.ok)
288      executed_names = set([f.name for f in executed_results])
289      logging.info('*' * 80)
290      logging.info(self.device)
291      logging.info('Executed: ' + str(len(executed_names)) + ' of ' +
292                   str(len(all_tests)))
293      logging.info('Failed so far: ' + str(len(failed_results)) + ' ' +
294                   str([f.name for f in failed_results]))
295      logging.info('Remaining: ' + str(len(all_tests - executed_names)) + ' ' +
296                   str(all_tests - executed_names))
297      logging.info('*' * 80)
298      if executed_names == all_tests:
299        break
300    self.test_results = TestResults.FromRun(
301        ok=list(executed_results - failed_results),
302        failed=list(failed_results))
303
304  def RunTests(self):
305    """Runs all tests (in rebaseline mode, runs each test in isolation).
306
307    Returns:
308      A TestResults object.
309    """
310    if self.test_package.rebaseline:
311      self.RebaselineTests()
312    else:
313      if not self._gtest_filter:
314        self._gtest_filter = ('-' + ':'.join(self.GetDisabledTests()) + ':' +
315                             ':'.join(['*.' + x + '*' for x in
316                                     self.test_package.GetDisabledPrefixes()]))
317      self.RunTestsWithFilter()
318    return self.test_results
319
320  def SetUp(self):
321    """Sets up necessary test enviroment for the test suite."""
322    super(SingleTestRunner, self).SetUp()
323    self.adb.ClearApplicationState(constants.CHROME_PACKAGE)
324    if self._performance_test_setup:
325      self._performance_test_setup.SetUp()
326    if self.dump_debug_info:
327      self.dump_debug_info.StartRecordingLog(True)
328    self.StripAndCopyFiles()
329    self.LaunchHelperToolsForTestSuite()
330    self.tool.SetupEnvironment()
331
332  def TearDown(self):
333    """Cleans up the test enviroment for the test suite."""
334    self.tool.CleanUpEnvironment()
335    if self.test_package.cleanup_test_files:
336      self.adb.RemovePushedFiles()
337    if self.dump_debug_info:
338      self.dump_debug_info.StopRecordingLog()
339    if self._performance_test_setup:
340      self._performance_test_setup.TearDown()
341    if self.dump_debug_info:
342      self.dump_debug_info.ArchiveNewCrashFiles()
343    super(SingleTestRunner, self).TearDown()
344