1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import contextlib
6import collections
7import itertools
8import logging
9import os
10import posixpath
11import subprocess
12import shutil
13import time
14
15from devil import base_error
16from devil.android import crash_handler
17from devil.android import device_errors
18from devil.android import device_temp_file
19from devil.android import logcat_monitor
20from devil.android import ports
21from devil.android.sdk import version_codes
22from devil.utils import reraiser_thread
23from incremental_install import installer
24from pylib import constants
25from pylib.base import base_test_result
26from pylib.gtest import gtest_test_instance
27from pylib.local import local_test_server_spawner
28from pylib.local.device import local_device_environment
29from pylib.local.device import local_device_test_run
30from pylib.utils import google_storage_helper
31from pylib.utils import logdog_helper
32from py_trace_event import trace_event
33from py_utils import contextlib_ext
34from py_utils import tempfile_ext
35import tombstones
36
37_MAX_INLINE_FLAGS_LENGTH = 50  # Arbitrarily chosen.
38_EXTRA_COMMAND_LINE_FILE = (
39    'org.chromium.native_test.NativeTest.CommandLineFile')
40_EXTRA_COMMAND_LINE_FLAGS = (
41    'org.chromium.native_test.NativeTest.CommandLineFlags')
42_EXTRA_COVERAGE_DEVICE_FILE = (
43    'org.chromium.native_test.NativeTest.CoverageDeviceFile')
44_EXTRA_STDOUT_FILE = (
45    'org.chromium.native_test.NativeTestInstrumentationTestRunner'
46        '.StdoutFile')
47_EXTRA_TEST = (
48    'org.chromium.native_test.NativeTestInstrumentationTestRunner'
49        '.Test')
50_EXTRA_TEST_LIST = (
51    'org.chromium.native_test.NativeTestInstrumentationTestRunner'
52        '.TestList')
53
54_SECONDS_TO_NANOS = int(1e9)
55
56# The amount of time a test executable may run before it gets killed.
57_TEST_TIMEOUT_SECONDS = 30*60
58
59# Tests that use SpawnedTestServer must run the LocalTestServerSpawner on the
60# host machine.
61# TODO(jbudorick): Move this up to the test instance if the net test server is
62# handled outside of the APK for the remote_device environment.
63_SUITE_REQUIRES_TEST_SERVER_SPAWNER = [
64  'components_browsertests', 'content_unittests', 'content_browsertests',
65  'net_unittests', 'services_unittests', 'unit_tests'
66]
67
68# No-op context manager. If we used Python 3, we could change this to
69# contextlib.ExitStack()
70class _NullContextManager(object):
71  def __enter__(self):
72    pass
73  def __exit__(self, *args):
74    pass
75
76
77def _GenerateSequentialFileNames(filename):
78  """Infinite generator of names: 'name.ext', 'name_1.ext', 'name_2.ext', ..."""
79  yield filename
80  base, ext = os.path.splitext(filename)
81  for i in itertools.count(1):
82    yield '%s_%d%s' % (base, i, ext)
83
84
85def _ExtractTestsFromFilter(gtest_filter):
86  """Returns the list of tests specified by the given filter.
87
88  Returns:
89    None if the device should be queried for the test list instead.
90  """
91  # Empty means all tests, - means exclude filter.
92  if not gtest_filter or '-' in gtest_filter:
93    return None
94
95  patterns = gtest_filter.split(':')
96  # For a single pattern, allow it even if it has a wildcard so long as the
97  # wildcard comes at the end and there is at least one . to prove the scope is
98  # not too large.
99  # This heuristic is not necessarily faster, but normally is.
100  if len(patterns) == 1 and patterns[0].endswith('*'):
101    no_suffix = patterns[0].rstrip('*')
102    if '*' not in no_suffix and '.' in no_suffix:
103      return patterns
104
105  if '*' in gtest_filter:
106    return None
107  return patterns
108
109
110def _PullCoverageFiles(device, device_coverage_dir, output_dir):
111  """Pulls coverage files on device to host directory.
112
113  Args:
114    device: The working device.
115    device_coverage_dir: The directory to store coverage data on device.
116    output_dir: The output directory on host.
117  """
118  try:
119    if not os.path.exists(output_dir):
120      os.makedirs(output_dir)
121    device.PullFile(device_coverage_dir, output_dir)
122    if not os.listdir(os.path.join(output_dir, 'profraw')):
123      logging.warning('No coverage data was generated for this run')
124  except (OSError, base_error.BaseError) as e:
125    logging.warning('Failed to handle coverage data after tests: %s', e)
126  finally:
127    device.RemovePath(device_coverage_dir, force=True, recursive=True)
128
129
130def _GetDeviceCoverageDir(device):
131  """Gets the directory to generate coverage data on device.
132
133  Args:
134    device: The working device.
135
136  Returns:
137    The directory path on the device.
138  """
139  return posixpath.join(device.GetExternalStoragePath(), 'chrome', 'test',
140                        'coverage', 'profraw')
141
142
143def _GetLLVMProfilePath(device_coverage_dir, suite, coverage_index):
144  """Gets 'LLVM_PROFILE_FILE' environment variable path.
145
146  Dumping data to ONLY 1 file may cause warning and data overwrite in
147  browsertests, so that pattern "%2m" is used to expand to 2 raw profiles
148  at runtime.
149
150  Args:
151    device_coverage_dir: The directory to generate data on device.
152    suite: Test suite name.
153    coverage_index: The incremental index for this test suite.
154
155  Returns:
156    The path pattern for environment variable 'LLVM_PROFILE_FILE'.
157  """
158  return posixpath.join(device_coverage_dir,
159                        '_'.join([suite,
160                                  str(coverage_index), '%2m.profraw']))
161
162
163class _ApkDelegate(object):
164  def __init__(self, test_instance, tool):
165    self._activity = test_instance.activity
166    self._apk_helper = test_instance.apk_helper
167    self._test_apk_incremental_install_json = (
168        test_instance.test_apk_incremental_install_json)
169    self._package = test_instance.package
170    self._runner = test_instance.runner
171    self._permissions = test_instance.permissions
172    self._suite = test_instance.suite
173    self._component = '%s/%s' % (self._package, self._runner)
174    self._extras = test_instance.extras
175    self._wait_for_java_debugger = test_instance.wait_for_java_debugger
176    self._tool = tool
177    self._coverage_dir = test_instance.coverage_dir
178    self._coverage_index = 0
179
180  def GetTestDataRoot(self, device):
181    # pylint: disable=no-self-use
182    return posixpath.join(device.GetExternalStoragePath(),
183                          'chromium_tests_root')
184
185  def Install(self, device):
186    if self._test_apk_incremental_install_json:
187      installer.Install(device, self._test_apk_incremental_install_json,
188                        apk=self._apk_helper, permissions=self._permissions)
189    else:
190      device.Install(
191          self._apk_helper,
192          allow_downgrade=True,
193          reinstall=True,
194          permissions=self._permissions)
195
196  def ResultsDirectory(self, device):
197    return device.GetApplicationDataDirectory(self._package)
198
199  def Run(self, test, device, flags=None, **kwargs):
200    extras = dict(self._extras)
201    device_api = device.build_version_sdk
202
203    if self._coverage_dir and device_api >= version_codes.LOLLIPOP:
204      device_coverage_dir = _GetDeviceCoverageDir(device)
205      extras[_EXTRA_COVERAGE_DEVICE_FILE] = _GetLLVMProfilePath(
206          device_coverage_dir, self._suite, self._coverage_index)
207      self._coverage_index += 1
208
209    if ('timeout' in kwargs
210        and gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT not in extras):
211      # Make sure the instrumentation doesn't kill the test before the
212      # scripts do. The provided timeout value is in seconds, but the
213      # instrumentation deals with nanoseconds because that's how Android
214      # handles time.
215      extras[gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT] = int(
216          kwargs['timeout'] * _SECONDS_TO_NANOS)
217
218    # pylint: disable=redefined-variable-type
219    command_line_file = _NullContextManager()
220    if flags:
221      if len(flags) > _MAX_INLINE_FLAGS_LENGTH:
222        command_line_file = device_temp_file.DeviceTempFile(device.adb)
223        device.WriteFile(command_line_file.name, '_ %s' % flags)
224        extras[_EXTRA_COMMAND_LINE_FILE] = command_line_file.name
225      else:
226        extras[_EXTRA_COMMAND_LINE_FLAGS] = flags
227
228    test_list_file = _NullContextManager()
229    if test:
230      if len(test) > 1:
231        test_list_file = device_temp_file.DeviceTempFile(device.adb)
232        device.WriteFile(test_list_file.name, '\n'.join(test))
233        extras[_EXTRA_TEST_LIST] = test_list_file.name
234      else:
235        extras[_EXTRA_TEST] = test[0]
236    # pylint: enable=redefined-variable-type
237
238    stdout_file = device_temp_file.DeviceTempFile(
239        device.adb, dir=device.GetExternalStoragePath(), suffix='.gtest_out')
240    extras[_EXTRA_STDOUT_FILE] = stdout_file.name
241
242    if self._wait_for_java_debugger:
243      cmd = ['am', 'set-debug-app', '-w', self._package]
244      device.RunShellCommand(cmd, check_return=True)
245      logging.warning('*' * 80)
246      logging.warning('Waiting for debugger to attach to process: %s',
247                      self._package)
248      logging.warning('*' * 80)
249
250    with command_line_file, test_list_file, stdout_file:
251      try:
252        device.StartInstrumentation(
253            self._component, extras=extras, raw=False, **kwargs)
254      except device_errors.CommandFailedError:
255        logging.exception('gtest shard failed.')
256      except device_errors.CommandTimeoutError:
257        logging.exception('gtest shard timed out.')
258      except device_errors.DeviceUnreachableError:
259        logging.exception('gtest shard device unreachable.')
260      except Exception:
261        device.ForceStop(self._package)
262        raise
263      finally:
264        if self._coverage_dir and device_api >= version_codes.LOLLIPOP:
265          _PullCoverageFiles(
266              device, device_coverage_dir,
267              os.path.join(self._coverage_dir, str(self._coverage_index)))
268
269      return device.ReadFile(stdout_file.name).splitlines()
270
271  def PullAppFiles(self, device, files, directory):
272    device_dir = device.GetApplicationDataDirectory(self._package)
273    host_dir = os.path.join(directory, str(device))
274    for f in files:
275      device_file = posixpath.join(device_dir, f)
276      host_file = os.path.join(host_dir, *f.split(posixpath.sep))
277      for host_file in _GenerateSequentialFileNames(host_file):
278        if not os.path.exists(host_file):
279          break
280      device.PullFile(device_file, host_file)
281
282  def Clear(self, device):
283    device.ClearApplicationState(self._package, permissions=self._permissions)
284
285
286class _ExeDelegate(object):
287
288  def __init__(self, tr, test_instance, tool):
289    self._host_dist_dir = test_instance.exe_dist_dir
290    self._exe_file_name = os.path.basename(
291        test_instance.exe_dist_dir)[:-len('__dist')]
292    self._device_dist_dir = posixpath.join(
293        constants.TEST_EXECUTABLE_DIR,
294        os.path.basename(test_instance.exe_dist_dir))
295    self._test_run = tr
296    self._tool = tool
297    self._suite = test_instance.suite
298    self._coverage_dir = test_instance.coverage_dir
299    self._coverage_index = 0
300
301  def GetTestDataRoot(self, device):
302    # pylint: disable=no-self-use
303    # pylint: disable=unused-argument
304    return posixpath.join(constants.TEST_EXECUTABLE_DIR, 'chromium_tests_root')
305
306  def Install(self, device):
307    # TODO(jbudorick): Look into merging this with normal data deps pushing if
308    # executables become supported on nonlocal environments.
309    device.PushChangedFiles([(self._host_dist_dir, self._device_dist_dir)],
310                            delete_device_stale=True)
311
312  def ResultsDirectory(self, device):
313    # pylint: disable=no-self-use
314    # pylint: disable=unused-argument
315    return constants.TEST_EXECUTABLE_DIR
316
317  def Run(self, test, device, flags=None, **kwargs):
318    tool = self._test_run.GetTool(device).GetTestWrapper()
319    if tool:
320      cmd = [tool]
321    else:
322      cmd = []
323    cmd.append(posixpath.join(self._device_dist_dir, self._exe_file_name))
324
325    if test:
326      cmd.append('--gtest_filter=%s' % ':'.join(test))
327    if flags:
328      # TODO(agrieve): This won't work if multiple flags are passed.
329      cmd.append(flags)
330    cwd = constants.TEST_EXECUTABLE_DIR
331
332    env = {
333      'LD_LIBRARY_PATH': self._device_dist_dir
334    }
335
336    if self._coverage_dir:
337      device_coverage_dir = _GetDeviceCoverageDir(device)
338      env['LLVM_PROFILE_FILE'] = _GetLLVMProfilePath(
339          device_coverage_dir, self._suite, self._coverage_index)
340      self._coverage_index += 1
341
342    if self._tool != 'asan':
343      env['UBSAN_OPTIONS'] = constants.UBSAN_OPTIONS
344
345    try:
346      gcov_strip_depth = os.environ['NATIVE_COVERAGE_DEPTH_STRIP']
347      external = device.GetExternalStoragePath()
348      env['GCOV_PREFIX'] = '%s/gcov' % external
349      env['GCOV_PREFIX_STRIP'] = gcov_strip_depth
350    except (device_errors.CommandFailedError, KeyError):
351      pass
352
353    # Executable tests return a nonzero exit code on test failure, which is
354    # fine from the test runner's perspective; thus check_return=False.
355    output = device.RunShellCommand(
356        cmd, cwd=cwd, env=env, check_return=False, large_output=True, **kwargs)
357
358    if self._coverage_dir:
359      _PullCoverageFiles(
360          device, device_coverage_dir,
361          os.path.join(self._coverage_dir, str(self._coverage_index)))
362
363    return output
364
365  def PullAppFiles(self, device, files, directory):
366    pass
367
368  def Clear(self, device):
369    device.KillAll(self._exe_file_name, blocking=True, timeout=30, quiet=True)
370
371
372class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun):
373
374  def __init__(self, env, test_instance):
375    assert isinstance(env, local_device_environment.LocalDeviceEnvironment)
376    assert isinstance(test_instance, gtest_test_instance.GtestTestInstance)
377    super(LocalDeviceGtestRun, self).__init__(env, test_instance)
378
379    # pylint: disable=redefined-variable-type
380    if self._test_instance.apk:
381      self._delegate = _ApkDelegate(self._test_instance, env.tool)
382    elif self._test_instance.exe_dist_dir:
383      self._delegate = _ExeDelegate(self, self._test_instance, self._env.tool)
384    if self._test_instance.isolated_script_test_perf_output:
385      self._test_perf_output_filenames = _GenerateSequentialFileNames(
386          self._test_instance.isolated_script_test_perf_output)
387    else:
388      self._test_perf_output_filenames = itertools.repeat(None)
389    # pylint: enable=redefined-variable-type
390    self._crashes = set()
391    self._servers = collections.defaultdict(list)
392
393  #override
394  def TestPackage(self):
395    return self._test_instance.suite
396
397  #override
398  def SetUp(self):
399    @local_device_environment.handle_shard_failures_with(
400        on_failure=self._env.DenylistDevice)
401    @trace_event.traced
402    def individual_device_set_up(device, host_device_tuples):
403      def install_apk(dev):
404        # Install test APK.
405        self._delegate.Install(dev)
406
407      def push_test_data(dev):
408        # Push data dependencies.
409        device_root = self._delegate.GetTestDataRoot(dev)
410        host_device_tuples_substituted = [
411            (h, local_device_test_run.SubstituteDeviceRoot(d, device_root))
412            for h, d in host_device_tuples]
413        local_device_environment.place_nomedia_on_device(dev, device_root)
414        dev.PushChangedFiles(
415            host_device_tuples_substituted,
416            delete_device_stale=True,
417            # Some gtest suites, e.g. unit_tests, have data dependencies that
418            # can take longer than the default timeout to push. See
419            # crbug.com/791632 for context.
420            timeout=600)
421        if not host_device_tuples:
422          dev.RemovePath(device_root, force=True, recursive=True, rename=True)
423          dev.RunShellCommand(['mkdir', '-p', device_root], check_return=True)
424
425      def init_tool_and_start_servers(dev):
426        tool = self.GetTool(dev)
427        tool.CopyFiles(dev)
428        tool.SetupEnvironment()
429
430        try:
431          # See https://crbug.com/1030827.
432          # This is a hack that may break in the future. We're relying on the
433          # fact that adb doesn't use ipv6 for it's server, and so doesn't
434          # listen on ipv6, but ssh remote forwarding does. 5037 is the port
435          # number adb uses for its server.
436          if "[::1]:5037" in subprocess.check_output(
437              "ss -o state listening 'sport = 5037'", shell=True):
438            logging.error(
439                'Test Server cannot be started with a remote-forwarded adb '
440                'server. Continuing anyways, but some tests may fail.')
441            return
442        except subprocess.CalledProcessError:
443          pass
444
445        self._servers[str(dev)] = []
446        if self.TestPackage() in _SUITE_REQUIRES_TEST_SERVER_SPAWNER:
447          self._servers[str(dev)].append(
448              local_test_server_spawner.LocalTestServerSpawner(
449                  ports.AllocateTestServerPort(), dev, tool))
450
451        for s in self._servers[str(dev)]:
452          s.SetUp()
453
454      def bind_crash_handler(step, dev):
455        return lambda: crash_handler.RetryOnSystemCrash(step, dev)
456
457      steps = [
458          bind_crash_handler(s, device)
459          for s in (install_apk, push_test_data, init_tool_and_start_servers)]
460      if self._env.concurrent_adb:
461        reraiser_thread.RunAsync(steps)
462      else:
463        for step in steps:
464          step()
465
466    self._env.parallel_devices.pMap(
467        individual_device_set_up,
468        self._test_instance.GetDataDependencies())
469
470  #override
471  def _ShouldShard(self):
472    return True
473
474  #override
475  def _CreateShards(self, tests):
476    # _crashes are tests that might crash and make the tests in the same shard
477    # following the crashed testcase not run.
478    # Thus we need to create separate shards for each crashed testcase,
479    # so that other tests can be run.
480    device_count = len(self._env.devices)
481    shards = []
482
483    # Add shards with only one suspect testcase.
484    shards += [[crash] for crash in self._crashes if crash in tests]
485
486    # Delete suspect testcase from tests.
487    tests = [test for test in tests if not test in self._crashes]
488
489    batch_size = self._test_instance.test_launcher_batch_limit
490
491    for i in xrange(0, device_count):
492      unbounded_shard = tests[i::device_count]
493      shards += [
494          unbounded_shard[j:j + batch_size]
495          for j in xrange(0, len(unbounded_shard), batch_size)
496      ]
497    return shards
498
499  #override
500  def _GetTests(self):
501    if self._test_instance.extract_test_list_from_filter:
502      # When the exact list of tests to run is given via command-line (e.g. when
503      # locally iterating on a specific test), skip querying the device (which
504      # takes ~3 seconds).
505      tests = _ExtractTestsFromFilter(self._test_instance.gtest_filter)
506      if tests:
507        return tests
508
509    # Even when there's only one device, it still makes sense to retrieve the
510    # test list so that tests can be split up and run in batches rather than all
511    # at once (since test output is not streamed).
512    @local_device_environment.handle_shard_failures_with(
513        on_failure=self._env.DenylistDevice)
514    def list_tests(dev):
515      timeout = 30
516      retries = 1
517      if self._test_instance.wait_for_java_debugger:
518        timeout = None
519
520      flags = [
521          f for f in self._test_instance.flags
522          if f not in ['--wait-for-debugger', '--wait-for-java-debugger']
523      ]
524      flags.append('--gtest_list_tests')
525
526      # TODO(crbug.com/726880): Remove retries when no longer necessary.
527      for i in range(0, retries+1):
528        logging.info('flags:')
529        for f in flags:
530          logging.info('  %s', f)
531
532        with self._ArchiveLogcat(dev, 'list_tests'):
533          raw_test_list = crash_handler.RetryOnSystemCrash(
534              lambda d: self._delegate.Run(
535                  None, d, flags=' '.join(flags), timeout=timeout),
536              device=dev)
537
538        tests = gtest_test_instance.ParseGTestListTests(raw_test_list)
539        if not tests:
540          logging.info('No tests found. Output:')
541          for l in raw_test_list:
542            logging.info('  %s', l)
543          if i < retries:
544            logging.info('Retrying...')
545        else:
546          break
547      return tests
548
549    # Query all devices in case one fails.
550    test_lists = self._env.parallel_devices.pMap(list_tests).pGet(None)
551
552    # If all devices failed to list tests, raise an exception.
553    # Check that tl is not None and is not empty.
554    if all(not tl for tl in test_lists):
555      raise device_errors.CommandFailedError(
556          'Failed to list tests on any device')
557    tests = list(sorted(set().union(*[set(tl) for tl in test_lists if tl])))
558    tests = self._test_instance.FilterTests(tests)
559    tests = self._ApplyExternalSharding(
560        tests, self._test_instance.external_shard_index,
561        self._test_instance.total_external_shards)
562    return tests
563
564  def _UploadTestArtifacts(self, device, test_artifacts_dir):
565    # TODO(jbudorick): Reconcile this with the output manager once
566    # https://codereview.chromium.org/2933993002/ lands.
567    if test_artifacts_dir:
568      with tempfile_ext.NamedTemporaryDirectory() as test_artifacts_host_dir:
569        device.PullFile(test_artifacts_dir.name, test_artifacts_host_dir)
570        with tempfile_ext.NamedTemporaryDirectory() as temp_zip_dir:
571          zip_base_name = os.path.join(temp_zip_dir, 'test_artifacts')
572          test_artifacts_zip = shutil.make_archive(
573              zip_base_name, 'zip', test_artifacts_host_dir)
574          link = google_storage_helper.upload(
575              google_storage_helper.unique_name(
576                  'test_artifacts', device=device),
577              test_artifacts_zip,
578              bucket='%s/test_artifacts' % (
579                  self._test_instance.gs_test_artifacts_bucket))
580          logging.info('Uploading test artifacts to %s.', link)
581          return link
582    return None
583
584  @contextlib.contextmanager
585  def _ArchiveLogcat(self, device, test):
586    if isinstance(test, str):
587      desc = test
588    else:
589      desc = hash(tuple(test))
590
591    stream_name = 'logcat_%s_%s_%s' % (
592        desc, time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial)
593
594    logcat_file = None
595    logmon = None
596    try:
597      with self._env.output_manager.ArchivedTempfile(stream_name,
598                                                     'logcat') as logcat_file:
599        with logcat_monitor.LogcatMonitor(
600            device.adb,
601            filter_specs=local_device_environment.LOGCAT_FILTERS,
602            output_file=logcat_file.name,
603            check_error=False) as logmon:
604          with contextlib_ext.Optional(trace_event.trace(str(test)),
605                                       self._env.trace_output):
606            yield logcat_file
607    finally:
608      if logmon:
609        logmon.Close()
610      if logcat_file and logcat_file.Link():
611        logging.info('Logcat saved to %s', logcat_file.Link())
612
613  #override
614  def _RunTest(self, device, test):
615    # Run the test.
616    timeout = (self._test_instance.shard_timeout
617               * self.GetTool(device).GetTimeoutScale())
618    if self._test_instance.wait_for_java_debugger:
619      timeout = None
620    if self._test_instance.store_tombstones:
621      tombstones.ClearAllTombstones(device)
622    test_perf_output_filename = next(self._test_perf_output_filenames)
623
624    if self._test_instance.isolated_script_test_output:
625      suffix = '.json'
626    else:
627      suffix = '.xml'
628
629    with device_temp_file.DeviceTempFile(
630        adb=device.adb,
631        dir=self._delegate.ResultsDirectory(device),
632        suffix=suffix) as device_tmp_results_file:
633      with contextlib_ext.Optional(
634          device_temp_file.NamedDeviceTemporaryDirectory(
635              adb=device.adb, dir='/sdcard/'),
636          self._test_instance.gs_test_artifacts_bucket) as test_artifacts_dir:
637        with (contextlib_ext.Optional(
638            device_temp_file.DeviceTempFile(
639                adb=device.adb, dir=self._delegate.ResultsDirectory(device)),
640            test_perf_output_filename)) as isolated_script_test_perf_output:
641
642          flags = list(self._test_instance.flags)
643          if self._test_instance.enable_xml_result_parsing:
644            flags.append('--gtest_output=xml:%s' % device_tmp_results_file.name)
645
646          if self._test_instance.gs_test_artifacts_bucket:
647            flags.append('--test_artifacts_dir=%s' % test_artifacts_dir.name)
648
649          if self._test_instance.isolated_script_test_output:
650            flags.append('--isolated-script-test-output=%s' %
651                         device_tmp_results_file.name)
652
653          if test_perf_output_filename:
654            flags.append('--isolated_script_test_perf_output=%s' %
655                         isolated_script_test_perf_output.name)
656
657          logging.info('flags:')
658          for f in flags:
659            logging.info('  %s', f)
660
661          with self._ArchiveLogcat(device, test) as logcat_file:
662            output = self._delegate.Run(test,
663                                        device,
664                                        flags=' '.join(flags),
665                                        timeout=timeout,
666                                        retries=0)
667
668          if self._test_instance.enable_xml_result_parsing:
669            try:
670              gtest_xml = device.ReadFile(device_tmp_results_file.name)
671            except device_errors.CommandFailedError:
672              logging.exception('Failed to pull gtest results XML file %s',
673                                device_tmp_results_file.name)
674              gtest_xml = None
675
676          if self._test_instance.isolated_script_test_output:
677            try:
678              gtest_json = device.ReadFile(device_tmp_results_file.name)
679            except device_errors.CommandFailedError:
680              logging.exception('Failed to pull gtest results JSON file %s',
681                                device_tmp_results_file.name)
682              gtest_json = None
683
684          if test_perf_output_filename:
685            try:
686              device.PullFile(isolated_script_test_perf_output.name,
687                              test_perf_output_filename)
688            except device_errors.CommandFailedError:
689              logging.exception('Failed to pull chartjson results %s',
690                                isolated_script_test_perf_output.name)
691
692          test_artifacts_url = self._UploadTestArtifacts(device,
693                                                         test_artifacts_dir)
694
695    for s in self._servers[str(device)]:
696      s.Reset()
697    if self._test_instance.app_files:
698      self._delegate.PullAppFiles(device, self._test_instance.app_files,
699                                  self._test_instance.app_file_dir)
700    if not self._env.skip_clear_data:
701      self._delegate.Clear(device)
702
703    for l in output:
704      logging.info(l)
705
706    # Parse the output.
707    # TODO(jbudorick): Transition test scripts away from parsing stdout.
708    if self._test_instance.enable_xml_result_parsing:
709      results = gtest_test_instance.ParseGTestXML(gtest_xml)
710    elif self._test_instance.isolated_script_test_output:
711      results = gtest_test_instance.ParseGTestJSON(gtest_json)
712    else:
713      results = gtest_test_instance.ParseGTestOutput(
714          output, self._test_instance.symbolizer, device.product_cpu_abi)
715
716    tombstones_url = None
717    for r in results:
718      if logcat_file:
719        r.SetLink('logcat', logcat_file.Link())
720
721      if self._test_instance.gs_test_artifacts_bucket:
722        r.SetLink('test_artifacts', test_artifacts_url)
723
724      if r.GetType() == base_test_result.ResultType.CRASH:
725        self._crashes.add(r.GetName())
726        if self._test_instance.store_tombstones:
727          if not tombstones_url:
728            resolved_tombstones = tombstones.ResolveTombstones(
729                device,
730                resolve_all_tombstones=True,
731                include_stack_symbols=False,
732                wipe_tombstones=True)
733            stream_name = 'tombstones_%s_%s' % (
734                time.strftime('%Y%m%dT%H%M%S', time.localtime()),
735                device.serial)
736            tombstones_url = logdog_helper.text(
737                stream_name, '\n'.join(resolved_tombstones))
738          r.SetLink('tombstones', tombstones_url)
739
740    tests_stripped_disabled_prefix = set()
741    for t in test:
742      tests_stripped_disabled_prefix.add(
743          gtest_test_instance.TestNameWithoutDisabledPrefix(t))
744    not_run_tests = tests_stripped_disabled_prefix.difference(
745        set(r.GetName() for r in results))
746    return results, list(not_run_tests) if results else None
747
748  #override
749  def TearDown(self):
750    # By default, teardown will invoke ADB. When receiving SIGTERM due to a
751    # timeout, there's a high probability that ADB is non-responsive. In these
752    # cases, sending an ADB command will potentially take a long time to time
753    # out. Before this happens, the process will be hard-killed for not
754    # responding to SIGTERM fast enough.
755    if self._received_sigterm:
756      return
757
758    @local_device_environment.handle_shard_failures
759    @trace_event.traced
760    def individual_device_tear_down(dev):
761      for s in self._servers.get(str(dev), []):
762        s.TearDown()
763
764      tool = self.GetTool(dev)
765      tool.CleanUpEnvironment()
766
767    self._env.parallel_devices.pMap(individual_device_tear_down)
768