1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import contextlib 6import collections 7import itertools 8import logging 9import os 10import posixpath 11import subprocess 12import shutil 13import time 14 15from devil import base_error 16from devil.android import crash_handler 17from devil.android import device_errors 18from devil.android import device_temp_file 19from devil.android import logcat_monitor 20from devil.android import ports 21from devil.android.sdk import version_codes 22from devil.utils import reraiser_thread 23from incremental_install import installer 24from pylib import constants 25from pylib.base import base_test_result 26from pylib.gtest import gtest_test_instance 27from pylib.local import local_test_server_spawner 28from pylib.local.device import local_device_environment 29from pylib.local.device import local_device_test_run 30from pylib.utils import google_storage_helper 31from pylib.utils import logdog_helper 32from py_trace_event import trace_event 33from py_utils import contextlib_ext 34from py_utils import tempfile_ext 35import tombstones 36 37_MAX_INLINE_FLAGS_LENGTH = 50 # Arbitrarily chosen. 38_EXTRA_COMMAND_LINE_FILE = ( 39 'org.chromium.native_test.NativeTest.CommandLineFile') 40_EXTRA_COMMAND_LINE_FLAGS = ( 41 'org.chromium.native_test.NativeTest.CommandLineFlags') 42_EXTRA_COVERAGE_DEVICE_FILE = ( 43 'org.chromium.native_test.NativeTest.CoverageDeviceFile') 44_EXTRA_STDOUT_FILE = ( 45 'org.chromium.native_test.NativeTestInstrumentationTestRunner' 46 '.StdoutFile') 47_EXTRA_TEST = ( 48 'org.chromium.native_test.NativeTestInstrumentationTestRunner' 49 '.Test') 50_EXTRA_TEST_LIST = ( 51 'org.chromium.native_test.NativeTestInstrumentationTestRunner' 52 '.TestList') 53 54_SECONDS_TO_NANOS = int(1e9) 55 56# The amount of time a test executable may run before it gets killed. 57_TEST_TIMEOUT_SECONDS = 30*60 58 59# Tests that use SpawnedTestServer must run the LocalTestServerSpawner on the 60# host machine. 61# TODO(jbudorick): Move this up to the test instance if the net test server is 62# handled outside of the APK for the remote_device environment. 63_SUITE_REQUIRES_TEST_SERVER_SPAWNER = [ 64 'components_browsertests', 'content_unittests', 'content_browsertests', 65 'net_unittests', 'services_unittests', 'unit_tests' 66] 67 68# No-op context manager. If we used Python 3, we could change this to 69# contextlib.ExitStack() 70class _NullContextManager(object): 71 def __enter__(self): 72 pass 73 def __exit__(self, *args): 74 pass 75 76 77def _GenerateSequentialFileNames(filename): 78 """Infinite generator of names: 'name.ext', 'name_1.ext', 'name_2.ext', ...""" 79 yield filename 80 base, ext = os.path.splitext(filename) 81 for i in itertools.count(1): 82 yield '%s_%d%s' % (base, i, ext) 83 84 85def _ExtractTestsFromFilter(gtest_filter): 86 """Returns the list of tests specified by the given filter. 87 88 Returns: 89 None if the device should be queried for the test list instead. 90 """ 91 # Empty means all tests, - means exclude filter. 92 if not gtest_filter or '-' in gtest_filter: 93 return None 94 95 patterns = gtest_filter.split(':') 96 # For a single pattern, allow it even if it has a wildcard so long as the 97 # wildcard comes at the end and there is at least one . to prove the scope is 98 # not too large. 99 # This heuristic is not necessarily faster, but normally is. 100 if len(patterns) == 1 and patterns[0].endswith('*'): 101 no_suffix = patterns[0].rstrip('*') 102 if '*' not in no_suffix and '.' in no_suffix: 103 return patterns 104 105 if '*' in gtest_filter: 106 return None 107 return patterns 108 109 110def _PullCoverageFiles(device, device_coverage_dir, output_dir): 111 """Pulls coverage files on device to host directory. 112 113 Args: 114 device: The working device. 115 device_coverage_dir: The directory to store coverage data on device. 116 output_dir: The output directory on host. 117 """ 118 try: 119 if not os.path.exists(output_dir): 120 os.makedirs(output_dir) 121 device.PullFile(device_coverage_dir, output_dir) 122 if not os.listdir(os.path.join(output_dir, 'profraw')): 123 logging.warning('No coverage data was generated for this run') 124 except (OSError, base_error.BaseError) as e: 125 logging.warning('Failed to handle coverage data after tests: %s', e) 126 finally: 127 device.RemovePath(device_coverage_dir, force=True, recursive=True) 128 129 130def _GetDeviceCoverageDir(device): 131 """Gets the directory to generate coverage data on device. 132 133 Args: 134 device: The working device. 135 136 Returns: 137 The directory path on the device. 138 """ 139 return posixpath.join(device.GetExternalStoragePath(), 'chrome', 'test', 140 'coverage', 'profraw') 141 142 143def _GetLLVMProfilePath(device_coverage_dir, suite, coverage_index): 144 """Gets 'LLVM_PROFILE_FILE' environment variable path. 145 146 Dumping data to ONLY 1 file may cause warning and data overwrite in 147 browsertests, so that pattern "%2m" is used to expand to 2 raw profiles 148 at runtime. 149 150 Args: 151 device_coverage_dir: The directory to generate data on device. 152 suite: Test suite name. 153 coverage_index: The incremental index for this test suite. 154 155 Returns: 156 The path pattern for environment variable 'LLVM_PROFILE_FILE'. 157 """ 158 return posixpath.join(device_coverage_dir, 159 '_'.join([suite, 160 str(coverage_index), '%2m.profraw'])) 161 162 163class _ApkDelegate(object): 164 def __init__(self, test_instance, tool): 165 self._activity = test_instance.activity 166 self._apk_helper = test_instance.apk_helper 167 self._test_apk_incremental_install_json = ( 168 test_instance.test_apk_incremental_install_json) 169 self._package = test_instance.package 170 self._runner = test_instance.runner 171 self._permissions = test_instance.permissions 172 self._suite = test_instance.suite 173 self._component = '%s/%s' % (self._package, self._runner) 174 self._extras = test_instance.extras 175 self._wait_for_java_debugger = test_instance.wait_for_java_debugger 176 self._tool = tool 177 self._coverage_dir = test_instance.coverage_dir 178 self._coverage_index = 0 179 180 def GetTestDataRoot(self, device): 181 # pylint: disable=no-self-use 182 return posixpath.join(device.GetExternalStoragePath(), 183 'chromium_tests_root') 184 185 def Install(self, device): 186 if self._test_apk_incremental_install_json: 187 installer.Install(device, self._test_apk_incremental_install_json, 188 apk=self._apk_helper, permissions=self._permissions) 189 else: 190 device.Install( 191 self._apk_helper, 192 allow_downgrade=True, 193 reinstall=True, 194 permissions=self._permissions) 195 196 def ResultsDirectory(self, device): 197 return device.GetApplicationDataDirectory(self._package) 198 199 def Run(self, test, device, flags=None, **kwargs): 200 extras = dict(self._extras) 201 device_api = device.build_version_sdk 202 203 if self._coverage_dir and device_api >= version_codes.LOLLIPOP: 204 device_coverage_dir = _GetDeviceCoverageDir(device) 205 extras[_EXTRA_COVERAGE_DEVICE_FILE] = _GetLLVMProfilePath( 206 device_coverage_dir, self._suite, self._coverage_index) 207 self._coverage_index += 1 208 209 if ('timeout' in kwargs 210 and gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT not in extras): 211 # Make sure the instrumentation doesn't kill the test before the 212 # scripts do. The provided timeout value is in seconds, but the 213 # instrumentation deals with nanoseconds because that's how Android 214 # handles time. 215 extras[gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT] = int( 216 kwargs['timeout'] * _SECONDS_TO_NANOS) 217 218 # pylint: disable=redefined-variable-type 219 command_line_file = _NullContextManager() 220 if flags: 221 if len(flags) > _MAX_INLINE_FLAGS_LENGTH: 222 command_line_file = device_temp_file.DeviceTempFile(device.adb) 223 device.WriteFile(command_line_file.name, '_ %s' % flags) 224 extras[_EXTRA_COMMAND_LINE_FILE] = command_line_file.name 225 else: 226 extras[_EXTRA_COMMAND_LINE_FLAGS] = flags 227 228 test_list_file = _NullContextManager() 229 if test: 230 if len(test) > 1: 231 test_list_file = device_temp_file.DeviceTempFile(device.adb) 232 device.WriteFile(test_list_file.name, '\n'.join(test)) 233 extras[_EXTRA_TEST_LIST] = test_list_file.name 234 else: 235 extras[_EXTRA_TEST] = test[0] 236 # pylint: enable=redefined-variable-type 237 238 stdout_file = device_temp_file.DeviceTempFile( 239 device.adb, dir=device.GetExternalStoragePath(), suffix='.gtest_out') 240 extras[_EXTRA_STDOUT_FILE] = stdout_file.name 241 242 if self._wait_for_java_debugger: 243 cmd = ['am', 'set-debug-app', '-w', self._package] 244 device.RunShellCommand(cmd, check_return=True) 245 logging.warning('*' * 80) 246 logging.warning('Waiting for debugger to attach to process: %s', 247 self._package) 248 logging.warning('*' * 80) 249 250 with command_line_file, test_list_file, stdout_file: 251 try: 252 device.StartInstrumentation( 253 self._component, extras=extras, raw=False, **kwargs) 254 except device_errors.CommandFailedError: 255 logging.exception('gtest shard failed.') 256 except device_errors.CommandTimeoutError: 257 logging.exception('gtest shard timed out.') 258 except device_errors.DeviceUnreachableError: 259 logging.exception('gtest shard device unreachable.') 260 except Exception: 261 device.ForceStop(self._package) 262 raise 263 finally: 264 if self._coverage_dir and device_api >= version_codes.LOLLIPOP: 265 _PullCoverageFiles( 266 device, device_coverage_dir, 267 os.path.join(self._coverage_dir, str(self._coverage_index))) 268 269 return device.ReadFile(stdout_file.name).splitlines() 270 271 def PullAppFiles(self, device, files, directory): 272 device_dir = device.GetApplicationDataDirectory(self._package) 273 host_dir = os.path.join(directory, str(device)) 274 for f in files: 275 device_file = posixpath.join(device_dir, f) 276 host_file = os.path.join(host_dir, *f.split(posixpath.sep)) 277 for host_file in _GenerateSequentialFileNames(host_file): 278 if not os.path.exists(host_file): 279 break 280 device.PullFile(device_file, host_file) 281 282 def Clear(self, device): 283 device.ClearApplicationState(self._package, permissions=self._permissions) 284 285 286class _ExeDelegate(object): 287 288 def __init__(self, tr, test_instance, tool): 289 self._host_dist_dir = test_instance.exe_dist_dir 290 self._exe_file_name = os.path.basename( 291 test_instance.exe_dist_dir)[:-len('__dist')] 292 self._device_dist_dir = posixpath.join( 293 constants.TEST_EXECUTABLE_DIR, 294 os.path.basename(test_instance.exe_dist_dir)) 295 self._test_run = tr 296 self._tool = tool 297 self._suite = test_instance.suite 298 self._coverage_dir = test_instance.coverage_dir 299 self._coverage_index = 0 300 301 def GetTestDataRoot(self, device): 302 # pylint: disable=no-self-use 303 # pylint: disable=unused-argument 304 return posixpath.join(constants.TEST_EXECUTABLE_DIR, 'chromium_tests_root') 305 306 def Install(self, device): 307 # TODO(jbudorick): Look into merging this with normal data deps pushing if 308 # executables become supported on nonlocal environments. 309 device.PushChangedFiles([(self._host_dist_dir, self._device_dist_dir)], 310 delete_device_stale=True) 311 312 def ResultsDirectory(self, device): 313 # pylint: disable=no-self-use 314 # pylint: disable=unused-argument 315 return constants.TEST_EXECUTABLE_DIR 316 317 def Run(self, test, device, flags=None, **kwargs): 318 tool = self._test_run.GetTool(device).GetTestWrapper() 319 if tool: 320 cmd = [tool] 321 else: 322 cmd = [] 323 cmd.append(posixpath.join(self._device_dist_dir, self._exe_file_name)) 324 325 if test: 326 cmd.append('--gtest_filter=%s' % ':'.join(test)) 327 if flags: 328 # TODO(agrieve): This won't work if multiple flags are passed. 329 cmd.append(flags) 330 cwd = constants.TEST_EXECUTABLE_DIR 331 332 env = { 333 'LD_LIBRARY_PATH': self._device_dist_dir 334 } 335 336 if self._coverage_dir: 337 device_coverage_dir = _GetDeviceCoverageDir(device) 338 env['LLVM_PROFILE_FILE'] = _GetLLVMProfilePath( 339 device_coverage_dir, self._suite, self._coverage_index) 340 self._coverage_index += 1 341 342 if self._tool != 'asan': 343 env['UBSAN_OPTIONS'] = constants.UBSAN_OPTIONS 344 345 try: 346 gcov_strip_depth = os.environ['NATIVE_COVERAGE_DEPTH_STRIP'] 347 external = device.GetExternalStoragePath() 348 env['GCOV_PREFIX'] = '%s/gcov' % external 349 env['GCOV_PREFIX_STRIP'] = gcov_strip_depth 350 except (device_errors.CommandFailedError, KeyError): 351 pass 352 353 # Executable tests return a nonzero exit code on test failure, which is 354 # fine from the test runner's perspective; thus check_return=False. 355 output = device.RunShellCommand( 356 cmd, cwd=cwd, env=env, check_return=False, large_output=True, **kwargs) 357 358 if self._coverage_dir: 359 _PullCoverageFiles( 360 device, device_coverage_dir, 361 os.path.join(self._coverage_dir, str(self._coverage_index))) 362 363 return output 364 365 def PullAppFiles(self, device, files, directory): 366 pass 367 368 def Clear(self, device): 369 device.KillAll(self._exe_file_name, blocking=True, timeout=30, quiet=True) 370 371 372class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun): 373 374 def __init__(self, env, test_instance): 375 assert isinstance(env, local_device_environment.LocalDeviceEnvironment) 376 assert isinstance(test_instance, gtest_test_instance.GtestTestInstance) 377 super(LocalDeviceGtestRun, self).__init__(env, test_instance) 378 379 # pylint: disable=redefined-variable-type 380 if self._test_instance.apk: 381 self._delegate = _ApkDelegate(self._test_instance, env.tool) 382 elif self._test_instance.exe_dist_dir: 383 self._delegate = _ExeDelegate(self, self._test_instance, self._env.tool) 384 if self._test_instance.isolated_script_test_perf_output: 385 self._test_perf_output_filenames = _GenerateSequentialFileNames( 386 self._test_instance.isolated_script_test_perf_output) 387 else: 388 self._test_perf_output_filenames = itertools.repeat(None) 389 # pylint: enable=redefined-variable-type 390 self._crashes = set() 391 self._servers = collections.defaultdict(list) 392 393 #override 394 def TestPackage(self): 395 return self._test_instance.suite 396 397 #override 398 def SetUp(self): 399 @local_device_environment.handle_shard_failures_with( 400 on_failure=self._env.DenylistDevice) 401 @trace_event.traced 402 def individual_device_set_up(device, host_device_tuples): 403 def install_apk(dev): 404 # Install test APK. 405 self._delegate.Install(dev) 406 407 def push_test_data(dev): 408 # Push data dependencies. 409 device_root = self._delegate.GetTestDataRoot(dev) 410 host_device_tuples_substituted = [ 411 (h, local_device_test_run.SubstituteDeviceRoot(d, device_root)) 412 for h, d in host_device_tuples] 413 local_device_environment.place_nomedia_on_device(dev, device_root) 414 dev.PushChangedFiles( 415 host_device_tuples_substituted, 416 delete_device_stale=True, 417 # Some gtest suites, e.g. unit_tests, have data dependencies that 418 # can take longer than the default timeout to push. See 419 # crbug.com/791632 for context. 420 timeout=600) 421 if not host_device_tuples: 422 dev.RemovePath(device_root, force=True, recursive=True, rename=True) 423 dev.RunShellCommand(['mkdir', '-p', device_root], check_return=True) 424 425 def init_tool_and_start_servers(dev): 426 tool = self.GetTool(dev) 427 tool.CopyFiles(dev) 428 tool.SetupEnvironment() 429 430 try: 431 # See https://crbug.com/1030827. 432 # This is a hack that may break in the future. We're relying on the 433 # fact that adb doesn't use ipv6 for it's server, and so doesn't 434 # listen on ipv6, but ssh remote forwarding does. 5037 is the port 435 # number adb uses for its server. 436 if "[::1]:5037" in subprocess.check_output( 437 "ss -o state listening 'sport = 5037'", shell=True): 438 logging.error( 439 'Test Server cannot be started with a remote-forwarded adb ' 440 'server. Continuing anyways, but some tests may fail.') 441 return 442 except subprocess.CalledProcessError: 443 pass 444 445 self._servers[str(dev)] = [] 446 if self.TestPackage() in _SUITE_REQUIRES_TEST_SERVER_SPAWNER: 447 self._servers[str(dev)].append( 448 local_test_server_spawner.LocalTestServerSpawner( 449 ports.AllocateTestServerPort(), dev, tool)) 450 451 for s in self._servers[str(dev)]: 452 s.SetUp() 453 454 def bind_crash_handler(step, dev): 455 return lambda: crash_handler.RetryOnSystemCrash(step, dev) 456 457 steps = [ 458 bind_crash_handler(s, device) 459 for s in (install_apk, push_test_data, init_tool_and_start_servers)] 460 if self._env.concurrent_adb: 461 reraiser_thread.RunAsync(steps) 462 else: 463 for step in steps: 464 step() 465 466 self._env.parallel_devices.pMap( 467 individual_device_set_up, 468 self._test_instance.GetDataDependencies()) 469 470 #override 471 def _ShouldShard(self): 472 return True 473 474 #override 475 def _CreateShards(self, tests): 476 # _crashes are tests that might crash and make the tests in the same shard 477 # following the crashed testcase not run. 478 # Thus we need to create separate shards for each crashed testcase, 479 # so that other tests can be run. 480 device_count = len(self._env.devices) 481 shards = [] 482 483 # Add shards with only one suspect testcase. 484 shards += [[crash] for crash in self._crashes if crash in tests] 485 486 # Delete suspect testcase from tests. 487 tests = [test for test in tests if not test in self._crashes] 488 489 batch_size = self._test_instance.test_launcher_batch_limit 490 491 for i in xrange(0, device_count): 492 unbounded_shard = tests[i::device_count] 493 shards += [ 494 unbounded_shard[j:j + batch_size] 495 for j in xrange(0, len(unbounded_shard), batch_size) 496 ] 497 return shards 498 499 #override 500 def _GetTests(self): 501 if self._test_instance.extract_test_list_from_filter: 502 # When the exact list of tests to run is given via command-line (e.g. when 503 # locally iterating on a specific test), skip querying the device (which 504 # takes ~3 seconds). 505 tests = _ExtractTestsFromFilter(self._test_instance.gtest_filter) 506 if tests: 507 return tests 508 509 # Even when there's only one device, it still makes sense to retrieve the 510 # test list so that tests can be split up and run in batches rather than all 511 # at once (since test output is not streamed). 512 @local_device_environment.handle_shard_failures_with( 513 on_failure=self._env.DenylistDevice) 514 def list_tests(dev): 515 timeout = 30 516 retries = 1 517 if self._test_instance.wait_for_java_debugger: 518 timeout = None 519 520 flags = [ 521 f for f in self._test_instance.flags 522 if f not in ['--wait-for-debugger', '--wait-for-java-debugger'] 523 ] 524 flags.append('--gtest_list_tests') 525 526 # TODO(crbug.com/726880): Remove retries when no longer necessary. 527 for i in range(0, retries+1): 528 logging.info('flags:') 529 for f in flags: 530 logging.info(' %s', f) 531 532 with self._ArchiveLogcat(dev, 'list_tests'): 533 raw_test_list = crash_handler.RetryOnSystemCrash( 534 lambda d: self._delegate.Run( 535 None, d, flags=' '.join(flags), timeout=timeout), 536 device=dev) 537 538 tests = gtest_test_instance.ParseGTestListTests(raw_test_list) 539 if not tests: 540 logging.info('No tests found. Output:') 541 for l in raw_test_list: 542 logging.info(' %s', l) 543 if i < retries: 544 logging.info('Retrying...') 545 else: 546 break 547 return tests 548 549 # Query all devices in case one fails. 550 test_lists = self._env.parallel_devices.pMap(list_tests).pGet(None) 551 552 # If all devices failed to list tests, raise an exception. 553 # Check that tl is not None and is not empty. 554 if all(not tl for tl in test_lists): 555 raise device_errors.CommandFailedError( 556 'Failed to list tests on any device') 557 tests = list(sorted(set().union(*[set(tl) for tl in test_lists if tl]))) 558 tests = self._test_instance.FilterTests(tests) 559 tests = self._ApplyExternalSharding( 560 tests, self._test_instance.external_shard_index, 561 self._test_instance.total_external_shards) 562 return tests 563 564 def _UploadTestArtifacts(self, device, test_artifacts_dir): 565 # TODO(jbudorick): Reconcile this with the output manager once 566 # https://codereview.chromium.org/2933993002/ lands. 567 if test_artifacts_dir: 568 with tempfile_ext.NamedTemporaryDirectory() as test_artifacts_host_dir: 569 device.PullFile(test_artifacts_dir.name, test_artifacts_host_dir) 570 with tempfile_ext.NamedTemporaryDirectory() as temp_zip_dir: 571 zip_base_name = os.path.join(temp_zip_dir, 'test_artifacts') 572 test_artifacts_zip = shutil.make_archive( 573 zip_base_name, 'zip', test_artifacts_host_dir) 574 link = google_storage_helper.upload( 575 google_storage_helper.unique_name( 576 'test_artifacts', device=device), 577 test_artifacts_zip, 578 bucket='%s/test_artifacts' % ( 579 self._test_instance.gs_test_artifacts_bucket)) 580 logging.info('Uploading test artifacts to %s.', link) 581 return link 582 return None 583 584 @contextlib.contextmanager 585 def _ArchiveLogcat(self, device, test): 586 if isinstance(test, str): 587 desc = test 588 else: 589 desc = hash(tuple(test)) 590 591 stream_name = 'logcat_%s_%s_%s' % ( 592 desc, time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial) 593 594 logcat_file = None 595 logmon = None 596 try: 597 with self._env.output_manager.ArchivedTempfile(stream_name, 598 'logcat') as logcat_file: 599 with logcat_monitor.LogcatMonitor( 600 device.adb, 601 filter_specs=local_device_environment.LOGCAT_FILTERS, 602 output_file=logcat_file.name, 603 check_error=False) as logmon: 604 with contextlib_ext.Optional(trace_event.trace(str(test)), 605 self._env.trace_output): 606 yield logcat_file 607 finally: 608 if logmon: 609 logmon.Close() 610 if logcat_file and logcat_file.Link(): 611 logging.info('Logcat saved to %s', logcat_file.Link()) 612 613 #override 614 def _RunTest(self, device, test): 615 # Run the test. 616 timeout = (self._test_instance.shard_timeout 617 * self.GetTool(device).GetTimeoutScale()) 618 if self._test_instance.wait_for_java_debugger: 619 timeout = None 620 if self._test_instance.store_tombstones: 621 tombstones.ClearAllTombstones(device) 622 test_perf_output_filename = next(self._test_perf_output_filenames) 623 624 if self._test_instance.isolated_script_test_output: 625 suffix = '.json' 626 else: 627 suffix = '.xml' 628 629 with device_temp_file.DeviceTempFile( 630 adb=device.adb, 631 dir=self._delegate.ResultsDirectory(device), 632 suffix=suffix) as device_tmp_results_file: 633 with contextlib_ext.Optional( 634 device_temp_file.NamedDeviceTemporaryDirectory( 635 adb=device.adb, dir='/sdcard/'), 636 self._test_instance.gs_test_artifacts_bucket) as test_artifacts_dir: 637 with (contextlib_ext.Optional( 638 device_temp_file.DeviceTempFile( 639 adb=device.adb, dir=self._delegate.ResultsDirectory(device)), 640 test_perf_output_filename)) as isolated_script_test_perf_output: 641 642 flags = list(self._test_instance.flags) 643 if self._test_instance.enable_xml_result_parsing: 644 flags.append('--gtest_output=xml:%s' % device_tmp_results_file.name) 645 646 if self._test_instance.gs_test_artifacts_bucket: 647 flags.append('--test_artifacts_dir=%s' % test_artifacts_dir.name) 648 649 if self._test_instance.isolated_script_test_output: 650 flags.append('--isolated-script-test-output=%s' % 651 device_tmp_results_file.name) 652 653 if test_perf_output_filename: 654 flags.append('--isolated_script_test_perf_output=%s' % 655 isolated_script_test_perf_output.name) 656 657 logging.info('flags:') 658 for f in flags: 659 logging.info(' %s', f) 660 661 with self._ArchiveLogcat(device, test) as logcat_file: 662 output = self._delegate.Run(test, 663 device, 664 flags=' '.join(flags), 665 timeout=timeout, 666 retries=0) 667 668 if self._test_instance.enable_xml_result_parsing: 669 try: 670 gtest_xml = device.ReadFile(device_tmp_results_file.name) 671 except device_errors.CommandFailedError: 672 logging.exception('Failed to pull gtest results XML file %s', 673 device_tmp_results_file.name) 674 gtest_xml = None 675 676 if self._test_instance.isolated_script_test_output: 677 try: 678 gtest_json = device.ReadFile(device_tmp_results_file.name) 679 except device_errors.CommandFailedError: 680 logging.exception('Failed to pull gtest results JSON file %s', 681 device_tmp_results_file.name) 682 gtest_json = None 683 684 if test_perf_output_filename: 685 try: 686 device.PullFile(isolated_script_test_perf_output.name, 687 test_perf_output_filename) 688 except device_errors.CommandFailedError: 689 logging.exception('Failed to pull chartjson results %s', 690 isolated_script_test_perf_output.name) 691 692 test_artifacts_url = self._UploadTestArtifacts(device, 693 test_artifacts_dir) 694 695 for s in self._servers[str(device)]: 696 s.Reset() 697 if self._test_instance.app_files: 698 self._delegate.PullAppFiles(device, self._test_instance.app_files, 699 self._test_instance.app_file_dir) 700 if not self._env.skip_clear_data: 701 self._delegate.Clear(device) 702 703 for l in output: 704 logging.info(l) 705 706 # Parse the output. 707 # TODO(jbudorick): Transition test scripts away from parsing stdout. 708 if self._test_instance.enable_xml_result_parsing: 709 results = gtest_test_instance.ParseGTestXML(gtest_xml) 710 elif self._test_instance.isolated_script_test_output: 711 results = gtest_test_instance.ParseGTestJSON(gtest_json) 712 else: 713 results = gtest_test_instance.ParseGTestOutput( 714 output, self._test_instance.symbolizer, device.product_cpu_abi) 715 716 tombstones_url = None 717 for r in results: 718 if logcat_file: 719 r.SetLink('logcat', logcat_file.Link()) 720 721 if self._test_instance.gs_test_artifacts_bucket: 722 r.SetLink('test_artifacts', test_artifacts_url) 723 724 if r.GetType() == base_test_result.ResultType.CRASH: 725 self._crashes.add(r.GetName()) 726 if self._test_instance.store_tombstones: 727 if not tombstones_url: 728 resolved_tombstones = tombstones.ResolveTombstones( 729 device, 730 resolve_all_tombstones=True, 731 include_stack_symbols=False, 732 wipe_tombstones=True) 733 stream_name = 'tombstones_%s_%s' % ( 734 time.strftime('%Y%m%dT%H%M%S', time.localtime()), 735 device.serial) 736 tombstones_url = logdog_helper.text( 737 stream_name, '\n'.join(resolved_tombstones)) 738 r.SetLink('tombstones', tombstones_url) 739 740 tests_stripped_disabled_prefix = set() 741 for t in test: 742 tests_stripped_disabled_prefix.add( 743 gtest_test_instance.TestNameWithoutDisabledPrefix(t)) 744 not_run_tests = tests_stripped_disabled_prefix.difference( 745 set(r.GetName() for r in results)) 746 return results, list(not_run_tests) if results else None 747 748 #override 749 def TearDown(self): 750 # By default, teardown will invoke ADB. When receiving SIGTERM due to a 751 # timeout, there's a high probability that ADB is non-responsive. In these 752 # cases, sending an ADB command will potentially take a long time to time 753 # out. Before this happens, the process will be hard-killed for not 754 # responding to SIGTERM fast enough. 755 if self._received_sigterm: 756 return 757 758 @local_device_environment.handle_shard_failures 759 @trace_event.traced 760 def individual_device_tear_down(dev): 761 for s in self._servers.get(str(dev), []): 762 s.TearDown() 763 764 tool = self.GetTool(dev) 765 tool.CleanUpEnvironment() 766 767 self._env.parallel_devices.pMap(individual_device_tear_down) 768