1#!/usr/bin/env python 2# 3# Copyright 2016 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Runs the CTS test APKs stored in CIPD.""" 8 9import argparse 10import contextlib 11import json 12import logging 13import os 14import shutil 15import sys 16import tempfile 17import zipfile 18 19 20sys.path.append(os.path.join( 21 os.path.dirname(__file__), os.pardir, os.pardir, 'build', 'android')) 22import devil_chromium # pylint: disable=unused-import 23from devil.android import device_utils 24from devil.android.ndk import abis 25from devil.android.sdk import version_codes 26from devil.android.tools import script_common 27from devil.utils import cmd_helper 28from devil.utils import logging_common 29from pylib.local.emulator import avd 30from pylib.utils import test_filter 31 32# cts test archives for all platforms are stored in this bucket 33# contents need to be updated if there is an important fix to any of 34# the tests 35 36_TEST_RUNNER_PATH = os.path.join( 37 os.path.dirname(__file__), os.pardir, os.pardir, 38 'build', 'android', 'test_runner.py') 39 40_WEBVIEW_CTS_GCS_PATH_FILE = os.path.join( 41 os.path.dirname(__file__), 'cts_config', 'webview_cts_gcs_path.json') 42_ARCH_SPECIFIC_CTS_INFO = ["filename", "unzip_dir", "_origin"] 43 44_CTS_ARCHIVE_DIR = os.path.join(os.path.dirname(__file__), 'cts_archive') 45 46_CTS_WEBKIT_PACKAGES = ["com.android.cts.webkit", "android.webkit.cts"] 47 48SDK_PLATFORM_DICT = { 49 version_codes.LOLLIPOP: 'L', 50 version_codes.LOLLIPOP_MR1: 'L', 51 version_codes.MARSHMALLOW: 'M', 52 version_codes.NOUGAT: 'N', 53 version_codes.NOUGAT_MR1: 'N', 54 version_codes.OREO: 'O', 55 version_codes.OREO_MR1: 'O', 56 version_codes.PIE: 'P', 57 version_codes.Q: 'Q', 58} 59 60# The test apks are apparently compatible across all architectures, the 61# arm vs x86 split is to match the current cts releases and in case things 62# start to diverge in the future. Keeping the arm64 (instead of arm) dict 63# key to avoid breaking the bots that specify --arch arm64 to invoke the tests. 64_SUPPORTED_ARCH_DICT = { 65 # TODO(aluo): Investigate how to force WebView abi on platforms supporting 66 # multiple abis. 67 # The test apks under 'arm64' support both arm and arm64 devices. 68 abis.ARM: 'arm64', 69 abis.ARM_64: 'arm64', 70 # The test apks under 'x86' support both x86 and x86_64 devices. 71 abis.X86: 'x86', 72 abis.X86_64: 'x86' 73} 74 75 76TEST_FILTER_OPT = '--test-filter' 77 78def GetCtsInfo(arch, cts_release, item): 79 """Gets contents of CTS Info for arch and cts_release. 80 81 See _WEBVIEW_CTS_GCS_PATH_FILE 82 """ 83 with open(_WEBVIEW_CTS_GCS_PATH_FILE) as f: 84 cts_gcs_path_info = json.load(f) 85 try: 86 if item in _ARCH_SPECIFIC_CTS_INFO: 87 return cts_gcs_path_info[cts_release]['arch'][arch][item] 88 else: 89 return cts_gcs_path_info[cts_release][item] 90 except KeyError: 91 raise Exception('No %s info available for arch:%s, android:%s' % 92 (item, arch, cts_release)) 93 94 95def GetCTSModuleNames(arch, cts_release): 96 """Gets the module apk name of the arch and cts_release""" 97 test_runs = GetCtsInfo(arch, cts_release, 'test_runs') 98 return [os.path.basename(r['apk']) for r in test_runs] 99 100 101def GetTestRunFilterArg(args, test_run): 102 """ Merges json file filters with cmdline filters using 103 test_filter.InitializeFilterFromArgs 104 """ 105 106 # Convert cmdline filters to test-filter style 107 filter_string = test_filter.InitializeFilterFromArgs(args) 108 109 # Only add inclusion filters if there's not already one specified, since 110 # they would conflict, see test_filter.ConflictingPositiveFiltersException. 111 if not test_filter.HasPositivePatterns(filter_string): 112 includes = test_run.get("includes", []) 113 filter_string = test_filter.AppendPatternsToFilter( 114 filter_string, 115 positive_patterns=[i["match"] for i in includes]) 116 117 if args.skip_expected_failures: 118 excludes = test_run.get("excludes", []) 119 filter_string = test_filter.AppendPatternsToFilter( 120 filter_string, 121 negative_patterns=[e["match"] for e in excludes]) 122 123 if filter_string: 124 return [TEST_FILTER_OPT + '=' + filter_string] 125 else: 126 return [] 127 128 129def RunCTS(test_runner_args, local_cts_dir, apk, json_results_file=None): 130 """Run tests in apk using test_runner script at _TEST_RUNNER_PATH. 131 132 Returns the script result code, test results will be stored in 133 the json_results_file file if specified. 134 """ 135 136 local_test_runner_args = test_runner_args + ['--test-apk', 137 os.path.join(local_cts_dir, apk)] 138 139 if json_results_file: 140 local_test_runner_args += ['--json-results-file=%s' % 141 json_results_file] 142 return cmd_helper.RunCmd( 143 [_TEST_RUNNER_PATH, 'instrumentation'] + local_test_runner_args) 144 145 146def MergeTestResults(existing_results_json, additional_results_json): 147 """Appends results in additional_results_json to existing_results_json.""" 148 for k, v in additional_results_json.iteritems(): 149 if k not in existing_results_json: 150 existing_results_json[k] = v 151 else: 152 if isinstance(v, dict): 153 if not isinstance(existing_results_json[k], dict): 154 raise NotImplementedError( 155 "Can't merge results field %s of different types" % v) 156 existing_results_json[k].update(v) 157 elif isinstance(v, list): 158 if not isinstance(existing_results_json[k], list): 159 raise NotImplementedError( 160 "Can't merge results field %s of different types" % v) 161 existing_results_json[k].extend(v) 162 else: 163 raise NotImplementedError( 164 "Can't merge results field %s that is not a list or dict" % v) 165 166 167def ExtractCTSZip(args, arch, cts_release): 168 """Extract the CTS tests for cts_release. 169 170 Extract the CTS zip file from _CTS_ARCHIVE_DIR to 171 apk_dir if specified, or a new temporary directory if not. 172 Returns following tuple (local_cts_dir, base_cts_dir, delete_cts_dir): 173 local_cts_dir - CTS extraction location for current arch and cts_release 174 base_cts_dir - Root directory for all the arches and platforms 175 delete_cts_dir - Set if the base_cts_dir was created as a temporary 176 directory 177 """ 178 base_cts_dir = None 179 delete_cts_dir = False 180 relative_cts_zip_path = GetCtsInfo(arch, cts_release, 'filename') 181 182 if args.apk_dir: 183 base_cts_dir = args.apk_dir 184 else: 185 base_cts_dir = tempfile.mkdtemp() 186 delete_cts_dir = True 187 188 cts_zip_path = os.path.join(_CTS_ARCHIVE_DIR, relative_cts_zip_path) 189 local_cts_dir = os.path.join(base_cts_dir, 190 GetCtsInfo(arch, cts_release, 191 'unzip_dir') 192 ) 193 zf = zipfile.ZipFile(cts_zip_path, 'r') 194 zf.extractall(local_cts_dir) 195 return (local_cts_dir, base_cts_dir, delete_cts_dir) 196 197 198def RunAllCTSTests(args, arch, cts_release, test_runner_args): 199 """Run CTS tests downloaded from _CTS_BUCKET. 200 201 Downloads CTS tests from bucket, runs them for the 202 specified cts_release+arch, then creates a single 203 results json file (if specified) 204 Returns 0 if all tests passed, otherwise 205 returns the failure code of the last failing 206 test. 207 """ 208 local_cts_dir, base_cts_dir, delete_cts_dir = ExtractCTSZip(args, arch, 209 cts_release) 210 cts_result = 0 211 json_results_file = args.json_results_file 212 try: 213 cts_test_runs = GetCtsInfo(arch, cts_release, 'test_runs') 214 cts_results_json = {} 215 for cts_test_run in cts_test_runs: 216 iteration_cts_result = 0 217 218 test_apk = cts_test_run['apk'] 219 # If --module-apk is specified then skip tests in all other modules 220 if args.module_apk and os.path.basename(test_apk) != args.module_apk: 221 continue 222 223 iter_test_runner_args = test_runner_args + GetTestRunFilterArg( 224 args, cts_test_run) 225 226 if json_results_file: 227 with tempfile.NamedTemporaryFile() as iteration_json_file: 228 iteration_cts_result = RunCTS(iter_test_runner_args, local_cts_dir, 229 test_apk, iteration_json_file.name) 230 with open(iteration_json_file.name) as f: 231 additional_results_json = json.load(f) 232 MergeTestResults(cts_results_json, additional_results_json) 233 else: 234 iteration_cts_result = RunCTS(iter_test_runner_args, local_cts_dir, 235 test_apk) 236 if iteration_cts_result: 237 cts_result = iteration_cts_result 238 if json_results_file: 239 with open(json_results_file, 'w') as f: 240 json.dump(cts_results_json, f, indent=2) 241 finally: 242 if delete_cts_dir and base_cts_dir: 243 shutil.rmtree(base_cts_dir) 244 245 return cts_result 246 247 248def DetermineCtsRelease(device): 249 """Determines the CTS release based on the Android SDK level 250 251 Args: 252 device: The DeviceUtils instance 253 Returns: 254 The first letter of the cts_release in uppercase. 255 Raises: 256 Exception: if we don't have the CTS tests for the device platform saved in 257 CIPD already. 258 """ 259 cts_release = SDK_PLATFORM_DICT.get(device.build_version_sdk) 260 if not cts_release: 261 # Check if we're above the supported version range. 262 max_supported_sdk = max(SDK_PLATFORM_DICT.keys()) 263 if device.build_version_sdk > max_supported_sdk: 264 raise Exception("We don't have tests for API level {api_level}, try " 265 "running the {release} tests with `--cts-release " 266 "{release}`".format( 267 api_level=device.build_version_sdk, 268 release=SDK_PLATFORM_DICT.get(max_supported_sdk), 269 )) 270 # Otherwise, we must be below the supported version range. 271 min_supported_sdk = min(SDK_PLATFORM_DICT.keys()) 272 raise Exception("We don't support running CTS tests on platforms less " 273 "than {release} as the WebView is not updatable".format( 274 release=SDK_PLATFORM_DICT.get(min_supported_sdk), 275 )) 276 logging.info(('Using test APKs from CTS release=%s because ' 277 'build.version.sdk=%s'), 278 cts_release, device.build_version_sdk) 279 return cts_release 280 281 282def DetermineArch(device): 283 """Determines which architecture to use based on the device properties 284 285 Args: 286 device: The DeviceUtils instance 287 Returns: 288 The formatted arch string (as expected by CIPD) 289 Raises: 290 Exception: if device architecture is not currently supported by this script. 291 """ 292 arch = _SUPPORTED_ARCH_DICT.get(device.product_cpu_abi) 293 if not arch: 294 raise Exception('Could not find CIPD bucket for your device arch (' + 295 device.product_cpu_abi + 296 '), please specify with --arch') 297 logging.info('Guessing arch=%s because product.cpu.abi=%s', arch, 298 device.product_cpu_abi) 299 return arch 300 301 302def UninstallAnyCtsWebkitPackages(device): 303 for package in _CTS_WEBKIT_PACKAGES: 304 device.Uninstall(package) 305 306 307def ForwardArgsToTestRunner(known_args): 308 """Convert any args that should be forwarded to test_runner.py""" 309 forwarded_args = [] 310 if known_args.devices: 311 # test_runner.py parses --device as nargs instead of append args 312 forwarded_args.extend(['--device'] + known_args.devices) 313 if known_args.denylist_file: 314 forwarded_args.extend(['--denylist-file', known_args.denylist_file]) 315 316 if known_args.verbose: 317 forwarded_args.extend(['-' + 'v' * known_args.verbose]) 318 #TODO: Pass quiet to test runner when it becomes supported 319 return forwarded_args 320 321 322@contextlib.contextmanager 323def GetDevice(args): 324 try: 325 emulator_instance = None 326 if args.avd_config: 327 avd_config = avd.AvdConfig(args.avd_config) 328 avd_config.Install() 329 emulator_instance = avd_config.CreateInstance() 330 # Start the emulator w/ -writable-system s.t. we can remount the system 331 # partition r/w and install our own webview provider. 332 emulator_instance.Start(writable_system=True) 333 device_utils.DeviceUtils(emulator_instance.serial).WaitUntilFullyBooted() 334 335 devices = script_common.GetDevices(args.devices, args.denylist_file) 336 device = devices[0] 337 if len(devices) > 1: 338 logging.warning('Detection of arch and cts-release will use 1st of %d ' 339 'devices: %s', len(devices), device.serial) 340 yield device 341 finally: 342 if emulator_instance: 343 emulator_instance.Stop() 344 345 346def main(): 347 parser = argparse.ArgumentParser() 348 parser.add_argument( 349 '--arch', 350 choices=list(set(_SUPPORTED_ARCH_DICT.values())), 351 default=None, 352 type=str, 353 help=('Architecture to for CTS tests. Will auto-determine based on ' 354 'the device ro.product.cpu.abi property.')) 355 parser.add_argument( 356 '--cts-release', 357 # TODO(aluo): --platform is deprecated (the meaning is unclear). 358 '--platform', 359 choices=sorted(set(SDK_PLATFORM_DICT.values())), 360 required=False, 361 default=None, 362 help='Which CTS release to use for the run. This should generally be <= ' 363 'device OS level (otherwise, the newer tests will fail). If ' 364 'unspecified, the script will auto-determine the release based on ' 365 'device OS level.') 366 parser.add_argument( 367 '--skip-expected-failures', 368 action='store_true', 369 help="Option to skip all tests that are expected to fail. Can't be used " 370 "with test filters.") 371 parser.add_argument( 372 '--apk-dir', 373 help='Directory to extract CTS APKs to. ' 374 'Will use temp directory by default.') 375 parser.add_argument( 376 '--test-launcher-summary-output', 377 '--json-results-file', 378 '--write-full-results-to', 379 '--isolated-script-test-output', 380 dest='json_results_file', type=os.path.realpath, 381 help='If set, will dump results in JSON form to the specified file. ' 382 'Note that this will also trigger saving per-test logcats to ' 383 'logdog.') 384 parser.add_argument( 385 '-m', 386 '--module-apk', 387 dest='module_apk', 388 help='CTS module apk name in ' + _WEBVIEW_CTS_GCS_PATH_FILE + 389 ' file, without the path prefix.') 390 parser.add_argument( 391 '--avd-config', 392 type=os.path.realpath, 393 help='Path to the avd config textpb. ' 394 '(See //tools/android/avd/proto for message definition' 395 ' and existing textpb files.)') 396 397 398 test_filter.AddFilterOptions(parser) 399 script_common.AddDeviceArguments(parser) 400 logging_common.AddLoggingArguments(parser) 401 402 args, test_runner_args = parser.parse_known_args() 403 logging_common.InitializeLogging(args) 404 devil_chromium.Initialize() 405 406 test_runner_args.extend(ForwardArgsToTestRunner(args)) 407 408 with GetDevice(args) as device: 409 arch = args.arch or DetermineArch(device) 410 cts_release = args.cts_release or DetermineCtsRelease(device) 411 412 if (args.test_filter_file or args.test_filter 413 or args.isolated_script_test_filter): 414 # TODO(aluo): auto-determine the module based on the test filter and the 415 # available tests in each module 416 if not args.module_apk: 417 args.module_apk = 'CtsWebkitTestCases.apk' 418 419 platform_modules = GetCTSModuleNames(arch, cts_release) 420 if args.module_apk and args.module_apk not in platform_modules: 421 raise Exception('--module-apk for arch==' + arch + 'and cts_release==' 422 + cts_release + ' must be one of: ' 423 + ', '.join(platform_modules)) 424 425 # Need to uninstall all previous cts webkit packages so that the 426 # MockContentProvider names won't conflict with a previously installed 427 # one under a different package name. This is due to CtsWebkitTestCases's 428 # package name change from M to N versions of the tests while keeping the 429 # MockContentProvider's authority string the same. 430 UninstallAnyCtsWebkitPackages(device) 431 432 return RunAllCTSTests(args, arch, cts_release, test_runner_args) 433 434 435if __name__ == '__main__': 436 sys.exit(main()) 437