1#!/usr/bin/env python
2#
3# Copyright 2016 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Runs the CTS test APKs stored in CIPD."""
8
9import argparse
10import contextlib
11import json
12import logging
13import os
14import shutil
15import sys
16import tempfile
17import zipfile
18
19
20sys.path.append(os.path.join(
21    os.path.dirname(__file__), os.pardir, os.pardir, 'build', 'android'))
22import devil_chromium  # pylint: disable=unused-import
23from devil.android import device_utils
24from devil.android.ndk import abis
25from devil.android.sdk import version_codes
26from devil.android.tools import script_common
27from devil.utils import cmd_helper
28from devil.utils import logging_common
29from pylib.local.emulator import avd
30from pylib.utils import test_filter
31
32# cts test archives for all platforms are stored in this bucket
33# contents need to be updated if there is an important fix to any of
34# the tests
35
36_TEST_RUNNER_PATH = os.path.join(
37    os.path.dirname(__file__), os.pardir, os.pardir,
38    'build', 'android', 'test_runner.py')
39
40_WEBVIEW_CTS_GCS_PATH_FILE = os.path.join(
41    os.path.dirname(__file__), 'cts_config', 'webview_cts_gcs_path.json')
42_ARCH_SPECIFIC_CTS_INFO = ["filename", "unzip_dir", "_origin"]
43
44_CTS_ARCHIVE_DIR = os.path.join(os.path.dirname(__file__), 'cts_archive')
45
46_CTS_WEBKIT_PACKAGES = ["com.android.cts.webkit", "android.webkit.cts"]
47
48SDK_PLATFORM_DICT = {
49    version_codes.LOLLIPOP: 'L',
50    version_codes.LOLLIPOP_MR1: 'L',
51    version_codes.MARSHMALLOW: 'M',
52    version_codes.NOUGAT: 'N',
53    version_codes.NOUGAT_MR1: 'N',
54    version_codes.OREO: 'O',
55    version_codes.OREO_MR1: 'O',
56    version_codes.PIE: 'P',
57    version_codes.Q: 'Q',
58}
59
60# The test apks are apparently compatible across all architectures, the
61# arm vs x86 split is to match the current cts releases and in case things
62# start to diverge in the future.  Keeping the arm64 (instead of arm) dict
63# key to avoid breaking the bots that specify --arch arm64 to invoke the tests.
64_SUPPORTED_ARCH_DICT = {
65    # TODO(aluo): Investigate how to force WebView abi on platforms supporting
66    # multiple abis.
67    # The test apks under 'arm64' support both arm and arm64 devices.
68    abis.ARM: 'arm64',
69    abis.ARM_64: 'arm64',
70    # The test apks under 'x86' support both x86 and x86_64 devices.
71    abis.X86: 'x86',
72    abis.X86_64: 'x86'
73}
74
75
76TEST_FILTER_OPT = '--test-filter'
77
78def GetCtsInfo(arch, cts_release, item):
79  """Gets contents of CTS Info for arch and cts_release.
80
81  See _WEBVIEW_CTS_GCS_PATH_FILE
82  """
83  with open(_WEBVIEW_CTS_GCS_PATH_FILE) as f:
84    cts_gcs_path_info = json.load(f)
85  try:
86    if item in _ARCH_SPECIFIC_CTS_INFO:
87      return cts_gcs_path_info[cts_release]['arch'][arch][item]
88    else:
89      return cts_gcs_path_info[cts_release][item]
90  except KeyError:
91    raise Exception('No %s info available for arch:%s, android:%s' %
92                    (item, arch, cts_release))
93
94
95def GetCTSModuleNames(arch, cts_release):
96  """Gets the module apk name of the arch and cts_release"""
97  test_runs = GetCtsInfo(arch, cts_release, 'test_runs')
98  return [os.path.basename(r['apk']) for r in test_runs]
99
100
101def GetTestRunFilterArg(args, test_run):
102  """ Merges json file filters with cmdline filters using
103      test_filter.InitializeFilterFromArgs
104  """
105
106  # Convert cmdline filters to test-filter style
107  filter_string = test_filter.InitializeFilterFromArgs(args)
108
109  # Only add inclusion filters if there's not already one specified, since
110  # they would conflict, see test_filter.ConflictingPositiveFiltersException.
111  if not test_filter.HasPositivePatterns(filter_string):
112    includes = test_run.get("includes", [])
113    filter_string = test_filter.AppendPatternsToFilter(
114        filter_string,
115        positive_patterns=[i["match"] for i in includes])
116
117  if args.skip_expected_failures:
118    excludes = test_run.get("excludes", [])
119    filter_string = test_filter.AppendPatternsToFilter(
120        filter_string,
121        negative_patterns=[e["match"] for e in excludes])
122
123  if filter_string:
124    return [TEST_FILTER_OPT + '=' + filter_string]
125  else:
126    return []
127
128
129def RunCTS(test_runner_args, local_cts_dir, apk, json_results_file=None):
130  """Run tests in apk using test_runner script at _TEST_RUNNER_PATH.
131
132  Returns the script result code, test results will be stored in
133  the json_results_file file if specified.
134  """
135
136  local_test_runner_args = test_runner_args + ['--test-apk',
137                                               os.path.join(local_cts_dir, apk)]
138
139  if json_results_file:
140    local_test_runner_args += ['--json-results-file=%s' %
141                               json_results_file]
142  return cmd_helper.RunCmd(
143      [_TEST_RUNNER_PATH, 'instrumentation'] + local_test_runner_args)
144
145
146def MergeTestResults(existing_results_json, additional_results_json):
147  """Appends results in additional_results_json to existing_results_json."""
148  for k, v in additional_results_json.iteritems():
149    if k not in existing_results_json:
150      existing_results_json[k] = v
151    else:
152      if isinstance(v, dict):
153        if not isinstance(existing_results_json[k], dict):
154          raise NotImplementedError(
155              "Can't merge results field %s of different types" % v)
156        existing_results_json[k].update(v)
157      elif isinstance(v, list):
158        if not isinstance(existing_results_json[k], list):
159          raise NotImplementedError(
160              "Can't merge results field %s of different types" % v)
161        existing_results_json[k].extend(v)
162      else:
163        raise NotImplementedError(
164            "Can't merge results field %s that is not a list or dict" % v)
165
166
167def ExtractCTSZip(args, arch, cts_release):
168  """Extract the CTS tests for cts_release.
169
170  Extract the CTS zip file from _CTS_ARCHIVE_DIR to
171  apk_dir if specified, or a new temporary directory if not.
172  Returns following tuple (local_cts_dir, base_cts_dir, delete_cts_dir):
173    local_cts_dir - CTS extraction location for current arch and cts_release
174    base_cts_dir - Root directory for all the arches and platforms
175    delete_cts_dir - Set if the base_cts_dir was created as a temporary
176    directory
177  """
178  base_cts_dir = None
179  delete_cts_dir = False
180  relative_cts_zip_path = GetCtsInfo(arch, cts_release, 'filename')
181
182  if args.apk_dir:
183    base_cts_dir = args.apk_dir
184  else:
185    base_cts_dir = tempfile.mkdtemp()
186    delete_cts_dir = True
187
188  cts_zip_path = os.path.join(_CTS_ARCHIVE_DIR, relative_cts_zip_path)
189  local_cts_dir = os.path.join(base_cts_dir,
190                               GetCtsInfo(arch, cts_release,
191                                          'unzip_dir')
192                              )
193  zf = zipfile.ZipFile(cts_zip_path, 'r')
194  zf.extractall(local_cts_dir)
195  return (local_cts_dir, base_cts_dir, delete_cts_dir)
196
197
198def RunAllCTSTests(args, arch, cts_release, test_runner_args):
199  """Run CTS tests downloaded from _CTS_BUCKET.
200
201  Downloads CTS tests from bucket, runs them for the
202  specified cts_release+arch, then creates a single
203  results json file (if specified)
204  Returns 0 if all tests passed, otherwise
205  returns the failure code of the last failing
206  test.
207  """
208  local_cts_dir, base_cts_dir, delete_cts_dir = ExtractCTSZip(args, arch,
209                                                              cts_release)
210  cts_result = 0
211  json_results_file = args.json_results_file
212  try:
213    cts_test_runs = GetCtsInfo(arch, cts_release, 'test_runs')
214    cts_results_json = {}
215    for cts_test_run in cts_test_runs:
216      iteration_cts_result = 0
217
218      test_apk = cts_test_run['apk']
219      # If --module-apk is specified then skip tests in all other modules
220      if args.module_apk and os.path.basename(test_apk) != args.module_apk:
221        continue
222
223      iter_test_runner_args = test_runner_args + GetTestRunFilterArg(
224          args, cts_test_run)
225
226      if json_results_file:
227        with tempfile.NamedTemporaryFile() as iteration_json_file:
228          iteration_cts_result = RunCTS(iter_test_runner_args, local_cts_dir,
229                                        test_apk, iteration_json_file.name)
230          with open(iteration_json_file.name) as f:
231            additional_results_json = json.load(f)
232            MergeTestResults(cts_results_json, additional_results_json)
233      else:
234        iteration_cts_result = RunCTS(iter_test_runner_args, local_cts_dir,
235                                      test_apk)
236      if iteration_cts_result:
237        cts_result = iteration_cts_result
238    if json_results_file:
239      with open(json_results_file, 'w') as f:
240        json.dump(cts_results_json, f, indent=2)
241  finally:
242    if delete_cts_dir and base_cts_dir:
243      shutil.rmtree(base_cts_dir)
244
245  return cts_result
246
247
248def DetermineCtsRelease(device):
249  """Determines the CTS release based on the Android SDK level
250
251  Args:
252    device: The DeviceUtils instance
253  Returns:
254    The first letter of the cts_release in uppercase.
255  Raises:
256    Exception: if we don't have the CTS tests for the device platform saved in
257      CIPD already.
258  """
259  cts_release = SDK_PLATFORM_DICT.get(device.build_version_sdk)
260  if not cts_release:
261    # Check if we're above the supported version range.
262    max_supported_sdk = max(SDK_PLATFORM_DICT.keys())
263    if device.build_version_sdk > max_supported_sdk:
264      raise Exception("We don't have tests for API level {api_level}, try "
265                      "running the {release} tests with `--cts-release "
266                      "{release}`".format(
267                          api_level=device.build_version_sdk,
268                          release=SDK_PLATFORM_DICT.get(max_supported_sdk),
269                      ))
270    # Otherwise, we must be below the supported version range.
271    min_supported_sdk = min(SDK_PLATFORM_DICT.keys())
272    raise Exception("We don't support running CTS tests on platforms less "
273                    "than {release} as the WebView is not updatable".format(
274                        release=SDK_PLATFORM_DICT.get(min_supported_sdk),
275                    ))
276  logging.info(('Using test APKs from CTS release=%s because '
277                'build.version.sdk=%s'),
278               cts_release, device.build_version_sdk)
279  return cts_release
280
281
282def DetermineArch(device):
283  """Determines which architecture to use based on the device properties
284
285  Args:
286    device: The DeviceUtils instance
287  Returns:
288    The formatted arch string (as expected by CIPD)
289  Raises:
290    Exception: if device architecture is not currently supported by this script.
291  """
292  arch = _SUPPORTED_ARCH_DICT.get(device.product_cpu_abi)
293  if not arch:
294    raise Exception('Could not find CIPD bucket for your device arch (' +
295                    device.product_cpu_abi +
296                    '), please specify with --arch')
297  logging.info('Guessing arch=%s because product.cpu.abi=%s', arch,
298               device.product_cpu_abi)
299  return arch
300
301
302def UninstallAnyCtsWebkitPackages(device):
303  for package in _CTS_WEBKIT_PACKAGES:
304    device.Uninstall(package)
305
306
307def ForwardArgsToTestRunner(known_args):
308  """Convert any args that should be forwarded to test_runner.py"""
309  forwarded_args = []
310  if known_args.devices:
311    # test_runner.py parses --device as nargs instead of append args
312    forwarded_args.extend(['--device'] + known_args.devices)
313  if known_args.denylist_file:
314    forwarded_args.extend(['--denylist-file', known_args.denylist_file])
315
316  if known_args.verbose:
317    forwarded_args.extend(['-' + 'v' * known_args.verbose])
318  #TODO: Pass quiet to test runner when it becomes supported
319  return forwarded_args
320
321
322@contextlib.contextmanager
323def GetDevice(args):
324  try:
325    emulator_instance = None
326    if args.avd_config:
327      avd_config = avd.AvdConfig(args.avd_config)
328      avd_config.Install()
329      emulator_instance = avd_config.CreateInstance()
330      # Start the emulator w/ -writable-system s.t. we can remount the system
331      # partition r/w and install our own webview provider.
332      emulator_instance.Start(writable_system=True)
333      device_utils.DeviceUtils(emulator_instance.serial).WaitUntilFullyBooted()
334
335    devices = script_common.GetDevices(args.devices, args.denylist_file)
336    device = devices[0]
337    if len(devices) > 1:
338      logging.warning('Detection of arch and cts-release will use 1st of %d '
339                      'devices: %s', len(devices), device.serial)
340    yield device
341  finally:
342    if emulator_instance:
343      emulator_instance.Stop()
344
345
346def main():
347  parser = argparse.ArgumentParser()
348  parser.add_argument(
349      '--arch',
350      choices=list(set(_SUPPORTED_ARCH_DICT.values())),
351      default=None,
352      type=str,
353      help=('Architecture to for CTS tests. Will auto-determine based on '
354            'the device ro.product.cpu.abi property.'))
355  parser.add_argument(
356      '--cts-release',
357      # TODO(aluo): --platform is deprecated (the meaning is unclear).
358      '--platform',
359      choices=sorted(set(SDK_PLATFORM_DICT.values())),
360      required=False,
361      default=None,
362      help='Which CTS release to use for the run. This should generally be <= '
363           'device OS level (otherwise, the newer tests will fail). If '
364           'unspecified, the script will auto-determine the release based on '
365           'device OS level.')
366  parser.add_argument(
367      '--skip-expected-failures',
368      action='store_true',
369      help="Option to skip all tests that are expected to fail.  Can't be used "
370           "with test filters.")
371  parser.add_argument(
372      '--apk-dir',
373      help='Directory to extract CTS APKs to. '
374           'Will use temp directory by default.')
375  parser.add_argument(
376      '--test-launcher-summary-output',
377      '--json-results-file',
378      '--write-full-results-to',
379      '--isolated-script-test-output',
380      dest='json_results_file', type=os.path.realpath,
381      help='If set, will dump results in JSON form to the specified file. '
382           'Note that this will also trigger saving per-test logcats to '
383           'logdog.')
384  parser.add_argument(
385      '-m',
386      '--module-apk',
387      dest='module_apk',
388      help='CTS module apk name in ' + _WEBVIEW_CTS_GCS_PATH_FILE +
389      ' file, without the path prefix.')
390  parser.add_argument(
391      '--avd-config',
392      type=os.path.realpath,
393      help='Path to the avd config textpb. '
394           '(See //tools/android/avd/proto for message definition'
395           ' and existing textpb files.)')
396
397
398  test_filter.AddFilterOptions(parser)
399  script_common.AddDeviceArguments(parser)
400  logging_common.AddLoggingArguments(parser)
401
402  args, test_runner_args = parser.parse_known_args()
403  logging_common.InitializeLogging(args)
404  devil_chromium.Initialize()
405
406  test_runner_args.extend(ForwardArgsToTestRunner(args))
407
408  with GetDevice(args) as device:
409    arch = args.arch or DetermineArch(device)
410    cts_release = args.cts_release or DetermineCtsRelease(device)
411
412    if (args.test_filter_file or args.test_filter
413        or args.isolated_script_test_filter):
414      # TODO(aluo): auto-determine the module based on the test filter and the
415      # available tests in each module
416      if not args.module_apk:
417        args.module_apk = 'CtsWebkitTestCases.apk'
418
419    platform_modules = GetCTSModuleNames(arch, cts_release)
420    if args.module_apk and args.module_apk not in platform_modules:
421      raise Exception('--module-apk for arch==' + arch + 'and cts_release=='
422                      + cts_release + ' must be one of: '
423                      + ', '.join(platform_modules))
424
425    # Need to uninstall all previous cts webkit packages so that the
426    # MockContentProvider names won't conflict with a previously installed
427    # one under a different package name.  This is due to CtsWebkitTestCases's
428    # package name change from M to N versions of the tests while keeping the
429    # MockContentProvider's authority string the same.
430    UninstallAnyCtsWebkitPackages(device)
431
432    return RunAllCTSTests(args, arch, cts_release, test_runner_args)
433
434
435if __name__ == '__main__':
436  sys.exit(main())
437