1#!/usr/bin/env vpython3
2#
3# Copyright 2015 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Install *_incremental.apk targets as well as their dependent files."""
8
9import argparse
10import collections
11import functools
12import glob
13import json
14import logging
15import os
16import posixpath
17import shutil
18import sys
19
20sys.path.append(
21    os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)))
22import devil_chromium
23from devil.android import apk_helper
24from devil.android import device_utils
25from devil.utils import reraiser_thread
26from devil.utils import run_tests_helper
27from pylib import constants
28from pylib.utils import time_profile
29
30prev_sys_path = list(sys.path)
31sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, 'gyp'))
32import dex
33from util import build_utils
34sys.path = prev_sys_path
35
36
37_R8_PATH = os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'r8', 'lib',
38                        'r8.jar')
39
40
41def _DeviceCachePath(device):
42  file_name = 'device_cache_%s.json' % device.adb.GetDeviceSerial()
43  return os.path.join(constants.GetOutDirectory(), file_name)
44
45
46def _Execute(concurrently, *funcs):
47  """Calls all functions in |funcs| concurrently or in sequence."""
48  timer = time_profile.TimeProfile()
49  if concurrently:
50    reraiser_thread.RunAsync(funcs)
51  else:
52    for f in funcs:
53      f()
54  timer.Stop(log=False)
55  return timer
56
57
58def _GetDeviceIncrementalDir(package):
59  """Returns the device path to put incremental files for the given package."""
60  return '/data/local/tmp/incremental-app-%s' % package
61
62
63def _IsStale(src_paths, dest):
64  """Returns if |dest| is older than any of |src_paths|, or missing."""
65  if not os.path.exists(dest):
66    return True
67  dest_time = os.path.getmtime(dest)
68  for path in src_paths:
69    if os.path.getmtime(path) > dest_time:
70      return True
71  return False
72
73
74def _AllocateDexShards(dex_files):
75  """Divides input dex files into buckets."""
76  # Goals:
77  # * Make shards small enough that they are fast to merge.
78  # * Minimize the number of shards so they load quickly on device.
79  # * Partition files into shards such that a change in one file results in only
80  #   one shard having to be re-created.
81  shards = collections.defaultdict(list)
82  # As of Oct 2019, 10 shards results in a min/max size of 582K/2.6M.
83  NUM_CORE_SHARDS = 10
84  # As of Oct 2019, 17 dex files are larger than 1M.
85  SHARD_THRESHOLD = 2**20
86  for src_path in dex_files:
87    if os.path.getsize(src_path) >= SHARD_THRESHOLD:
88      # Use the path as the name rather than an incrementing number to ensure
89      # that it shards to the same name every time.
90      name = os.path.relpath(src_path, constants.GetOutDirectory()).replace(
91          os.sep, '.')
92      shards[name].append(src_path)
93    else:
94      name = 'shard{}.dex.jar'.format(hash(src_path) % NUM_CORE_SHARDS)
95      shards[name].append(src_path)
96  logging.info('Sharding %d dex files into %d buckets', len(dex_files),
97               len(shards))
98  return shards
99
100
101def _CreateDexFiles(shards, dex_staging_dir, min_api, use_concurrency):
102  """Creates dex files within |dex_staging_dir| defined by |shards|."""
103  tasks = []
104  for name, src_paths in shards.items():
105    dest_path = os.path.join(dex_staging_dir, name)
106    if _IsStale(src_paths, dest_path):
107      tasks.append(
108          functools.partial(dex.MergeDexForIncrementalInstall, _R8_PATH,
109                            src_paths, dest_path, min_api))
110
111  # TODO(agrieve): It would be more performant to write a custom d8.jar
112  #     wrapper in java that would process these in bulk, rather than spinning
113  #     up a new process for each one.
114  _Execute(use_concurrency, *tasks)
115
116  # Remove any stale shards.
117  for name in os.listdir(dex_staging_dir):
118    if name not in shards:
119      os.unlink(os.path.join(dex_staging_dir, name))
120
121
122def Uninstall(device, package, enable_device_cache=False):
123  """Uninstalls and removes all incremental files for the given package."""
124  main_timer = time_profile.TimeProfile()
125  device.Uninstall(package)
126  if enable_device_cache:
127    # Uninstall is rare, so just wipe the cache in this case.
128    cache_path = _DeviceCachePath(device)
129    if os.path.exists(cache_path):
130      os.unlink(cache_path)
131  device.RunShellCommand(['rm', '-rf', _GetDeviceIncrementalDir(package)],
132                         check_return=True)
133  logging.info('Uninstall took %s seconds.', main_timer.GetDelta())
134
135
136def Install(device, install_json, apk=None, enable_device_cache=False,
137            use_concurrency=True, permissions=()):
138  """Installs the given incremental apk and all required supporting files.
139
140  Args:
141    device: A DeviceUtils instance (to install to).
142    install_json: Path to .json file or already parsed .json object.
143    apk: An existing ApkHelper instance for the apk (optional).
144    enable_device_cache: Whether to enable on-device caching of checksums.
145    use_concurrency: Whether to speed things up using multiple threads.
146    permissions: A list of the permissions to grant, or None to grant all
147                 non-denylisted permissions in the manifest.
148  """
149  if isinstance(install_json, str):
150    with open(install_json) as f:
151      install_dict = json.load(f)
152  else:
153    install_dict = install_json
154
155  main_timer = time_profile.TimeProfile()
156  install_timer = time_profile.TimeProfile()
157  push_native_timer = time_profile.TimeProfile()
158  merge_dex_timer = time_profile.TimeProfile()
159  push_dex_timer = time_profile.TimeProfile()
160
161  def fix_path(p):
162    return os.path.normpath(os.path.join(constants.GetOutDirectory(), p))
163
164  if not apk:
165    apk = apk_helper.ToHelper(fix_path(install_dict['apk_path']))
166  split_globs = [fix_path(p) for p in install_dict['split_globs']]
167  native_libs = [fix_path(p) for p in install_dict['native_libs']]
168  dex_files = [fix_path(p) for p in install_dict['dex_files']]
169  show_proguard_warning = install_dict.get('show_proguard_warning')
170
171  apk_package = apk.GetPackageName()
172  device_incremental_dir = _GetDeviceIncrementalDir(apk_package)
173  dex_staging_dir = os.path.join(constants.GetOutDirectory(),
174                                 'incremental-install',
175                                 install_dict['apk_path'])
176  device_dex_dir = posixpath.join(device_incremental_dir, 'dex')
177
178  # Install .apk(s) if any of them have changed.
179  def do_install():
180    install_timer.Start()
181    if split_globs:
182      splits = []
183      for split_glob in split_globs:
184        splits.extend((f for f in glob.glob(split_glob)))
185      device.InstallSplitApk(
186          apk,
187          splits,
188          allow_downgrade=True,
189          reinstall=True,
190          allow_cached_props=True,
191          permissions=permissions)
192    else:
193      device.Install(
194          apk, allow_downgrade=True, reinstall=True, permissions=permissions)
195    install_timer.Stop(log=False)
196
197  # Push .so and .dex files to the device (if they have changed).
198  def do_push_files():
199
200    def do_push_native():
201      push_native_timer.Start()
202      if native_libs:
203        with build_utils.TempDir() as temp_dir:
204          device_lib_dir = posixpath.join(device_incremental_dir, 'lib')
205          for path in native_libs:
206            # Note: Can't use symlinks as they don't work when
207            # "adb push parent_dir" is used (like we do here).
208            shutil.copy(path, os.path.join(temp_dir, os.path.basename(path)))
209          device.PushChangedFiles([(temp_dir, device_lib_dir)],
210                                  delete_device_stale=True)
211      push_native_timer.Stop(log=False)
212
213    def do_merge_dex():
214      merge_dex_timer.Start()
215      shards = _AllocateDexShards(dex_files)
216      build_utils.MakeDirectory(dex_staging_dir)
217      _CreateDexFiles(shards, dex_staging_dir, apk.GetMinSdkVersion(),
218                      use_concurrency)
219      merge_dex_timer.Stop(log=False)
220
221    def do_push_dex():
222      push_dex_timer.Start()
223      device.PushChangedFiles([(dex_staging_dir, device_dex_dir)],
224                              delete_device_stale=True)
225      push_dex_timer.Stop(log=False)
226
227    _Execute(use_concurrency, do_push_native, do_merge_dex)
228    do_push_dex()
229
230  def check_device_configured():
231    if apk.GetTargetSdkVersion().isalpha():
232      # Assume pre-release SDK is always really new.
233      target_sdk_version = 99
234    else:
235      target_sdk_version = int(apk.GetTargetSdkVersion())
236    # Beta Q builds apply allowlist to targetSdk=28 as well.
237    if target_sdk_version >= 28 and device.build_version_sdk >= 28:
238      # In P, there are two settings:
239      #  * hidden_api_policy_p_apps
240      #  * hidden_api_policy_pre_p_apps
241      # In Q, there is just one:
242      #  * hidden_api_policy
243      if device.build_version_sdk == 28:
244        setting_name = 'hidden_api_policy_p_apps'
245      else:
246        setting_name = 'hidden_api_policy'
247      apis_allowed = ''.join(
248          device.RunShellCommand(['settings', 'get', 'global', setting_name],
249                                 check_return=True))
250      if apis_allowed.strip() not in '01':
251        msg = """\
252Cannot use incremental installs on Android P+ without first enabling access to
253non-SDK interfaces (https://developer.android.com/preview/non-sdk-q).
254
255To enable access:
256   adb -s {0} shell settings put global {1} 0
257To restore back to default:
258   adb -s {0} shell settings delete global {1}"""
259        raise Exception(msg.format(device.serial, setting_name))
260
261  cache_path = _DeviceCachePath(device)
262  def restore_cache():
263    if not enable_device_cache:
264      return
265    if os.path.exists(cache_path):
266      logging.info('Using device cache: %s', cache_path)
267      with open(cache_path) as f:
268        device.LoadCacheData(f.read())
269      # Delete the cached file so that any exceptions cause it to be cleared.
270      os.unlink(cache_path)
271    else:
272      logging.info('No device cache present: %s', cache_path)
273
274  def save_cache():
275    if not enable_device_cache:
276      return
277    with open(cache_path, 'w') as f:
278      f.write(device.DumpCacheData())
279      logging.info('Wrote device cache: %s', cache_path)
280
281  # Create 2 lock files:
282  # * install.lock tells the app to pause on start-up (until we release it).
283  # * firstrun.lock is used by the app to pause all secondary processes until
284  #   the primary process finishes loading the .dex / .so files.
285  def create_lock_files():
286    # Creates or zeros out lock files.
287    cmd = ('D="%s";'
288           'mkdir -p $D &&'
289           'echo -n >$D/install.lock 2>$D/firstrun.lock')
290    device.RunShellCommand(
291        cmd % device_incremental_dir, shell=True, check_return=True)
292
293  # The firstrun.lock is released by the app itself.
294  def release_installer_lock():
295    device.RunShellCommand('echo > %s/install.lock' % device_incremental_dir,
296                           check_return=True, shell=True)
297
298  # Concurrency here speeds things up quite a bit, but DeviceUtils hasn't
299  # been designed for multi-threading. Enabling only because this is a
300  # developer-only tool.
301  setup_timer = _Execute(use_concurrency, create_lock_files, restore_cache,
302                         check_device_configured)
303
304  _Execute(use_concurrency, do_install, do_push_files)
305
306  finalize_timer = _Execute(use_concurrency, release_installer_lock, save_cache)
307
308  logging.info(
309      'Install of %s took %s seconds (setup=%s, install=%s, lib_push=%s, '
310      'dex_merge=%s dex_push=%s, finalize=%s)', os.path.basename(apk.path),
311      main_timer.GetDelta(), setup_timer.GetDelta(), install_timer.GetDelta(),
312      push_native_timer.GetDelta(), merge_dex_timer.GetDelta(),
313      push_dex_timer.GetDelta(), finalize_timer.GetDelta())
314  if show_proguard_warning:
315    logging.warning('Target had proguard enabled, but incremental install uses '
316                    'non-proguarded .dex files. Performance characteristics '
317                    'may differ.')
318
319
320def main():
321  parser = argparse.ArgumentParser()
322  parser.add_argument('json_path',
323                      help='The path to the generated incremental apk .json.')
324  parser.add_argument('-d', '--device', dest='device',
325                      help='Target device for apk to install on.')
326  parser.add_argument('--uninstall',
327                      action='store_true',
328                      default=False,
329                      help='Remove the app and all side-loaded files.')
330  parser.add_argument('--output-directory',
331                      help='Path to the root build directory.')
332  parser.add_argument('--no-threading',
333                      action='store_false',
334                      default=True,
335                      dest='threading',
336                      help='Do not install and push concurrently')
337  parser.add_argument('--no-cache',
338                      action='store_false',
339                      default=True,
340                      dest='cache',
341                      help='Do not use cached information about what files are '
342                           'currently on the target device.')
343  parser.add_argument('-v',
344                      '--verbose',
345                      dest='verbose_count',
346                      default=0,
347                      action='count',
348                      help='Verbose level (multiple times for more)')
349
350  args = parser.parse_args()
351
352  run_tests_helper.SetLogLevel(args.verbose_count)
353  if args.output_directory:
354    constants.SetOutputDirectory(args.output_directory)
355
356  devil_chromium.Initialize(output_directory=constants.GetOutDirectory())
357
358  # Retries are annoying when commands fail for legitimate reasons. Might want
359  # to enable them if this is ever used on bots though.
360  device = device_utils.DeviceUtils.HealthyDevices(
361      device_arg=args.device,
362      default_retries=0,
363      enable_device_files_cache=True)[0]
364
365  if args.uninstall:
366    with open(args.json_path) as f:
367      install_dict = json.load(f)
368    apk = apk_helper.ToHelper(install_dict['apk_path'])
369    Uninstall(device, apk.GetPackageName(), enable_device_cache=args.cache)
370  else:
371    Install(device, args.json_path, enable_device_cache=args.cache,
372            use_concurrency=args.threading)
373
374
375if __name__ == '__main__':
376  sys.exit(main())
377