1#!/usr/bin/env python 2# 3# Copyright 2015 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Install *_incremental.apk targets as well as their dependent files.""" 8 9import argparse 10import collections 11import functools 12import glob 13import json 14import logging 15import os 16import posixpath 17import shutil 18import sys 19 20sys.path.append( 21 os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) 22import devil_chromium 23from devil.android import apk_helper 24from devil.android import device_utils 25from devil.utils import reraiser_thread 26from devil.utils import run_tests_helper 27from pylib import constants 28from pylib.utils import time_profile 29 30prev_sys_path = list(sys.path) 31sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, 'gyp')) 32import dex 33from util import build_utils 34sys.path = prev_sys_path 35 36 37_R8_PATH = os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'r8', 'lib', 38 'r8.jar') 39 40 41def _DeviceCachePath(device): 42 file_name = 'device_cache_%s.json' % device.adb.GetDeviceSerial() 43 return os.path.join(constants.GetOutDirectory(), file_name) 44 45 46def _Execute(concurrently, *funcs): 47 """Calls all functions in |funcs| concurrently or in sequence.""" 48 timer = time_profile.TimeProfile() 49 if concurrently: 50 reraiser_thread.RunAsync(funcs) 51 else: 52 for f in funcs: 53 f() 54 timer.Stop(log=False) 55 return timer 56 57 58def _GetDeviceIncrementalDir(package): 59 """Returns the device path to put incremental files for the given package.""" 60 return '/data/local/tmp/incremental-app-%s' % package 61 62 63def _IsStale(src_paths, dest): 64 """Returns if |dest| is older than any of |src_paths|, or missing.""" 65 if not os.path.exists(dest): 66 return True 67 dest_time = os.path.getmtime(dest) 68 for path in src_paths: 69 if os.path.getmtime(path) > dest_time: 70 return True 71 return False 72 73 74def _AllocateDexShards(dex_files): 75 """Divides input dex files into buckets.""" 76 # Goals: 77 # * Make shards small enough that they are fast to merge. 78 # * Minimize the number of shards so they load quickly on device. 79 # * Partition files into shards such that a change in one file results in only 80 # one shard having to be re-created. 81 shards = collections.defaultdict(list) 82 # As of Oct 2019, 10 shards results in a min/max size of 582K/2.6M. 83 NUM_CORE_SHARDS = 10 84 # As of Oct 2019, 17 dex files are larger than 1M. 85 SHARD_THRESHOLD = 2**20 86 for src_path in dex_files: 87 if os.path.getsize(src_path) >= SHARD_THRESHOLD: 88 # Use the path as the name rather than an incrementing number to ensure 89 # that it shards to the same name every time. 90 name = os.path.relpath(src_path, constants.GetOutDirectory()).replace( 91 os.sep, '.') 92 shards[name].append(src_path) 93 else: 94 name = 'shard{}.dex.jar'.format(hash(src_path) % NUM_CORE_SHARDS) 95 shards[name].append(src_path) 96 logging.info('Sharding %d dex files into %d buckets', len(dex_files), 97 len(shards)) 98 return shards 99 100 101def _CreateDexFiles(shards, dex_staging_dir, use_concurrency): 102 """Creates dex files within |dex_staging_dir| defined by |shards|.""" 103 tasks = [] 104 for name, src_paths in shards.iteritems(): 105 dest_path = os.path.join(dex_staging_dir, name) 106 if _IsStale(src_paths, dest_path): 107 tasks.append( 108 functools.partial(dex.MergeDexForIncrementalInstall, _R8_PATH, 109 src_paths, dest_path)) 110 111 # TODO(agrieve): It would be more performant to write a custom d8.jar 112 # wrapper in java that would process these in bulk, rather than spinning 113 # up a new process for each one. 114 _Execute(use_concurrency, *tasks) 115 116 # Remove any stale shards. 117 for name in os.listdir(dex_staging_dir): 118 if name not in shards: 119 os.unlink(os.path.join(dex_staging_dir, name)) 120 121 122def Uninstall(device, package, enable_device_cache=False): 123 """Uninstalls and removes all incremental files for the given package.""" 124 main_timer = time_profile.TimeProfile() 125 device.Uninstall(package) 126 if enable_device_cache: 127 # Uninstall is rare, so just wipe the cache in this case. 128 cache_path = _DeviceCachePath(device) 129 if os.path.exists(cache_path): 130 os.unlink(cache_path) 131 device.RunShellCommand(['rm', '-rf', _GetDeviceIncrementalDir(package)], 132 check_return=True) 133 logging.info('Uninstall took %s seconds.', main_timer.GetDelta()) 134 135 136def Install(device, install_json, apk=None, enable_device_cache=False, 137 use_concurrency=True, permissions=()): 138 """Installs the given incremental apk and all required supporting files. 139 140 Args: 141 device: A DeviceUtils instance (to install to). 142 install_json: Path to .json file or already parsed .json object. 143 apk: An existing ApkHelper instance for the apk (optional). 144 enable_device_cache: Whether to enable on-device caching of checksums. 145 use_concurrency: Whether to speed things up using multiple threads. 146 permissions: A list of the permissions to grant, or None to grant all 147 non-blacklisted permissions in the manifest. 148 """ 149 if isinstance(install_json, basestring): 150 with open(install_json) as f: 151 install_dict = json.load(f) 152 else: 153 install_dict = install_json 154 155 if install_dict.get('dont_even_try'): 156 raise Exception(install_dict['dont_even_try']) 157 158 main_timer = time_profile.TimeProfile() 159 install_timer = time_profile.TimeProfile() 160 push_native_timer = time_profile.TimeProfile() 161 merge_dex_timer = time_profile.TimeProfile() 162 push_dex_timer = time_profile.TimeProfile() 163 164 def fix_path(p): 165 return os.path.normpath(os.path.join(constants.GetOutDirectory(), p)) 166 167 if not apk: 168 apk = apk_helper.ToHelper(fix_path(install_dict['apk_path'])) 169 split_globs = [fix_path(p) for p in install_dict['split_globs']] 170 native_libs = [fix_path(p) for p in install_dict['native_libs']] 171 dex_files = [fix_path(p) for p in install_dict['dex_files']] 172 show_proguard_warning = install_dict.get('show_proguard_warning') 173 174 apk_package = apk.GetPackageName() 175 device_incremental_dir = _GetDeviceIncrementalDir(apk_package) 176 dex_staging_dir = os.path.join(constants.GetOutDirectory(), 177 'incremental-install', 178 install_dict['apk_path']) 179 device_dex_dir = posixpath.join(device_incremental_dir, 'dex') 180 181 # Install .apk(s) if any of them have changed. 182 def do_install(): 183 install_timer.Start() 184 if split_globs: 185 splits = [] 186 for split_glob in split_globs: 187 splits.extend((f for f in glob.glob(split_glob))) 188 device.InstallSplitApk( 189 apk, 190 splits, 191 allow_downgrade=True, 192 reinstall=True, 193 allow_cached_props=True, 194 permissions=permissions) 195 else: 196 device.Install( 197 apk, allow_downgrade=True, reinstall=True, permissions=permissions) 198 install_timer.Stop(log=False) 199 200 # Push .so and .dex files to the device (if they have changed). 201 def do_push_files(): 202 203 def do_push_native(): 204 push_native_timer.Start() 205 if native_libs: 206 with build_utils.TempDir() as temp_dir: 207 device_lib_dir = posixpath.join(device_incremental_dir, 'lib') 208 for path in native_libs: 209 # Note: Can't use symlinks as they don't work when 210 # "adb push parent_dir" is used (like we do here). 211 shutil.copy(path, os.path.join(temp_dir, os.path.basename(path))) 212 device.PushChangedFiles([(temp_dir, device_lib_dir)], 213 delete_device_stale=True) 214 push_native_timer.Stop(log=False) 215 216 def do_merge_dex(): 217 merge_dex_timer.Start() 218 shards = _AllocateDexShards(dex_files) 219 build_utils.MakeDirectory(dex_staging_dir) 220 _CreateDexFiles(shards, dex_staging_dir, use_concurrency) 221 merge_dex_timer.Stop(log=False) 222 223 def do_push_dex(): 224 push_dex_timer.Start() 225 device.PushChangedFiles([(dex_staging_dir, device_dex_dir)], 226 delete_device_stale=True) 227 push_dex_timer.Stop(log=False) 228 229 _Execute(use_concurrency, do_push_native, do_merge_dex) 230 do_push_dex() 231 232 def check_device_configured(): 233 target_sdk_version = int(apk.GetTargetSdkVersion()) 234 # Beta Q builds apply whitelist to targetSdk=28 as well. 235 if target_sdk_version >= 28 and device.build_version_sdk >= 28: 236 apis_allowed = ''.join( 237 device.RunShellCommand( 238 ['settings', 'get', 'global', 'hidden_api_policy'], 239 check_return=True)) 240 if apis_allowed.strip() not in '01': 241 msg = """\ 242Cannot use incremental installs on Android Q+ without first enabling access to 243non-SDK interfaces (https://developer.android.com/preview/non-sdk-q). 244 245To enable access: 246 adb -s {0} shell settings put global hidden_api_policy 0 247To restore back to default: 248 adb -s {0} shell settings delete global hidden_api_policy""" 249 raise Exception(msg.format(device.serial)) 250 251 cache_path = _DeviceCachePath(device) 252 def restore_cache(): 253 if not enable_device_cache: 254 return 255 if os.path.exists(cache_path): 256 logging.info('Using device cache: %s', cache_path) 257 with open(cache_path) as f: 258 device.LoadCacheData(f.read()) 259 # Delete the cached file so that any exceptions cause it to be cleared. 260 os.unlink(cache_path) 261 else: 262 logging.info('No device cache present: %s', cache_path) 263 264 def save_cache(): 265 if not enable_device_cache: 266 return 267 with open(cache_path, 'w') as f: 268 f.write(device.DumpCacheData()) 269 logging.info('Wrote device cache: %s', cache_path) 270 271 # Create 2 lock files: 272 # * install.lock tells the app to pause on start-up (until we release it). 273 # * firstrun.lock is used by the app to pause all secondary processes until 274 # the primary process finishes loading the .dex / .so files. 275 def create_lock_files(): 276 # Creates or zeros out lock files. 277 cmd = ('D="%s";' 278 'mkdir -p $D &&' 279 'echo -n >$D/install.lock 2>$D/firstrun.lock') 280 device.RunShellCommand( 281 cmd % device_incremental_dir, shell=True, check_return=True) 282 283 # The firstrun.lock is released by the app itself. 284 def release_installer_lock(): 285 device.RunShellCommand('echo > %s/install.lock' % device_incremental_dir, 286 check_return=True, shell=True) 287 288 # Concurrency here speeds things up quite a bit, but DeviceUtils hasn't 289 # been designed for multi-threading. Enabling only because this is a 290 # developer-only tool. 291 setup_timer = _Execute(use_concurrency, create_lock_files, restore_cache, 292 check_device_configured) 293 294 _Execute(use_concurrency, do_install, do_push_files) 295 296 finalize_timer = _Execute(use_concurrency, release_installer_lock, save_cache) 297 298 logging.info( 299 'Install of %s took %s seconds (setup=%s, install=%s, lib_push=%s, ' 300 'dex_merge=%s dex_push=%s, finalize=%s)', os.path.basename(apk.path), 301 main_timer.GetDelta(), setup_timer.GetDelta(), install_timer.GetDelta(), 302 push_native_timer.GetDelta(), merge_dex_timer.GetDelta(), 303 push_dex_timer.GetDelta(), finalize_timer.GetDelta()) 304 if show_proguard_warning: 305 logging.warning('Target had proguard enabled, but incremental install uses ' 306 'non-proguarded .dex files. Performance characteristics ' 307 'may differ.') 308 309 310def main(): 311 parser = argparse.ArgumentParser() 312 parser.add_argument('json_path', 313 help='The path to the generated incremental apk .json.') 314 parser.add_argument('-d', '--device', dest='device', 315 help='Target device for apk to install on.') 316 parser.add_argument('--uninstall', 317 action='store_true', 318 default=False, 319 help='Remove the app and all side-loaded files.') 320 parser.add_argument('--output-directory', 321 help='Path to the root build directory.') 322 parser.add_argument('--no-threading', 323 action='store_false', 324 default=True, 325 dest='threading', 326 help='Do not install and push concurrently') 327 parser.add_argument('--no-cache', 328 action='store_false', 329 default=True, 330 dest='cache', 331 help='Do not use cached information about what files are ' 332 'currently on the target device.') 333 parser.add_argument('-v', 334 '--verbose', 335 dest='verbose_count', 336 default=0, 337 action='count', 338 help='Verbose level (multiple times for more)') 339 340 args = parser.parse_args() 341 342 run_tests_helper.SetLogLevel(args.verbose_count) 343 if args.output_directory: 344 constants.SetOutputDirectory(args.output_directory) 345 346 devil_chromium.Initialize(output_directory=constants.GetOutDirectory()) 347 348 # Retries are annoying when commands fail for legitimate reasons. Might want 349 # to enable them if this is ever used on bots though. 350 device = device_utils.DeviceUtils.HealthyDevices( 351 device_arg=args.device, 352 default_retries=0, 353 enable_device_files_cache=True)[0] 354 355 if args.uninstall: 356 with open(args.json_path) as f: 357 install_dict = json.load(f) 358 apk = apk_helper.ToHelper(install_dict['apk_path']) 359 Uninstall(device, apk.GetPackageName(), enable_device_cache=args.cache) 360 else: 361 Install(device, args.json_path, enable_device_cache=args.cache, 362 use_concurrency=args.threading) 363 364 365if __name__ == '__main__': 366 sys.exit(main()) 367