1#!/usr/bin/env vpython3 2# 3# Copyright 2015 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Install *_incremental.apk targets as well as their dependent files.""" 8 9import argparse 10import collections 11import functools 12import glob 13import json 14import logging 15import os 16import posixpath 17import shutil 18import sys 19 20sys.path.append( 21 os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) 22import devil_chromium 23from devil.android import apk_helper 24from devil.android import device_utils 25from devil.utils import reraiser_thread 26from devil.utils import run_tests_helper 27from pylib import constants 28from pylib.utils import time_profile 29 30prev_sys_path = list(sys.path) 31sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, 'gyp')) 32import dex 33from util import build_utils 34sys.path = prev_sys_path 35 36 37_R8_PATH = os.path.join(build_utils.DIR_SOURCE_ROOT, 'third_party', 'r8', 'lib', 38 'r8.jar') 39 40 41def _DeviceCachePath(device): 42 file_name = 'device_cache_%s.json' % device.adb.GetDeviceSerial() 43 return os.path.join(constants.GetOutDirectory(), file_name) 44 45 46def _Execute(concurrently, *funcs): 47 """Calls all functions in |funcs| concurrently or in sequence.""" 48 timer = time_profile.TimeProfile() 49 if concurrently: 50 reraiser_thread.RunAsync(funcs) 51 else: 52 for f in funcs: 53 f() 54 timer.Stop(log=False) 55 return timer 56 57 58def _GetDeviceIncrementalDir(package): 59 """Returns the device path to put incremental files for the given package.""" 60 return '/data/local/tmp/incremental-app-%s' % package 61 62 63def _IsStale(src_paths, dest): 64 """Returns if |dest| is older than any of |src_paths|, or missing.""" 65 if not os.path.exists(dest): 66 return True 67 dest_time = os.path.getmtime(dest) 68 for path in src_paths: 69 if os.path.getmtime(path) > dest_time: 70 return True 71 return False 72 73 74def _AllocateDexShards(dex_files): 75 """Divides input dex files into buckets.""" 76 # Goals: 77 # * Make shards small enough that they are fast to merge. 78 # * Minimize the number of shards so they load quickly on device. 79 # * Partition files into shards such that a change in one file results in only 80 # one shard having to be re-created. 81 shards = collections.defaultdict(list) 82 # As of Oct 2019, 10 shards results in a min/max size of 582K/2.6M. 83 NUM_CORE_SHARDS = 10 84 # As of Oct 2019, 17 dex files are larger than 1M. 85 SHARD_THRESHOLD = 2**20 86 for src_path in dex_files: 87 if os.path.getsize(src_path) >= SHARD_THRESHOLD: 88 # Use the path as the name rather than an incrementing number to ensure 89 # that it shards to the same name every time. 90 name = os.path.relpath(src_path, constants.GetOutDirectory()).replace( 91 os.sep, '.') 92 shards[name].append(src_path) 93 else: 94 name = 'shard{}.dex.jar'.format(hash(src_path) % NUM_CORE_SHARDS) 95 shards[name].append(src_path) 96 logging.info('Sharding %d dex files into %d buckets', len(dex_files), 97 len(shards)) 98 return shards 99 100 101def _CreateDexFiles(shards, dex_staging_dir, min_api, use_concurrency): 102 """Creates dex files within |dex_staging_dir| defined by |shards|.""" 103 tasks = [] 104 for name, src_paths in shards.items(): 105 dest_path = os.path.join(dex_staging_dir, name) 106 if _IsStale(src_paths, dest_path): 107 tasks.append( 108 functools.partial(dex.MergeDexForIncrementalInstall, _R8_PATH, 109 src_paths, dest_path, min_api)) 110 111 # TODO(agrieve): It would be more performant to write a custom d8.jar 112 # wrapper in java that would process these in bulk, rather than spinning 113 # up a new process for each one. 114 _Execute(use_concurrency, *tasks) 115 116 # Remove any stale shards. 117 for name in os.listdir(dex_staging_dir): 118 if name not in shards: 119 os.unlink(os.path.join(dex_staging_dir, name)) 120 121 122def Uninstall(device, package, enable_device_cache=False): 123 """Uninstalls and removes all incremental files for the given package.""" 124 main_timer = time_profile.TimeProfile() 125 device.Uninstall(package) 126 if enable_device_cache: 127 # Uninstall is rare, so just wipe the cache in this case. 128 cache_path = _DeviceCachePath(device) 129 if os.path.exists(cache_path): 130 os.unlink(cache_path) 131 device.RunShellCommand(['rm', '-rf', _GetDeviceIncrementalDir(package)], 132 check_return=True) 133 logging.info('Uninstall took %s seconds.', main_timer.GetDelta()) 134 135 136def Install(device, install_json, apk=None, enable_device_cache=False, 137 use_concurrency=True, permissions=()): 138 """Installs the given incremental apk and all required supporting files. 139 140 Args: 141 device: A DeviceUtils instance (to install to). 142 install_json: Path to .json file or already parsed .json object. 143 apk: An existing ApkHelper instance for the apk (optional). 144 enable_device_cache: Whether to enable on-device caching of checksums. 145 use_concurrency: Whether to speed things up using multiple threads. 146 permissions: A list of the permissions to grant, or None to grant all 147 non-denylisted permissions in the manifest. 148 """ 149 if isinstance(install_json, str): 150 with open(install_json) as f: 151 install_dict = json.load(f) 152 else: 153 install_dict = install_json 154 155 main_timer = time_profile.TimeProfile() 156 install_timer = time_profile.TimeProfile() 157 push_native_timer = time_profile.TimeProfile() 158 merge_dex_timer = time_profile.TimeProfile() 159 push_dex_timer = time_profile.TimeProfile() 160 161 def fix_path(p): 162 return os.path.normpath(os.path.join(constants.GetOutDirectory(), p)) 163 164 if not apk: 165 apk = apk_helper.ToHelper(fix_path(install_dict['apk_path'])) 166 split_globs = [fix_path(p) for p in install_dict['split_globs']] 167 native_libs = [fix_path(p) for p in install_dict['native_libs']] 168 dex_files = [fix_path(p) for p in install_dict['dex_files']] 169 show_proguard_warning = install_dict.get('show_proguard_warning') 170 171 apk_package = apk.GetPackageName() 172 device_incremental_dir = _GetDeviceIncrementalDir(apk_package) 173 dex_staging_dir = os.path.join(constants.GetOutDirectory(), 174 'incremental-install', 175 install_dict['apk_path']) 176 device_dex_dir = posixpath.join(device_incremental_dir, 'dex') 177 178 # Install .apk(s) if any of them have changed. 179 def do_install(): 180 install_timer.Start() 181 if split_globs: 182 splits = [] 183 for split_glob in split_globs: 184 splits.extend((f for f in glob.glob(split_glob))) 185 device.InstallSplitApk( 186 apk, 187 splits, 188 allow_downgrade=True, 189 reinstall=True, 190 allow_cached_props=True, 191 permissions=permissions) 192 else: 193 device.Install( 194 apk, allow_downgrade=True, reinstall=True, permissions=permissions) 195 install_timer.Stop(log=False) 196 197 # Push .so and .dex files to the device (if they have changed). 198 def do_push_files(): 199 200 def do_push_native(): 201 push_native_timer.Start() 202 if native_libs: 203 with build_utils.TempDir() as temp_dir: 204 device_lib_dir = posixpath.join(device_incremental_dir, 'lib') 205 for path in native_libs: 206 # Note: Can't use symlinks as they don't work when 207 # "adb push parent_dir" is used (like we do here). 208 shutil.copy(path, os.path.join(temp_dir, os.path.basename(path))) 209 device.PushChangedFiles([(temp_dir, device_lib_dir)], 210 delete_device_stale=True) 211 push_native_timer.Stop(log=False) 212 213 def do_merge_dex(): 214 merge_dex_timer.Start() 215 shards = _AllocateDexShards(dex_files) 216 build_utils.MakeDirectory(dex_staging_dir) 217 _CreateDexFiles(shards, dex_staging_dir, apk.GetMinSdkVersion(), 218 use_concurrency) 219 merge_dex_timer.Stop(log=False) 220 221 def do_push_dex(): 222 push_dex_timer.Start() 223 device.PushChangedFiles([(dex_staging_dir, device_dex_dir)], 224 delete_device_stale=True) 225 push_dex_timer.Stop(log=False) 226 227 _Execute(use_concurrency, do_push_native, do_merge_dex) 228 do_push_dex() 229 230 def check_device_configured(): 231 if apk.GetTargetSdkVersion().isalpha(): 232 # Assume pre-release SDK is always really new. 233 target_sdk_version = 99 234 else: 235 target_sdk_version = int(apk.GetTargetSdkVersion()) 236 # Beta Q builds apply allowlist to targetSdk=28 as well. 237 if target_sdk_version >= 28 and device.build_version_sdk >= 28: 238 # In P, there are two settings: 239 # * hidden_api_policy_p_apps 240 # * hidden_api_policy_pre_p_apps 241 # In Q, there is just one: 242 # * hidden_api_policy 243 if device.build_version_sdk == 28: 244 setting_name = 'hidden_api_policy_p_apps' 245 else: 246 setting_name = 'hidden_api_policy' 247 apis_allowed = ''.join( 248 device.RunShellCommand(['settings', 'get', 'global', setting_name], 249 check_return=True)) 250 if apis_allowed.strip() not in '01': 251 msg = """\ 252Cannot use incremental installs on Android P+ without first enabling access to 253non-SDK interfaces (https://developer.android.com/preview/non-sdk-q). 254 255To enable access: 256 adb -s {0} shell settings put global {1} 0 257To restore back to default: 258 adb -s {0} shell settings delete global {1}""" 259 raise Exception(msg.format(device.serial, setting_name)) 260 261 cache_path = _DeviceCachePath(device) 262 def restore_cache(): 263 if not enable_device_cache: 264 return 265 if os.path.exists(cache_path): 266 logging.info('Using device cache: %s', cache_path) 267 with open(cache_path) as f: 268 device.LoadCacheData(f.read()) 269 # Delete the cached file so that any exceptions cause it to be cleared. 270 os.unlink(cache_path) 271 else: 272 logging.info('No device cache present: %s', cache_path) 273 274 def save_cache(): 275 if not enable_device_cache: 276 return 277 with open(cache_path, 'w') as f: 278 f.write(device.DumpCacheData()) 279 logging.info('Wrote device cache: %s', cache_path) 280 281 # Create 2 lock files: 282 # * install.lock tells the app to pause on start-up (until we release it). 283 # * firstrun.lock is used by the app to pause all secondary processes until 284 # the primary process finishes loading the .dex / .so files. 285 def create_lock_files(): 286 # Creates or zeros out lock files. 287 cmd = ('D="%s";' 288 'mkdir -p $D &&' 289 'echo -n >$D/install.lock 2>$D/firstrun.lock') 290 device.RunShellCommand( 291 cmd % device_incremental_dir, shell=True, check_return=True) 292 293 # The firstrun.lock is released by the app itself. 294 def release_installer_lock(): 295 device.RunShellCommand('echo > %s/install.lock' % device_incremental_dir, 296 check_return=True, shell=True) 297 298 # Concurrency here speeds things up quite a bit, but DeviceUtils hasn't 299 # been designed for multi-threading. Enabling only because this is a 300 # developer-only tool. 301 setup_timer = _Execute(use_concurrency, create_lock_files, restore_cache, 302 check_device_configured) 303 304 _Execute(use_concurrency, do_install, do_push_files) 305 306 finalize_timer = _Execute(use_concurrency, release_installer_lock, save_cache) 307 308 logging.info( 309 'Install of %s took %s seconds (setup=%s, install=%s, lib_push=%s, ' 310 'dex_merge=%s dex_push=%s, finalize=%s)', os.path.basename(apk.path), 311 main_timer.GetDelta(), setup_timer.GetDelta(), install_timer.GetDelta(), 312 push_native_timer.GetDelta(), merge_dex_timer.GetDelta(), 313 push_dex_timer.GetDelta(), finalize_timer.GetDelta()) 314 if show_proguard_warning: 315 logging.warning('Target had proguard enabled, but incremental install uses ' 316 'non-proguarded .dex files. Performance characteristics ' 317 'may differ.') 318 319 320def main(): 321 parser = argparse.ArgumentParser() 322 parser.add_argument('json_path', 323 help='The path to the generated incremental apk .json.') 324 parser.add_argument('-d', '--device', dest='device', 325 help='Target device for apk to install on.') 326 parser.add_argument('--uninstall', 327 action='store_true', 328 default=False, 329 help='Remove the app and all side-loaded files.') 330 parser.add_argument('--output-directory', 331 help='Path to the root build directory.') 332 parser.add_argument('--no-threading', 333 action='store_false', 334 default=True, 335 dest='threading', 336 help='Do not install and push concurrently') 337 parser.add_argument('--no-cache', 338 action='store_false', 339 default=True, 340 dest='cache', 341 help='Do not use cached information about what files are ' 342 'currently on the target device.') 343 parser.add_argument('-v', 344 '--verbose', 345 dest='verbose_count', 346 default=0, 347 action='count', 348 help='Verbose level (multiple times for more)') 349 350 args = parser.parse_args() 351 352 run_tests_helper.SetLogLevel(args.verbose_count) 353 if args.output_directory: 354 constants.SetOutputDirectory(args.output_directory) 355 356 devil_chromium.Initialize(output_directory=constants.GetOutDirectory()) 357 358 # Retries are annoying when commands fail for legitimate reasons. Might want 359 # to enable them if this is ever used on bots though. 360 device = device_utils.DeviceUtils.HealthyDevices( 361 device_arg=args.device, 362 default_retries=0, 363 enable_device_files_cache=True)[0] 364 365 if args.uninstall: 366 with open(args.json_path) as f: 367 install_dict = json.load(f) 368 apk = apk_helper.ToHelper(install_dict['apk_path']) 369 Uninstall(device, apk.GetPackageName(), enable_device_cache=args.cache) 370 else: 371 Install(device, args.json_path, enable_device_cache=args.cache, 372 use_concurrency=args.threading) 373 374 375if __name__ == '__main__': 376 sys.exit(main()) 377