1#!/usr/bin/env python 2# 3# Copyright (c) 2015 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Adds the code parts to a resource APK.""" 8 9import argparse 10import logging 11import os 12import shutil 13import sys 14import tempfile 15import zipfile 16import zlib 17 18import finalize_apk 19 20from util import build_utils 21from util import diff_utils 22from util import zipalign 23 24# Input dex.jar files are zipaligned. 25zipalign.ApplyZipFileZipAlignFix() 26 27 28# Taken from aapt's Package.cpp: 29_NO_COMPRESS_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.gif', '.wav', '.mp2', 30 '.mp3', '.ogg', '.aac', '.mpg', '.mpeg', '.mid', 31 '.midi', '.smf', '.jet', '.rtttl', '.imy', '.xmf', 32 '.mp4', '.m4a', '.m4v', '.3gp', '.3gpp', '.3g2', 33 '.3gpp2', '.amr', '.awb', '.wma', '.wmv', '.webm') 34 35 36def _ParseArgs(args): 37 parser = argparse.ArgumentParser() 38 build_utils.AddDepfileOption(parser) 39 parser.add_argument( 40 '--assets', 41 help='GYP-list of files to add as assets in the form ' 42 '"srcPath:zipPath", where ":zipPath" is optional.') 43 parser.add_argument( 44 '--java-resources', help='GYP-list of java_resources JARs to include.') 45 parser.add_argument('--write-asset-list', 46 action='store_true', 47 help='Whether to create an assets/assets_list file.') 48 parser.add_argument( 49 '--uncompressed-assets', 50 help='Same as --assets, except disables compression.') 51 parser.add_argument('--resource-apk', 52 help='An .ap_ file built using aapt', 53 required=True) 54 parser.add_argument('--output-apk', 55 help='Path to the output file', 56 required=True) 57 parser.add_argument('--format', choices=['apk', 'bundle-module'], 58 default='apk', help='Specify output format.') 59 parser.add_argument('--dex-file', 60 help='Path to the classes.dex to use') 61 parser.add_argument( 62 '--jdk-libs-dex-file', 63 help='Path to classes.dex created by dex_jdk_libs.py') 64 parser.add_argument('--uncompress-dex', action='store_true', 65 help='Store .dex files uncompressed in the APK') 66 parser.add_argument('--native-libs', 67 action='append', 68 help='GYP-list of native libraries to include. ' 69 'Can be specified multiple times.', 70 default=[]) 71 parser.add_argument('--secondary-native-libs', 72 action='append', 73 help='GYP-list of native libraries for secondary ' 74 'android-abi. Can be specified multiple times.', 75 default=[]) 76 parser.add_argument('--android-abi', 77 help='Android architecture to use for native libraries') 78 parser.add_argument('--secondary-android-abi', 79 help='The secondary Android architecture to use for' 80 'secondary native libraries') 81 parser.add_argument( 82 '--is-multi-abi', 83 action='store_true', 84 help='Will add a placeholder for the missing ABI if no native libs or ' 85 'placeholders are set for either the primary or secondary ABI. Can only ' 86 'be set if both --android-abi and --secondary-android-abi are set.') 87 parser.add_argument( 88 '--native-lib-placeholders', 89 help='GYP-list of native library placeholders to add.') 90 parser.add_argument( 91 '--secondary-native-lib-placeholders', 92 help='GYP-list of native library placeholders to add ' 93 'for the secondary ABI') 94 parser.add_argument('--uncompress-shared-libraries', default='False', 95 choices=['true', 'True', 'false', 'False'], 96 help='Whether to uncompress native shared libraries. Argument must be ' 97 'a boolean value.') 98 parser.add_argument( 99 '--apksigner-jar', help='Path to the apksigner executable.') 100 parser.add_argument('--zipalign-path', 101 help='Path to the zipalign executable.') 102 parser.add_argument('--key-path', 103 help='Path to keystore for signing.') 104 parser.add_argument('--key-passwd', 105 help='Keystore password') 106 parser.add_argument('--key-name', 107 help='Keystore name') 108 parser.add_argument( 109 '--min-sdk-version', required=True, help='Value of APK\'s minSdkVersion') 110 parser.add_argument( 111 '--best-compression', 112 action='store_true', 113 help='Use zip -9 rather than zip -1') 114 parser.add_argument( 115 '--library-always-compress', 116 action='append', 117 help='The list of library files that we always compress.') 118 parser.add_argument( 119 '--library-renames', 120 action='append', 121 help='The list of library files that we prepend crazy. to their names.') 122 parser.add_argument('--warnings-as-errors', 123 action='store_true', 124 help='Treat all warnings as errors.') 125 diff_utils.AddCommandLineFlags(parser) 126 options = parser.parse_args(args) 127 options.assets = build_utils.ParseGnList(options.assets) 128 options.uncompressed_assets = build_utils.ParseGnList( 129 options.uncompressed_assets) 130 options.native_lib_placeholders = build_utils.ParseGnList( 131 options.native_lib_placeholders) 132 options.secondary_native_lib_placeholders = build_utils.ParseGnList( 133 options.secondary_native_lib_placeholders) 134 options.java_resources = build_utils.ParseGnList(options.java_resources) 135 options.native_libs = build_utils.ParseGnList(options.native_libs) 136 options.secondary_native_libs = build_utils.ParseGnList( 137 options.secondary_native_libs) 138 options.library_always_compress = build_utils.ParseGnList( 139 options.library_always_compress) 140 options.library_renames = build_utils.ParseGnList(options.library_renames) 141 142 # --apksigner-jar, --zipalign-path, --key-xxx arguments are 143 # required when building an APK, but not a bundle module. 144 if options.format == 'apk': 145 required_args = [ 146 'apksigner_jar', 'zipalign_path', 'key_path', 'key_passwd', 'key_name' 147 ] 148 for required in required_args: 149 if not vars(options)[required]: 150 raise Exception('Argument --%s is required for APKs.' % ( 151 required.replace('_', '-'))) 152 153 options.uncompress_shared_libraries = \ 154 options.uncompress_shared_libraries in [ 'true', 'True' ] 155 156 if not options.android_abi and (options.native_libs or 157 options.native_lib_placeholders): 158 raise Exception('Must specify --android-abi with --native-libs') 159 if not options.secondary_android_abi and (options.secondary_native_libs or 160 options.secondary_native_lib_placeholders): 161 raise Exception('Must specify --secondary-android-abi with' 162 ' --secondary-native-libs') 163 if options.is_multi_abi and not (options.android_abi 164 and options.secondary_android_abi): 165 raise Exception('Must specify --is-multi-abi with both --android-abi ' 166 'and --secondary-android-abi.') 167 return options 168 169 170def _SplitAssetPath(path): 171 """Returns (src, dest) given an asset path in the form src[:dest].""" 172 path_parts = path.split(':') 173 src_path = path_parts[0] 174 if len(path_parts) > 1: 175 dest_path = path_parts[1] 176 else: 177 dest_path = os.path.basename(src_path) 178 return src_path, dest_path 179 180 181def _ExpandPaths(paths): 182 """Converts src:dst into tuples and enumerates files within directories. 183 184 Args: 185 paths: Paths in the form "src_path:dest_path" 186 187 Returns: 188 A list of (src_path, dest_path) tuples sorted by dest_path (for stable 189 ordering within output .apk). 190 """ 191 ret = [] 192 for path in paths: 193 src_path, dest_path = _SplitAssetPath(path) 194 if os.path.isdir(src_path): 195 for f in build_utils.FindInDirectory(src_path, '*'): 196 ret.append((f, os.path.join(dest_path, f[len(src_path) + 1:]))) 197 else: 198 ret.append((src_path, dest_path)) 199 ret.sort(key=lambda t:t[1]) 200 return ret 201 202 203def _GetAssetsToAdd(path_tuples, 204 fast_align, 205 disable_compression=False, 206 allow_reads=True): 207 """Returns the list of file_detail tuples for assets in the apk. 208 209 Args: 210 path_tuples: List of src_path, dest_path tuples to add. 211 fast_align: Whether to perform alignment in python zipfile (alternatively 212 alignment can be done using the zipalign utility out of band). 213 disable_compression: Whether to disable compression. 214 allow_reads: If false, we do not try to read the files from disk (to find 215 their size for example). 216 217 Returns: A list of (src_path, apk_path, compress, alignment) tuple 218 representing what and how assets are added. 219 """ 220 assets_to_add = [] 221 222 # Group all uncompressed assets together in the hope that it will increase 223 # locality of mmap'ed files. 224 for target_compress in (False, True): 225 for src_path, dest_path in path_tuples: 226 compress = not disable_compression and ( 227 os.path.splitext(src_path)[1] not in _NO_COMPRESS_EXTENSIONS) 228 229 if target_compress == compress: 230 # AddToZipHermetic() uses this logic to avoid growing small files. 231 # We need it here in order to set alignment correctly. 232 if allow_reads and compress and os.path.getsize(src_path) < 16: 233 compress = False 234 235 apk_path = 'assets/' + dest_path 236 alignment = 0 if compress and not fast_align else 4 237 assets_to_add.append((apk_path, src_path, compress, alignment)) 238 return assets_to_add 239 240 241def _AddFiles(apk, details): 242 """Adds files to the apk. 243 244 Args: 245 apk: path to APK to add to. 246 details: A list of file detail tuples (src_path, apk_path, compress, 247 alignment) representing what and how files are added to the APK. 248 """ 249 for apk_path, src_path, compress, alignment in details: 250 # This check is only relevant for assets, but it should not matter if it is 251 # checked for the whole list of files. 252 try: 253 apk.getinfo(apk_path) 254 # Should never happen since write_build_config.py handles merging. 255 raise Exception( 256 'Multiple targets specified the asset path: %s' % apk_path) 257 except KeyError: 258 zipalign.AddToZipHermetic( 259 apk, 260 apk_path, 261 src_path=src_path, 262 compress=compress, 263 alignment=alignment) 264 265 266def _GetNativeLibrariesToAdd(native_libs, android_abi, uncompress, fast_align, 267 lib_always_compress, lib_renames): 268 """Returns the list of file_detail tuples for native libraries in the apk. 269 270 Returns: A list of (src_path, apk_path, compress, alignment) tuple 271 representing what and how native libraries are added. 272 """ 273 libraries_to_add = [] 274 275 276 for path in native_libs: 277 basename = os.path.basename(path) 278 compress = not uncompress or any(lib_name in basename 279 for lib_name in lib_always_compress) 280 rename = any(lib_name in basename for lib_name in lib_renames) 281 if rename: 282 basename = 'crazy.' + basename 283 284 lib_android_abi = android_abi 285 if path.startswith('android_clang_arm64_hwasan/'): 286 lib_android_abi = 'arm64-v8a-hwasan' 287 288 apk_path = 'lib/%s/%s' % (lib_android_abi, basename) 289 alignment = 0 if compress and not fast_align else 0x1000 290 libraries_to_add.append((apk_path, path, compress, alignment)) 291 292 return libraries_to_add 293 294 295def _CreateExpectationsData(native_libs, assets): 296 """Creates list of native libraries and assets.""" 297 native_libs = sorted(native_libs) 298 assets = sorted(assets) 299 300 ret = [] 301 for apk_path, _, compress, alignment in native_libs + assets: 302 ret.append('apk_path=%s, compress=%s, alignment=%s\n' % 303 (apk_path, compress, alignment)) 304 return ''.join(ret) 305 306 307def main(args): 308 build_utils.InitLogging('APKBUILDER_DEBUG') 309 args = build_utils.ExpandFileArgs(args) 310 options = _ParseArgs(args) 311 312 # Until Python 3.7, there's no better way to set compression level. 313 # The default is 6. 314 if options.best_compression: 315 # Compresses about twice as slow as the default. 316 zlib.Z_DEFAULT_COMPRESSION = 9 317 else: 318 # Compresses about twice as fast as the default. 319 zlib.Z_DEFAULT_COMPRESSION = 1 320 321 # Manually align only when alignment is necessary. 322 # Python's zip implementation duplicates file comments in the central 323 # directory, whereas zipalign does not, so use zipalign for official builds. 324 fast_align = options.format == 'apk' and not options.best_compression 325 326 native_libs = sorted(options.native_libs) 327 328 # Include native libs in the depfile_deps since GN doesn't know about the 329 # dependencies when is_component_build=true. 330 depfile_deps = list(native_libs) 331 332 # For targets that depend on static library APKs, dex paths are created by 333 # the static library's dexsplitter target and GN doesn't know about these 334 # paths. 335 if options.dex_file: 336 depfile_deps.append(options.dex_file) 337 338 secondary_native_libs = [] 339 if options.secondary_native_libs: 340 secondary_native_libs = sorted(options.secondary_native_libs) 341 depfile_deps += secondary_native_libs 342 343 if options.java_resources: 344 # Included via .build_config, so need to write it to depfile. 345 depfile_deps.extend(options.java_resources) 346 347 assets = _ExpandPaths(options.assets) 348 uncompressed_assets = _ExpandPaths(options.uncompressed_assets) 349 350 # Included via .build_config, so need to write it to depfile. 351 depfile_deps.extend(x[0] for x in assets) 352 depfile_deps.extend(x[0] for x in uncompressed_assets) 353 354 # Bundle modules have a structure similar to APKs, except that resources 355 # are compiled in protobuf format (instead of binary xml), and that some 356 # files are located into different top-level directories, e.g.: 357 # AndroidManifest.xml -> manifest/AndroidManifest.xml 358 # classes.dex -> dex/classes.dex 359 # res/ -> res/ (unchanged) 360 # assets/ -> assets/ (unchanged) 361 # <other-file> -> root/<other-file> 362 # 363 # Hence, the following variables are used to control the location of files in 364 # the final archive. 365 if options.format == 'bundle-module': 366 apk_manifest_dir = 'manifest/' 367 apk_root_dir = 'root/' 368 apk_dex_dir = 'dex/' 369 else: 370 apk_manifest_dir = '' 371 apk_root_dir = '' 372 apk_dex_dir = '' 373 374 def _GetAssetDetails(assets, uncompressed_assets, fast_align, allow_reads): 375 ret = _GetAssetsToAdd(assets, 376 fast_align, 377 disable_compression=False, 378 allow_reads=allow_reads) 379 ret.extend( 380 _GetAssetsToAdd(uncompressed_assets, 381 fast_align, 382 disable_compression=True, 383 allow_reads=allow_reads)) 384 return ret 385 386 libs_to_add = _GetNativeLibrariesToAdd( 387 native_libs, options.android_abi, options.uncompress_shared_libraries, 388 fast_align, options.library_always_compress, options.library_renames) 389 if options.secondary_android_abi: 390 libs_to_add.extend( 391 _GetNativeLibrariesToAdd( 392 secondary_native_libs, options.secondary_android_abi, 393 options.uncompress_shared_libraries, fast_align, 394 options.library_always_compress, options.library_renames)) 395 396 if options.expected_file: 397 # We compute expectations without reading the files. This allows us to check 398 # expectations for different targets by just generating their build_configs 399 # and not have to first generate all the actual files and all their 400 # dependencies (for example by just passing --only-verify-expectations). 401 asset_details = _GetAssetDetails(assets, 402 uncompressed_assets, 403 fast_align, 404 allow_reads=False) 405 406 actual_data = _CreateExpectationsData(libs_to_add, asset_details) 407 diff_utils.CheckExpectations(actual_data, options) 408 409 if options.only_verify_expectations: 410 if options.depfile: 411 build_utils.WriteDepfile(options.depfile, 412 options.actual_file, 413 inputs=depfile_deps) 414 return 415 416 # If we are past this point, we are going to actually create the final apk so 417 # we should recompute asset details again but maybe perform some optimizations 418 # based on the size of the files on disk. 419 assets_to_add = _GetAssetDetails( 420 assets, uncompressed_assets, fast_align, allow_reads=True) 421 422 # Targets generally do not depend on apks, so no need for only_if_changed. 423 with build_utils.AtomicOutput(options.output_apk, only_if_changed=False) as f: 424 with zipfile.ZipFile(options.resource_apk) as resource_apk, \ 425 zipfile.ZipFile(f, 'w') as out_apk: 426 427 def add_to_zip(zip_path, data, compress=True, alignment=4): 428 zipalign.AddToZipHermetic( 429 out_apk, 430 zip_path, 431 data=data, 432 compress=compress, 433 alignment=0 if compress and not fast_align else alignment) 434 435 def copy_resource(zipinfo, out_dir=''): 436 add_to_zip( 437 out_dir + zipinfo.filename, 438 resource_apk.read(zipinfo.filename), 439 compress=zipinfo.compress_type != zipfile.ZIP_STORED) 440 441 # Make assets come before resources in order to maintain the same file 442 # ordering as GYP / aapt. http://crbug.com/561862 443 resource_infos = resource_apk.infolist() 444 445 # 1. AndroidManifest.xml 446 logging.debug('Adding AndroidManifest.xml') 447 copy_resource( 448 resource_apk.getinfo('AndroidManifest.xml'), out_dir=apk_manifest_dir) 449 450 # 2. Assets 451 logging.debug('Adding assets/') 452 _AddFiles(out_apk, assets_to_add) 453 454 # 3. Dex files 455 logging.debug('Adding classes.dex') 456 if options.dex_file: 457 with open(options.dex_file) as dex_file_obj: 458 if options.dex_file.endswith('.dex'): 459 max_dex_number = 1 460 # This is the case for incremental_install=true. 461 add_to_zip( 462 apk_dex_dir + 'classes.dex', 463 dex_file_obj.read(), 464 compress=not options.uncompress_dex) 465 else: 466 max_dex_number = 0 467 with zipfile.ZipFile(dex_file_obj) as dex_zip: 468 for dex in (d for d in dex_zip.namelist() if d.endswith('.dex')): 469 max_dex_number += 1 470 add_to_zip( 471 apk_dex_dir + dex, 472 dex_zip.read(dex), 473 compress=not options.uncompress_dex) 474 475 if options.jdk_libs_dex_file: 476 with open(options.jdk_libs_dex_file) as dex_file_obj: 477 add_to_zip( 478 apk_dex_dir + 'classes{}.dex'.format(max_dex_number + 1), 479 dex_file_obj.read(), 480 compress=not options.uncompress_dex) 481 482 # 4. Native libraries. 483 logging.debug('Adding lib/') 484 _AddFiles(out_apk, libs_to_add) 485 486 # Add a placeholder lib if the APK should be multi ABI but is missing libs 487 # for one of the ABIs. 488 native_lib_placeholders = options.native_lib_placeholders 489 secondary_native_lib_placeholders = ( 490 options.secondary_native_lib_placeholders) 491 if options.is_multi_abi: 492 if ((secondary_native_libs or secondary_native_lib_placeholders) 493 and not native_libs and not native_lib_placeholders): 494 native_lib_placeholders += ['libplaceholder.so'] 495 if ((native_libs or native_lib_placeholders) 496 and not secondary_native_libs 497 and not secondary_native_lib_placeholders): 498 secondary_native_lib_placeholders += ['libplaceholder.so'] 499 500 # Add placeholder libs. 501 for name in sorted(native_lib_placeholders): 502 # Note: Empty libs files are ignored by md5check (can cause issues 503 # with stale builds when the only change is adding/removing 504 # placeholders). 505 apk_path = 'lib/%s/%s' % (options.android_abi, name) 506 add_to_zip(apk_path, '', alignment=0x1000) 507 508 for name in sorted(secondary_native_lib_placeholders): 509 # Note: Empty libs files are ignored by md5check (can cause issues 510 # with stale builds when the only change is adding/removing 511 # placeholders). 512 apk_path = 'lib/%s/%s' % (options.secondary_android_abi, name) 513 add_to_zip(apk_path, '', alignment=0x1000) 514 515 # 5. Resources 516 logging.debug('Adding res/') 517 for info in sorted(resource_infos, key=lambda i: i.filename): 518 if info.filename != 'AndroidManifest.xml': 519 copy_resource(info) 520 521 # 6. Java resources that should be accessible via 522 # Class.getResourceAsStream(), in particular parts of Emma jar. 523 # Prebuilt jars may contain class files which we shouldn't include. 524 logging.debug('Adding Java resources') 525 for java_resource in options.java_resources: 526 with zipfile.ZipFile(java_resource, 'r') as java_resource_jar: 527 for apk_path in sorted(java_resource_jar.namelist()): 528 apk_path_lower = apk_path.lower() 529 530 if apk_path_lower.startswith('meta-inf/'): 531 continue 532 if apk_path_lower.endswith('/'): 533 continue 534 if apk_path_lower.endswith('.class'): 535 continue 536 537 add_to_zip(apk_root_dir + apk_path, 538 java_resource_jar.read(apk_path)) 539 540 if options.format == 'apk': 541 zipalign_path = None if fast_align else options.zipalign_path 542 finalize_apk.FinalizeApk(options.apksigner_jar, 543 zipalign_path, 544 f.name, 545 f.name, 546 options.key_path, 547 options.key_passwd, 548 options.key_name, 549 int(options.min_sdk_version), 550 warnings_as_errors=options.warnings_as_errors) 551 logging.debug('Moving file into place') 552 553 if options.depfile: 554 build_utils.WriteDepfile(options.depfile, 555 options.output_apk, 556 inputs=depfile_deps) 557 558 559if __name__ == '__main__': 560 main(sys.argv[1:]) 561