1#!/usr/bin/env python
2#
3# Copyright (c) 2015 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Adds the code parts to a resource APK."""
8
9import argparse
10import logging
11import os
12import shutil
13import sys
14import tempfile
15import zipfile
16import zlib
17
18import finalize_apk
19
20from util import build_utils
21from util import diff_utils
22from util import zipalign
23
24# Input dex.jar files are zipaligned.
25zipalign.ApplyZipFileZipAlignFix()
26
27
28# Taken from aapt's Package.cpp:
29_NO_COMPRESS_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.gif', '.wav', '.mp2',
30                           '.mp3', '.ogg', '.aac', '.mpg', '.mpeg', '.mid',
31                           '.midi', '.smf', '.jet', '.rtttl', '.imy', '.xmf',
32                           '.mp4', '.m4a', '.m4v', '.3gp', '.3gpp', '.3g2',
33                           '.3gpp2', '.amr', '.awb', '.wma', '.wmv', '.webm')
34
35
36def _ParseArgs(args):
37  parser = argparse.ArgumentParser()
38  build_utils.AddDepfileOption(parser)
39  parser.add_argument(
40      '--assets',
41      help='GYP-list of files to add as assets in the form '
42      '"srcPath:zipPath", where ":zipPath" is optional.')
43  parser.add_argument(
44      '--java-resources', help='GYP-list of java_resources JARs to include.')
45  parser.add_argument('--write-asset-list',
46                      action='store_true',
47                      help='Whether to create an assets/assets_list file.')
48  parser.add_argument(
49      '--uncompressed-assets',
50      help='Same as --assets, except disables compression.')
51  parser.add_argument('--resource-apk',
52                      help='An .ap_ file built using aapt',
53                      required=True)
54  parser.add_argument('--output-apk',
55                      help='Path to the output file',
56                      required=True)
57  parser.add_argument('--format', choices=['apk', 'bundle-module'],
58                      default='apk', help='Specify output format.')
59  parser.add_argument('--dex-file',
60                      help='Path to the classes.dex to use')
61  parser.add_argument(
62      '--jdk-libs-dex-file',
63      help='Path to classes.dex created by dex_jdk_libs.py')
64  parser.add_argument('--uncompress-dex', action='store_true',
65                      help='Store .dex files uncompressed in the APK')
66  parser.add_argument('--native-libs',
67                      action='append',
68                      help='GYP-list of native libraries to include. '
69                           'Can be specified multiple times.',
70                      default=[])
71  parser.add_argument('--secondary-native-libs',
72                      action='append',
73                      help='GYP-list of native libraries for secondary '
74                           'android-abi. Can be specified multiple times.',
75                      default=[])
76  parser.add_argument('--android-abi',
77                      help='Android architecture to use for native libraries')
78  parser.add_argument('--secondary-android-abi',
79                      help='The secondary Android architecture to use for'
80                           'secondary native libraries')
81  parser.add_argument(
82      '--is-multi-abi',
83      action='store_true',
84      help='Will add a placeholder for the missing ABI if no native libs or '
85      'placeholders are set for either the primary or secondary ABI. Can only '
86      'be set if both --android-abi and --secondary-android-abi are set.')
87  parser.add_argument(
88      '--native-lib-placeholders',
89      help='GYP-list of native library placeholders to add.')
90  parser.add_argument(
91      '--secondary-native-lib-placeholders',
92      help='GYP-list of native library placeholders to add '
93      'for the secondary ABI')
94  parser.add_argument('--uncompress-shared-libraries', default='False',
95      choices=['true', 'True', 'false', 'False'],
96      help='Whether to uncompress native shared libraries. Argument must be '
97           'a boolean value.')
98  parser.add_argument(
99      '--apksigner-jar', help='Path to the apksigner executable.')
100  parser.add_argument('--zipalign-path',
101                      help='Path to the zipalign executable.')
102  parser.add_argument('--key-path',
103                      help='Path to keystore for signing.')
104  parser.add_argument('--key-passwd',
105                      help='Keystore password')
106  parser.add_argument('--key-name',
107                      help='Keystore name')
108  parser.add_argument(
109      '--min-sdk-version', required=True, help='Value of APK\'s minSdkVersion')
110  parser.add_argument(
111      '--best-compression',
112      action='store_true',
113      help='Use zip -9 rather than zip -1')
114  parser.add_argument(
115      '--library-always-compress',
116      action='append',
117      help='The list of library files that we always compress.')
118  parser.add_argument(
119      '--library-renames',
120      action='append',
121      help='The list of library files that we prepend crazy. to their names.')
122  parser.add_argument('--warnings-as-errors',
123                      action='store_true',
124                      help='Treat all warnings as errors.')
125  diff_utils.AddCommandLineFlags(parser)
126  options = parser.parse_args(args)
127  options.assets = build_utils.ParseGnList(options.assets)
128  options.uncompressed_assets = build_utils.ParseGnList(
129      options.uncompressed_assets)
130  options.native_lib_placeholders = build_utils.ParseGnList(
131      options.native_lib_placeholders)
132  options.secondary_native_lib_placeholders = build_utils.ParseGnList(
133      options.secondary_native_lib_placeholders)
134  options.java_resources = build_utils.ParseGnList(options.java_resources)
135  options.native_libs = build_utils.ParseGnList(options.native_libs)
136  options.secondary_native_libs = build_utils.ParseGnList(
137      options.secondary_native_libs)
138  options.library_always_compress = build_utils.ParseGnList(
139      options.library_always_compress)
140  options.library_renames = build_utils.ParseGnList(options.library_renames)
141
142  # --apksigner-jar, --zipalign-path, --key-xxx arguments are
143  # required when building an APK, but not a bundle module.
144  if options.format == 'apk':
145    required_args = [
146        'apksigner_jar', 'zipalign_path', 'key_path', 'key_passwd', 'key_name'
147    ]
148    for required in required_args:
149      if not vars(options)[required]:
150        raise Exception('Argument --%s is required for APKs.' % (
151            required.replace('_', '-')))
152
153  options.uncompress_shared_libraries = \
154      options.uncompress_shared_libraries in [ 'true', 'True' ]
155
156  if not options.android_abi and (options.native_libs or
157                                  options.native_lib_placeholders):
158    raise Exception('Must specify --android-abi with --native-libs')
159  if not options.secondary_android_abi and (options.secondary_native_libs or
160      options.secondary_native_lib_placeholders):
161    raise Exception('Must specify --secondary-android-abi with'
162                    ' --secondary-native-libs')
163  if options.is_multi_abi and not (options.android_abi
164                                   and options.secondary_android_abi):
165    raise Exception('Must specify --is-multi-abi with both --android-abi '
166                    'and --secondary-android-abi.')
167  return options
168
169
170def _SplitAssetPath(path):
171  """Returns (src, dest) given an asset path in the form src[:dest]."""
172  path_parts = path.split(':')
173  src_path = path_parts[0]
174  if len(path_parts) > 1:
175    dest_path = path_parts[1]
176  else:
177    dest_path = os.path.basename(src_path)
178  return src_path, dest_path
179
180
181def _ExpandPaths(paths):
182  """Converts src:dst into tuples and enumerates files within directories.
183
184  Args:
185    paths: Paths in the form "src_path:dest_path"
186
187  Returns:
188    A list of (src_path, dest_path) tuples sorted by dest_path (for stable
189    ordering within output .apk).
190  """
191  ret = []
192  for path in paths:
193    src_path, dest_path = _SplitAssetPath(path)
194    if os.path.isdir(src_path):
195      for f in build_utils.FindInDirectory(src_path, '*'):
196        ret.append((f, os.path.join(dest_path, f[len(src_path) + 1:])))
197    else:
198      ret.append((src_path, dest_path))
199  ret.sort(key=lambda t:t[1])
200  return ret
201
202
203def _GetAssetsToAdd(path_tuples,
204                    fast_align,
205                    disable_compression=False,
206                    allow_reads=True):
207  """Returns the list of file_detail tuples for assets in the apk.
208
209  Args:
210    path_tuples: List of src_path, dest_path tuples to add.
211    fast_align: Whether to perform alignment in python zipfile (alternatively
212                alignment can be done using the zipalign utility out of band).
213    disable_compression: Whether to disable compression.
214    allow_reads: If false, we do not try to read the files from disk (to find
215                 their size for example).
216
217  Returns: A list of (src_path, apk_path, compress, alignment) tuple
218  representing what and how assets are added.
219  """
220  assets_to_add = []
221
222  # Group all uncompressed assets together in the hope that it will increase
223  # locality of mmap'ed files.
224  for target_compress in (False, True):
225    for src_path, dest_path in path_tuples:
226      compress = not disable_compression and (
227          os.path.splitext(src_path)[1] not in _NO_COMPRESS_EXTENSIONS)
228
229      if target_compress == compress:
230        # AddToZipHermetic() uses this logic to avoid growing small files.
231        # We need it here in order to set alignment correctly.
232        if allow_reads and compress and os.path.getsize(src_path) < 16:
233          compress = False
234
235        apk_path = 'assets/' + dest_path
236        alignment = 0 if compress and not fast_align else 4
237        assets_to_add.append((apk_path, src_path, compress, alignment))
238  return assets_to_add
239
240
241def _AddFiles(apk, details):
242  """Adds files to the apk.
243
244  Args:
245    apk: path to APK to add to.
246    details: A list of file detail tuples (src_path, apk_path, compress,
247    alignment) representing what and how files are added to the APK.
248  """
249  for apk_path, src_path, compress, alignment in details:
250    # This check is only relevant for assets, but it should not matter if it is
251    # checked for the whole list of files.
252    try:
253      apk.getinfo(apk_path)
254      # Should never happen since write_build_config.py handles merging.
255      raise Exception(
256          'Multiple targets specified the asset path: %s' % apk_path)
257    except KeyError:
258      zipalign.AddToZipHermetic(
259          apk,
260          apk_path,
261          src_path=src_path,
262          compress=compress,
263          alignment=alignment)
264
265
266def _GetNativeLibrariesToAdd(native_libs, android_abi, uncompress, fast_align,
267                             lib_always_compress, lib_renames):
268  """Returns the list of file_detail tuples for native libraries in the apk.
269
270  Returns: A list of (src_path, apk_path, compress, alignment) tuple
271  representing what and how native libraries are added.
272  """
273  libraries_to_add = []
274
275
276  for path in native_libs:
277    basename = os.path.basename(path)
278    compress = not uncompress or any(lib_name in basename
279                                     for lib_name in lib_always_compress)
280    rename = any(lib_name in basename for lib_name in lib_renames)
281    if rename:
282      basename = 'crazy.' + basename
283
284    lib_android_abi = android_abi
285    if path.startswith('android_clang_arm64_hwasan/'):
286      lib_android_abi = 'arm64-v8a-hwasan'
287
288    apk_path = 'lib/%s/%s' % (lib_android_abi, basename)
289    alignment = 0 if compress and not fast_align else 0x1000
290    libraries_to_add.append((apk_path, path, compress, alignment))
291
292  return libraries_to_add
293
294
295def _CreateExpectationsData(native_libs, assets):
296  """Creates list of native libraries and assets."""
297  native_libs = sorted(native_libs)
298  assets = sorted(assets)
299
300  ret = []
301  for apk_path, _, compress, alignment in native_libs + assets:
302    ret.append('apk_path=%s, compress=%s, alignment=%s\n' %
303               (apk_path, compress, alignment))
304  return ''.join(ret)
305
306
307def main(args):
308  build_utils.InitLogging('APKBUILDER_DEBUG')
309  args = build_utils.ExpandFileArgs(args)
310  options = _ParseArgs(args)
311
312  # Until Python 3.7, there's no better way to set compression level.
313  # The default is 6.
314  if options.best_compression:
315    # Compresses about twice as slow as the default.
316    zlib.Z_DEFAULT_COMPRESSION = 9
317  else:
318    # Compresses about twice as fast as the default.
319    zlib.Z_DEFAULT_COMPRESSION = 1
320
321  # Manually align only when alignment is necessary.
322  # Python's zip implementation duplicates file comments in the central
323  # directory, whereas zipalign does not, so use zipalign for official builds.
324  fast_align = options.format == 'apk' and not options.best_compression
325
326  native_libs = sorted(options.native_libs)
327
328  # Include native libs in the depfile_deps since GN doesn't know about the
329  # dependencies when is_component_build=true.
330  depfile_deps = list(native_libs)
331
332  # For targets that depend on static library APKs, dex paths are created by
333  # the static library's dexsplitter target and GN doesn't know about these
334  # paths.
335  if options.dex_file:
336    depfile_deps.append(options.dex_file)
337
338  secondary_native_libs = []
339  if options.secondary_native_libs:
340    secondary_native_libs = sorted(options.secondary_native_libs)
341    depfile_deps += secondary_native_libs
342
343  if options.java_resources:
344    # Included via .build_config, so need to write it to depfile.
345    depfile_deps.extend(options.java_resources)
346
347  assets = _ExpandPaths(options.assets)
348  uncompressed_assets = _ExpandPaths(options.uncompressed_assets)
349
350  # Included via .build_config, so need to write it to depfile.
351  depfile_deps.extend(x[0] for x in assets)
352  depfile_deps.extend(x[0] for x in uncompressed_assets)
353
354  # Bundle modules have a structure similar to APKs, except that resources
355  # are compiled in protobuf format (instead of binary xml), and that some
356  # files are located into different top-level directories, e.g.:
357  #  AndroidManifest.xml -> manifest/AndroidManifest.xml
358  #  classes.dex -> dex/classes.dex
359  #  res/ -> res/  (unchanged)
360  #  assets/ -> assets/  (unchanged)
361  #  <other-file> -> root/<other-file>
362  #
363  # Hence, the following variables are used to control the location of files in
364  # the final archive.
365  if options.format == 'bundle-module':
366    apk_manifest_dir = 'manifest/'
367    apk_root_dir = 'root/'
368    apk_dex_dir = 'dex/'
369  else:
370    apk_manifest_dir = ''
371    apk_root_dir = ''
372    apk_dex_dir = ''
373
374  def _GetAssetDetails(assets, uncompressed_assets, fast_align, allow_reads):
375    ret = _GetAssetsToAdd(assets,
376                          fast_align,
377                          disable_compression=False,
378                          allow_reads=allow_reads)
379    ret.extend(
380        _GetAssetsToAdd(uncompressed_assets,
381                        fast_align,
382                        disable_compression=True,
383                        allow_reads=allow_reads))
384    return ret
385
386  libs_to_add = _GetNativeLibrariesToAdd(
387      native_libs, options.android_abi, options.uncompress_shared_libraries,
388      fast_align, options.library_always_compress, options.library_renames)
389  if options.secondary_android_abi:
390    libs_to_add.extend(
391        _GetNativeLibrariesToAdd(
392            secondary_native_libs, options.secondary_android_abi,
393            options.uncompress_shared_libraries, fast_align,
394            options.library_always_compress, options.library_renames))
395
396  if options.expected_file:
397    # We compute expectations without reading the files. This allows us to check
398    # expectations for different targets by just generating their build_configs
399    # and not have to first generate all the actual files and all their
400    # dependencies (for example by just passing --only-verify-expectations).
401    asset_details = _GetAssetDetails(assets,
402                                     uncompressed_assets,
403                                     fast_align,
404                                     allow_reads=False)
405
406    actual_data = _CreateExpectationsData(libs_to_add, asset_details)
407    diff_utils.CheckExpectations(actual_data, options)
408
409    if options.only_verify_expectations:
410      if options.depfile:
411        build_utils.WriteDepfile(options.depfile,
412                                 options.actual_file,
413                                 inputs=depfile_deps)
414      return
415
416  # If we are past this point, we are going to actually create the final apk so
417  # we should recompute asset details again but maybe perform some optimizations
418  # based on the size of the files on disk.
419  assets_to_add = _GetAssetDetails(
420      assets, uncompressed_assets, fast_align, allow_reads=True)
421
422  # Targets generally do not depend on apks, so no need for only_if_changed.
423  with build_utils.AtomicOutput(options.output_apk, only_if_changed=False) as f:
424    with zipfile.ZipFile(options.resource_apk) as resource_apk, \
425         zipfile.ZipFile(f, 'w') as out_apk:
426
427      def add_to_zip(zip_path, data, compress=True, alignment=4):
428        zipalign.AddToZipHermetic(
429            out_apk,
430            zip_path,
431            data=data,
432            compress=compress,
433            alignment=0 if compress and not fast_align else alignment)
434
435      def copy_resource(zipinfo, out_dir=''):
436        add_to_zip(
437            out_dir + zipinfo.filename,
438            resource_apk.read(zipinfo.filename),
439            compress=zipinfo.compress_type != zipfile.ZIP_STORED)
440
441      # Make assets come before resources in order to maintain the same file
442      # ordering as GYP / aapt. http://crbug.com/561862
443      resource_infos = resource_apk.infolist()
444
445      # 1. AndroidManifest.xml
446      logging.debug('Adding AndroidManifest.xml')
447      copy_resource(
448          resource_apk.getinfo('AndroidManifest.xml'), out_dir=apk_manifest_dir)
449
450      # 2. Assets
451      logging.debug('Adding assets/')
452      _AddFiles(out_apk, assets_to_add)
453
454      # 3. Dex files
455      logging.debug('Adding classes.dex')
456      if options.dex_file:
457        with open(options.dex_file) as dex_file_obj:
458          if options.dex_file.endswith('.dex'):
459            max_dex_number = 1
460            # This is the case for incremental_install=true.
461            add_to_zip(
462                apk_dex_dir + 'classes.dex',
463                dex_file_obj.read(),
464                compress=not options.uncompress_dex)
465          else:
466            max_dex_number = 0
467            with zipfile.ZipFile(dex_file_obj) as dex_zip:
468              for dex in (d for d in dex_zip.namelist() if d.endswith('.dex')):
469                max_dex_number += 1
470                add_to_zip(
471                    apk_dex_dir + dex,
472                    dex_zip.read(dex),
473                    compress=not options.uncompress_dex)
474
475      if options.jdk_libs_dex_file:
476        with open(options.jdk_libs_dex_file) as dex_file_obj:
477          add_to_zip(
478              apk_dex_dir + 'classes{}.dex'.format(max_dex_number + 1),
479              dex_file_obj.read(),
480              compress=not options.uncompress_dex)
481
482      # 4. Native libraries.
483      logging.debug('Adding lib/')
484      _AddFiles(out_apk, libs_to_add)
485
486      # Add a placeholder lib if the APK should be multi ABI but is missing libs
487      # for one of the ABIs.
488      native_lib_placeholders = options.native_lib_placeholders
489      secondary_native_lib_placeholders = (
490          options.secondary_native_lib_placeholders)
491      if options.is_multi_abi:
492        if ((secondary_native_libs or secondary_native_lib_placeholders)
493            and not native_libs and not native_lib_placeholders):
494          native_lib_placeholders += ['libplaceholder.so']
495        if ((native_libs or native_lib_placeholders)
496            and not secondary_native_libs
497            and not secondary_native_lib_placeholders):
498          secondary_native_lib_placeholders += ['libplaceholder.so']
499
500      # Add placeholder libs.
501      for name in sorted(native_lib_placeholders):
502        # Note: Empty libs files are ignored by md5check (can cause issues
503        # with stale builds when the only change is adding/removing
504        # placeholders).
505        apk_path = 'lib/%s/%s' % (options.android_abi, name)
506        add_to_zip(apk_path, '', alignment=0x1000)
507
508      for name in sorted(secondary_native_lib_placeholders):
509        # Note: Empty libs files are ignored by md5check (can cause issues
510        # with stale builds when the only change is adding/removing
511        # placeholders).
512        apk_path = 'lib/%s/%s' % (options.secondary_android_abi, name)
513        add_to_zip(apk_path, '', alignment=0x1000)
514
515      # 5. Resources
516      logging.debug('Adding res/')
517      for info in sorted(resource_infos, key=lambda i: i.filename):
518        if info.filename != 'AndroidManifest.xml':
519          copy_resource(info)
520
521      # 6. Java resources that should be accessible via
522      # Class.getResourceAsStream(), in particular parts of Emma jar.
523      # Prebuilt jars may contain class files which we shouldn't include.
524      logging.debug('Adding Java resources')
525      for java_resource in options.java_resources:
526        with zipfile.ZipFile(java_resource, 'r') as java_resource_jar:
527          for apk_path in sorted(java_resource_jar.namelist()):
528            apk_path_lower = apk_path.lower()
529
530            if apk_path_lower.startswith('meta-inf/'):
531              continue
532            if apk_path_lower.endswith('/'):
533              continue
534            if apk_path_lower.endswith('.class'):
535              continue
536
537            add_to_zip(apk_root_dir + apk_path,
538                       java_resource_jar.read(apk_path))
539
540    if options.format == 'apk':
541      zipalign_path = None if fast_align else options.zipalign_path
542      finalize_apk.FinalizeApk(options.apksigner_jar,
543                               zipalign_path,
544                               f.name,
545                               f.name,
546                               options.key_path,
547                               options.key_passwd,
548                               options.key_name,
549                               int(options.min_sdk_version),
550                               warnings_as_errors=options.warnings_as_errors)
551    logging.debug('Moving file into place')
552
553    if options.depfile:
554      build_utils.WriteDepfile(options.depfile,
555                               options.output_apk,
556                               inputs=depfile_deps)
557
558
559if __name__ == '__main__':
560  main(sys.argv[1:])
561