1#!/usr/bin/env python
2# encoding: utf-8
3# Copyright (c) 2012 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Compile Android resources into an intermediate APK.
8
9This can also generate an R.txt, and an .srcjar file containing the proper
10final R.java class for all resource packages the APK depends on.
11
12This will crunch images with aapt2.
13"""
14
15import argparse
16import collections
17import contextlib
18import filecmp
19import hashlib
20import logging
21import os
22import re
23import shutil
24import subprocess
25import sys
26import tempfile
27import textwrap
28import zipfile
29from xml.etree import ElementTree
30
31from util import build_utils
32from util import diff_utils
33from util import manifest_utils
34from util import md5_check
35from util import parallel
36from util import protoresources
37from util import resource_utils
38
39
40# Pngs that we shouldn't convert to webp. Please add rationale when updating.
41_PNG_WEBP_EXCLUSION_PATTERN = re.compile('|'.join([
42    # Crashes on Galaxy S5 running L (https://crbug.com/807059).
43    r'.*star_gray\.png',
44    # Android requires pngs for 9-patch images.
45    r'.*\.9\.png',
46    # Daydream requires pngs for icon files.
47    r'.*daydream_icon_.*\.png'
48]))
49
50
51def _ParseArgs(args):
52  """Parses command line options.
53
54  Returns:
55    An options object as from argparse.ArgumentParser.parse_args()
56  """
57  parser, input_opts, output_opts = resource_utils.ResourceArgsParser()
58
59  input_opts.add_argument(
60      '--aapt2-path', required=True, help='Path to the Android aapt2 tool.')
61  input_opts.add_argument(
62      '--android-manifest', required=True, help='AndroidManifest.xml path.')
63  input_opts.add_argument(
64      '--r-java-root-package-name',
65      default='base',
66      help='Short package name for this target\'s root R java file (ex. '
67      'input of "base" would become gen.base_module). Defaults to "base".')
68  group = input_opts.add_mutually_exclusive_group()
69  group.add_argument(
70      '--shared-resources',
71      action='store_true',
72      help='Make all resources in R.java non-final and allow the resource IDs '
73      'to be reset to a different package index when the apk is loaded by '
74      'another application at runtime.')
75  group.add_argument(
76      '--app-as-shared-lib',
77      action='store_true',
78      help='Same as --shared-resources, but also ensures all resource IDs are '
79      'directly usable from the APK loaded as an application.')
80
81  input_opts.add_argument(
82      '--package-id',
83      type=int,
84      help='Decimal integer representing custom package ID for resources '
85      '(instead of 127==0x7f). Cannot be used with --shared-resources.')
86
87  input_opts.add_argument(
88      '--package-name',
89      help='Package name that will be used to create R class.')
90
91  input_opts.add_argument(
92      '--rename-manifest-package', help='Package name to force AAPT to use.')
93
94  input_opts.add_argument(
95      '--arsc-package-name',
96      help='Package name to set in manifest of resources.arsc file. This is '
97      'only used for apks under test.')
98
99  input_opts.add_argument(
100      '--shared-resources-allowlist',
101      help='An R.txt file acting as a allowlist for resources that should be '
102      'non-final and have their package ID changed at runtime in R.java. '
103      'Implies and overrides --shared-resources.')
104
105  input_opts.add_argument(
106      '--shared-resources-allowlist-locales',
107      default='[]',
108      help='Optional GN-list of locales. If provided, all strings corresponding'
109      ' to this locale list will be kept in the final output for the '
110      'resources identified through --shared-resources-allowlist, even '
111      'if --locale-allowlist is being used.')
112
113  input_opts.add_argument(
114      '--use-resource-ids-path',
115      help='Use resource IDs generated by aapt --emit-ids.')
116
117  input_opts.add_argument(
118      '--extra-main-r-text-files',
119      help='Additional R.txt files that will be added to the root R.java file, '
120      'but not packaged in the generated resources.arsc. If these resources '
121      'entries contain duplicate resources with the generated R.txt file, they '
122      'must be identical.')
123
124  input_opts.add_argument(
125      '--support-zh-hk',
126      action='store_true',
127      help='Use zh-rTW resources for zh-rHK.')
128
129  input_opts.add_argument(
130      '--debuggable',
131      action='store_true',
132      help='Whether to add android:debuggable="true".')
133
134  input_opts.add_argument('--version-code', help='Version code for apk.')
135  input_opts.add_argument('--version-name', help='Version name for apk.')
136  input_opts.add_argument(
137      '--min-sdk-version', required=True, help='android:minSdkVersion for APK.')
138  input_opts.add_argument(
139      '--target-sdk-version',
140      required=True,
141      help="android:targetSdkVersion for APK.")
142  input_opts.add_argument(
143      '--max-sdk-version',
144      help="android:maxSdkVersion expected in AndroidManifest.xml.")
145  input_opts.add_argument(
146      '--manifest-package', help='Package name of the AndroidManifest.xml.')
147
148  input_opts.add_argument(
149      '--locale-allowlist',
150      default='[]',
151      help='GN list of languages to include. All other language configs will '
152      'be stripped out. List may include a combination of Android locales '
153      'or Chrome locales.')
154  input_opts.add_argument(
155      '--resource-exclusion-regex',
156      default='',
157      help='File-based filter for resources (applied before compiling)')
158  input_opts.add_argument(
159      '--resource-exclusion-exceptions',
160      default='[]',
161      help='GN list of globs that say which files to include even '
162      'when --resource-exclusion-regex is set.')
163
164  input_opts.add_argument(
165      '--dependencies-res-zip-overlays',
166      help='GN list with subset of --dependencies-res-zips to use overlay '
167      'semantics for.')
168
169  input_opts.add_argument(
170      '--values-filter-rules',
171      help='GN list of source_glob:regex for filtering resources after they '
172      'are compiled. Use this to filter out entries within values/ files.')
173
174  input_opts.add_argument('--png-to-webp', action='store_true',
175                          help='Convert png files to webp format.')
176
177  input_opts.add_argument('--webp-binary', default='',
178                          help='Path to the cwebp binary.')
179  input_opts.add_argument(
180      '--webp-cache-dir', help='The directory to store webp image cache.')
181
182  input_opts.add_argument(
183      '--no-xml-namespaces',
184      action='store_true',
185      help='Whether to strip xml namespaces from processed xml resources.')
186  input_opts.add_argument(
187      '--short-resource-paths',
188      action='store_true',
189      help='Whether to shorten resource paths inside the apk or module.')
190  input_opts.add_argument(
191      '--strip-resource-names',
192      action='store_true',
193      help='Whether to strip resource names from the resource table of the apk '
194      'or module.')
195
196  output_opts.add_argument('--arsc-path', help='Apk output for arsc format.')
197  output_opts.add_argument('--proto-path', help='Apk output for proto format.')
198  group = input_opts.add_mutually_exclusive_group()
199  group.add_argument(
200      '--optimized-arsc-path',
201      help='Output for `aapt2 optimize` for arsc format (enables the step).')
202  group.add_argument(
203      '--optimized-proto-path',
204      help='Output for `aapt2 optimize` for proto format (enables the step).')
205  input_opts.add_argument(
206      '--resources-config-paths',
207      default='[]',
208      help='GN list of paths to aapt2 resources config files.')
209
210  output_opts.add_argument(
211      '--info-path', help='Path to output info file for the partial apk.')
212
213  output_opts.add_argument(
214      '--srcjar-out',
215      required=True,
216      help='Path to srcjar to contain generated R.java.')
217
218  output_opts.add_argument('--r-text-out',
219                           help='Path to store the generated R.txt file.')
220
221  output_opts.add_argument(
222      '--proguard-file', help='Path to proguard.txt generated file.')
223
224  output_opts.add_argument(
225      '--proguard-file-main-dex',
226      help='Path to proguard.txt generated file for main dex.')
227
228  output_opts.add_argument(
229      '--emit-ids-out', help='Path to file produced by aapt2 --emit-ids.')
230
231  output_opts.add_argument(
232      '--resources-path-map-out-path',
233      help='Path to file produced by aapt2 that maps original resource paths '
234      'to shortened resource paths inside the apk or module.')
235
236  input_opts.add_argument(
237      '--is-bundle-module',
238      action='store_true',
239      help='Whether resources are being generated for a bundle module.')
240
241  input_opts.add_argument(
242      '--uses-split',
243      help='Value to set uses-split to in the AndroidManifest.xml.')
244
245  diff_utils.AddCommandLineFlags(parser)
246  options = parser.parse_args(args)
247
248  resource_utils.HandleCommonOptions(options)
249
250  options.locale_allowlist = build_utils.ParseGnList(options.locale_allowlist)
251  options.shared_resources_allowlist_locales = build_utils.ParseGnList(
252      options.shared_resources_allowlist_locales)
253  options.resource_exclusion_exceptions = build_utils.ParseGnList(
254      options.resource_exclusion_exceptions)
255  options.dependencies_res_zip_overlays = build_utils.ParseGnList(
256      options.dependencies_res_zip_overlays)
257  options.values_filter_rules = build_utils.ParseGnList(
258      options.values_filter_rules)
259  options.extra_main_r_text_files = build_utils.ParseGnList(
260      options.extra_main_r_text_files)
261  options.resources_config_paths = build_utils.ParseGnList(
262      options.resources_config_paths)
263
264  if options.optimized_proto_path and not options.proto_path:
265    # We could write to a temp file, but it's simpler to require it.
266    parser.error('--optimized-proto-path requires --proto-path')
267
268  if not options.arsc_path and not options.proto_path:
269    parser.error('One of --arsc-path or --proto-path is required.')
270
271  if options.resources_path_map_out_path and not options.short_resource_paths:
272    parser.error(
273        '--resources-path-map-out-path requires --short-resource-paths')
274
275  if options.package_id and options.shared_resources:
276    parser.error('--package-id and --shared-resources are mutually exclusive')
277
278  return options
279
280
281def _IterFiles(root_dir):
282  for root, _, files in os.walk(root_dir):
283    for f in files:
284      yield os.path.join(root, f)
285
286
287def _DuplicateZhResources(resource_dirs, path_info):
288  """Duplicate Taiwanese resources into Hong-Kong specific directory."""
289  for resource_dir in resource_dirs:
290    # We use zh-TW resources for zh-HK (if we have zh-TW resources).
291    for path in _IterFiles(resource_dir):
292      if 'zh-rTW' in path:
293        hk_path = path.replace('zh-rTW', 'zh-rHK')
294        build_utils.MakeDirectory(os.path.dirname(hk_path))
295        shutil.copyfile(path, hk_path)
296        path_info.RegisterRename(
297            os.path.relpath(path, resource_dir),
298            os.path.relpath(hk_path, resource_dir))
299
300
301def _RenameLocaleResourceDirs(resource_dirs, path_info):
302  """Rename locale resource directories into standard names when necessary.
303
304  This is necessary to deal with the fact that older Android releases only
305  support ISO 639-1 two-letter codes, and sometimes even obsolete versions
306  of them.
307
308  In practice it means:
309    * 3-letter ISO 639-2 qualifiers are renamed under a corresponding
310      2-letter one. E.g. for Filipino, strings under values-fil/ will be moved
311      to a new corresponding values-tl/ sub-directory.
312
313    * Modern ISO 639-1 codes will be renamed to their obsolete variant
314      for Indonesian, Hebrew and Yiddish (e.g. 'values-in/ -> values-id/).
315
316    * Norwegian macrolanguage strings will be renamed to Bokmål (main
317      Norway language). See http://crbug.com/920960. In practice this
318      means that 'values-no/ -> values-nb/' unless 'values-nb/' already
319      exists.
320
321    * BCP 47 langauge tags will be renamed to an equivalent ISO 639-1
322      locale qualifier if possible (e.g. 'values-b+en+US/ -> values-en-rUS').
323      Though this is not necessary at the moment, because no third-party
324      package that Chromium links against uses these for the current list of
325      supported locales, this may change when the list is extended in the
326      future).
327
328  Args:
329    resource_dirs: list of top-level resource directories.
330  """
331  for resource_dir in resource_dirs:
332    for path in _IterFiles(resource_dir):
333      locale = resource_utils.FindLocaleInStringResourceFilePath(path)
334      if not locale:
335        continue
336      cr_locale = resource_utils.ToChromiumLocaleName(locale)
337      if not cr_locale:
338        continue  # Unsupported Android locale qualifier!?
339      locale2 = resource_utils.ToAndroidLocaleName(cr_locale)
340      if locale != locale2:
341        path2 = path.replace('/values-%s/' % locale, '/values-%s/' % locale2)
342        if path == path2:
343          raise Exception('Could not substitute locale %s for %s in %s' %
344                          (locale, locale2, path))
345        if os.path.exists(path2):
346          # This happens sometimes, e.g. some libraries provide both
347          # values-nb/ and values-no/ with the same content.
348          continue
349        build_utils.MakeDirectory(os.path.dirname(path2))
350        shutil.move(path, path2)
351        path_info.RegisterRename(
352            os.path.relpath(path, resource_dir),
353            os.path.relpath(path2, resource_dir))
354
355
356def _ToAndroidLocales(locale_allowlist, support_zh_hk):
357  """Converts the list of Chrome locales to Android config locale qualifiers.
358
359  Args:
360    locale_allowlist: A list of Chromium locale names.
361    support_zh_hk: True if we need to support zh-HK by duplicating
362      the zh-TW strings.
363  Returns:
364    A set of matching Android config locale qualifier names.
365  """
366  ret = set()
367  for locale in locale_allowlist:
368    locale = resource_utils.ToAndroidLocaleName(locale)
369    if locale is None or ('-' in locale and '-r' not in locale):
370      raise Exception('Unsupported Chromium locale name: %s' % locale)
371    ret.add(locale)
372    # Always keep non-regional fall-backs.
373    language = locale.split('-')[0]
374    ret.add(language)
375
376  # We don't actually support zh-HK in Chrome on Android, but we mimic the
377  # native side behavior where we use zh-TW resources when the locale is set to
378  # zh-HK. See https://crbug.com/780847.
379  if support_zh_hk:
380    assert not any('HK' in l for l in locale_allowlist), (
381        'Remove special logic if zh-HK is now supported (crbug.com/780847).')
382    ret.add('zh-rHK')
383  return set(ret)
384
385
386def _MoveImagesToNonMdpiFolders(res_root, path_info):
387  """Move images from drawable-*-mdpi-* folders to drawable-* folders.
388
389  Why? http://crbug.com/289843
390  """
391  for src_dir_name in os.listdir(res_root):
392    src_components = src_dir_name.split('-')
393    if src_components[0] != 'drawable' or 'mdpi' not in src_components:
394      continue
395    src_dir = os.path.join(res_root, src_dir_name)
396    if not os.path.isdir(src_dir):
397      continue
398    dst_components = [c for c in src_components if c != 'mdpi']
399    assert dst_components != src_components
400    dst_dir_name = '-'.join(dst_components)
401    dst_dir = os.path.join(res_root, dst_dir_name)
402    build_utils.MakeDirectory(dst_dir)
403    for src_file_name in os.listdir(src_dir):
404      if not os.path.splitext(src_file_name)[1] in ('.png', '.webp', ''):
405        continue
406      src_file = os.path.join(src_dir, src_file_name)
407      dst_file = os.path.join(dst_dir, src_file_name)
408      assert not os.path.lexists(dst_file)
409      shutil.move(src_file, dst_file)
410      path_info.RegisterRename(
411          os.path.relpath(src_file, res_root),
412          os.path.relpath(dst_file, res_root))
413
414
415def _FixManifest(options, temp_dir):
416  """Fix the APK's AndroidManifest.xml.
417
418  This adds any missing namespaces for 'android' and 'tools', and
419  sets certains elements like 'platformBuildVersionCode' or
420  'android:debuggable' depending on the content of |options|.
421
422  Args:
423    options: The command-line arguments tuple.
424    temp_dir: A temporary directory where the fixed manifest will be written to.
425  Returns:
426    Tuple of:
427     * Manifest path within |temp_dir|.
428     * Original package_name.
429  """
430  def maybe_extract_version(j):
431    try:
432      return resource_utils.ExtractBinaryManifestValues(options.aapt2_path, j)
433    except build_utils.CalledProcessError:
434      return None
435
436  android_sdk_jars = [j for j in options.include_resources
437                      if os.path.basename(j) in ('android.jar',
438                                                 'android_system.jar')]
439  extract_all = [maybe_extract_version(j) for j in android_sdk_jars]
440  successful_extractions = [x for x in extract_all if x]
441  if len(successful_extractions) == 0:
442    raise Exception(
443        'Unable to find android SDK jar among candidates: %s'
444            % ', '.join(android_sdk_jars))
445  elif len(successful_extractions) > 1:
446    raise Exception(
447        'Found multiple android SDK jars among candidates: %s'
448            % ', '.join(android_sdk_jars))
449  version_code, version_name = successful_extractions.pop()[:2]
450
451  debug_manifest_path = os.path.join(temp_dir, 'AndroidManifest.xml')
452  doc, manifest_node, app_node = manifest_utils.ParseManifest(
453      options.android_manifest)
454
455  manifest_utils.AssertUsesSdk(manifest_node, options.min_sdk_version,
456                               options.target_sdk_version)
457  # We explicitly check that maxSdkVersion is set in the manifest since we don't
458  # add it later like minSdkVersion and targetSdkVersion.
459  manifest_utils.AssertUsesSdk(
460      manifest_node,
461      max_sdk_version=options.max_sdk_version,
462      fail_if_not_exist=True)
463  manifest_utils.AssertPackage(manifest_node, options.manifest_package)
464
465  manifest_node.set('platformBuildVersionCode', version_code)
466  manifest_node.set('platformBuildVersionName', version_name)
467
468  orig_package = manifest_node.get('package')
469  if options.arsc_package_name:
470    manifest_node.set('package', options.arsc_package_name)
471
472  if options.debuggable:
473    app_node.set('{%s}%s' % (manifest_utils.ANDROID_NAMESPACE, 'debuggable'),
474                 'true')
475
476  if options.uses_split:
477    uses_split = ElementTree.SubElement(manifest_node, 'uses-split')
478    uses_split.set('{%s}name' % manifest_utils.ANDROID_NAMESPACE,
479                   options.uses_split)
480
481  manifest_utils.SaveManifest(doc, debug_manifest_path)
482  return debug_manifest_path, orig_package
483
484
485def _CreateKeepPredicate(resource_exclusion_regex,
486                         resource_exclusion_exceptions):
487  """Return a predicate lambda to determine which resource files to keep.
488
489  Args:
490    resource_exclusion_regex: A regular expression describing all resources
491      to exclude, except if they are mip-maps, or if they are listed
492      in |resource_exclusion_exceptions|.
493    resource_exclusion_exceptions: A list of glob patterns corresponding
494      to exceptions to the |resource_exclusion_regex|.
495  Returns:
496    A lambda that takes a path, and returns true if the corresponding file
497    must be kept.
498  """
499  predicate = lambda path: os.path.basename(path)[0] != '.'
500  if resource_exclusion_regex == '':
501    # Do not extract dotfiles (e.g. ".gitkeep"). aapt ignores them anyways.
502    return predicate
503
504  # A simple predicate that only removes (returns False for) paths covered by
505  # the exclusion regex or listed as exceptions.
506  return lambda path: (
507      not re.search(resource_exclusion_regex, path) or
508      build_utils.MatchesGlob(path, resource_exclusion_exceptions))
509
510
511def _ComputeSha1(path):
512  with open(path, 'rb') as f:
513    data = f.read()
514  return hashlib.sha1(data).hexdigest()
515
516
517def _ConvertToWebPSingle(png_path, cwebp_binary, cwebp_version, webp_cache_dir):
518  sha1_hash = _ComputeSha1(png_path)
519
520  # The set of arguments that will appear in the cache key.
521  quality_args = ['-m', '6', '-q', '100', '-lossless']
522
523  webp_cache_path = os.path.join(
524      webp_cache_dir, '{}-{}-{}'.format(sha1_hash, cwebp_version,
525                                        ''.join(quality_args)))
526  # No need to add .webp. Android can load images fine without them.
527  webp_path = os.path.splitext(png_path)[0]
528
529  cache_hit = os.path.exists(webp_cache_path)
530  if cache_hit:
531    os.link(webp_cache_path, webp_path)
532  else:
533    # We place the generated webp image to webp_path, instead of in the
534    # webp_cache_dir to avoid concurrency issues.
535    args = [cwebp_binary, png_path, '-o', webp_path, '-quiet'] + quality_args
536    subprocess.check_call(args)
537
538    try:
539      os.link(webp_path, webp_cache_path)
540    except OSError:
541      # Because of concurrent run, a webp image may already exists in
542      # webp_cache_path.
543      pass
544
545  os.remove(png_path)
546  original_dir = os.path.dirname(os.path.dirname(png_path))
547  rename_tuple = (os.path.relpath(png_path, original_dir),
548                  os.path.relpath(webp_path, original_dir))
549  return rename_tuple, cache_hit
550
551
552def _ConvertToWebP(cwebp_binary, png_paths, path_info, webp_cache_dir):
553  cwebp_version = subprocess.check_output([cwebp_binary, '-version']).rstrip()
554  shard_args = [(f, ) for f in png_paths
555                if not _PNG_WEBP_EXCLUSION_PATTERN.match(f)]
556
557  build_utils.MakeDirectory(webp_cache_dir)
558  results = parallel.BulkForkAndCall(_ConvertToWebPSingle,
559                                     shard_args,
560                                     cwebp_binary=cwebp_binary,
561                                     cwebp_version=cwebp_version,
562                                     webp_cache_dir=webp_cache_dir)
563  total_cache_hits = 0
564  for rename_tuple, cache_hit in results:
565    path_info.RegisterRename(*rename_tuple)
566    total_cache_hits += int(cache_hit)
567
568  logging.debug('png->webp cache: %d/%d', total_cache_hits, len(shard_args))
569
570
571def _RemoveImageExtensions(directory, path_info):
572  """Remove extensions from image files in the passed directory.
573
574  This reduces binary size but does not affect android's ability to load the
575  images.
576  """
577  for f in _IterFiles(directory):
578    if (f.endswith('.png') or f.endswith('.webp')) and not f.endswith('.9.png'):
579      path_with_extension = f
580      path_no_extension = os.path.splitext(path_with_extension)[0]
581      if path_no_extension != path_with_extension:
582        shutil.move(path_with_extension, path_no_extension)
583        path_info.RegisterRename(
584            os.path.relpath(path_with_extension, directory),
585            os.path.relpath(path_no_extension, directory))
586
587
588def _CompileSingleDep(index, dep_subdir, keep_predicate, aapt2_path,
589                      partials_dir):
590  unique_name = '{}_{}'.format(index, os.path.basename(dep_subdir))
591  partial_path = os.path.join(partials_dir, '{}.zip'.format(unique_name))
592
593  compile_command = [
594      aapt2_path,
595      'compile',
596      # TODO(wnwen): Turn this on once aapt2 forces 9-patch to be crunched.
597      # '--no-crunch',
598      '--dir',
599      dep_subdir,
600      '-o',
601      partial_path
602  ]
603
604  # There are resources targeting API-versions lower than our minapi. For
605  # various reasons it's easier to let aapt2 ignore these than for us to
606  # remove them from our build (e.g. it's from a 3rd party library).
607  build_utils.CheckOutput(
608      compile_command,
609      stderr_filter=lambda output: build_utils.FilterLines(
610          output, r'ignoring configuration .* for (styleable|attribute)'))
611
612  # Filtering these files is expensive, so only apply filters to the partials
613  # that have been explicitly targeted.
614  if keep_predicate:
615    logging.debug('Applying .arsc filtering to %s', dep_subdir)
616    protoresources.StripUnwantedResources(partial_path, keep_predicate)
617  return partial_path
618
619
620def _CreateValuesKeepPredicate(exclusion_rules, dep_subdir):
621  patterns = [
622      x[1] for x in exclusion_rules
623      if build_utils.MatchesGlob(dep_subdir, [x[0]])
624  ]
625  if not patterns:
626    return None
627
628  regexes = [re.compile(p) for p in patterns]
629  return lambda x: not any(r.search(x) for r in regexes)
630
631
632def _CompileDeps(aapt2_path, dep_subdirs, dep_subdir_overlay_set, temp_dir,
633                 exclusion_rules):
634  partials_dir = os.path.join(temp_dir, 'partials')
635  build_utils.MakeDirectory(partials_dir)
636
637  job_params = [(i, dep_subdir,
638                 _CreateValuesKeepPredicate(exclusion_rules, dep_subdir))
639                for i, dep_subdir in enumerate(dep_subdirs)]
640
641  # Filtering is slow, so ensure jobs with keep_predicate are started first.
642  job_params.sort(key=lambda x: not x[2])
643  partials = list(
644      parallel.BulkForkAndCall(_CompileSingleDep,
645                               job_params,
646                               aapt2_path=aapt2_path,
647                               partials_dir=partials_dir))
648
649  partials_cmd = list()
650  for i, partial in enumerate(partials):
651    dep_subdir = job_params[i][1]
652    if dep_subdir in dep_subdir_overlay_set:
653      partials_cmd += ['-R']
654    partials_cmd += [partial]
655  return partials_cmd
656
657
658def _CreateResourceInfoFile(path_info, info_path, dependencies_res_zips):
659  for zip_file in dependencies_res_zips:
660    zip_info_file_path = zip_file + '.info'
661    if os.path.exists(zip_info_file_path):
662      path_info.MergeInfoFile(zip_info_file_path)
663  path_info.Write(info_path)
664
665
666def _RemoveUnwantedLocalizedStrings(dep_subdirs, options):
667  """Remove localized strings that should not go into the final output.
668
669  Args:
670    dep_subdirs: List of resource dependency directories.
671    options: Command-line options namespace.
672  """
673  # Collect locale and file paths from the existing subdirs.
674  # The following variable maps Android locale names to
675  # sets of corresponding xml file paths.
676  locale_to_files_map = collections.defaultdict(set)
677  for directory in dep_subdirs:
678    for f in _IterFiles(directory):
679      locale = resource_utils.FindLocaleInStringResourceFilePath(f)
680      if locale:
681        locale_to_files_map[locale].add(f)
682
683  all_locales = set(locale_to_files_map)
684
685  # Set A: wanted locales, either all of them or the
686  # list provided by --locale-allowlist.
687  wanted_locales = all_locales
688  if options.locale_allowlist:
689    wanted_locales = _ToAndroidLocales(options.locale_allowlist,
690                                       options.support_zh_hk)
691
692  # Set B: shared resources locales, which is either set A
693  # or the list provided by --shared-resources-allowlist-locales
694  shared_resources_locales = wanted_locales
695  shared_names_allowlist = set()
696  if options.shared_resources_allowlist_locales:
697    shared_names_allowlist = set(
698        resource_utils.GetRTxtStringResourceNames(
699            options.shared_resources_allowlist))
700
701    shared_resources_locales = _ToAndroidLocales(
702        options.shared_resources_allowlist_locales, options.support_zh_hk)
703
704  # Remove any file that belongs to a locale not covered by
705  # either A or B.
706  removable_locales = (all_locales - wanted_locales - shared_resources_locales)
707  for locale in removable_locales:
708    for path in locale_to_files_map[locale]:
709      os.remove(path)
710
711  # For any locale in B but not in A, only keep the shared
712  # resource strings in each file.
713  for locale in shared_resources_locales - wanted_locales:
714    for path in locale_to_files_map[locale]:
715      resource_utils.FilterAndroidResourceStringsXml(
716          path, lambda x: x in shared_names_allowlist)
717
718  # For any locale in A but not in B, only keep the strings
719  # that are _not_ from shared resources in the file.
720  for locale in wanted_locales - shared_resources_locales:
721    for path in locale_to_files_map[locale]:
722      resource_utils.FilterAndroidResourceStringsXml(
723          path, lambda x: x not in shared_names_allowlist)
724
725
726def _FilterResourceFiles(dep_subdirs, keep_predicate):
727  # Create a function that selects which resource files should be packaged
728  # into the final output. Any file that does not pass the predicate will
729  # be removed below.
730  png_paths = []
731  for directory in dep_subdirs:
732    for f in _IterFiles(directory):
733      if not keep_predicate(f):
734        os.remove(f)
735      elif f.endswith('.png'):
736        png_paths.append(f)
737
738  return png_paths
739
740
741def _PackageApk(options, build):
742  """Compile and link resources with aapt2.
743
744  Args:
745    options: The command-line options.
746    build: BuildContext object.
747  Returns:
748    The manifest package name for the APK.
749  """
750  logging.debug('Extracting resource .zips')
751  dep_subdirs = []
752  dep_subdir_overlay_set = set()
753  for dependency_res_zip in options.dependencies_res_zips:
754    extracted_dep_subdirs = resource_utils.ExtractDeps([dependency_res_zip],
755                                                       build.deps_dir)
756    dep_subdirs += extracted_dep_subdirs
757    if dependency_res_zip in options.dependencies_res_zip_overlays:
758      dep_subdir_overlay_set.update(extracted_dep_subdirs)
759
760  logging.debug('Applying locale transformations')
761  path_info = resource_utils.ResourceInfoFile()
762  if options.support_zh_hk:
763    _DuplicateZhResources(dep_subdirs, path_info)
764  _RenameLocaleResourceDirs(dep_subdirs, path_info)
765
766  logging.debug('Applying file-based exclusions')
767  keep_predicate = _CreateKeepPredicate(options.resource_exclusion_regex,
768                                        options.resource_exclusion_exceptions)
769  png_paths = _FilterResourceFiles(dep_subdirs, keep_predicate)
770
771  if options.locale_allowlist or options.shared_resources_allowlist_locales:
772    logging.debug('Applying locale-based string exclusions')
773    _RemoveUnwantedLocalizedStrings(dep_subdirs, options)
774
775  if png_paths and options.png_to_webp:
776    logging.debug('Converting png->webp')
777    _ConvertToWebP(options.webp_binary, png_paths, path_info,
778                   options.webp_cache_dir)
779  logging.debug('Applying drawable transformations')
780  for directory in dep_subdirs:
781    _MoveImagesToNonMdpiFolders(directory, path_info)
782    _RemoveImageExtensions(directory, path_info)
783
784  logging.debug('Running aapt2 compile')
785  exclusion_rules = [x.split(':', 1) for x in options.values_filter_rules]
786  partials = _CompileDeps(options.aapt2_path, dep_subdirs,
787                          dep_subdir_overlay_set, build.temp_dir,
788                          exclusion_rules)
789
790  link_command = [
791      options.aapt2_path,
792      'link',
793      '--auto-add-overlay',
794      '--no-version-vectors',
795      # Set SDK versions in case they are not set in the Android manifest.
796      '--min-sdk-version',
797      options.min_sdk_version,
798      '--target-sdk-version',
799      options.target_sdk_version,
800  ]
801
802  for j in options.include_resources:
803    link_command += ['-I', j]
804  if options.version_code:
805    link_command += ['--version-code', options.version_code]
806  if options.version_name:
807    link_command += ['--version-name', options.version_name]
808  if options.proguard_file:
809    link_command += ['--proguard', build.proguard_path]
810    link_command += ['--proguard-minimal-keep-rules']
811  if options.proguard_file_main_dex:
812    link_command += ['--proguard-main-dex', build.proguard_main_dex_path]
813  if options.emit_ids_out:
814    link_command += ['--emit-ids', build.emit_ids_path]
815  if options.r_text_in:
816    shutil.copyfile(options.r_text_in, build.r_txt_path)
817  else:
818    link_command += ['--output-text-symbols', build.r_txt_path]
819
820  # Note: only one of --proto-format, --shared-lib or --app-as-shared-lib
821  #       can be used with recent versions of aapt2.
822  if options.shared_resources:
823    link_command.append('--shared-lib')
824
825  if options.no_xml_namespaces:
826    link_command.append('--no-xml-namespaces')
827
828  if options.package_id:
829    link_command += [
830        '--package-id',
831        hex(options.package_id),
832        '--allow-reserved-package-id',
833    ]
834
835  fixed_manifest, desired_manifest_package_name = _FixManifest(
836      options, build.temp_dir)
837  if options.rename_manifest_package:
838    desired_manifest_package_name = options.rename_manifest_package
839
840  link_command += [
841      '--manifest', fixed_manifest, '--rename-manifest-package',
842      desired_manifest_package_name
843  ]
844
845  # Creates a .zip with AndroidManifest.xml, resources.arsc, res/*
846  # Also creates R.txt
847  if options.use_resource_ids_path:
848    _CreateStableIdsFile(options.use_resource_ids_path, build.stable_ids_path,
849                         desired_manifest_package_name)
850    link_command += ['--stable-ids', build.stable_ids_path]
851
852  link_command += partials
853
854  # We always create a binary arsc file first, then convert to proto, so flags
855  # such as --shared-lib can be supported.
856  arsc_path = build.arsc_path
857  if arsc_path is None:
858    _, arsc_path = tempfile.mkstmp()
859  link_command += ['-o', build.arsc_path]
860
861  logging.debug('Starting: aapt2 link')
862  link_proc = subprocess.Popen(link_command)
863
864  # Create .res.info file in parallel.
865  _CreateResourceInfoFile(path_info, build.info_path,
866                          options.dependencies_res_zips)
867  logging.debug('Created .res.info file')
868
869  exit_code = link_proc.wait()
870  logging.debug('Finished: aapt2 link')
871  if exit_code:
872    raise subprocess.CalledProcessError(exit_code, link_command)
873
874  if options.proguard_file and (options.shared_resources
875                                or options.app_as_shared_lib):
876    # Make sure the R class associated with the manifest package does not have
877    # its onResourcesLoaded method obfuscated or removed, so that the framework
878    # can call it in the case where the APK is being loaded as a library.
879    with open(build.proguard_path, 'a') as proguard_file:
880      keep_rule = '''
881                  -keep class {package}.R {{
882                    public static void onResourcesLoaded(int);
883                  }}
884                  '''.format(package=desired_manifest_package_name)
885      proguard_file.write(textwrap.dedent(keep_rule))
886
887  logging.debug('Running aapt2 convert')
888  build_utils.CheckOutput([
889      options.aapt2_path, 'convert', '--output-format', 'proto', '-o',
890      build.proto_path, build.arsc_path
891  ])
892
893  # Workaround for b/147674078. This is only needed for WebLayer and does not
894  # affect WebView usage, since WebView does not used dynamic attributes.
895  if options.shared_resources:
896    logging.debug('Hardcoding dynamic attributes')
897    protoresources.HardcodeSharedLibraryDynamicAttributes(
898        build.proto_path, options.is_bundle_module,
899        options.shared_resources_allowlist)
900
901    build_utils.CheckOutput([
902        options.aapt2_path, 'convert', '--output-format', 'binary', '-o',
903        build.arsc_path, build.proto_path
904    ])
905
906  if build.arsc_path is None:
907    os.remove(arsc_path)
908
909  if options.optimized_proto_path:
910    _OptimizeApk(build.optimized_proto_path, options, build.temp_dir,
911                 build.proto_path, build.r_txt_path)
912  elif options.optimized_arsc_path:
913    _OptimizeApk(build.optimized_arsc_path, options, build.temp_dir,
914                 build.arsc_path, build.r_txt_path)
915
916  return desired_manifest_package_name
917
918
919def _CombineResourceConfigs(resources_config_paths, out_config_path):
920  with open(out_config_path, 'w') as out_config:
921    for config_path in resources_config_paths:
922      with open(config_path) as config:
923        out_config.write(config.read())
924        out_config.write('\n')
925
926
927def _OptimizeApk(output, options, temp_dir, unoptimized_path, r_txt_path):
928  """Optimize intermediate .ap_ file with aapt2.
929
930  Args:
931    output: Path to write to.
932    options: The command-line options.
933    temp_dir: A temporary directory.
934    unoptimized_path: path of the apk to optimize.
935    r_txt_path: path to the R.txt file of the unoptimized apk.
936  """
937  optimize_command = [
938      options.aapt2_path,
939      'optimize',
940      unoptimized_path,
941      '-o',
942      output,
943  ]
944
945  # Optimize the resources.arsc file by obfuscating resource names and only
946  # allow usage via R.java constant.
947  if options.strip_resource_names:
948    no_collapse_resources = _ExtractNonCollapsableResources(r_txt_path)
949    gen_config_path = os.path.join(temp_dir, 'aapt2.config')
950    if options.resources_config_paths:
951      _CombineResourceConfigs(options.resources_config_paths, gen_config_path)
952    with open(gen_config_path, 'a') as config:
953      for resource in no_collapse_resources:
954        config.write('{}#no_collapse\n'.format(resource))
955
956    optimize_command += [
957        '--collapse-resource-names',
958        '--resources-config-path',
959        gen_config_path,
960    ]
961
962  if options.short_resource_paths:
963    optimize_command += ['--shorten-resource-paths']
964  if options.resources_path_map_out_path:
965    optimize_command += [
966        '--resource-path-shortening-map', options.resources_path_map_out_path
967    ]
968
969  logging.debug('Running aapt2 optimize')
970  build_utils.CheckOutput(
971      optimize_command, print_stdout=False, print_stderr=False)
972
973
974def _ExtractNonCollapsableResources(rtxt_path):
975  """Extract resources that should not be collapsed from the R.txt file
976
977  Resources of type ID are references to UI elements/views. They are used by
978  UI automation testing frameworks. They are kept in so that they don't break
979  tests, even though they may not actually be used during runtime. See
980  https://crbug.com/900993
981  App icons (aka mipmaps) are sometimes referenced by other apps by name so must
982  be keps as well. See https://b/161564466
983
984  Args:
985    rtxt_path: Path to R.txt file with all the resources
986  Returns:
987    List of resources in the form of <resource_type>/<resource_name>
988  """
989  resources = []
990  _NO_COLLAPSE_TYPES = ['id', 'mipmap']
991  with open(rtxt_path) as rtxt:
992    for line in rtxt:
993      for resource_type in _NO_COLLAPSE_TYPES:
994        if ' {} '.format(resource_type) in line:
995          resource_name = line.split()[2]
996          resources.append('{}/{}'.format(resource_type, resource_name))
997  return resources
998
999
1000@contextlib.contextmanager
1001def _CreateStableIdsFile(in_path, out_path, package_name):
1002  """Transforms a file generated by --emit-ids from another package.
1003
1004  --stable-ids is generally meant to be used by different versions of the same
1005  package. To make it work for other packages, we need to transform the package
1006  name references to match the package that resources are being generated for.
1007
1008  Note: This will fail if the package ID of the resources in
1009  |options.use_resource_ids_path| does not match the package ID of the
1010  resources being linked.
1011  """
1012  with open(in_path) as stable_ids_file:
1013    with open(out_path, 'w') as output_ids_file:
1014      output_stable_ids = re.sub(
1015          r'^.*?:',
1016          package_name + ':',
1017          stable_ids_file.read(),
1018          flags=re.MULTILINE)
1019      output_ids_file.write(output_stable_ids)
1020
1021
1022def _WriteOutputs(options, build):
1023  possible_outputs = [
1024      (options.srcjar_out, build.srcjar_path),
1025      (options.r_text_out, build.r_txt_path),
1026      (options.arsc_path, build.arsc_path),
1027      (options.proto_path, build.proto_path),
1028      (options.optimized_arsc_path, build.optimized_arsc_path),
1029      (options.optimized_proto_path, build.optimized_proto_path),
1030      (options.proguard_file, build.proguard_path),
1031      (options.proguard_file_main_dex, build.proguard_main_dex_path),
1032      (options.emit_ids_out, build.emit_ids_path),
1033      (options.info_path, build.info_path),
1034  ]
1035
1036  for final, temp in possible_outputs:
1037    # Write file only if it's changed.
1038    if final and not (os.path.exists(final) and filecmp.cmp(final, temp)):
1039      shutil.move(temp, final)
1040
1041
1042def _CreateNormalizedManifest(options):
1043  with build_utils.TempDir() as tempdir:
1044    fixed_manifest, _ = _FixManifest(options, tempdir)
1045    with open(fixed_manifest) as f:
1046      return manifest_utils.NormalizeManifest(f.read())
1047
1048
1049def _OnStaleMd5(options):
1050  path = options.arsc_path or options.proto_path
1051  debug_temp_resources_dir = os.environ.get('TEMP_RESOURCES_DIR')
1052  if debug_temp_resources_dir:
1053    path = os.path.join(debug_temp_resources_dir, os.path.basename(path))
1054  else:
1055    # Use a deterministic temp directory since .pb files embed the absolute
1056    # path of resources: crbug.com/939984
1057    path = path + '.tmpdir'
1058  build_utils.DeleteDirectory(path)
1059
1060  with resource_utils.BuildContext(
1061      temp_dir=path, keep_files=bool(debug_temp_resources_dir)) as build:
1062
1063    manifest_package_name = _PackageApk(options, build)
1064
1065    # If --shared-resources-allowlist is used, all the resources listed in the
1066    # corresponding R.txt file will be non-final, and an onResourcesLoaded()
1067    # will be generated to adjust them at runtime.
1068    #
1069    # Otherwise, if --shared-resources is used, the all resources will be
1070    # non-final, and an onResourcesLoaded() method will be generated too.
1071    #
1072    # Otherwise, all resources will be final, and no method will be generated.
1073    #
1074    rjava_build_options = resource_utils.RJavaBuildOptions()
1075    if options.shared_resources_allowlist:
1076      rjava_build_options.ExportSomeResources(
1077          options.shared_resources_allowlist)
1078      rjava_build_options.GenerateOnResourcesLoaded()
1079      if options.shared_resources:
1080        # The final resources will only be used in WebLayer, so hardcode the
1081        # package ID to be what WebLayer expects.
1082        rjava_build_options.SetFinalPackageId(
1083            protoresources.SHARED_LIBRARY_HARDCODED_ID)
1084    elif options.shared_resources or options.app_as_shared_lib:
1085      rjava_build_options.ExportAllResources()
1086      rjava_build_options.GenerateOnResourcesLoaded()
1087
1088    custom_root_package_name = options.r_java_root_package_name
1089    grandparent_custom_package_name = None
1090
1091    # Always generate an R.java file for the package listed in
1092    # AndroidManifest.xml because this is where Android framework looks to find
1093    # onResourcesLoaded() for shared library apks. While not actually necessary
1094    # for application apks, it also doesn't hurt.
1095    apk_package_name = manifest_package_name
1096
1097    if options.package_name and not options.arsc_package_name:
1098      # Feature modules have their own custom root package name and should
1099      # inherit from the appropriate base module package. This behaviour should
1100      # not be present for test apks with an apk under test. Thus,
1101      # arsc_package_name is used as it is only defined for test apks with an
1102      # apk under test.
1103      custom_root_package_name = options.package_name
1104      grandparent_custom_package_name = options.r_java_root_package_name
1105      # Feature modules have the same manifest package as the base module but
1106      # they should not create an R.java for said manifest package because it
1107      # will be created in the base module.
1108      apk_package_name = None
1109
1110    logging.debug('Creating R.srcjar')
1111    resource_utils.CreateRJavaFiles(
1112        build.srcjar_dir, apk_package_name, build.r_txt_path,
1113        options.extra_res_packages, rjava_build_options, options.srcjar_out,
1114        custom_root_package_name, grandparent_custom_package_name,
1115        options.extra_main_r_text_files)
1116    build_utils.ZipDir(build.srcjar_path, build.srcjar_dir)
1117
1118    # Sanity check that the created resources have the expected package ID.
1119    logging.debug('Performing sanity check')
1120    if options.package_id:
1121      expected_id = options.package_id
1122    elif options.shared_resources:
1123      expected_id = 0
1124    else:
1125      expected_id = 127  # == '0x7f'.
1126    _, package_id = resource_utils.ExtractArscPackage(
1127        options.aapt2_path,
1128        build.arsc_path if options.arsc_path else build.proto_path)
1129    if package_id != expected_id:
1130      raise Exception(
1131          'Invalid package ID 0x%x (expected 0x%x)' % (package_id, expected_id))
1132
1133    logging.debug('Copying outputs')
1134    _WriteOutputs(options, build)
1135
1136
1137def main(args):
1138  build_utils.InitLogging('RESOURCE_DEBUG')
1139  args = build_utils.ExpandFileArgs(args)
1140  options = _ParseArgs(args)
1141
1142  if options.expected_file:
1143    actual_data = _CreateNormalizedManifest(options)
1144    diff_utils.CheckExpectations(actual_data, options)
1145    if options.only_verify_expectations:
1146      return
1147
1148  depfile_deps = (options.dependencies_res_zips +
1149                  options.dependencies_res_zip_overlays +
1150                  options.extra_main_r_text_files + options.include_resources)
1151
1152  possible_input_paths = depfile_deps + options.resources_config_paths + [
1153      options.aapt2_path,
1154      options.android_manifest,
1155      options.expected_file,
1156      options.expected_file_base,
1157      options.shared_resources_allowlist,
1158      options.use_resource_ids_path,
1159      options.webp_binary,
1160  ]
1161  input_paths = [p for p in possible_input_paths if p]
1162  input_strings = [
1163      options.app_as_shared_lib,
1164      options.arsc_package_name,
1165      options.debuggable,
1166      options.extra_res_packages,
1167      options.failure_file,
1168      options.include_resources,
1169      options.locale_allowlist,
1170      options.manifest_package,
1171      options.max_sdk_version,
1172      options.min_sdk_version,
1173      options.no_xml_namespaces,
1174      options.package_id,
1175      options.package_name,
1176      options.png_to_webp,
1177      options.rename_manifest_package,
1178      options.resource_exclusion_exceptions,
1179      options.resource_exclusion_regex,
1180      options.r_java_root_package_name,
1181      options.shared_resources,
1182      options.shared_resources_allowlist_locales,
1183      options.short_resource_paths,
1184      options.strip_resource_names,
1185      options.support_zh_hk,
1186      options.target_sdk_version,
1187      options.values_filter_rules,
1188      options.version_code,
1189      options.version_name,
1190      options.webp_cache_dir,
1191  ]
1192  output_paths = [options.srcjar_out]
1193  possible_output_paths = [
1194      options.actual_file,
1195      options.arsc_path,
1196      options.emit_ids_out,
1197      options.info_path,
1198      options.optimized_arsc_path,
1199      options.optimized_proto_path,
1200      options.proguard_file,
1201      options.proguard_file_main_dex,
1202      options.proto_path,
1203      options.resources_path_map_out_path,
1204      options.r_text_out,
1205  ]
1206  output_paths += [p for p in possible_output_paths if p]
1207
1208  # Since we overspecify deps, this target depends on java deps that are not
1209  # going to change its output. This target is also slow (6-12 seconds) and
1210  # blocking the critical path. We want changes to java_library targets to not
1211  # trigger re-compilation of resources, thus we need to use md5_check.
1212  md5_check.CallAndWriteDepfileIfStale(
1213      lambda: _OnStaleMd5(options),
1214      options,
1215      input_paths=input_paths,
1216      input_strings=input_strings,
1217      output_paths=output_paths,
1218      depfile_deps=depfile_deps)
1219
1220
1221if __name__ == '__main__':
1222  main(sys.argv[1:])
1223