1#!/usr/bin/env python 2# encoding: utf-8 3# Copyright (c) 2012 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Compile Android resources into an intermediate APK. 8 9This can also generate an R.txt, and an .srcjar file containing the proper 10final R.java class for all resource packages the APK depends on. 11 12This will crunch images with aapt2. 13""" 14 15import argparse 16import collections 17import contextlib 18import filecmp 19import hashlib 20import logging 21import os 22import re 23import shutil 24import subprocess 25import sys 26import tempfile 27import textwrap 28import zipfile 29from xml.etree import ElementTree 30 31from util import build_utils 32from util import diff_utils 33from util import manifest_utils 34from util import md5_check 35from util import parallel 36from util import protoresources 37from util import resource_utils 38 39 40# Pngs that we shouldn't convert to webp. Please add rationale when updating. 41_PNG_WEBP_EXCLUSION_PATTERN = re.compile('|'.join([ 42 # Crashes on Galaxy S5 running L (https://crbug.com/807059). 43 r'.*star_gray\.png', 44 # Android requires pngs for 9-patch images. 45 r'.*\.9\.png', 46 # Daydream requires pngs for icon files. 47 r'.*daydream_icon_.*\.png' 48])) 49 50 51def _ParseArgs(args): 52 """Parses command line options. 53 54 Returns: 55 An options object as from argparse.ArgumentParser.parse_args() 56 """ 57 parser, input_opts, output_opts = resource_utils.ResourceArgsParser() 58 59 input_opts.add_argument( 60 '--aapt2-path', required=True, help='Path to the Android aapt2 tool.') 61 input_opts.add_argument( 62 '--android-manifest', required=True, help='AndroidManifest.xml path.') 63 input_opts.add_argument( 64 '--r-java-root-package-name', 65 default='base', 66 help='Short package name for this target\'s root R java file (ex. ' 67 'input of "base" would become gen.base_module). Defaults to "base".') 68 group = input_opts.add_mutually_exclusive_group() 69 group.add_argument( 70 '--shared-resources', 71 action='store_true', 72 help='Make all resources in R.java non-final and allow the resource IDs ' 73 'to be reset to a different package index when the apk is loaded by ' 74 'another application at runtime.') 75 group.add_argument( 76 '--app-as-shared-lib', 77 action='store_true', 78 help='Same as --shared-resources, but also ensures all resource IDs are ' 79 'directly usable from the APK loaded as an application.') 80 81 input_opts.add_argument( 82 '--package-id', 83 type=int, 84 help='Decimal integer representing custom package ID for resources ' 85 '(instead of 127==0x7f). Cannot be used with --shared-resources.') 86 87 input_opts.add_argument( 88 '--package-name', 89 help='Package name that will be used to create R class.') 90 91 input_opts.add_argument( 92 '--rename-manifest-package', help='Package name to force AAPT to use.') 93 94 input_opts.add_argument( 95 '--arsc-package-name', 96 help='Package name to set in manifest of resources.arsc file. This is ' 97 'only used for apks under test.') 98 99 input_opts.add_argument( 100 '--shared-resources-allowlist', 101 help='An R.txt file acting as a allowlist for resources that should be ' 102 'non-final and have their package ID changed at runtime in R.java. ' 103 'Implies and overrides --shared-resources.') 104 105 input_opts.add_argument( 106 '--shared-resources-allowlist-locales', 107 default='[]', 108 help='Optional GN-list of locales. If provided, all strings corresponding' 109 ' to this locale list will be kept in the final output for the ' 110 'resources identified through --shared-resources-allowlist, even ' 111 'if --locale-allowlist is being used.') 112 113 input_opts.add_argument( 114 '--use-resource-ids-path', 115 help='Use resource IDs generated by aapt --emit-ids.') 116 117 input_opts.add_argument( 118 '--extra-main-r-text-files', 119 help='Additional R.txt files that will be added to the root R.java file, ' 120 'but not packaged in the generated resources.arsc. If these resources ' 121 'entries contain duplicate resources with the generated R.txt file, they ' 122 'must be identical.') 123 124 input_opts.add_argument( 125 '--support-zh-hk', 126 action='store_true', 127 help='Use zh-rTW resources for zh-rHK.') 128 129 input_opts.add_argument( 130 '--debuggable', 131 action='store_true', 132 help='Whether to add android:debuggable="true".') 133 134 input_opts.add_argument('--version-code', help='Version code for apk.') 135 input_opts.add_argument('--version-name', help='Version name for apk.') 136 input_opts.add_argument( 137 '--min-sdk-version', required=True, help='android:minSdkVersion for APK.') 138 input_opts.add_argument( 139 '--target-sdk-version', 140 required=True, 141 help="android:targetSdkVersion for APK.") 142 input_opts.add_argument( 143 '--max-sdk-version', 144 help="android:maxSdkVersion expected in AndroidManifest.xml.") 145 input_opts.add_argument( 146 '--manifest-package', help='Package name of the AndroidManifest.xml.') 147 148 input_opts.add_argument( 149 '--locale-allowlist', 150 default='[]', 151 help='GN list of languages to include. All other language configs will ' 152 'be stripped out. List may include a combination of Android locales ' 153 'or Chrome locales.') 154 input_opts.add_argument( 155 '--resource-exclusion-regex', 156 default='', 157 help='File-based filter for resources (applied before compiling)') 158 input_opts.add_argument( 159 '--resource-exclusion-exceptions', 160 default='[]', 161 help='GN list of globs that say which files to include even ' 162 'when --resource-exclusion-regex is set.') 163 164 input_opts.add_argument( 165 '--dependencies-res-zip-overlays', 166 help='GN list with subset of --dependencies-res-zips to use overlay ' 167 'semantics for.') 168 169 input_opts.add_argument( 170 '--values-filter-rules', 171 help='GN list of source_glob:regex for filtering resources after they ' 172 'are compiled. Use this to filter out entries within values/ files.') 173 174 input_opts.add_argument('--png-to-webp', action='store_true', 175 help='Convert png files to webp format.') 176 177 input_opts.add_argument('--webp-binary', default='', 178 help='Path to the cwebp binary.') 179 input_opts.add_argument( 180 '--webp-cache-dir', help='The directory to store webp image cache.') 181 182 input_opts.add_argument( 183 '--no-xml-namespaces', 184 action='store_true', 185 help='Whether to strip xml namespaces from processed xml resources.') 186 input_opts.add_argument( 187 '--short-resource-paths', 188 action='store_true', 189 help='Whether to shorten resource paths inside the apk or module.') 190 input_opts.add_argument( 191 '--strip-resource-names', 192 action='store_true', 193 help='Whether to strip resource names from the resource table of the apk ' 194 'or module.') 195 196 output_opts.add_argument('--arsc-path', help='Apk output for arsc format.') 197 output_opts.add_argument('--proto-path', help='Apk output for proto format.') 198 group = input_opts.add_mutually_exclusive_group() 199 group.add_argument( 200 '--optimized-arsc-path', 201 help='Output for `aapt2 optimize` for arsc format (enables the step).') 202 group.add_argument( 203 '--optimized-proto-path', 204 help='Output for `aapt2 optimize` for proto format (enables the step).') 205 input_opts.add_argument( 206 '--resources-config-paths', 207 default='[]', 208 help='GN list of paths to aapt2 resources config files.') 209 210 output_opts.add_argument( 211 '--info-path', help='Path to output info file for the partial apk.') 212 213 output_opts.add_argument( 214 '--srcjar-out', 215 required=True, 216 help='Path to srcjar to contain generated R.java.') 217 218 output_opts.add_argument('--r-text-out', 219 help='Path to store the generated R.txt file.') 220 221 output_opts.add_argument( 222 '--proguard-file', help='Path to proguard.txt generated file.') 223 224 output_opts.add_argument( 225 '--proguard-file-main-dex', 226 help='Path to proguard.txt generated file for main dex.') 227 228 output_opts.add_argument( 229 '--emit-ids-out', help='Path to file produced by aapt2 --emit-ids.') 230 231 output_opts.add_argument( 232 '--resources-path-map-out-path', 233 help='Path to file produced by aapt2 that maps original resource paths ' 234 'to shortened resource paths inside the apk or module.') 235 236 input_opts.add_argument( 237 '--is-bundle-module', 238 action='store_true', 239 help='Whether resources are being generated for a bundle module.') 240 241 input_opts.add_argument( 242 '--uses-split', 243 help='Value to set uses-split to in the AndroidManifest.xml.') 244 245 diff_utils.AddCommandLineFlags(parser) 246 options = parser.parse_args(args) 247 248 resource_utils.HandleCommonOptions(options) 249 250 options.locale_allowlist = build_utils.ParseGnList(options.locale_allowlist) 251 options.shared_resources_allowlist_locales = build_utils.ParseGnList( 252 options.shared_resources_allowlist_locales) 253 options.resource_exclusion_exceptions = build_utils.ParseGnList( 254 options.resource_exclusion_exceptions) 255 options.dependencies_res_zip_overlays = build_utils.ParseGnList( 256 options.dependencies_res_zip_overlays) 257 options.values_filter_rules = build_utils.ParseGnList( 258 options.values_filter_rules) 259 options.extra_main_r_text_files = build_utils.ParseGnList( 260 options.extra_main_r_text_files) 261 options.resources_config_paths = build_utils.ParseGnList( 262 options.resources_config_paths) 263 264 if options.optimized_proto_path and not options.proto_path: 265 # We could write to a temp file, but it's simpler to require it. 266 parser.error('--optimized-proto-path requires --proto-path') 267 268 if not options.arsc_path and not options.proto_path: 269 parser.error('One of --arsc-path or --proto-path is required.') 270 271 if options.resources_path_map_out_path and not options.short_resource_paths: 272 parser.error( 273 '--resources-path-map-out-path requires --short-resource-paths') 274 275 if options.package_id and options.shared_resources: 276 parser.error('--package-id and --shared-resources are mutually exclusive') 277 278 return options 279 280 281def _IterFiles(root_dir): 282 for root, _, files in os.walk(root_dir): 283 for f in files: 284 yield os.path.join(root, f) 285 286 287def _DuplicateZhResources(resource_dirs, path_info): 288 """Duplicate Taiwanese resources into Hong-Kong specific directory.""" 289 for resource_dir in resource_dirs: 290 # We use zh-TW resources for zh-HK (if we have zh-TW resources). 291 for path in _IterFiles(resource_dir): 292 if 'zh-rTW' in path: 293 hk_path = path.replace('zh-rTW', 'zh-rHK') 294 build_utils.MakeDirectory(os.path.dirname(hk_path)) 295 shutil.copyfile(path, hk_path) 296 path_info.RegisterRename( 297 os.path.relpath(path, resource_dir), 298 os.path.relpath(hk_path, resource_dir)) 299 300 301def _RenameLocaleResourceDirs(resource_dirs, path_info): 302 """Rename locale resource directories into standard names when necessary. 303 304 This is necessary to deal with the fact that older Android releases only 305 support ISO 639-1 two-letter codes, and sometimes even obsolete versions 306 of them. 307 308 In practice it means: 309 * 3-letter ISO 639-2 qualifiers are renamed under a corresponding 310 2-letter one. E.g. for Filipino, strings under values-fil/ will be moved 311 to a new corresponding values-tl/ sub-directory. 312 313 * Modern ISO 639-1 codes will be renamed to their obsolete variant 314 for Indonesian, Hebrew and Yiddish (e.g. 'values-in/ -> values-id/). 315 316 * Norwegian macrolanguage strings will be renamed to Bokmål (main 317 Norway language). See http://crbug.com/920960. In practice this 318 means that 'values-no/ -> values-nb/' unless 'values-nb/' already 319 exists. 320 321 * BCP 47 langauge tags will be renamed to an equivalent ISO 639-1 322 locale qualifier if possible (e.g. 'values-b+en+US/ -> values-en-rUS'). 323 Though this is not necessary at the moment, because no third-party 324 package that Chromium links against uses these for the current list of 325 supported locales, this may change when the list is extended in the 326 future). 327 328 Args: 329 resource_dirs: list of top-level resource directories. 330 """ 331 for resource_dir in resource_dirs: 332 for path in _IterFiles(resource_dir): 333 locale = resource_utils.FindLocaleInStringResourceFilePath(path) 334 if not locale: 335 continue 336 cr_locale = resource_utils.ToChromiumLocaleName(locale) 337 if not cr_locale: 338 continue # Unsupported Android locale qualifier!? 339 locale2 = resource_utils.ToAndroidLocaleName(cr_locale) 340 if locale != locale2: 341 path2 = path.replace('/values-%s/' % locale, '/values-%s/' % locale2) 342 if path == path2: 343 raise Exception('Could not substitute locale %s for %s in %s' % 344 (locale, locale2, path)) 345 if os.path.exists(path2): 346 # This happens sometimes, e.g. some libraries provide both 347 # values-nb/ and values-no/ with the same content. 348 continue 349 build_utils.MakeDirectory(os.path.dirname(path2)) 350 shutil.move(path, path2) 351 path_info.RegisterRename( 352 os.path.relpath(path, resource_dir), 353 os.path.relpath(path2, resource_dir)) 354 355 356def _ToAndroidLocales(locale_allowlist, support_zh_hk): 357 """Converts the list of Chrome locales to Android config locale qualifiers. 358 359 Args: 360 locale_allowlist: A list of Chromium locale names. 361 support_zh_hk: True if we need to support zh-HK by duplicating 362 the zh-TW strings. 363 Returns: 364 A set of matching Android config locale qualifier names. 365 """ 366 ret = set() 367 for locale in locale_allowlist: 368 locale = resource_utils.ToAndroidLocaleName(locale) 369 if locale is None or ('-' in locale and '-r' not in locale): 370 raise Exception('Unsupported Chromium locale name: %s' % locale) 371 ret.add(locale) 372 # Always keep non-regional fall-backs. 373 language = locale.split('-')[0] 374 ret.add(language) 375 376 # We don't actually support zh-HK in Chrome on Android, but we mimic the 377 # native side behavior where we use zh-TW resources when the locale is set to 378 # zh-HK. See https://crbug.com/780847. 379 if support_zh_hk: 380 assert not any('HK' in l for l in locale_allowlist), ( 381 'Remove special logic if zh-HK is now supported (crbug.com/780847).') 382 ret.add('zh-rHK') 383 return set(ret) 384 385 386def _MoveImagesToNonMdpiFolders(res_root, path_info): 387 """Move images from drawable-*-mdpi-* folders to drawable-* folders. 388 389 Why? http://crbug.com/289843 390 """ 391 for src_dir_name in os.listdir(res_root): 392 src_components = src_dir_name.split('-') 393 if src_components[0] != 'drawable' or 'mdpi' not in src_components: 394 continue 395 src_dir = os.path.join(res_root, src_dir_name) 396 if not os.path.isdir(src_dir): 397 continue 398 dst_components = [c for c in src_components if c != 'mdpi'] 399 assert dst_components != src_components 400 dst_dir_name = '-'.join(dst_components) 401 dst_dir = os.path.join(res_root, dst_dir_name) 402 build_utils.MakeDirectory(dst_dir) 403 for src_file_name in os.listdir(src_dir): 404 if not os.path.splitext(src_file_name)[1] in ('.png', '.webp', ''): 405 continue 406 src_file = os.path.join(src_dir, src_file_name) 407 dst_file = os.path.join(dst_dir, src_file_name) 408 assert not os.path.lexists(dst_file) 409 shutil.move(src_file, dst_file) 410 path_info.RegisterRename( 411 os.path.relpath(src_file, res_root), 412 os.path.relpath(dst_file, res_root)) 413 414 415def _FixManifest(options, temp_dir): 416 """Fix the APK's AndroidManifest.xml. 417 418 This adds any missing namespaces for 'android' and 'tools', and 419 sets certains elements like 'platformBuildVersionCode' or 420 'android:debuggable' depending on the content of |options|. 421 422 Args: 423 options: The command-line arguments tuple. 424 temp_dir: A temporary directory where the fixed manifest will be written to. 425 Returns: 426 Tuple of: 427 * Manifest path within |temp_dir|. 428 * Original package_name. 429 """ 430 def maybe_extract_version(j): 431 try: 432 return resource_utils.ExtractBinaryManifestValues(options.aapt2_path, j) 433 except build_utils.CalledProcessError: 434 return None 435 436 android_sdk_jars = [j for j in options.include_resources 437 if os.path.basename(j) in ('android.jar', 438 'android_system.jar')] 439 extract_all = [maybe_extract_version(j) for j in android_sdk_jars] 440 successful_extractions = [x for x in extract_all if x] 441 if len(successful_extractions) == 0: 442 raise Exception( 443 'Unable to find android SDK jar among candidates: %s' 444 % ', '.join(android_sdk_jars)) 445 elif len(successful_extractions) > 1: 446 raise Exception( 447 'Found multiple android SDK jars among candidates: %s' 448 % ', '.join(android_sdk_jars)) 449 version_code, version_name = successful_extractions.pop()[:2] 450 451 debug_manifest_path = os.path.join(temp_dir, 'AndroidManifest.xml') 452 doc, manifest_node, app_node = manifest_utils.ParseManifest( 453 options.android_manifest) 454 455 manifest_utils.AssertUsesSdk(manifest_node, options.min_sdk_version, 456 options.target_sdk_version) 457 # We explicitly check that maxSdkVersion is set in the manifest since we don't 458 # add it later like minSdkVersion and targetSdkVersion. 459 manifest_utils.AssertUsesSdk( 460 manifest_node, 461 max_sdk_version=options.max_sdk_version, 462 fail_if_not_exist=True) 463 manifest_utils.AssertPackage(manifest_node, options.manifest_package) 464 465 manifest_node.set('platformBuildVersionCode', version_code) 466 manifest_node.set('platformBuildVersionName', version_name) 467 468 orig_package = manifest_node.get('package') 469 if options.arsc_package_name: 470 manifest_node.set('package', options.arsc_package_name) 471 472 if options.debuggable: 473 app_node.set('{%s}%s' % (manifest_utils.ANDROID_NAMESPACE, 'debuggable'), 474 'true') 475 476 if options.uses_split: 477 uses_split = ElementTree.SubElement(manifest_node, 'uses-split') 478 uses_split.set('{%s}name' % manifest_utils.ANDROID_NAMESPACE, 479 options.uses_split) 480 481 manifest_utils.SaveManifest(doc, debug_manifest_path) 482 return debug_manifest_path, orig_package 483 484 485def _CreateKeepPredicate(resource_exclusion_regex, 486 resource_exclusion_exceptions): 487 """Return a predicate lambda to determine which resource files to keep. 488 489 Args: 490 resource_exclusion_regex: A regular expression describing all resources 491 to exclude, except if they are mip-maps, or if they are listed 492 in |resource_exclusion_exceptions|. 493 resource_exclusion_exceptions: A list of glob patterns corresponding 494 to exceptions to the |resource_exclusion_regex|. 495 Returns: 496 A lambda that takes a path, and returns true if the corresponding file 497 must be kept. 498 """ 499 predicate = lambda path: os.path.basename(path)[0] != '.' 500 if resource_exclusion_regex == '': 501 # Do not extract dotfiles (e.g. ".gitkeep"). aapt ignores them anyways. 502 return predicate 503 504 # A simple predicate that only removes (returns False for) paths covered by 505 # the exclusion regex or listed as exceptions. 506 return lambda path: ( 507 not re.search(resource_exclusion_regex, path) or 508 build_utils.MatchesGlob(path, resource_exclusion_exceptions)) 509 510 511def _ComputeSha1(path): 512 with open(path, 'rb') as f: 513 data = f.read() 514 return hashlib.sha1(data).hexdigest() 515 516 517def _ConvertToWebPSingle(png_path, cwebp_binary, cwebp_version, webp_cache_dir): 518 sha1_hash = _ComputeSha1(png_path) 519 520 # The set of arguments that will appear in the cache key. 521 quality_args = ['-m', '6', '-q', '100', '-lossless'] 522 523 webp_cache_path = os.path.join( 524 webp_cache_dir, '{}-{}-{}'.format(sha1_hash, cwebp_version, 525 ''.join(quality_args))) 526 # No need to add .webp. Android can load images fine without them. 527 webp_path = os.path.splitext(png_path)[0] 528 529 cache_hit = os.path.exists(webp_cache_path) 530 if cache_hit: 531 os.link(webp_cache_path, webp_path) 532 else: 533 # We place the generated webp image to webp_path, instead of in the 534 # webp_cache_dir to avoid concurrency issues. 535 args = [cwebp_binary, png_path, '-o', webp_path, '-quiet'] + quality_args 536 subprocess.check_call(args) 537 538 try: 539 os.link(webp_path, webp_cache_path) 540 except OSError: 541 # Because of concurrent run, a webp image may already exists in 542 # webp_cache_path. 543 pass 544 545 os.remove(png_path) 546 original_dir = os.path.dirname(os.path.dirname(png_path)) 547 rename_tuple = (os.path.relpath(png_path, original_dir), 548 os.path.relpath(webp_path, original_dir)) 549 return rename_tuple, cache_hit 550 551 552def _ConvertToWebP(cwebp_binary, png_paths, path_info, webp_cache_dir): 553 cwebp_version = subprocess.check_output([cwebp_binary, '-version']).rstrip() 554 shard_args = [(f, ) for f in png_paths 555 if not _PNG_WEBP_EXCLUSION_PATTERN.match(f)] 556 557 build_utils.MakeDirectory(webp_cache_dir) 558 results = parallel.BulkForkAndCall(_ConvertToWebPSingle, 559 shard_args, 560 cwebp_binary=cwebp_binary, 561 cwebp_version=cwebp_version, 562 webp_cache_dir=webp_cache_dir) 563 total_cache_hits = 0 564 for rename_tuple, cache_hit in results: 565 path_info.RegisterRename(*rename_tuple) 566 total_cache_hits += int(cache_hit) 567 568 logging.debug('png->webp cache: %d/%d', total_cache_hits, len(shard_args)) 569 570 571def _RemoveImageExtensions(directory, path_info): 572 """Remove extensions from image files in the passed directory. 573 574 This reduces binary size but does not affect android's ability to load the 575 images. 576 """ 577 for f in _IterFiles(directory): 578 if (f.endswith('.png') or f.endswith('.webp')) and not f.endswith('.9.png'): 579 path_with_extension = f 580 path_no_extension = os.path.splitext(path_with_extension)[0] 581 if path_no_extension != path_with_extension: 582 shutil.move(path_with_extension, path_no_extension) 583 path_info.RegisterRename( 584 os.path.relpath(path_with_extension, directory), 585 os.path.relpath(path_no_extension, directory)) 586 587 588def _CompileSingleDep(index, dep_subdir, keep_predicate, aapt2_path, 589 partials_dir): 590 unique_name = '{}_{}'.format(index, os.path.basename(dep_subdir)) 591 partial_path = os.path.join(partials_dir, '{}.zip'.format(unique_name)) 592 593 compile_command = [ 594 aapt2_path, 595 'compile', 596 # TODO(wnwen): Turn this on once aapt2 forces 9-patch to be crunched. 597 # '--no-crunch', 598 '--dir', 599 dep_subdir, 600 '-o', 601 partial_path 602 ] 603 604 # There are resources targeting API-versions lower than our minapi. For 605 # various reasons it's easier to let aapt2 ignore these than for us to 606 # remove them from our build (e.g. it's from a 3rd party library). 607 build_utils.CheckOutput( 608 compile_command, 609 stderr_filter=lambda output: build_utils.FilterLines( 610 output, r'ignoring configuration .* for (styleable|attribute)')) 611 612 # Filtering these files is expensive, so only apply filters to the partials 613 # that have been explicitly targeted. 614 if keep_predicate: 615 logging.debug('Applying .arsc filtering to %s', dep_subdir) 616 protoresources.StripUnwantedResources(partial_path, keep_predicate) 617 return partial_path 618 619 620def _CreateValuesKeepPredicate(exclusion_rules, dep_subdir): 621 patterns = [ 622 x[1] for x in exclusion_rules 623 if build_utils.MatchesGlob(dep_subdir, [x[0]]) 624 ] 625 if not patterns: 626 return None 627 628 regexes = [re.compile(p) for p in patterns] 629 return lambda x: not any(r.search(x) for r in regexes) 630 631 632def _CompileDeps(aapt2_path, dep_subdirs, dep_subdir_overlay_set, temp_dir, 633 exclusion_rules): 634 partials_dir = os.path.join(temp_dir, 'partials') 635 build_utils.MakeDirectory(partials_dir) 636 637 job_params = [(i, dep_subdir, 638 _CreateValuesKeepPredicate(exclusion_rules, dep_subdir)) 639 for i, dep_subdir in enumerate(dep_subdirs)] 640 641 # Filtering is slow, so ensure jobs with keep_predicate are started first. 642 job_params.sort(key=lambda x: not x[2]) 643 partials = list( 644 parallel.BulkForkAndCall(_CompileSingleDep, 645 job_params, 646 aapt2_path=aapt2_path, 647 partials_dir=partials_dir)) 648 649 partials_cmd = list() 650 for i, partial in enumerate(partials): 651 dep_subdir = job_params[i][1] 652 if dep_subdir in dep_subdir_overlay_set: 653 partials_cmd += ['-R'] 654 partials_cmd += [partial] 655 return partials_cmd 656 657 658def _CreateResourceInfoFile(path_info, info_path, dependencies_res_zips): 659 for zip_file in dependencies_res_zips: 660 zip_info_file_path = zip_file + '.info' 661 if os.path.exists(zip_info_file_path): 662 path_info.MergeInfoFile(zip_info_file_path) 663 path_info.Write(info_path) 664 665 666def _RemoveUnwantedLocalizedStrings(dep_subdirs, options): 667 """Remove localized strings that should not go into the final output. 668 669 Args: 670 dep_subdirs: List of resource dependency directories. 671 options: Command-line options namespace. 672 """ 673 # Collect locale and file paths from the existing subdirs. 674 # The following variable maps Android locale names to 675 # sets of corresponding xml file paths. 676 locale_to_files_map = collections.defaultdict(set) 677 for directory in dep_subdirs: 678 for f in _IterFiles(directory): 679 locale = resource_utils.FindLocaleInStringResourceFilePath(f) 680 if locale: 681 locale_to_files_map[locale].add(f) 682 683 all_locales = set(locale_to_files_map) 684 685 # Set A: wanted locales, either all of them or the 686 # list provided by --locale-allowlist. 687 wanted_locales = all_locales 688 if options.locale_allowlist: 689 wanted_locales = _ToAndroidLocales(options.locale_allowlist, 690 options.support_zh_hk) 691 692 # Set B: shared resources locales, which is either set A 693 # or the list provided by --shared-resources-allowlist-locales 694 shared_resources_locales = wanted_locales 695 shared_names_allowlist = set() 696 if options.shared_resources_allowlist_locales: 697 shared_names_allowlist = set( 698 resource_utils.GetRTxtStringResourceNames( 699 options.shared_resources_allowlist)) 700 701 shared_resources_locales = _ToAndroidLocales( 702 options.shared_resources_allowlist_locales, options.support_zh_hk) 703 704 # Remove any file that belongs to a locale not covered by 705 # either A or B. 706 removable_locales = (all_locales - wanted_locales - shared_resources_locales) 707 for locale in removable_locales: 708 for path in locale_to_files_map[locale]: 709 os.remove(path) 710 711 # For any locale in B but not in A, only keep the shared 712 # resource strings in each file. 713 for locale in shared_resources_locales - wanted_locales: 714 for path in locale_to_files_map[locale]: 715 resource_utils.FilterAndroidResourceStringsXml( 716 path, lambda x: x in shared_names_allowlist) 717 718 # For any locale in A but not in B, only keep the strings 719 # that are _not_ from shared resources in the file. 720 for locale in wanted_locales - shared_resources_locales: 721 for path in locale_to_files_map[locale]: 722 resource_utils.FilterAndroidResourceStringsXml( 723 path, lambda x: x not in shared_names_allowlist) 724 725 726def _FilterResourceFiles(dep_subdirs, keep_predicate): 727 # Create a function that selects which resource files should be packaged 728 # into the final output. Any file that does not pass the predicate will 729 # be removed below. 730 png_paths = [] 731 for directory in dep_subdirs: 732 for f in _IterFiles(directory): 733 if not keep_predicate(f): 734 os.remove(f) 735 elif f.endswith('.png'): 736 png_paths.append(f) 737 738 return png_paths 739 740 741def _PackageApk(options, build): 742 """Compile and link resources with aapt2. 743 744 Args: 745 options: The command-line options. 746 build: BuildContext object. 747 Returns: 748 The manifest package name for the APK. 749 """ 750 logging.debug('Extracting resource .zips') 751 dep_subdirs = [] 752 dep_subdir_overlay_set = set() 753 for dependency_res_zip in options.dependencies_res_zips: 754 extracted_dep_subdirs = resource_utils.ExtractDeps([dependency_res_zip], 755 build.deps_dir) 756 dep_subdirs += extracted_dep_subdirs 757 if dependency_res_zip in options.dependencies_res_zip_overlays: 758 dep_subdir_overlay_set.update(extracted_dep_subdirs) 759 760 logging.debug('Applying locale transformations') 761 path_info = resource_utils.ResourceInfoFile() 762 if options.support_zh_hk: 763 _DuplicateZhResources(dep_subdirs, path_info) 764 _RenameLocaleResourceDirs(dep_subdirs, path_info) 765 766 logging.debug('Applying file-based exclusions') 767 keep_predicate = _CreateKeepPredicate(options.resource_exclusion_regex, 768 options.resource_exclusion_exceptions) 769 png_paths = _FilterResourceFiles(dep_subdirs, keep_predicate) 770 771 if options.locale_allowlist or options.shared_resources_allowlist_locales: 772 logging.debug('Applying locale-based string exclusions') 773 _RemoveUnwantedLocalizedStrings(dep_subdirs, options) 774 775 if png_paths and options.png_to_webp: 776 logging.debug('Converting png->webp') 777 _ConvertToWebP(options.webp_binary, png_paths, path_info, 778 options.webp_cache_dir) 779 logging.debug('Applying drawable transformations') 780 for directory in dep_subdirs: 781 _MoveImagesToNonMdpiFolders(directory, path_info) 782 _RemoveImageExtensions(directory, path_info) 783 784 logging.debug('Running aapt2 compile') 785 exclusion_rules = [x.split(':', 1) for x in options.values_filter_rules] 786 partials = _CompileDeps(options.aapt2_path, dep_subdirs, 787 dep_subdir_overlay_set, build.temp_dir, 788 exclusion_rules) 789 790 link_command = [ 791 options.aapt2_path, 792 'link', 793 '--auto-add-overlay', 794 '--no-version-vectors', 795 # Set SDK versions in case they are not set in the Android manifest. 796 '--min-sdk-version', 797 options.min_sdk_version, 798 '--target-sdk-version', 799 options.target_sdk_version, 800 ] 801 802 for j in options.include_resources: 803 link_command += ['-I', j] 804 if options.version_code: 805 link_command += ['--version-code', options.version_code] 806 if options.version_name: 807 link_command += ['--version-name', options.version_name] 808 if options.proguard_file: 809 link_command += ['--proguard', build.proguard_path] 810 link_command += ['--proguard-minimal-keep-rules'] 811 if options.proguard_file_main_dex: 812 link_command += ['--proguard-main-dex', build.proguard_main_dex_path] 813 if options.emit_ids_out: 814 link_command += ['--emit-ids', build.emit_ids_path] 815 if options.r_text_in: 816 shutil.copyfile(options.r_text_in, build.r_txt_path) 817 else: 818 link_command += ['--output-text-symbols', build.r_txt_path] 819 820 # Note: only one of --proto-format, --shared-lib or --app-as-shared-lib 821 # can be used with recent versions of aapt2. 822 if options.shared_resources: 823 link_command.append('--shared-lib') 824 825 if options.no_xml_namespaces: 826 link_command.append('--no-xml-namespaces') 827 828 if options.package_id: 829 link_command += [ 830 '--package-id', 831 hex(options.package_id), 832 '--allow-reserved-package-id', 833 ] 834 835 fixed_manifest, desired_manifest_package_name = _FixManifest( 836 options, build.temp_dir) 837 if options.rename_manifest_package: 838 desired_manifest_package_name = options.rename_manifest_package 839 840 link_command += [ 841 '--manifest', fixed_manifest, '--rename-manifest-package', 842 desired_manifest_package_name 843 ] 844 845 # Creates a .zip with AndroidManifest.xml, resources.arsc, res/* 846 # Also creates R.txt 847 if options.use_resource_ids_path: 848 _CreateStableIdsFile(options.use_resource_ids_path, build.stable_ids_path, 849 desired_manifest_package_name) 850 link_command += ['--stable-ids', build.stable_ids_path] 851 852 link_command += partials 853 854 # We always create a binary arsc file first, then convert to proto, so flags 855 # such as --shared-lib can be supported. 856 arsc_path = build.arsc_path 857 if arsc_path is None: 858 _, arsc_path = tempfile.mkstmp() 859 link_command += ['-o', build.arsc_path] 860 861 logging.debug('Starting: aapt2 link') 862 link_proc = subprocess.Popen(link_command) 863 864 # Create .res.info file in parallel. 865 _CreateResourceInfoFile(path_info, build.info_path, 866 options.dependencies_res_zips) 867 logging.debug('Created .res.info file') 868 869 exit_code = link_proc.wait() 870 logging.debug('Finished: aapt2 link') 871 if exit_code: 872 raise subprocess.CalledProcessError(exit_code, link_command) 873 874 if options.proguard_file and (options.shared_resources 875 or options.app_as_shared_lib): 876 # Make sure the R class associated with the manifest package does not have 877 # its onResourcesLoaded method obfuscated or removed, so that the framework 878 # can call it in the case where the APK is being loaded as a library. 879 with open(build.proguard_path, 'a') as proguard_file: 880 keep_rule = ''' 881 -keep class {package}.R {{ 882 public static void onResourcesLoaded(int); 883 }} 884 '''.format(package=desired_manifest_package_name) 885 proguard_file.write(textwrap.dedent(keep_rule)) 886 887 logging.debug('Running aapt2 convert') 888 build_utils.CheckOutput([ 889 options.aapt2_path, 'convert', '--output-format', 'proto', '-o', 890 build.proto_path, build.arsc_path 891 ]) 892 893 # Workaround for b/147674078. This is only needed for WebLayer and does not 894 # affect WebView usage, since WebView does not used dynamic attributes. 895 if options.shared_resources: 896 logging.debug('Hardcoding dynamic attributes') 897 protoresources.HardcodeSharedLibraryDynamicAttributes( 898 build.proto_path, options.is_bundle_module, 899 options.shared_resources_allowlist) 900 901 build_utils.CheckOutput([ 902 options.aapt2_path, 'convert', '--output-format', 'binary', '-o', 903 build.arsc_path, build.proto_path 904 ]) 905 906 if build.arsc_path is None: 907 os.remove(arsc_path) 908 909 if options.optimized_proto_path: 910 _OptimizeApk(build.optimized_proto_path, options, build.temp_dir, 911 build.proto_path, build.r_txt_path) 912 elif options.optimized_arsc_path: 913 _OptimizeApk(build.optimized_arsc_path, options, build.temp_dir, 914 build.arsc_path, build.r_txt_path) 915 916 return desired_manifest_package_name 917 918 919def _CombineResourceConfigs(resources_config_paths, out_config_path): 920 with open(out_config_path, 'w') as out_config: 921 for config_path in resources_config_paths: 922 with open(config_path) as config: 923 out_config.write(config.read()) 924 out_config.write('\n') 925 926 927def _OptimizeApk(output, options, temp_dir, unoptimized_path, r_txt_path): 928 """Optimize intermediate .ap_ file with aapt2. 929 930 Args: 931 output: Path to write to. 932 options: The command-line options. 933 temp_dir: A temporary directory. 934 unoptimized_path: path of the apk to optimize. 935 r_txt_path: path to the R.txt file of the unoptimized apk. 936 """ 937 optimize_command = [ 938 options.aapt2_path, 939 'optimize', 940 unoptimized_path, 941 '-o', 942 output, 943 ] 944 945 # Optimize the resources.arsc file by obfuscating resource names and only 946 # allow usage via R.java constant. 947 if options.strip_resource_names: 948 no_collapse_resources = _ExtractNonCollapsableResources(r_txt_path) 949 gen_config_path = os.path.join(temp_dir, 'aapt2.config') 950 if options.resources_config_paths: 951 _CombineResourceConfigs(options.resources_config_paths, gen_config_path) 952 with open(gen_config_path, 'a') as config: 953 for resource in no_collapse_resources: 954 config.write('{}#no_collapse\n'.format(resource)) 955 956 optimize_command += [ 957 '--collapse-resource-names', 958 '--resources-config-path', 959 gen_config_path, 960 ] 961 962 if options.short_resource_paths: 963 optimize_command += ['--shorten-resource-paths'] 964 if options.resources_path_map_out_path: 965 optimize_command += [ 966 '--resource-path-shortening-map', options.resources_path_map_out_path 967 ] 968 969 logging.debug('Running aapt2 optimize') 970 build_utils.CheckOutput( 971 optimize_command, print_stdout=False, print_stderr=False) 972 973 974def _ExtractNonCollapsableResources(rtxt_path): 975 """Extract resources that should not be collapsed from the R.txt file 976 977 Resources of type ID are references to UI elements/views. They are used by 978 UI automation testing frameworks. They are kept in so that they don't break 979 tests, even though they may not actually be used during runtime. See 980 https://crbug.com/900993 981 App icons (aka mipmaps) are sometimes referenced by other apps by name so must 982 be keps as well. See https://b/161564466 983 984 Args: 985 rtxt_path: Path to R.txt file with all the resources 986 Returns: 987 List of resources in the form of <resource_type>/<resource_name> 988 """ 989 resources = [] 990 _NO_COLLAPSE_TYPES = ['id', 'mipmap'] 991 with open(rtxt_path) as rtxt: 992 for line in rtxt: 993 for resource_type in _NO_COLLAPSE_TYPES: 994 if ' {} '.format(resource_type) in line: 995 resource_name = line.split()[2] 996 resources.append('{}/{}'.format(resource_type, resource_name)) 997 return resources 998 999 1000@contextlib.contextmanager 1001def _CreateStableIdsFile(in_path, out_path, package_name): 1002 """Transforms a file generated by --emit-ids from another package. 1003 1004 --stable-ids is generally meant to be used by different versions of the same 1005 package. To make it work for other packages, we need to transform the package 1006 name references to match the package that resources are being generated for. 1007 1008 Note: This will fail if the package ID of the resources in 1009 |options.use_resource_ids_path| does not match the package ID of the 1010 resources being linked. 1011 """ 1012 with open(in_path) as stable_ids_file: 1013 with open(out_path, 'w') as output_ids_file: 1014 output_stable_ids = re.sub( 1015 r'^.*?:', 1016 package_name + ':', 1017 stable_ids_file.read(), 1018 flags=re.MULTILINE) 1019 output_ids_file.write(output_stable_ids) 1020 1021 1022def _WriteOutputs(options, build): 1023 possible_outputs = [ 1024 (options.srcjar_out, build.srcjar_path), 1025 (options.r_text_out, build.r_txt_path), 1026 (options.arsc_path, build.arsc_path), 1027 (options.proto_path, build.proto_path), 1028 (options.optimized_arsc_path, build.optimized_arsc_path), 1029 (options.optimized_proto_path, build.optimized_proto_path), 1030 (options.proguard_file, build.proguard_path), 1031 (options.proguard_file_main_dex, build.proguard_main_dex_path), 1032 (options.emit_ids_out, build.emit_ids_path), 1033 (options.info_path, build.info_path), 1034 ] 1035 1036 for final, temp in possible_outputs: 1037 # Write file only if it's changed. 1038 if final and not (os.path.exists(final) and filecmp.cmp(final, temp)): 1039 shutil.move(temp, final) 1040 1041 1042def _CreateNormalizedManifest(options): 1043 with build_utils.TempDir() as tempdir: 1044 fixed_manifest, _ = _FixManifest(options, tempdir) 1045 with open(fixed_manifest) as f: 1046 return manifest_utils.NormalizeManifest(f.read()) 1047 1048 1049def _OnStaleMd5(options): 1050 path = options.arsc_path or options.proto_path 1051 debug_temp_resources_dir = os.environ.get('TEMP_RESOURCES_DIR') 1052 if debug_temp_resources_dir: 1053 path = os.path.join(debug_temp_resources_dir, os.path.basename(path)) 1054 else: 1055 # Use a deterministic temp directory since .pb files embed the absolute 1056 # path of resources: crbug.com/939984 1057 path = path + '.tmpdir' 1058 build_utils.DeleteDirectory(path) 1059 1060 with resource_utils.BuildContext( 1061 temp_dir=path, keep_files=bool(debug_temp_resources_dir)) as build: 1062 1063 manifest_package_name = _PackageApk(options, build) 1064 1065 # If --shared-resources-allowlist is used, all the resources listed in the 1066 # corresponding R.txt file will be non-final, and an onResourcesLoaded() 1067 # will be generated to adjust them at runtime. 1068 # 1069 # Otherwise, if --shared-resources is used, the all resources will be 1070 # non-final, and an onResourcesLoaded() method will be generated too. 1071 # 1072 # Otherwise, all resources will be final, and no method will be generated. 1073 # 1074 rjava_build_options = resource_utils.RJavaBuildOptions() 1075 if options.shared_resources_allowlist: 1076 rjava_build_options.ExportSomeResources( 1077 options.shared_resources_allowlist) 1078 rjava_build_options.GenerateOnResourcesLoaded() 1079 if options.shared_resources: 1080 # The final resources will only be used in WebLayer, so hardcode the 1081 # package ID to be what WebLayer expects. 1082 rjava_build_options.SetFinalPackageId( 1083 protoresources.SHARED_LIBRARY_HARDCODED_ID) 1084 elif options.shared_resources or options.app_as_shared_lib: 1085 rjava_build_options.ExportAllResources() 1086 rjava_build_options.GenerateOnResourcesLoaded() 1087 1088 custom_root_package_name = options.r_java_root_package_name 1089 grandparent_custom_package_name = None 1090 1091 # Always generate an R.java file for the package listed in 1092 # AndroidManifest.xml because this is where Android framework looks to find 1093 # onResourcesLoaded() for shared library apks. While not actually necessary 1094 # for application apks, it also doesn't hurt. 1095 apk_package_name = manifest_package_name 1096 1097 if options.package_name and not options.arsc_package_name: 1098 # Feature modules have their own custom root package name and should 1099 # inherit from the appropriate base module package. This behaviour should 1100 # not be present for test apks with an apk under test. Thus, 1101 # arsc_package_name is used as it is only defined for test apks with an 1102 # apk under test. 1103 custom_root_package_name = options.package_name 1104 grandparent_custom_package_name = options.r_java_root_package_name 1105 # Feature modules have the same manifest package as the base module but 1106 # they should not create an R.java for said manifest package because it 1107 # will be created in the base module. 1108 apk_package_name = None 1109 1110 logging.debug('Creating R.srcjar') 1111 resource_utils.CreateRJavaFiles( 1112 build.srcjar_dir, apk_package_name, build.r_txt_path, 1113 options.extra_res_packages, rjava_build_options, options.srcjar_out, 1114 custom_root_package_name, grandparent_custom_package_name, 1115 options.extra_main_r_text_files) 1116 build_utils.ZipDir(build.srcjar_path, build.srcjar_dir) 1117 1118 # Sanity check that the created resources have the expected package ID. 1119 logging.debug('Performing sanity check') 1120 if options.package_id: 1121 expected_id = options.package_id 1122 elif options.shared_resources: 1123 expected_id = 0 1124 else: 1125 expected_id = 127 # == '0x7f'. 1126 _, package_id = resource_utils.ExtractArscPackage( 1127 options.aapt2_path, 1128 build.arsc_path if options.arsc_path else build.proto_path) 1129 if package_id != expected_id: 1130 raise Exception( 1131 'Invalid package ID 0x%x (expected 0x%x)' % (package_id, expected_id)) 1132 1133 logging.debug('Copying outputs') 1134 _WriteOutputs(options, build) 1135 1136 1137def main(args): 1138 build_utils.InitLogging('RESOURCE_DEBUG') 1139 args = build_utils.ExpandFileArgs(args) 1140 options = _ParseArgs(args) 1141 1142 if options.expected_file: 1143 actual_data = _CreateNormalizedManifest(options) 1144 diff_utils.CheckExpectations(actual_data, options) 1145 if options.only_verify_expectations: 1146 return 1147 1148 depfile_deps = (options.dependencies_res_zips + 1149 options.dependencies_res_zip_overlays + 1150 options.extra_main_r_text_files + options.include_resources) 1151 1152 possible_input_paths = depfile_deps + options.resources_config_paths + [ 1153 options.aapt2_path, 1154 options.android_manifest, 1155 options.expected_file, 1156 options.expected_file_base, 1157 options.shared_resources_allowlist, 1158 options.use_resource_ids_path, 1159 options.webp_binary, 1160 ] 1161 input_paths = [p for p in possible_input_paths if p] 1162 input_strings = [ 1163 options.app_as_shared_lib, 1164 options.arsc_package_name, 1165 options.debuggable, 1166 options.extra_res_packages, 1167 options.failure_file, 1168 options.include_resources, 1169 options.locale_allowlist, 1170 options.manifest_package, 1171 options.max_sdk_version, 1172 options.min_sdk_version, 1173 options.no_xml_namespaces, 1174 options.package_id, 1175 options.package_name, 1176 options.png_to_webp, 1177 options.rename_manifest_package, 1178 options.resource_exclusion_exceptions, 1179 options.resource_exclusion_regex, 1180 options.r_java_root_package_name, 1181 options.shared_resources, 1182 options.shared_resources_allowlist_locales, 1183 options.short_resource_paths, 1184 options.strip_resource_names, 1185 options.support_zh_hk, 1186 options.target_sdk_version, 1187 options.values_filter_rules, 1188 options.version_code, 1189 options.version_name, 1190 options.webp_cache_dir, 1191 ] 1192 output_paths = [options.srcjar_out] 1193 possible_output_paths = [ 1194 options.actual_file, 1195 options.arsc_path, 1196 options.emit_ids_out, 1197 options.info_path, 1198 options.optimized_arsc_path, 1199 options.optimized_proto_path, 1200 options.proguard_file, 1201 options.proguard_file_main_dex, 1202 options.proto_path, 1203 options.resources_path_map_out_path, 1204 options.r_text_out, 1205 ] 1206 output_paths += [p for p in possible_output_paths if p] 1207 1208 # Since we overspecify deps, this target depends on java deps that are not 1209 # going to change its output. This target is also slow (6-12 seconds) and 1210 # blocking the critical path. We want changes to java_library targets to not 1211 # trigger re-compilation of resources, thus we need to use md5_check. 1212 md5_check.CallAndWriteDepfileIfStale( 1213 lambda: _OnStaleMd5(options), 1214 options, 1215 input_paths=input_paths, 1216 input_strings=input_strings, 1217 output_paths=output_paths, 1218 depfile_deps=depfile_deps) 1219 1220 1221if __name__ == '__main__': 1222 main(sys.argv[1:]) 1223