1#!/usr/bin/env python 2# 3# Copyright (c) 2013 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6"""Runs Android's lint tool.""" 7 8from __future__ import print_function 9 10import argparse 11import functools 12import logging 13import os 14import re 15import shutil 16import sys 17import time 18import traceback 19from xml.dom import minidom 20from xml.etree import ElementTree 21 22from util import build_utils 23from util import manifest_utils 24 25_LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/master/build/android/docs/lint.md' # pylint: disable=line-too-long 26 27# These checks are not useful for chromium. 28_DISABLED_ALWAYS = [ 29 "AppCompatResource", # Lint does not correctly detect our appcompat lib. 30 "Assert", # R8 --force-enable-assertions is used to enable java asserts. 31 "InflateParams", # Null is ok when inflating views for dialogs. 32 "InlinedApi", # Constants are copied so they are always available. 33 "LintBaseline", # Don't warn about using baseline.xml files. 34 "MissingApplicationIcon", # False positive for non-production targets. 35 "SwitchIntDef", # Many C++ enums are not used at all in java. 36 "UniqueConstants", # Chromium enums allow aliases. 37 "UnusedAttribute", # Chromium apks have various minSdkVersion values. 38 "ObsoleteLintCustomCheck", # We have no control over custom lint checks. 39] 40 41# These checks are not useful for test targets and adds an unnecessary burden 42# to suppress them. 43_DISABLED_FOR_TESTS = [ 44 # We should not require test strings.xml files to explicitly add 45 # translatable=false since they are not translated and not used in 46 # production. 47 "MissingTranslation", 48 # Test strings.xml files often have simple names and are not translatable, 49 # so it may conflict with a production string and cause this error. 50 "Untranslatable", 51 # Test targets often use the same strings target and resources target as the 52 # production targets but may not use all of them. 53 "UnusedResources", 54 # TODO(wnwen): Turn this back on since to crash it would require running on 55 # a device with all the various minSdkVersions. 56 # Real NewApi violations crash the app, so the only ones that lint catches 57 # but tests still succeed are false positives. 58 "NewApi", 59 # Tests should be allowed to access these methods/classes. 60 "VisibleForTests", 61] 62 63_RES_ZIP_DIR = 'RESZIPS' 64_SRCJAR_DIR = 'SRCJARS' 65_AAR_DIR = 'AARS' 66 67 68def _SrcRelative(path): 69 """Returns relative path to top-level src dir.""" 70 return os.path.relpath(path, build_utils.DIR_SOURCE_ROOT) 71 72 73def _GenerateProjectFile(android_manifest, 74 android_sdk_root, 75 cache_dir, 76 sources=None, 77 classpath=None, 78 srcjar_sources=None, 79 resource_sources=None, 80 custom_lint_jars=None, 81 custom_annotation_zips=None, 82 android_sdk_version=None): 83 project = ElementTree.Element('project') 84 root = ElementTree.SubElement(project, 'root') 85 # Run lint from output directory: crbug.com/1115594 86 root.set('dir', os.getcwd()) 87 sdk = ElementTree.SubElement(project, 'sdk') 88 # Lint requires that the sdk path be an absolute path. 89 sdk.set('dir', os.path.abspath(android_sdk_root)) 90 cache = ElementTree.SubElement(project, 'cache') 91 cache.set('dir', cache_dir) 92 main_module = ElementTree.SubElement(project, 'module') 93 main_module.set('name', 'main') 94 main_module.set('android', 'true') 95 main_module.set('library', 'false') 96 if android_sdk_version: 97 main_module.set('compile_sdk_version', android_sdk_version) 98 manifest = ElementTree.SubElement(main_module, 'manifest') 99 manifest.set('file', android_manifest) 100 if srcjar_sources: 101 for srcjar_file in srcjar_sources: 102 src = ElementTree.SubElement(main_module, 'src') 103 src.set('file', srcjar_file) 104 if sources: 105 for source in sources: 106 src = ElementTree.SubElement(main_module, 'src') 107 src.set('file', source) 108 if classpath: 109 for file_path in classpath: 110 classpath_element = ElementTree.SubElement(main_module, 'classpath') 111 classpath_element.set('file', file_path) 112 if resource_sources: 113 for resource_file in resource_sources: 114 resource = ElementTree.SubElement(main_module, 'resource') 115 resource.set('file', resource_file) 116 if custom_lint_jars: 117 for lint_jar in custom_lint_jars: 118 lint = ElementTree.SubElement(main_module, 'lint-checks') 119 lint.set('file', lint_jar) 120 if custom_annotation_zips: 121 for annotation_zip in custom_annotation_zips: 122 annotation = ElementTree.SubElement(main_module, 'annotations') 123 annotation.set('file', annotation_zip) 124 return project 125 126 127def _RetrieveBackportedMethods(backported_methods_path): 128 with open(backported_methods_path) as f: 129 methods = f.read().splitlines() 130 # Methods look like: 131 # java/util/Set#of(Ljava/lang/Object;)Ljava/util/Set; 132 # But error message looks like: 133 # Call requires API level R (current min is 21): java.util.Set#of [NewApi] 134 methods = (m.replace('/', '\\.') for m in methods) 135 methods = (m[:m.index('(')] for m in methods) 136 return sorted(set(methods)) 137 138 139def _GenerateConfigXmlTree(orig_config_path, backported_methods): 140 if orig_config_path: 141 root_node = ElementTree.parse(orig_config_path).getroot() 142 else: 143 root_node = ElementTree.fromstring('<lint/>') 144 145 issue_node = ElementTree.SubElement(root_node, 'issue') 146 issue_node.attrib['id'] = 'NewApi' 147 ignore_node = ElementTree.SubElement(issue_node, 'ignore') 148 ignore_node.attrib['regexp'] = '|'.join(backported_methods) 149 return root_node 150 151 152def _GenerateAndroidManifest(original_manifest_path, extra_manifest_paths, 153 min_sdk_version, android_sdk_version): 154 # Set minSdkVersion in the manifest to the correct value. 155 doc, manifest, app_node = manifest_utils.ParseManifest(original_manifest_path) 156 157 # TODO(crbug.com/1126301): Should this be done using manifest merging? 158 # Add anything in the application node of the extra manifests to the main 159 # manifest to prevent unused resource errors. 160 for path in extra_manifest_paths: 161 _, _, extra_app_node = manifest_utils.ParseManifest(path) 162 for node in extra_app_node: 163 app_node.append(node) 164 165 if app_node.find( 166 '{%s}allowBackup' % manifest_utils.ANDROID_NAMESPACE) is None: 167 # Assume no backup is intended, appeases AllowBackup lint check and keeping 168 # it working for manifests that do define android:allowBackup. 169 app_node.set('{%s}allowBackup' % manifest_utils.ANDROID_NAMESPACE, 'false') 170 171 uses_sdk = manifest.find('./uses-sdk') 172 if uses_sdk is None: 173 uses_sdk = ElementTree.Element('uses-sdk') 174 manifest.insert(0, uses_sdk) 175 uses_sdk.set('{%s}minSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 176 min_sdk_version) 177 uses_sdk.set('{%s}targetSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 178 android_sdk_version) 179 return doc 180 181 182def _WriteXmlFile(root, path): 183 logging.info('Writing xml file %s', path) 184 build_utils.MakeDirectory(os.path.dirname(path)) 185 with build_utils.AtomicOutput(path) as f: 186 # Although we can write it just with ElementTree.tostring, using minidom 187 # makes it a lot easier to read as a human (also on code search). 188 f.write( 189 minidom.parseString(ElementTree.tostring( 190 root, encoding='utf-8')).toprettyxml(indent=' ')) 191 192 193def _CheckLintWarning(expected_warnings, lint_output): 194 for expected_warning in expected_warnings.split(','): 195 expected_str = '[{}]'.format(expected_warning) 196 if expected_str not in lint_output: 197 raise Exception('Expected {!r} warning in lint output:\n{}.'.format( 198 expected_str, lint_output)) 199 200 # Do not print warning 201 return '' 202 203 204def _RunLint(lint_binary_path, 205 backported_methods_path, 206 config_path, 207 manifest_path, 208 extra_manifest_paths, 209 sources, 210 classpath, 211 cache_dir, 212 android_sdk_version, 213 aars, 214 srcjars, 215 min_sdk_version, 216 resource_sources, 217 resource_zips, 218 android_sdk_root, 219 lint_gen_dir, 220 baseline, 221 expected_warnings, 222 testonly_target=False, 223 warnings_as_errors=False): 224 logging.info('Lint starting') 225 226 cmd = [ 227 lint_binary_path, 228 '--quiet', # Silences lint's "." progress updates. 229 '--disable', 230 ','.join(_DISABLED_ALWAYS), 231 ] 232 if baseline: 233 cmd.extend(['--baseline', baseline]) 234 if testonly_target: 235 cmd.extend(['--disable', ','.join(_DISABLED_FOR_TESTS)]) 236 237 if not manifest_path: 238 manifest_path = os.path.join(build_utils.DIR_SOURCE_ROOT, 'build', 239 'android', 'AndroidManifest.xml') 240 241 logging.info('Generating config.xml') 242 backported_methods = _RetrieveBackportedMethods(backported_methods_path) 243 config_xml_node = _GenerateConfigXmlTree(config_path, backported_methods) 244 generated_config_path = os.path.join(lint_gen_dir, 'config.xml') 245 _WriteXmlFile(config_xml_node, generated_config_path) 246 cmd.extend(['--config', generated_config_path]) 247 248 logging.info('Generating Android manifest file') 249 android_manifest_tree = _GenerateAndroidManifest(manifest_path, 250 extra_manifest_paths, 251 min_sdk_version, 252 android_sdk_version) 253 # Include the rebased manifest_path in the lint generated path so that it is 254 # clear in error messages where the original AndroidManifest.xml came from. 255 lint_android_manifest_path = os.path.join(lint_gen_dir, manifest_path) 256 _WriteXmlFile(android_manifest_tree.getroot(), lint_android_manifest_path) 257 258 resource_root_dir = os.path.join(lint_gen_dir, _RES_ZIP_DIR) 259 # These are zip files with generated resources (e. g. strings from GRD). 260 logging.info('Extracting resource zips') 261 for resource_zip in resource_zips: 262 # Use a consistent root and name rather than a temporary file so that 263 # suppressions can be local to the lint target and the resource target. 264 resource_dir = os.path.join(resource_root_dir, resource_zip) 265 shutil.rmtree(resource_dir, True) 266 os.makedirs(resource_dir) 267 resource_sources.extend( 268 build_utils.ExtractAll(resource_zip, path=resource_dir)) 269 270 logging.info('Extracting aars') 271 aar_root_dir = os.path.join(lint_gen_dir, _AAR_DIR) 272 custom_lint_jars = [] 273 custom_annotation_zips = [] 274 if aars: 275 for aar in aars: 276 # Use relative source for aar files since they are not generated. 277 aar_dir = os.path.join(aar_root_dir, 278 os.path.splitext(_SrcRelative(aar))[0]) 279 shutil.rmtree(aar_dir, True) 280 os.makedirs(aar_dir) 281 aar_files = build_utils.ExtractAll(aar, path=aar_dir) 282 for f in aar_files: 283 if f.endswith('lint.jar'): 284 custom_lint_jars.append(f) 285 elif f.endswith('annotations.zip'): 286 custom_annotation_zips.append(f) 287 288 logging.info('Extracting srcjars') 289 srcjar_root_dir = os.path.join(lint_gen_dir, _SRCJAR_DIR) 290 srcjar_sources = [] 291 if srcjars: 292 for srcjar in srcjars: 293 # Use path without extensions since otherwise the file name includes 294 # .srcjar and lint treats it as a srcjar. 295 srcjar_dir = os.path.join(srcjar_root_dir, os.path.splitext(srcjar)[0]) 296 shutil.rmtree(srcjar_dir, True) 297 os.makedirs(srcjar_dir) 298 # Sadly lint's srcjar support is broken since it only considers the first 299 # srcjar. Until we roll a lint version with that fixed, we need to extract 300 # it ourselves. 301 srcjar_sources.extend(build_utils.ExtractAll(srcjar, path=srcjar_dir)) 302 303 logging.info('Generating project file') 304 project_file_root = _GenerateProjectFile(lint_android_manifest_path, 305 android_sdk_root, cache_dir, sources, 306 classpath, srcjar_sources, 307 resource_sources, custom_lint_jars, 308 custom_annotation_zips, 309 android_sdk_version) 310 311 project_xml_path = os.path.join(lint_gen_dir, 'project.xml') 312 _WriteXmlFile(project_file_root, project_xml_path) 313 cmd += ['--project', project_xml_path] 314 315 logging.info('Preparing environment variables') 316 env = os.environ.copy() 317 # It is important that lint uses the checked-in JDK11 as it is almost 50% 318 # faster than JDK8. 319 env['JAVA_HOME'] = build_utils.JAVA_HOME 320 # This is necessary so that lint errors print stack traces in stdout. 321 env['LINT_PRINT_STACKTRACE'] = 'true' 322 if baseline and not os.path.exists(baseline): 323 # Generating new baselines is only done locally, and requires more memory to 324 # avoid OOMs. 325 env['LINT_OPTS'] = '-Xmx4g' 326 else: 327 # The default set in the wrapper script is 1g, but it seems not enough :( 328 env['LINT_OPTS'] = '-Xmx2g' 329 330 # This filter is necessary for JDK11. 331 stderr_filter = build_utils.FilterReflectiveAccessJavaWarnings 332 stdout_filter = lambda x: build_utils.FilterLines(x, 'No issues found') 333 if expected_warnings: 334 stdout_filter = functools.partial(_CheckLintWarning, expected_warnings) 335 336 start = time.time() 337 logging.debug('Lint command %s', ' '.join(cmd)) 338 failed = True 339 try: 340 failed = bool( 341 build_utils.CheckOutput(cmd, 342 env=env, 343 print_stdout=True, 344 stdout_filter=stdout_filter, 345 stderr_filter=stderr_filter, 346 fail_on_output=warnings_as_errors)) 347 finally: 348 # When not treating warnings as errors, display the extra footer. 349 is_debug = os.environ.get('LINT_DEBUG', '0') != '0' 350 351 if failed: 352 print('- For more help with lint in Chrome:', _LINT_MD_URL) 353 if is_debug: 354 print('- DEBUG MODE: Here is the project.xml: {}'.format( 355 _SrcRelative(project_xml_path))) 356 else: 357 print('- Run with LINT_DEBUG=1 to enable lint configuration debugging') 358 359 end = time.time() - start 360 logging.info('Lint command took %ss', end) 361 if not is_debug: 362 shutil.rmtree(aar_root_dir, ignore_errors=True) 363 shutil.rmtree(resource_root_dir, ignore_errors=True) 364 shutil.rmtree(srcjar_root_dir, ignore_errors=True) 365 os.unlink(project_xml_path) 366 367 logging.info('Lint completed') 368 369 370def _ParseArgs(argv): 371 parser = argparse.ArgumentParser() 372 build_utils.AddDepfileOption(parser) 373 parser.add_argument('--lint-binary-path', 374 required=True, 375 help='Path to lint executable.') 376 parser.add_argument('--backported-methods', 377 help='Path to backported methods file created by R8.') 378 parser.add_argument('--cache-dir', 379 required=True, 380 help='Path to the directory in which the android cache ' 381 'directory tree should be stored.') 382 parser.add_argument('--config-path', help='Path to lint suppressions file.') 383 parser.add_argument('--lint-gen-dir', 384 required=True, 385 help='Path to store generated xml files.') 386 parser.add_argument('--stamp', help='Path to stamp upon success.') 387 parser.add_argument('--android-sdk-version', 388 help='Version (API level) of the Android SDK used for ' 389 'building.') 390 parser.add_argument('--min-sdk-version', 391 required=True, 392 help='Minimal SDK version to lint against.') 393 parser.add_argument('--android-sdk-root', 394 required=True, 395 help='Lint needs an explicit path to the android sdk.') 396 parser.add_argument('--testonly', 397 action='store_true', 398 help='If set, some checks like UnusedResources will be ' 399 'disabled since they are not helpful for test ' 400 'targets.') 401 parser.add_argument('--warnings-as-errors', 402 action='store_true', 403 help='Treat all warnings as errors.') 404 parser.add_argument('--java-sources', 405 help='File containing a list of java sources files.') 406 parser.add_argument('--aars', help='GN list of included aars.') 407 parser.add_argument('--srcjars', help='GN list of included srcjars.') 408 parser.add_argument('--manifest-path', 409 help='Path to original AndroidManifest.xml') 410 parser.add_argument('--extra-manifest-paths', 411 action='append', 412 help='GYP-list of manifest paths to merge into the ' 413 'original AndroidManifest.xml') 414 parser.add_argument('--resource-sources', 415 default=[], 416 action='append', 417 help='GYP-list of resource sources files, similar to ' 418 'java sources files, but for resource files.') 419 parser.add_argument('--resource-zips', 420 default=[], 421 action='append', 422 help='GYP-list of resource zips, zip files of generated ' 423 'resource files.') 424 parser.add_argument('--classpath', 425 help='List of jars to add to the classpath.') 426 parser.add_argument('--baseline', 427 help='Baseline file to ignore existing errors and fail ' 428 'on new errors.') 429 parser.add_argument('--expected-warnings', 430 help='Comma separated list of warnings to test for in ' 431 'the output, failing if not found.') 432 433 args = parser.parse_args(build_utils.ExpandFileArgs(argv)) 434 args.java_sources = build_utils.ParseGnList(args.java_sources) 435 args.aars = build_utils.ParseGnList(args.aars) 436 args.srcjars = build_utils.ParseGnList(args.srcjars) 437 args.resource_sources = build_utils.ParseGnList(args.resource_sources) 438 args.extra_manifest_paths = build_utils.ParseGnList(args.extra_manifest_paths) 439 args.resource_zips = build_utils.ParseGnList(args.resource_zips) 440 args.classpath = build_utils.ParseGnList(args.classpath) 441 return args 442 443 444def main(): 445 build_utils.InitLogging('LINT_DEBUG') 446 args = _ParseArgs(sys.argv[1:]) 447 448 sources = [] 449 for java_sources_file in args.java_sources: 450 sources.extend(build_utils.ReadSourcesList(java_sources_file)) 451 resource_sources = [] 452 for resource_sources_file in args.resource_sources: 453 resource_sources.extend(build_utils.ReadSourcesList(resource_sources_file)) 454 455 possible_depfile_deps = (args.srcjars + args.resource_zips + sources + 456 resource_sources + [ 457 args.baseline, 458 args.manifest_path, 459 ]) 460 depfile_deps = [p for p in possible_depfile_deps if p] 461 462 _RunLint(args.lint_binary_path, 463 args.backported_methods, 464 args.config_path, 465 args.manifest_path, 466 args.extra_manifest_paths, 467 sources, 468 args.classpath, 469 args.cache_dir, 470 args.android_sdk_version, 471 args.aars, 472 args.srcjars, 473 args.min_sdk_version, 474 resource_sources, 475 args.resource_zips, 476 args.android_sdk_root, 477 args.lint_gen_dir, 478 args.baseline, 479 args.expected_warnings, 480 testonly_target=args.testonly, 481 warnings_as_errors=args.warnings_as_errors) 482 logging.info('Creating stamp file') 483 build_utils.Touch(args.stamp) 484 485 if args.depfile: 486 build_utils.WriteDepfile(args.depfile, args.stamp, depfile_deps) 487 488 489if __name__ == '__main__': 490 sys.exit(main()) 491