1#!/usr/bin/env python
2#
3# Copyright (c) 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Runs Android's lint tool."""
7
8from __future__ import print_function
9
10import argparse
11import functools
12import logging
13import os
14import re
15import shutil
16import sys
17import time
18import traceback
19from xml.dom import minidom
20from xml.etree import ElementTree
21
22from util import build_utils
23from util import manifest_utils
24
25_LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/master/build/android/docs/lint.md'  # pylint: disable=line-too-long
26
27# These checks are not useful for chromium.
28_DISABLED_ALWAYS = [
29    "AppCompatResource",  # Lint does not correctly detect our appcompat lib.
30    "Assert",  # R8 --force-enable-assertions is used to enable java asserts.
31    "InflateParams",  # Null is ok when inflating views for dialogs.
32    "InlinedApi",  # Constants are copied so they are always available.
33    "LintBaseline",  # Don't warn about using baseline.xml files.
34    "MissingApplicationIcon",  # False positive for non-production targets.
35    "SwitchIntDef",  # Many C++ enums are not used at all in java.
36    "UniqueConstants",  # Chromium enums allow aliases.
37    "UnusedAttribute",  # Chromium apks have various minSdkVersion values.
38    "ObsoleteLintCustomCheck",  # We have no control over custom lint checks.
39]
40
41# These checks are not useful for test targets and adds an unnecessary burden
42# to suppress them.
43_DISABLED_FOR_TESTS = [
44    # We should not require test strings.xml files to explicitly add
45    # translatable=false since they are not translated and not used in
46    # production.
47    "MissingTranslation",
48    # Test strings.xml files often have simple names and are not translatable,
49    # so it may conflict with a production string and cause this error.
50    "Untranslatable",
51    # Test targets often use the same strings target and resources target as the
52    # production targets but may not use all of them.
53    "UnusedResources",
54    # TODO(wnwen): Turn this back on since to crash it would require running on
55    #     a device with all the various minSdkVersions.
56    # Real NewApi violations crash the app, so the only ones that lint catches
57    # but tests still succeed are false positives.
58    "NewApi",
59    # Tests should be allowed to access these methods/classes.
60    "VisibleForTests",
61]
62
63_RES_ZIP_DIR = 'RESZIPS'
64_SRCJAR_DIR = 'SRCJARS'
65_AAR_DIR = 'AARS'
66
67
68def _SrcRelative(path):
69  """Returns relative path to top-level src dir."""
70  return os.path.relpath(path, build_utils.DIR_SOURCE_ROOT)
71
72
73def _GenerateProjectFile(android_manifest,
74                         android_sdk_root,
75                         cache_dir,
76                         sources=None,
77                         classpath=None,
78                         srcjar_sources=None,
79                         resource_sources=None,
80                         custom_lint_jars=None,
81                         custom_annotation_zips=None,
82                         android_sdk_version=None):
83  project = ElementTree.Element('project')
84  root = ElementTree.SubElement(project, 'root')
85  # Run lint from output directory: crbug.com/1115594
86  root.set('dir', os.getcwd())
87  sdk = ElementTree.SubElement(project, 'sdk')
88  # Lint requires that the sdk path be an absolute path.
89  sdk.set('dir', os.path.abspath(android_sdk_root))
90  cache = ElementTree.SubElement(project, 'cache')
91  cache.set('dir', cache_dir)
92  main_module = ElementTree.SubElement(project, 'module')
93  main_module.set('name', 'main')
94  main_module.set('android', 'true')
95  main_module.set('library', 'false')
96  if android_sdk_version:
97    main_module.set('compile_sdk_version', android_sdk_version)
98  manifest = ElementTree.SubElement(main_module, 'manifest')
99  manifest.set('file', android_manifest)
100  if srcjar_sources:
101    for srcjar_file in srcjar_sources:
102      src = ElementTree.SubElement(main_module, 'src')
103      src.set('file', srcjar_file)
104  if sources:
105    for source in sources:
106      src = ElementTree.SubElement(main_module, 'src')
107      src.set('file', source)
108  if classpath:
109    for file_path in classpath:
110      classpath_element = ElementTree.SubElement(main_module, 'classpath')
111      classpath_element.set('file', file_path)
112  if resource_sources:
113    for resource_file in resource_sources:
114      resource = ElementTree.SubElement(main_module, 'resource')
115      resource.set('file', resource_file)
116  if custom_lint_jars:
117    for lint_jar in custom_lint_jars:
118      lint = ElementTree.SubElement(main_module, 'lint-checks')
119      lint.set('file', lint_jar)
120  if custom_annotation_zips:
121    for annotation_zip in custom_annotation_zips:
122      annotation = ElementTree.SubElement(main_module, 'annotations')
123      annotation.set('file', annotation_zip)
124  return project
125
126
127def _RetrieveBackportedMethods(backported_methods_path):
128  with open(backported_methods_path) as f:
129    methods = f.read().splitlines()
130  # Methods look like:
131  #   java/util/Set#of(Ljava/lang/Object;)Ljava/util/Set;
132  # But error message looks like:
133  #   Call requires API level R (current min is 21): java.util.Set#of [NewApi]
134  methods = (m.replace('/', '\\.') for m in methods)
135  methods = (m[:m.index('(')] for m in methods)
136  return sorted(set(methods))
137
138
139def _GenerateConfigXmlTree(orig_config_path, backported_methods):
140  if orig_config_path:
141    root_node = ElementTree.parse(orig_config_path).getroot()
142  else:
143    root_node = ElementTree.fromstring('<lint/>')
144
145  issue_node = ElementTree.SubElement(root_node, 'issue')
146  issue_node.attrib['id'] = 'NewApi'
147  ignore_node = ElementTree.SubElement(issue_node, 'ignore')
148  ignore_node.attrib['regexp'] = '|'.join(backported_methods)
149  return root_node
150
151
152def _GenerateAndroidManifest(original_manifest_path, extra_manifest_paths,
153                             min_sdk_version, android_sdk_version):
154  # Set minSdkVersion in the manifest to the correct value.
155  doc, manifest, app_node = manifest_utils.ParseManifest(original_manifest_path)
156
157  # TODO(crbug.com/1126301): Should this be done using manifest merging?
158  # Add anything in the application node of the extra manifests to the main
159  # manifest to prevent unused resource errors.
160  for path in extra_manifest_paths:
161    _, _, extra_app_node = manifest_utils.ParseManifest(path)
162    for node in extra_app_node:
163      app_node.append(node)
164
165  if app_node.find(
166      '{%s}allowBackup' % manifest_utils.ANDROID_NAMESPACE) is None:
167    # Assume no backup is intended, appeases AllowBackup lint check and keeping
168    # it working for manifests that do define android:allowBackup.
169    app_node.set('{%s}allowBackup' % manifest_utils.ANDROID_NAMESPACE, 'false')
170
171  uses_sdk = manifest.find('./uses-sdk')
172  if uses_sdk is None:
173    uses_sdk = ElementTree.Element('uses-sdk')
174    manifest.insert(0, uses_sdk)
175  uses_sdk.set('{%s}minSdkVersion' % manifest_utils.ANDROID_NAMESPACE,
176               min_sdk_version)
177  uses_sdk.set('{%s}targetSdkVersion' % manifest_utils.ANDROID_NAMESPACE,
178               android_sdk_version)
179  return doc
180
181
182def _WriteXmlFile(root, path):
183  logging.info('Writing xml file %s', path)
184  build_utils.MakeDirectory(os.path.dirname(path))
185  with build_utils.AtomicOutput(path) as f:
186    # Although we can write it just with ElementTree.tostring, using minidom
187    # makes it a lot easier to read as a human (also on code search).
188    f.write(
189        minidom.parseString(ElementTree.tostring(
190            root, encoding='utf-8')).toprettyxml(indent='  '))
191
192
193def _CheckLintWarning(expected_warnings, lint_output):
194  for expected_warning in expected_warnings.split(','):
195    expected_str = '[{}]'.format(expected_warning)
196    if expected_str not in lint_output:
197      raise Exception('Expected {!r} warning in lint output:\n{}.'.format(
198          expected_str, lint_output))
199
200  # Do not print warning
201  return ''
202
203
204def _RunLint(lint_binary_path,
205             backported_methods_path,
206             config_path,
207             manifest_path,
208             extra_manifest_paths,
209             sources,
210             classpath,
211             cache_dir,
212             android_sdk_version,
213             aars,
214             srcjars,
215             min_sdk_version,
216             resource_sources,
217             resource_zips,
218             android_sdk_root,
219             lint_gen_dir,
220             baseline,
221             expected_warnings,
222             testonly_target=False,
223             warnings_as_errors=False):
224  logging.info('Lint starting')
225
226  cmd = [
227      lint_binary_path,
228      '--quiet',  # Silences lint's "." progress updates.
229      '--disable',
230      ','.join(_DISABLED_ALWAYS),
231  ]
232  if baseline:
233    cmd.extend(['--baseline', baseline])
234  if testonly_target:
235    cmd.extend(['--disable', ','.join(_DISABLED_FOR_TESTS)])
236
237  if not manifest_path:
238    manifest_path = os.path.join(build_utils.DIR_SOURCE_ROOT, 'build',
239                                 'android', 'AndroidManifest.xml')
240
241  logging.info('Generating config.xml')
242  backported_methods = _RetrieveBackportedMethods(backported_methods_path)
243  config_xml_node = _GenerateConfigXmlTree(config_path, backported_methods)
244  generated_config_path = os.path.join(lint_gen_dir, 'config.xml')
245  _WriteXmlFile(config_xml_node, generated_config_path)
246  cmd.extend(['--config', generated_config_path])
247
248  logging.info('Generating Android manifest file')
249  android_manifest_tree = _GenerateAndroidManifest(manifest_path,
250                                                   extra_manifest_paths,
251                                                   min_sdk_version,
252                                                   android_sdk_version)
253  # Include the rebased manifest_path in the lint generated path so that it is
254  # clear in error messages where the original AndroidManifest.xml came from.
255  lint_android_manifest_path = os.path.join(lint_gen_dir, manifest_path)
256  _WriteXmlFile(android_manifest_tree.getroot(), lint_android_manifest_path)
257
258  resource_root_dir = os.path.join(lint_gen_dir, _RES_ZIP_DIR)
259  # These are zip files with generated resources (e. g. strings from GRD).
260  logging.info('Extracting resource zips')
261  for resource_zip in resource_zips:
262    # Use a consistent root and name rather than a temporary file so that
263    # suppressions can be local to the lint target and the resource target.
264    resource_dir = os.path.join(resource_root_dir, resource_zip)
265    shutil.rmtree(resource_dir, True)
266    os.makedirs(resource_dir)
267    resource_sources.extend(
268        build_utils.ExtractAll(resource_zip, path=resource_dir))
269
270  logging.info('Extracting aars')
271  aar_root_dir = os.path.join(lint_gen_dir, _AAR_DIR)
272  custom_lint_jars = []
273  custom_annotation_zips = []
274  if aars:
275    for aar in aars:
276      # Use relative source for aar files since they are not generated.
277      aar_dir = os.path.join(aar_root_dir,
278                             os.path.splitext(_SrcRelative(aar))[0])
279      shutil.rmtree(aar_dir, True)
280      os.makedirs(aar_dir)
281      aar_files = build_utils.ExtractAll(aar, path=aar_dir)
282      for f in aar_files:
283        if f.endswith('lint.jar'):
284          custom_lint_jars.append(f)
285        elif f.endswith('annotations.zip'):
286          custom_annotation_zips.append(f)
287
288  logging.info('Extracting srcjars')
289  srcjar_root_dir = os.path.join(lint_gen_dir, _SRCJAR_DIR)
290  srcjar_sources = []
291  if srcjars:
292    for srcjar in srcjars:
293      # Use path without extensions since otherwise the file name includes
294      # .srcjar and lint treats it as a srcjar.
295      srcjar_dir = os.path.join(srcjar_root_dir, os.path.splitext(srcjar)[0])
296      shutil.rmtree(srcjar_dir, True)
297      os.makedirs(srcjar_dir)
298      # Sadly lint's srcjar support is broken since it only considers the first
299      # srcjar. Until we roll a lint version with that fixed, we need to extract
300      # it ourselves.
301      srcjar_sources.extend(build_utils.ExtractAll(srcjar, path=srcjar_dir))
302
303  logging.info('Generating project file')
304  project_file_root = _GenerateProjectFile(lint_android_manifest_path,
305                                           android_sdk_root, cache_dir, sources,
306                                           classpath, srcjar_sources,
307                                           resource_sources, custom_lint_jars,
308                                           custom_annotation_zips,
309                                           android_sdk_version)
310
311  project_xml_path = os.path.join(lint_gen_dir, 'project.xml')
312  _WriteXmlFile(project_file_root, project_xml_path)
313  cmd += ['--project', project_xml_path]
314
315  logging.info('Preparing environment variables')
316  env = os.environ.copy()
317  # It is important that lint uses the checked-in JDK11 as it is almost 50%
318  # faster than JDK8.
319  env['JAVA_HOME'] = build_utils.JAVA_HOME
320  # This is necessary so that lint errors print stack traces in stdout.
321  env['LINT_PRINT_STACKTRACE'] = 'true'
322  if baseline and not os.path.exists(baseline):
323    # Generating new baselines is only done locally, and requires more memory to
324    # avoid OOMs.
325    env['LINT_OPTS'] = '-Xmx4g'
326  else:
327    # The default set in the wrapper script is 1g, but it seems not enough :(
328    env['LINT_OPTS'] = '-Xmx2g'
329
330  # This filter is necessary for JDK11.
331  stderr_filter = build_utils.FilterReflectiveAccessJavaWarnings
332  stdout_filter = lambda x: build_utils.FilterLines(x, 'No issues found')
333  if expected_warnings:
334    stdout_filter = functools.partial(_CheckLintWarning, expected_warnings)
335
336  start = time.time()
337  logging.debug('Lint command %s', ' '.join(cmd))
338  failed = True
339  try:
340    failed = bool(
341        build_utils.CheckOutput(cmd,
342                                env=env,
343                                print_stdout=True,
344                                stdout_filter=stdout_filter,
345                                stderr_filter=stderr_filter,
346                                fail_on_output=warnings_as_errors))
347  finally:
348    # When not treating warnings as errors, display the extra footer.
349    is_debug = os.environ.get('LINT_DEBUG', '0') != '0'
350
351    if failed:
352      print('- For more help with lint in Chrome:', _LINT_MD_URL)
353      if is_debug:
354        print('- DEBUG MODE: Here is the project.xml: {}'.format(
355            _SrcRelative(project_xml_path)))
356      else:
357        print('- Run with LINT_DEBUG=1 to enable lint configuration debugging')
358
359    end = time.time() - start
360    logging.info('Lint command took %ss', end)
361    if not is_debug:
362      shutil.rmtree(aar_root_dir, ignore_errors=True)
363      shutil.rmtree(resource_root_dir, ignore_errors=True)
364      shutil.rmtree(srcjar_root_dir, ignore_errors=True)
365      os.unlink(project_xml_path)
366
367  logging.info('Lint completed')
368
369
370def _ParseArgs(argv):
371  parser = argparse.ArgumentParser()
372  build_utils.AddDepfileOption(parser)
373  parser.add_argument('--lint-binary-path',
374                      required=True,
375                      help='Path to lint executable.')
376  parser.add_argument('--backported-methods',
377                      help='Path to backported methods file created by R8.')
378  parser.add_argument('--cache-dir',
379                      required=True,
380                      help='Path to the directory in which the android cache '
381                      'directory tree should be stored.')
382  parser.add_argument('--config-path', help='Path to lint suppressions file.')
383  parser.add_argument('--lint-gen-dir',
384                      required=True,
385                      help='Path to store generated xml files.')
386  parser.add_argument('--stamp', help='Path to stamp upon success.')
387  parser.add_argument('--android-sdk-version',
388                      help='Version (API level) of the Android SDK used for '
389                      'building.')
390  parser.add_argument('--min-sdk-version',
391                      required=True,
392                      help='Minimal SDK version to lint against.')
393  parser.add_argument('--android-sdk-root',
394                      required=True,
395                      help='Lint needs an explicit path to the android sdk.')
396  parser.add_argument('--testonly',
397                      action='store_true',
398                      help='If set, some checks like UnusedResources will be '
399                      'disabled since they are not helpful for test '
400                      'targets.')
401  parser.add_argument('--warnings-as-errors',
402                      action='store_true',
403                      help='Treat all warnings as errors.')
404  parser.add_argument('--java-sources',
405                      help='File containing a list of java sources files.')
406  parser.add_argument('--aars', help='GN list of included aars.')
407  parser.add_argument('--srcjars', help='GN list of included srcjars.')
408  parser.add_argument('--manifest-path',
409                      help='Path to original AndroidManifest.xml')
410  parser.add_argument('--extra-manifest-paths',
411                      action='append',
412                      help='GYP-list of manifest paths to merge into the '
413                      'original AndroidManifest.xml')
414  parser.add_argument('--resource-sources',
415                      default=[],
416                      action='append',
417                      help='GYP-list of resource sources files, similar to '
418                      'java sources files, but for resource files.')
419  parser.add_argument('--resource-zips',
420                      default=[],
421                      action='append',
422                      help='GYP-list of resource zips, zip files of generated '
423                      'resource files.')
424  parser.add_argument('--classpath',
425                      help='List of jars to add to the classpath.')
426  parser.add_argument('--baseline',
427                      help='Baseline file to ignore existing errors and fail '
428                      'on new errors.')
429  parser.add_argument('--expected-warnings',
430                      help='Comma separated list of warnings to test for in '
431                      'the output, failing if not found.')
432
433  args = parser.parse_args(build_utils.ExpandFileArgs(argv))
434  args.java_sources = build_utils.ParseGnList(args.java_sources)
435  args.aars = build_utils.ParseGnList(args.aars)
436  args.srcjars = build_utils.ParseGnList(args.srcjars)
437  args.resource_sources = build_utils.ParseGnList(args.resource_sources)
438  args.extra_manifest_paths = build_utils.ParseGnList(args.extra_manifest_paths)
439  args.resource_zips = build_utils.ParseGnList(args.resource_zips)
440  args.classpath = build_utils.ParseGnList(args.classpath)
441  return args
442
443
444def main():
445  build_utils.InitLogging('LINT_DEBUG')
446  args = _ParseArgs(sys.argv[1:])
447
448  sources = []
449  for java_sources_file in args.java_sources:
450    sources.extend(build_utils.ReadSourcesList(java_sources_file))
451  resource_sources = []
452  for resource_sources_file in args.resource_sources:
453    resource_sources.extend(build_utils.ReadSourcesList(resource_sources_file))
454
455  possible_depfile_deps = (args.srcjars + args.resource_zips + sources +
456                           resource_sources + [
457                               args.baseline,
458                               args.manifest_path,
459                           ])
460  depfile_deps = [p for p in possible_depfile_deps if p]
461
462  _RunLint(args.lint_binary_path,
463           args.backported_methods,
464           args.config_path,
465           args.manifest_path,
466           args.extra_manifest_paths,
467           sources,
468           args.classpath,
469           args.cache_dir,
470           args.android_sdk_version,
471           args.aars,
472           args.srcjars,
473           args.min_sdk_version,
474           resource_sources,
475           args.resource_zips,
476           args.android_sdk_root,
477           args.lint_gen_dir,
478           args.baseline,
479           args.expected_warnings,
480           testonly_target=args.testonly,
481           warnings_as_errors=args.warnings_as_errors)
482  logging.info('Creating stamp file')
483  build_utils.Touch(args.stamp)
484
485  if args.depfile:
486    build_utils.WriteDepfile(args.depfile, args.stamp, depfile_deps)
487
488
489if __name__ == '__main__':
490  sys.exit(main())
491