1#!/usr/bin/env python
2#
3# Copyright (c) 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Runs Android's lint tool."""
8
9from __future__ import print_function
10
11import argparse
12import os
13import re
14import shutil
15import sys
16import traceback
17from xml.dom import minidom
18from xml.etree import ElementTree
19
20from util import build_utils
21from util import manifest_utils
22from util import md5_check
23
24_LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/master/build/android/docs/lint.md' # pylint: disable=line-too-long
25
26
27def _RunLint(lint_path,
28             config_path,
29             processed_config_path,
30             manifest_path,
31             result_path,
32             product_dir,
33             sources,
34             jar_path,
35             cache_dir,
36             android_sdk_version,
37             srcjars,
38             min_sdk_version,
39             manifest_package,
40             resource_sources,
41             disable=None,
42             classpath=None,
43             can_fail_build=False,
44             include_unexpected=False,
45             silent=False):
46
47  def _RebasePath(path):
48    """Returns relative path to top-level src dir.
49
50    Args:
51      path: A path relative to cwd.
52    """
53    ret = os.path.relpath(os.path.abspath(path), build_utils.DIR_SOURCE_ROOT)
54    # If it's outside of src/, just use abspath.
55    if ret.startswith('..'):
56      ret = os.path.abspath(path)
57    return ret
58
59  def _ProcessConfigFile():
60    if not config_path or not processed_config_path:
61      return
62    if not build_utils.IsTimeStale(processed_config_path, [config_path]):
63      return
64
65    with open(config_path, 'rb') as f:
66      content = f.read().replace(
67          'PRODUCT_DIR', _RebasePath(product_dir))
68
69    with open(processed_config_path, 'wb') as f:
70      f.write(content)
71
72  def _ProcessResultFile():
73    with open(result_path, 'rb') as f:
74      content = f.read().replace(
75          _RebasePath(product_dir), 'PRODUCT_DIR')
76
77    with open(result_path, 'wb') as f:
78      f.write(content)
79
80  def _ParseAndShowResultFile():
81    dom = minidom.parse(result_path)
82    issues = dom.getElementsByTagName('issue')
83    if not silent:
84      print(file=sys.stderr)
85      for issue in issues:
86        issue_id = issue.attributes['id'].value
87        message = issue.attributes['message'].value
88        location_elem = issue.getElementsByTagName('location')[0]
89        path = location_elem.attributes['file'].value
90        line = location_elem.getAttribute('line')
91        if line:
92          error = '%s:%s %s: %s [warning]' % (path, line, message, issue_id)
93        else:
94          # Issues in class files don't have a line number.
95          error = '%s %s: %s [warning]' % (path, message, issue_id)
96        print(error.encode('utf-8'), file=sys.stderr)
97        for attr in ['errorLine1', 'errorLine2']:
98          error_line = issue.getAttribute(attr)
99          if error_line:
100            print(error_line.encode('utf-8'), file=sys.stderr)
101    return len(issues)
102
103  with build_utils.TempDir() as temp_dir:
104    _ProcessConfigFile()
105
106    cmd = [
107        _RebasePath(lint_path), '-Werror', '--exitcode', '--showall',
108        '--xml', _RebasePath(result_path),
109    ]
110    if jar_path:
111      # --classpath is just for .class files for this one target.
112      cmd.extend(['--classpath', _RebasePath(jar_path)])
113    if processed_config_path:
114      cmd.extend(['--config', _RebasePath(processed_config_path)])
115
116    tmp_dir_counter = [0]
117    def _NewTempSubdir(prefix, append_digit=True):
118      # Helper function to create a new sub directory based on the number of
119      # subdirs created earlier.
120      if append_digit:
121        tmp_dir_counter[0] += 1
122        prefix += str(tmp_dir_counter[0])
123      new_dir = os.path.join(temp_dir, prefix)
124      os.makedirs(new_dir)
125      return new_dir
126
127    resource_dirs = []
128    for resource_source in resource_sources:
129      if os.path.isdir(resource_source):
130        resource_dirs.append(resource_source)
131      else:
132        # This is a zip file with generated resources (e. g. strings from GRD).
133        # Extract it to temporary folder.
134        resource_dir = _NewTempSubdir(resource_source, append_digit=False)
135        resource_dirs.append(resource_dir)
136        build_utils.ExtractAll(resource_source, path=resource_dir)
137
138    for resource_dir in resource_dirs:
139      cmd.extend(['--resources', _RebasePath(resource_dir)])
140
141    if classpath:
142      # --libraries is the classpath (excluding active target).
143      cp = ':'.join(_RebasePath(p) for p in classpath)
144      cmd.extend(['--libraries', cp])
145
146    # There may be multiple source files with the same basename (but in
147    # different directories). It is difficult to determine what part of the path
148    # corresponds to the java package, and so instead just link the source files
149    # into temporary directories (creating a new one whenever there is a name
150    # conflict).
151    def PathInDir(d, src):
152      subpath = os.path.join(d, _RebasePath(src))
153      subdir = os.path.dirname(subpath)
154      if not os.path.exists(subdir):
155        os.makedirs(subdir)
156      return subpath
157
158    src_dirs = []
159    for src in sources:
160      src_dir = None
161      for d in src_dirs:
162        if not os.path.exists(PathInDir(d, src)):
163          src_dir = d
164          break
165      if not src_dir:
166        src_dir = _NewTempSubdir('SRC_ROOT')
167        src_dirs.append(src_dir)
168        cmd.extend(['--sources', _RebasePath(src_dir)])
169      # In cases where the build dir is outside of the src dir, this can
170      # result in trying to symlink a file to itself for this file:
171      # gen/components/version_info/android/java/org/chromium/
172      #   components/version_info/VersionConstants.java
173      src = os.path.abspath(src)
174      dst = PathInDir(src_dir, src)
175      if src == dst:
176        continue
177      os.symlink(src, dst)
178
179    if srcjars:
180      srcjar_paths = build_utils.ParseGnList(srcjars)
181      if srcjar_paths:
182        srcjar_dir = _NewTempSubdir('SRC_ROOT')
183        cmd.extend(['--sources', _RebasePath(srcjar_dir)])
184        for srcjar in srcjar_paths:
185          build_utils.ExtractAll(srcjar, path=srcjar_dir)
186
187    if disable:
188      cmd.extend(['--disable', ','.join(disable)])
189
190    project_dir = _NewTempSubdir('SRC_ROOT')
191    if android_sdk_version:
192      # Create dummy project.properies file in a temporary "project" directory.
193      # It is the only way to add Android SDK to the Lint's classpath. Proper
194      # classpath is necessary for most source-level checks.
195      with open(os.path.join(project_dir, 'project.properties'), 'w') \
196          as propfile:
197        print('target=android-{}'.format(android_sdk_version), file=propfile)
198
199    # Put the manifest in a temporary directory in order to avoid lint detecting
200    # sibling res/ and src/ directories (which should be pass explicitly if they
201    # are to be included).
202    if not manifest_path:
203      manifest_path = os.path.join(
204          build_utils.DIR_SOURCE_ROOT, 'build', 'android',
205          'AndroidManifest.xml')
206    lint_manifest_path = os.path.join(project_dir, 'AndroidManifest.xml')
207    shutil.copyfile(os.path.abspath(manifest_path), lint_manifest_path)
208
209    # Check that minSdkVersion and package is correct and add it to the manifest
210    # in case it does not exist.
211    doc, manifest, _ = manifest_utils.ParseManifest(lint_manifest_path)
212    manifest_utils.AssertUsesSdk(manifest, min_sdk_version)
213    manifest_utils.AssertPackage(manifest, manifest_package)
214    uses_sdk = manifest.find('./uses-sdk')
215    if uses_sdk is None:
216      uses_sdk = ElementTree.Element('uses-sdk')
217      manifest.insert(0, uses_sdk)
218    uses_sdk.set('{%s}minSdkVersion' % manifest_utils.ANDROID_NAMESPACE,
219                 min_sdk_version)
220    if manifest_package:
221      manifest.set('package', manifest_package)
222    manifest_utils.SaveManifest(doc, lint_manifest_path)
223
224    cmd.append(project_dir)
225
226    if os.path.exists(result_path):
227      os.remove(result_path)
228
229    env = os.environ.copy()
230    stderr_filter = build_utils.FilterReflectiveAccessJavaWarnings
231    if cache_dir:
232      env['_JAVA_OPTIONS'] = '-Duser.home=%s' % _RebasePath(cache_dir)
233      # When _JAVA_OPTIONS is set, java prints to stderr:
234      # Picked up _JAVA_OPTIONS: ...
235      #
236      # We drop all lines that contain _JAVA_OPTIONS from the output
237      stderr_filter = lambda l: re.sub(
238          r'.*_JAVA_OPTIONS.*\n?',
239          '',
240          build_utils.FilterReflectiveAccessJavaWarnings(l))
241
242    def fail_func(returncode, stderr):
243      if returncode != 0:
244        return True
245      if (include_unexpected and
246          'Unexpected failure during lint analysis' in stderr):
247        return True
248      return False
249
250    try:
251      env['JAVA_HOME'] = os.path.relpath(build_utils.JAVA_HOME,
252                                         build_utils.DIR_SOURCE_ROOT)
253      build_utils.CheckOutput(cmd, cwd=build_utils.DIR_SOURCE_ROOT,
254                              env=env or None, stderr_filter=stderr_filter,
255                              fail_func=fail_func)
256    except build_utils.CalledProcessError:
257      # There is a problem with lint usage
258      if not os.path.exists(result_path):
259        raise
260
261      # Sometimes produces empty (almost) files:
262      if os.path.getsize(result_path) < 10:
263        if can_fail_build:
264          raise
265        elif not silent:
266          traceback.print_exc()
267        return
268
269      # There are actual lint issues
270      try:
271        num_issues = _ParseAndShowResultFile()
272      except Exception: # pylint: disable=broad-except
273        if not silent:
274          print('Lint created unparseable xml file...')
275          print('File contents:')
276          with open(result_path) as f:
277            print(f.read())
278          if can_fail_build:
279            traceback.print_exc()
280        if can_fail_build:
281          raise
282        else:
283          return
284
285      _ProcessResultFile()
286      if num_issues == 0 and include_unexpected:
287        msg = 'Please refer to output above for unexpected lint failures.\n'
288      else:
289        msg = ('\nLint found %d new issues.\n'
290               ' - For full explanation, please refer to %s\n'
291               ' - For more information about lint and how to fix lint issues,'
292               ' please refer to %s\n' %
293               (num_issues, _RebasePath(result_path), _LINT_MD_URL))
294      if not silent:
295        print(msg, file=sys.stderr)
296      if can_fail_build:
297        raise Exception('Lint failed.')
298
299
300def _FindInDirectories(directories, filename_filter):
301  all_files = []
302  for directory in directories:
303    all_files.extend(build_utils.FindInDirectory(directory, filename_filter))
304  return all_files
305
306
307def main():
308  parser = argparse.ArgumentParser()
309  build_utils.AddDepfileOption(parser)
310
311  parser.add_argument('--lint-path', required=True,
312                      help='Path to lint executable.')
313  parser.add_argument('--product-dir', required=True,
314                      help='Path to product dir.')
315  parser.add_argument('--result-path', required=True,
316                      help='Path to XML lint result file.')
317  parser.add_argument('--cache-dir', required=True,
318                      help='Path to the directory in which the android cache '
319                           'directory tree should be stored.')
320  parser.add_argument('--platform-xml-path', required=True,
321                      help='Path to api-platforms.xml')
322  parser.add_argument('--android-sdk-version',
323                      help='Version (API level) of the Android SDK used for '
324                           'building.')
325  parser.add_argument('--can-fail-build', action='store_true',
326                      help='If set, script will exit with nonzero exit status'
327                           ' if lint errors are present')
328  parser.add_argument('--include-unexpected-failures', action='store_true',
329                      help='If set, script will exit with nonzero exit status'
330                           ' if lint itself crashes with unexpected failures.')
331  parser.add_argument('--config-path',
332                      help='Path to lint suppressions file.')
333  parser.add_argument('--disable',
334                      help='List of checks to disable.')
335  parser.add_argument('--jar-path',
336                      help='Jar file containing class files.')
337  parser.add_argument('--java-sources-file',
338                      help='File containing a list of java files.')
339  parser.add_argument('--manifest-path',
340                      help='Path to AndroidManifest.xml')
341  parser.add_argument('--classpath', default=[], action='append',
342                      help='GYP-list of classpath .jar files')
343  parser.add_argument('--processed-config-path',
344                      help='Path to processed lint suppressions file.')
345  parser.add_argument('--resource-dir',
346                      help='Path to resource dir.')
347  parser.add_argument('--resource-sources', default=[], action='append',
348                      help='GYP-list of resource sources (directories with '
349                      'resources or archives created by resource-generating '
350                      'tasks.')
351  parser.add_argument('--silent', action='store_true',
352                      help='If set, script will not log anything.')
353  parser.add_argument('--src-dirs',
354                      help='Directories containing java files.')
355  parser.add_argument('--srcjars',
356                      help='GN list of included srcjars.')
357  parser.add_argument('--stamp', help='Path to stamp upon success.')
358  parser.add_argument(
359      '--min-sdk-version',
360      required=True,
361      help='Minimal SDK version to lint against.')
362  parser.add_argument(
363      '--manifest-package', help='Package name of the AndroidManifest.xml.')
364
365  args = parser.parse_args(build_utils.ExpandFileArgs(sys.argv[1:]))
366
367  sources = []
368  if args.src_dirs:
369    src_dirs = build_utils.ParseGnList(args.src_dirs)
370    sources = _FindInDirectories(src_dirs, '*.java')
371  elif args.java_sources_file:
372    sources.extend(build_utils.ReadSourcesList(args.java_sources_file))
373
374  if args.config_path and not args.processed_config_path:
375    parser.error('--config-path specified without --processed-config-path')
376  elif args.processed_config_path and not args.config_path:
377    parser.error('--processed-config-path specified without --config-path')
378
379  input_paths = [
380      args.lint_path,
381      args.platform_xml_path,
382  ]
383  if args.config_path:
384    input_paths.append(args.config_path)
385  if args.jar_path:
386    input_paths.append(args.jar_path)
387  if args.manifest_path:
388    input_paths.append(args.manifest_path)
389  if sources:
390    input_paths.extend(sources)
391  classpath = []
392  for gyp_list in args.classpath:
393    classpath.extend(build_utils.ParseGnList(gyp_list))
394  input_paths.extend(classpath)
395
396  resource_sources = []
397  if args.resource_dir:
398    # Backward compatibility with GYP
399    resource_sources += [ args.resource_dir ]
400
401  for gyp_list in args.resource_sources:
402    resource_sources += build_utils.ParseGnList(gyp_list)
403
404  for resource_source in resource_sources:
405    if os.path.isdir(resource_source):
406      input_paths.extend(build_utils.FindInDirectory(resource_source, '*'))
407    else:
408      input_paths.append(resource_source)
409
410  input_strings = [
411    args.can_fail_build,
412    args.include_unexpected_failures,
413    args.silent,
414  ]
415  if args.android_sdk_version:
416    input_strings.append(args.android_sdk_version)
417  if args.processed_config_path:
418    input_strings.append(args.processed_config_path)
419
420  disable = []
421  if args.disable:
422    disable = build_utils.ParseGnList(args.disable)
423    input_strings.extend(disable)
424
425  output_paths = [args.stamp]
426
427  def on_stale_md5():
428    _RunLint(
429        args.lint_path,
430        args.config_path,
431        args.processed_config_path,
432        args.manifest_path,
433        args.result_path,
434        args.product_dir,
435        sources,
436        args.jar_path,
437        args.cache_dir,
438        args.android_sdk_version,
439        args.srcjars,
440        args.min_sdk_version,
441        args.manifest_package,
442        resource_sources,
443        disable=disable,
444        classpath=classpath,
445        can_fail_build=args.can_fail_build,
446        include_unexpected=args.include_unexpected_failures,
447        silent=args.silent)
448
449    build_utils.Touch(args.stamp)
450
451  md5_check.CallAndWriteDepfileIfStale(
452      on_stale_md5,
453      args,
454      input_paths=input_paths,
455      input_strings=input_strings,
456      output_paths=output_paths,
457      depfile_deps=classpath)
458
459
460if __name__ == '__main__':
461  sys.exit(main())
462