1#!/usr/bin/env python
2#
3# Copyright 2018 the V8 project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# for py2/py3 compatibility
8from __future__ import print_function
9
10import json
11import multiprocessing
12import optparse
13import os
14import re
15import subprocess
16import sys
17
18CLANG_TIDY_WARNING = re.compile(r'(\/.*?)\ .*\[(.*)\]$')
19CLANG_TIDY_CMDLINE_OUT = re.compile(r'^clang-tidy.*\ .*|^\./\.\*')
20FILE_REGEXS = ['../src/*', '../test/*']
21HEADER_REGEX = ['\.\.\/src\/.*|\.\.\/include\/.*|\.\.\/test\/.*']
22
23THREADS = multiprocessing.cpu_count()
24
25
26class ClangTidyWarning(object):
27  """
28  Wraps up a clang-tidy warning to present aggregated information.
29  """
30
31  def __init__(self, warning_type):
32    self.warning_type = warning_type
33    self.occurrences = set()
34
35  def add_occurrence(self, file_path):
36    self.occurrences.add(file_path.lstrip())
37
38  def __hash__(self):
39    return hash(self.warning_type)
40
41  def to_string(self, file_loc):
42    s = '[%s] #%d\n' % (self.warning_type, len(self.occurrences))
43    if file_loc:
44      s += ' ' + '\n  '.join(self.occurrences)
45      s += '\n'
46    return s
47
48  def __str__(self):
49    return self.to_string(False)
50
51  def __lt__(self, other):
52    return len(self.occurrences) < len(other.occurrences)
53
54
55def GenerateCompileCommands(build_folder):
56  """
57  Generate a compilation database.
58
59  Currently clang-tidy-4 does not understand all flags that are passed
60  by the build system, therefore, we remove them from the generated file.
61  """
62  ninja_ps = subprocess.Popen(
63    ['ninja', '-t', 'compdb', 'cxx', 'cc'],
64    stdout=subprocess.PIPE,
65    cwd=build_folder)
66
67  out_filepath = os.path.join(build_folder, 'compile_commands.json')
68  with open(out_filepath, 'w') as cc_file:
69    while True:
70        line = ninja_ps.stdout.readline()
71
72        if line == '':
73            break
74
75        line = line.replace('-fcomplete-member-pointers', '')
76        line = line.replace('-Wno-enum-compare-switch', '')
77        line = line.replace('-Wno-ignored-pragma-optimize', '')
78        line = line.replace('-Wno-null-pointer-arithmetic', '')
79        line = line.replace('-Wno-unused-lambda-capture', '')
80        line = line.replace('-Wno-defaulted-function-deleted', '')
81        cc_file.write(line)
82
83
84def skip_line(line):
85  """
86  Check if a clang-tidy output line should be skipped.
87  """
88  return bool(CLANG_TIDY_CMDLINE_OUT.search(line))
89
90
91def ClangTidyRunFull(build_folder, skip_output_filter, checks, auto_fix):
92  """
93  Run clang-tidy on the full codebase and print warnings.
94  """
95  extra_args = []
96  if auto_fix:
97    extra_args.append('-fix')
98
99  if checks is not None:
100    extra_args.append('-checks')
101    extra_args.append('-*, ' + checks)
102
103  with open(os.devnull, 'w') as DEVNULL:
104    ct_process = subprocess.Popen(
105      ['run-clang-tidy', '-j' + str(THREADS), '-p', '.']
106       + ['-header-filter'] + HEADER_REGEX + extra_args
107       + FILE_REGEXS,
108      cwd=build_folder,
109      stdout=subprocess.PIPE,
110      stderr=DEVNULL)
111  removing_check_header = False
112  empty_lines = 0
113
114  while True:
115    line = ct_process.stdout.readline()
116    if line == '':
117      break
118
119    # Skip all lines after Enbale checks and before two newlines,
120    # i.e., skip clang-tidy check list.
121    if line.startswith('Enabled checks'):
122      removing_check_header = True
123    if removing_check_header and not skip_output_filter:
124      if line == '\n':
125        empty_lines += 1
126      if empty_lines == 2:
127        removing_check_header = False
128      continue
129
130    # Different lines get removed to ease output reading.
131    if not skip_output_filter and skip_line(line):
132      continue
133
134    # Print line, because no filter was matched.
135    if line != '\n':
136        sys.stdout.write(line)
137
138
139def ClangTidyRunAggregate(build_folder, print_files):
140  """
141  Run clang-tidy on the full codebase and aggregate warnings into categories.
142  """
143  with open(os.devnull, 'w') as DEVNULL:
144    ct_process = subprocess.Popen(
145      ['run-clang-tidy', '-j' + str(THREADS), '-p', '.'] +
146        ['-header-filter'] + HEADER_REGEX +
147        FILE_REGEXS,
148      cwd=build_folder,
149      stdout=subprocess.PIPE,
150      stderr=DEVNULL)
151  warnings = dict()
152  while True:
153    line = ct_process.stdout.readline()
154    if line == '':
155      break
156
157    res = CLANG_TIDY_WARNING.search(line)
158    if res is not None:
159      warnings.setdefault(
160          res.group(2),
161          ClangTidyWarning(res.group(2))).add_occurrence(res.group(1))
162
163  for warning in sorted(warnings.values(), reverse=True):
164    sys.stdout.write(warning.to_string(print_files))
165
166
167def ClangTidyRunDiff(build_folder, diff_branch, auto_fix):
168  """
169  Run clang-tidy on the diff between current and the diff_branch.
170  """
171  if diff_branch is None:
172    diff_branch = subprocess.check_output(['git', 'merge-base',
173                                           'HEAD', 'origin/master']).strip()
174
175  git_ps = subprocess.Popen(
176    ['git', 'diff', '-U0', diff_branch], stdout=subprocess.PIPE)
177
178  extra_args = []
179  if auto_fix:
180    extra_args.append('-fix')
181
182  with open(os.devnull, 'w') as DEVNULL:
183    """
184    The script `clang-tidy-diff` does not provide support to add header-
185    filters. To still analyze headers we use the build path option `-path` to
186    inject our header-filter option. This works because the script just adds
187    the passed path string to the commandline of clang-tidy.
188    """
189    modified_build_folder = build_folder
190    modified_build_folder += ' -header-filter='
191    modified_build_folder += '\'' + ''.join(HEADER_REGEX) + '\''
192
193    ct_ps = subprocess.Popen(
194      ['clang-tidy-diff.py', '-path', modified_build_folder, '-p1'] +
195        extra_args,
196      stdin=git_ps.stdout,
197      stdout=subprocess.PIPE,
198      stderr=DEVNULL)
199  git_ps.wait()
200  while True:
201    line = ct_ps.stdout.readline()
202    if line == '':
203      break
204
205    if skip_line(line):
206      continue
207
208    sys.stdout.write(line)
209
210
211def rm_prefix(string, prefix):
212  """
213  Removes prefix from a string until the new string
214  no longer starts with the prefix.
215  """
216  while string.startswith(prefix):
217    string = string[len(prefix):]
218  return string
219
220
221def ClangTidyRunSingleFile(build_folder, filename_to_check, auto_fix,
222                           line_ranges=[]):
223  """
224  Run clang-tidy on a single file.
225  """
226  files_with_relative_path = []
227
228  compdb_filepath = os.path.join(build_folder, 'compile_commands.json')
229  with open(compdb_filepath) as raw_json_file:
230    compdb = json.load(raw_json_file)
231
232  for db_entry in compdb:
233    if db_entry['file'].endswith(filename_to_check):
234      files_with_relative_path.append(db_entry['file'])
235
236  with open(os.devnull, 'w') as DEVNULL:
237    for file_with_relative_path in files_with_relative_path:
238      line_filter = None
239      if len(line_ranges) != 0:
240        line_filter = '['
241        line_filter += '{ \"lines\":[' + ', '.join(line_ranges)
242        line_filter += '], \"name\":\"'
243        line_filter += rm_prefix(file_with_relative_path,
244                                 '../') + '\"}'
245        line_filter += ']'
246
247      extra_args = ['-line-filter=' + line_filter] if line_filter else []
248
249      if auto_fix:
250        extra_args.append('-fix')
251
252      subprocess.call(['clang-tidy', '-p', '.'] +
253                      extra_args +
254                      [file_with_relative_path],
255                      cwd=build_folder,
256                      stderr=DEVNULL)
257
258
259def CheckClangTidy():
260  """
261  Checks if a clang-tidy binary exists.
262  """
263  with open(os.devnull, 'w') as DEVNULL:
264    return subprocess.call(['which', 'clang-tidy'], stdout=DEVNULL) == 0
265
266
267def CheckCompDB(build_folder):
268  """
269  Checks if a compilation database exists in the build_folder.
270  """
271  return os.path.isfile(os.path.join(build_folder, 'compile_commands.json'))
272
273
274def DetectBuildFolder():
275    """
276    Tries to auto detect the last used build folder in out/
277    """
278    outdirs_folder = 'out/'
279    last_used = None
280    last_timestamp = -1
281    for outdir in [outdirs_folder + folder_name
282                   for folder_name in os.listdir(outdirs_folder)
283                   if os.path.isdir(outdirs_folder + folder_name)]:
284        outdir_modified_timestamp = os.path.getmtime(outdir)
285        if  outdir_modified_timestamp > last_timestamp:
286            last_timestamp = outdir_modified_timestamp
287            last_used = outdir
288
289    return last_used
290
291
292def GetOptions():
293  """
294  Generate the option parser for this script.
295  """
296  result = optparse.OptionParser()
297  result.add_option(
298    '-b',
299    '--build-folder',
300    help='Set V8 build folder',
301    dest='build_folder',
302    default=None)
303  result.add_option(
304    '-j',
305    help='Set the amount of threads that should be used',
306    dest='threads',
307    default=None)
308  result.add_option(
309    '--gen-compdb',
310    help='Generate a compilation database for clang-tidy',
311    default=False,
312    action='store_true')
313  result.add_option(
314    '--no-output-filter',
315    help='Done use any output filterning',
316    default=False,
317    action='store_true')
318  result.add_option(
319    '--fix',
320    help='Fix auto fixable issues',
321    default=False,
322    dest='auto_fix',
323    action='store_true'
324  )
325
326  # Full clang-tidy.
327  full_run_g = optparse.OptionGroup(result, 'Clang-tidy full', '')
328  full_run_g.add_option(
329    '--full',
330    help='Run clang-tidy on the whole codebase',
331    default=False,
332    action='store_true')
333  full_run_g.add_option('--checks',
334                        help='Clang-tidy checks to use.',
335                        default=None)
336  result.add_option_group(full_run_g)
337
338  # Aggregate clang-tidy.
339  agg_run_g = optparse.OptionGroup(result, 'Clang-tidy aggregate', '')
340  agg_run_g.add_option('--aggregate', help='Run clang-tidy on the whole '\
341             'codebase and aggregate the warnings',
342             default=False, action='store_true')
343  agg_run_g.add_option('--show-loc', help='Show file locations when running '\
344             'in aggregate mode', default=False,
345             action='store_true')
346  result.add_option_group(agg_run_g)
347
348  # Diff clang-tidy.
349  diff_run_g = optparse.OptionGroup(result, 'Clang-tidy diff', '')
350  diff_run_g.add_option('--branch', help='Run clang-tidy on the diff '\
351             'between HEAD and the merge-base between HEAD '\
352             'and DIFF_BRANCH (origin/master by default).',
353             default=None, dest='diff_branch')
354  result.add_option_group(diff_run_g)
355
356  # Single clang-tidy.
357  single_run_g = optparse.OptionGroup(result, 'Clang-tidy single', '')
358  single_run_g.add_option(
359    '--single', help='', default=False, action='store_true')
360  single_run_g.add_option(
361    '--file', help='File name to check', default=None, dest='file_name')
362  single_run_g.add_option('--lines', help='Limit checks to a line range. '\
363              'For example: --lines="[2,4], [5,6]"',
364              default=[], dest='line_ranges')
365
366  result.add_option_group(single_run_g)
367  return result
368
369
370def main():
371  parser = GetOptions()
372  (options, _) = parser.parse_args()
373
374  if options.threads is not None:
375    global THREADS
376    THREADS = options.threads
377
378  if options.build_folder is None:
379    options.build_folder = DetectBuildFolder()
380
381  if not CheckClangTidy():
382    print('Could not find clang-tidy')
383  elif options.build_folder is None or not os.path.isdir(options.build_folder):
384    print('Please provide a build folder with -b')
385  elif options.gen_compdb:
386    GenerateCompileCommands(options.build_folder)
387  elif not CheckCompDB(options.build_folder):
388    print('Could not find compilation database, ' \
389      'please generate it with --gen-compdb')
390  else:
391    print('Using build folder:', options.build_folder)
392    if options.full:
393      print('Running clang-tidy - full')
394      ClangTidyRunFull(options.build_folder,
395                       options.no_output_filter,
396                       options.checks,
397                       options.auto_fix)
398    elif options.aggregate:
399      print('Running clang-tidy - aggregating warnings')
400      if options.auto_fix:
401        print('Auto fix not working in aggregate mode, running without.')
402      ClangTidyRunAggregate(options.build_folder, options.show_loc)
403    elif options.single:
404      print('Running clang-tidy - single on ' + options.file_name)
405      if options.file_name is not None:
406        line_ranges = []
407        for match in re.findall(r'(\[.*?\])', options.line_ranges):
408          if match is not []:
409            line_ranges.append(match)
410        ClangTidyRunSingleFile(options.build_folder,
411                               options.file_name,
412                               options.auto_fix,
413                               line_ranges)
414      else:
415        print('Filename provided, please specify a filename with --file')
416    else:
417      print('Running clang-tidy')
418      ClangTidyRunDiff(options.build_folder,
419                       options.diff_branch,
420                       options.auto_fix)
421
422
423if __name__ == '__main__':
424  main()
425