1#!/usr/bin/env python 2# 3# Copyright 2018 the V8 project authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7# for py2/py3 compatibility 8from __future__ import print_function 9 10import json 11import multiprocessing 12import optparse 13import os 14import re 15import subprocess 16import sys 17 18CLANG_TIDY_WARNING = re.compile(r'(\/.*?)\ .*\[(.*)\]$') 19CLANG_TIDY_CMDLINE_OUT = re.compile(r'^clang-tidy.*\ .*|^\./\.\*') 20FILE_REGEXS = ['../src/*', '../test/*'] 21HEADER_REGEX = ['\.\.\/src\/.*|\.\.\/include\/.*|\.\.\/test\/.*'] 22 23THREADS = multiprocessing.cpu_count() 24 25 26class ClangTidyWarning(object): 27 """ 28 Wraps up a clang-tidy warning to present aggregated information. 29 """ 30 31 def __init__(self, warning_type): 32 self.warning_type = warning_type 33 self.occurrences = set() 34 35 def add_occurrence(self, file_path): 36 self.occurrences.add(file_path.lstrip()) 37 38 def __hash__(self): 39 return hash(self.warning_type) 40 41 def to_string(self, file_loc): 42 s = '[%s] #%d\n' % (self.warning_type, len(self.occurrences)) 43 if file_loc: 44 s += ' ' + '\n '.join(self.occurrences) 45 s += '\n' 46 return s 47 48 def __str__(self): 49 return self.to_string(False) 50 51 def __lt__(self, other): 52 return len(self.occurrences) < len(other.occurrences) 53 54 55def GenerateCompileCommands(build_folder): 56 """ 57 Generate a compilation database. 58 59 Currently clang-tidy-4 does not understand all flags that are passed 60 by the build system, therefore, we remove them from the generated file. 61 """ 62 ninja_ps = subprocess.Popen( 63 ['ninja', '-t', 'compdb', 'cxx', 'cc'], 64 stdout=subprocess.PIPE, 65 cwd=build_folder) 66 67 out_filepath = os.path.join(build_folder, 'compile_commands.json') 68 with open(out_filepath, 'w') as cc_file: 69 while True: 70 line = ninja_ps.stdout.readline() 71 72 if line == '': 73 break 74 75 line = line.replace('-fcomplete-member-pointers', '') 76 line = line.replace('-Wno-enum-compare-switch', '') 77 line = line.replace('-Wno-ignored-pragma-optimize', '') 78 line = line.replace('-Wno-null-pointer-arithmetic', '') 79 line = line.replace('-Wno-unused-lambda-capture', '') 80 line = line.replace('-Wno-defaulted-function-deleted', '') 81 cc_file.write(line) 82 83 84def skip_line(line): 85 """ 86 Check if a clang-tidy output line should be skipped. 87 """ 88 return bool(CLANG_TIDY_CMDLINE_OUT.search(line)) 89 90 91def ClangTidyRunFull(build_folder, skip_output_filter, checks, auto_fix): 92 """ 93 Run clang-tidy on the full codebase and print warnings. 94 """ 95 extra_args = [] 96 if auto_fix: 97 extra_args.append('-fix') 98 99 if checks is not None: 100 extra_args.append('-checks') 101 extra_args.append('-*, ' + checks) 102 103 with open(os.devnull, 'w') as DEVNULL: 104 ct_process = subprocess.Popen( 105 ['run-clang-tidy', '-j' + str(THREADS), '-p', '.'] 106 + ['-header-filter'] + HEADER_REGEX + extra_args 107 + FILE_REGEXS, 108 cwd=build_folder, 109 stdout=subprocess.PIPE, 110 stderr=DEVNULL) 111 removing_check_header = False 112 empty_lines = 0 113 114 while True: 115 line = ct_process.stdout.readline() 116 if line == '': 117 break 118 119 # Skip all lines after Enbale checks and before two newlines, 120 # i.e., skip clang-tidy check list. 121 if line.startswith('Enabled checks'): 122 removing_check_header = True 123 if removing_check_header and not skip_output_filter: 124 if line == '\n': 125 empty_lines += 1 126 if empty_lines == 2: 127 removing_check_header = False 128 continue 129 130 # Different lines get removed to ease output reading. 131 if not skip_output_filter and skip_line(line): 132 continue 133 134 # Print line, because no filter was matched. 135 if line != '\n': 136 sys.stdout.write(line) 137 138 139def ClangTidyRunAggregate(build_folder, print_files): 140 """ 141 Run clang-tidy on the full codebase and aggregate warnings into categories. 142 """ 143 with open(os.devnull, 'w') as DEVNULL: 144 ct_process = subprocess.Popen( 145 ['run-clang-tidy', '-j' + str(THREADS), '-p', '.'] + 146 ['-header-filter'] + HEADER_REGEX + 147 FILE_REGEXS, 148 cwd=build_folder, 149 stdout=subprocess.PIPE, 150 stderr=DEVNULL) 151 warnings = dict() 152 while True: 153 line = ct_process.stdout.readline() 154 if line == '': 155 break 156 157 res = CLANG_TIDY_WARNING.search(line) 158 if res is not None: 159 warnings.setdefault( 160 res.group(2), 161 ClangTidyWarning(res.group(2))).add_occurrence(res.group(1)) 162 163 for warning in sorted(warnings.values(), reverse=True): 164 sys.stdout.write(warning.to_string(print_files)) 165 166 167def ClangTidyRunDiff(build_folder, diff_branch, auto_fix): 168 """ 169 Run clang-tidy on the diff between current and the diff_branch. 170 """ 171 if diff_branch is None: 172 diff_branch = subprocess.check_output(['git', 'merge-base', 173 'HEAD', 'origin/master']).strip() 174 175 git_ps = subprocess.Popen( 176 ['git', 'diff', '-U0', diff_branch], stdout=subprocess.PIPE) 177 178 extra_args = [] 179 if auto_fix: 180 extra_args.append('-fix') 181 182 with open(os.devnull, 'w') as DEVNULL: 183 """ 184 The script `clang-tidy-diff` does not provide support to add header- 185 filters. To still analyze headers we use the build path option `-path` to 186 inject our header-filter option. This works because the script just adds 187 the passed path string to the commandline of clang-tidy. 188 """ 189 modified_build_folder = build_folder 190 modified_build_folder += ' -header-filter=' 191 modified_build_folder += '\'' + ''.join(HEADER_REGEX) + '\'' 192 193 ct_ps = subprocess.Popen( 194 ['clang-tidy-diff.py', '-path', modified_build_folder, '-p1'] + 195 extra_args, 196 stdin=git_ps.stdout, 197 stdout=subprocess.PIPE, 198 stderr=DEVNULL) 199 git_ps.wait() 200 while True: 201 line = ct_ps.stdout.readline() 202 if line == '': 203 break 204 205 if skip_line(line): 206 continue 207 208 sys.stdout.write(line) 209 210 211def rm_prefix(string, prefix): 212 """ 213 Removes prefix from a string until the new string 214 no longer starts with the prefix. 215 """ 216 while string.startswith(prefix): 217 string = string[len(prefix):] 218 return string 219 220 221def ClangTidyRunSingleFile(build_folder, filename_to_check, auto_fix, 222 line_ranges=[]): 223 """ 224 Run clang-tidy on a single file. 225 """ 226 files_with_relative_path = [] 227 228 compdb_filepath = os.path.join(build_folder, 'compile_commands.json') 229 with open(compdb_filepath) as raw_json_file: 230 compdb = json.load(raw_json_file) 231 232 for db_entry in compdb: 233 if db_entry['file'].endswith(filename_to_check): 234 files_with_relative_path.append(db_entry['file']) 235 236 with open(os.devnull, 'w') as DEVNULL: 237 for file_with_relative_path in files_with_relative_path: 238 line_filter = None 239 if len(line_ranges) != 0: 240 line_filter = '[' 241 line_filter += '{ \"lines\":[' + ', '.join(line_ranges) 242 line_filter += '], \"name\":\"' 243 line_filter += rm_prefix(file_with_relative_path, 244 '../') + '\"}' 245 line_filter += ']' 246 247 extra_args = ['-line-filter=' + line_filter] if line_filter else [] 248 249 if auto_fix: 250 extra_args.append('-fix') 251 252 subprocess.call(['clang-tidy', '-p', '.'] + 253 extra_args + 254 [file_with_relative_path], 255 cwd=build_folder, 256 stderr=DEVNULL) 257 258 259def CheckClangTidy(): 260 """ 261 Checks if a clang-tidy binary exists. 262 """ 263 with open(os.devnull, 'w') as DEVNULL: 264 return subprocess.call(['which', 'clang-tidy'], stdout=DEVNULL) == 0 265 266 267def CheckCompDB(build_folder): 268 """ 269 Checks if a compilation database exists in the build_folder. 270 """ 271 return os.path.isfile(os.path.join(build_folder, 'compile_commands.json')) 272 273 274def DetectBuildFolder(): 275 """ 276 Tries to auto detect the last used build folder in out/ 277 """ 278 outdirs_folder = 'out/' 279 last_used = None 280 last_timestamp = -1 281 for outdir in [outdirs_folder + folder_name 282 for folder_name in os.listdir(outdirs_folder) 283 if os.path.isdir(outdirs_folder + folder_name)]: 284 outdir_modified_timestamp = os.path.getmtime(outdir) 285 if outdir_modified_timestamp > last_timestamp: 286 last_timestamp = outdir_modified_timestamp 287 last_used = outdir 288 289 return last_used 290 291 292def GetOptions(): 293 """ 294 Generate the option parser for this script. 295 """ 296 result = optparse.OptionParser() 297 result.add_option( 298 '-b', 299 '--build-folder', 300 help='Set V8 build folder', 301 dest='build_folder', 302 default=None) 303 result.add_option( 304 '-j', 305 help='Set the amount of threads that should be used', 306 dest='threads', 307 default=None) 308 result.add_option( 309 '--gen-compdb', 310 help='Generate a compilation database for clang-tidy', 311 default=False, 312 action='store_true') 313 result.add_option( 314 '--no-output-filter', 315 help='Done use any output filterning', 316 default=False, 317 action='store_true') 318 result.add_option( 319 '--fix', 320 help='Fix auto fixable issues', 321 default=False, 322 dest='auto_fix', 323 action='store_true' 324 ) 325 326 # Full clang-tidy. 327 full_run_g = optparse.OptionGroup(result, 'Clang-tidy full', '') 328 full_run_g.add_option( 329 '--full', 330 help='Run clang-tidy on the whole codebase', 331 default=False, 332 action='store_true') 333 full_run_g.add_option('--checks', 334 help='Clang-tidy checks to use.', 335 default=None) 336 result.add_option_group(full_run_g) 337 338 # Aggregate clang-tidy. 339 agg_run_g = optparse.OptionGroup(result, 'Clang-tidy aggregate', '') 340 agg_run_g.add_option('--aggregate', help='Run clang-tidy on the whole '\ 341 'codebase and aggregate the warnings', 342 default=False, action='store_true') 343 agg_run_g.add_option('--show-loc', help='Show file locations when running '\ 344 'in aggregate mode', default=False, 345 action='store_true') 346 result.add_option_group(agg_run_g) 347 348 # Diff clang-tidy. 349 diff_run_g = optparse.OptionGroup(result, 'Clang-tidy diff', '') 350 diff_run_g.add_option('--branch', help='Run clang-tidy on the diff '\ 351 'between HEAD and the merge-base between HEAD '\ 352 'and DIFF_BRANCH (origin/master by default).', 353 default=None, dest='diff_branch') 354 result.add_option_group(diff_run_g) 355 356 # Single clang-tidy. 357 single_run_g = optparse.OptionGroup(result, 'Clang-tidy single', '') 358 single_run_g.add_option( 359 '--single', help='', default=False, action='store_true') 360 single_run_g.add_option( 361 '--file', help='File name to check', default=None, dest='file_name') 362 single_run_g.add_option('--lines', help='Limit checks to a line range. '\ 363 'For example: --lines="[2,4], [5,6]"', 364 default=[], dest='line_ranges') 365 366 result.add_option_group(single_run_g) 367 return result 368 369 370def main(): 371 parser = GetOptions() 372 (options, _) = parser.parse_args() 373 374 if options.threads is not None: 375 global THREADS 376 THREADS = options.threads 377 378 if options.build_folder is None: 379 options.build_folder = DetectBuildFolder() 380 381 if not CheckClangTidy(): 382 print('Could not find clang-tidy') 383 elif options.build_folder is None or not os.path.isdir(options.build_folder): 384 print('Please provide a build folder with -b') 385 elif options.gen_compdb: 386 GenerateCompileCommands(options.build_folder) 387 elif not CheckCompDB(options.build_folder): 388 print('Could not find compilation database, ' \ 389 'please generate it with --gen-compdb') 390 else: 391 print('Using build folder:', options.build_folder) 392 if options.full: 393 print('Running clang-tidy - full') 394 ClangTidyRunFull(options.build_folder, 395 options.no_output_filter, 396 options.checks, 397 options.auto_fix) 398 elif options.aggregate: 399 print('Running clang-tidy - aggregating warnings') 400 if options.auto_fix: 401 print('Auto fix not working in aggregate mode, running without.') 402 ClangTidyRunAggregate(options.build_folder, options.show_loc) 403 elif options.single: 404 print('Running clang-tidy - single on ' + options.file_name) 405 if options.file_name is not None: 406 line_ranges = [] 407 for match in re.findall(r'(\[.*?\])', options.line_ranges): 408 if match is not []: 409 line_ranges.append(match) 410 ClangTidyRunSingleFile(options.build_folder, 411 options.file_name, 412 options.auto_fix, 413 line_ranges) 414 else: 415 print('Filename provided, please specify a filename with --file') 416 else: 417 print('Running clang-tidy') 418 ClangTidyRunDiff(options.build_folder, 419 options.diff_branch, 420 options.auto_fix) 421 422 423if __name__ == '__main__': 424 main() 425