1#!/usr/bin/env python 2"""A wrapper script around clang-format, suitable for linting multiple files 3and to use for continuous integration. 4 5This is an alternative API for the clang-format command line. 6It runs over multiple files and directories in parallel. 7A diff output is produced and a sensible exit code is returned. 8 9""" 10 11from __future__ import print_function, unicode_literals 12 13import argparse 14import codecs 15import difflib 16import fnmatch 17import io 18import errno 19import multiprocessing 20import os 21import signal 22import subprocess 23import sys 24import traceback 25 26from functools import partial 27 28try: 29 from subprocess import DEVNULL # py3k 30except ImportError: 31 DEVNULL = open(os.devnull, "wb") 32 33 34DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx' 35DEFAULT_CLANG_FORMAT_IGNORE = '.clang-format-ignore' 36 37 38class ExitStatus: 39 SUCCESS = 0 40 DIFF = 1 41 TROUBLE = 2 42 43def excludes_from_file(ignore_file): 44 excludes = [] 45 try: 46 with io.open(ignore_file, 'r', encoding='utf-8') as f: 47 for line in f: 48 if line.startswith('#'): 49 # ignore comments 50 continue 51 pattern = line.rstrip() 52 if not pattern: 53 # allow empty lines 54 continue 55 excludes.append(pattern) 56 except EnvironmentError as e: 57 if e.errno != errno.ENOENT: 58 raise 59 return excludes; 60 61def list_files(files, recursive=False, extensions=None, exclude=None): 62 if extensions is None: 63 extensions = [] 64 if exclude is None: 65 exclude = [] 66 67 out = [] 68 for file in files: 69 if recursive and os.path.isdir(file): 70 for dirpath, dnames, fnames in os.walk(file): 71 fpaths = [os.path.join(dirpath, fname) for fname in fnames] 72 for pattern in exclude: 73 # os.walk() supports trimming down the dnames list 74 # by modifying it in-place, 75 # to avoid unnecessary directory listings. 76 dnames[:] = [ 77 x for x in dnames 78 if 79 not fnmatch.fnmatch(os.path.join(dirpath, x), pattern) 80 ] 81 fpaths = [ 82 x for x in fpaths if not fnmatch.fnmatch(x, pattern) 83 ] 84 for f in fpaths: 85 ext = os.path.splitext(f)[1][1:] 86 if ext in extensions: 87 out.append(f) 88 else: 89 out.append(file) 90 return out 91 92 93def make_diff(file, original, reformatted): 94 return list( 95 difflib.unified_diff( 96 original, 97 reformatted, 98 fromfile='{}\t(original)'.format(file), 99 tofile='{}\t(reformatted)'.format(file), 100 n=3)) 101 102 103class DiffError(Exception): 104 def __init__(self, message, errs=None): 105 super(DiffError, self).__init__(message) 106 self.errs = errs or [] 107 108 109class UnexpectedError(Exception): 110 def __init__(self, message, exc=None): 111 super(UnexpectedError, self).__init__(message) 112 self.formatted_traceback = traceback.format_exc() 113 self.exc = exc 114 115 116def run_clang_format_diff_wrapper(args, file): 117 try: 118 ret = run_clang_format_diff(args, file) 119 return ret 120 except DiffError: 121 raise 122 except Exception as e: 123 raise UnexpectedError('{}: {}: {}'.format(file, e.__class__.__name__, 124 e), e) 125 126 127def run_clang_format_diff(args, file): 128 try: 129 with io.open(file, 'r', encoding='utf-8') as f: 130 original = f.readlines() 131 except IOError as exc: 132 raise DiffError(str(exc)) 133 invocation = [args.clang_format_executable, file] 134 135 # Use of utf-8 to decode the process output. 136 # 137 # Hopefully, this is the correct thing to do. 138 # 139 # It's done due to the following assumptions (which may be incorrect): 140 # - clang-format will returns the bytes read from the files as-is, 141 # without conversion, and it is already assumed that the files use utf-8. 142 # - if the diagnostics were internationalized, they would use utf-8: 143 # > Adding Translations to Clang 144 # > 145 # > Not possible yet! 146 # > Diagnostic strings should be written in UTF-8, 147 # > the client can translate to the relevant code page if needed. 148 # > Each translation completely replaces the format string 149 # > for the diagnostic. 150 # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation 151 # 152 # It's not pretty, due to Python 2 & 3 compatibility. 153 encoding_py3 = {} 154 if sys.version_info[0] >= 3: 155 encoding_py3['encoding'] = 'utf-8' 156 157 try: 158 proc = subprocess.Popen( 159 invocation, 160 stdout=subprocess.PIPE, 161 stderr=subprocess.PIPE, 162 universal_newlines=True, 163 **encoding_py3) 164 except OSError as exc: 165 raise DiffError( 166 "Command '{}' failed to start: {}".format( 167 subprocess.list2cmdline(invocation), exc 168 ) 169 ) 170 proc_stdout = proc.stdout 171 proc_stderr = proc.stderr 172 if sys.version_info[0] < 3: 173 # make the pipes compatible with Python 3, 174 # reading lines should output unicode 175 encoding = 'utf-8' 176 proc_stdout = codecs.getreader(encoding)(proc_stdout) 177 proc_stderr = codecs.getreader(encoding)(proc_stderr) 178 # hopefully the stderr pipe won't get full and block the process 179 outs = list(proc_stdout.readlines()) 180 errs = list(proc_stderr.readlines()) 181 proc.wait() 182 if proc.returncode: 183 raise DiffError( 184 "Command '{}' returned non-zero exit status {}".format( 185 subprocess.list2cmdline(invocation), proc.returncode 186 ), 187 errs, 188 ) 189 return make_diff(file, original, outs), errs 190 191 192def bold_red(s): 193 return '\x1b[1m\x1b[31m' + s + '\x1b[0m' 194 195 196def colorize(diff_lines): 197 def bold(s): 198 return '\x1b[1m' + s + '\x1b[0m' 199 200 def cyan(s): 201 return '\x1b[36m' + s + '\x1b[0m' 202 203 def green(s): 204 return '\x1b[32m' + s + '\x1b[0m' 205 206 def red(s): 207 return '\x1b[31m' + s + '\x1b[0m' 208 209 for line in diff_lines: 210 if line[:4] in ['--- ', '+++ ']: 211 yield bold(line) 212 elif line.startswith('@@ '): 213 yield cyan(line) 214 elif line.startswith('+'): 215 yield green(line) 216 elif line.startswith('-'): 217 yield red(line) 218 else: 219 yield line 220 221 222def print_diff(diff_lines, use_color): 223 if use_color: 224 diff_lines = colorize(diff_lines) 225 if sys.version_info[0] < 3: 226 sys.stdout.writelines((l.encode('utf-8') for l in diff_lines)) 227 else: 228 sys.stdout.writelines(diff_lines) 229 230 231def print_trouble(prog, message, use_colors): 232 error_text = 'error:' 233 if use_colors: 234 error_text = bold_red(error_text) 235 print("{}: {} {}".format(prog, error_text, message), file=sys.stderr) 236 237 238def main(): 239 parser = argparse.ArgumentParser(description=__doc__) 240 parser.add_argument( 241 '--clang-format-executable', 242 metavar='EXECUTABLE', 243 help='path to the clang-format executable', 244 default='clang-format') 245 parser.add_argument( 246 '--extensions', 247 help='comma separated list of file extensions (default: {})'.format( 248 DEFAULT_EXTENSIONS), 249 default=DEFAULT_EXTENSIONS) 250 parser.add_argument( 251 '-r', 252 '--recursive', 253 action='store_true', 254 help='run recursively over directories') 255 parser.add_argument('files', metavar='file', nargs='+') 256 parser.add_argument( 257 '-q', 258 '--quiet', 259 action='store_true', 260 help="disable output, useful for the exit code") 261 parser.add_argument( 262 '-j', 263 metavar='N', 264 type=int, 265 default=0, 266 help='run N clang-format jobs in parallel' 267 ' (default number of cpus + 1)') 268 parser.add_argument( 269 '--color', 270 default='auto', 271 choices=['auto', 'always', 'never'], 272 help='show colored diff (default: auto)') 273 parser.add_argument( 274 '-e', 275 '--exclude', 276 metavar='PATTERN', 277 action='append', 278 default=[], 279 help='exclude paths matching the given glob-like pattern(s)' 280 ' from recursive search') 281 282 args = parser.parse_args() 283 284 # use default signal handling, like diff return SIGINT value on ^C 285 # https://bugs.python.org/issue14229#msg156446 286 signal.signal(signal.SIGINT, signal.SIG_DFL) 287 try: 288 signal.SIGPIPE 289 except AttributeError: 290 # compatibility, SIGPIPE does not exist on Windows 291 pass 292 else: 293 signal.signal(signal.SIGPIPE, signal.SIG_DFL) 294 295 colored_stdout = False 296 colored_stderr = False 297 if args.color == 'always': 298 colored_stdout = True 299 colored_stderr = True 300 elif args.color == 'auto': 301 colored_stdout = sys.stdout.isatty() 302 colored_stderr = sys.stderr.isatty() 303 304 version_invocation = [args.clang_format_executable, str("--version")] 305 try: 306 subprocess.check_call(version_invocation, stdout=DEVNULL) 307 except subprocess.CalledProcessError as e: 308 print_trouble(parser.prog, str(e), use_colors=colored_stderr) 309 return ExitStatus.TROUBLE 310 except OSError as e: 311 print_trouble( 312 parser.prog, 313 "Command '{}' failed to start: {}".format( 314 subprocess.list2cmdline(version_invocation), e 315 ), 316 use_colors=colored_stderr, 317 ) 318 return ExitStatus.TROUBLE 319 320 retcode = ExitStatus.SUCCESS 321 322 excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE) 323 excludes.extend(args.exclude) 324 325 files = list_files( 326 args.files, 327 recursive=args.recursive, 328 exclude=excludes, 329 extensions=args.extensions.split(',')) 330 331 if not files: 332 return 333 334 njobs = args.j 335 if njobs == 0: 336 njobs = multiprocessing.cpu_count() + 1 337 njobs = min(len(files), njobs) 338 339 if njobs == 1: 340 # execute directly instead of in a pool, 341 # less overhead, simpler stacktraces 342 it = (run_clang_format_diff_wrapper(args, file) for file in files) 343 pool = None 344 else: 345 pool = multiprocessing.Pool(njobs) 346 it = pool.imap_unordered( 347 partial(run_clang_format_diff_wrapper, args), files) 348 while True: 349 try: 350 outs, errs = next(it) 351 except StopIteration: 352 break 353 except DiffError as e: 354 print_trouble(parser.prog, str(e), use_colors=colored_stderr) 355 retcode = ExitStatus.TROUBLE 356 sys.stderr.writelines(e.errs) 357 except UnexpectedError as e: 358 print_trouble(parser.prog, str(e), use_colors=colored_stderr) 359 sys.stderr.write(e.formatted_traceback) 360 retcode = ExitStatus.TROUBLE 361 # stop at the first unexpected error, 362 # something could be very wrong, 363 # don't process all files unnecessarily 364 if pool: 365 pool.terminate() 366 break 367 else: 368 sys.stderr.writelines(errs) 369 if outs == []: 370 continue 371 if not args.quiet: 372 print_diff(outs, use_color=colored_stdout) 373 if retcode == ExitStatus.SUCCESS: 374 retcode = ExitStatus.DIFF 375 return retcode 376 377 378if __name__ == '__main__': 379 sys.exit(main()) 380