1#!/usr/bin/env python
2"""A wrapper script around clang-format, suitable for linting multiple files
3and to use for continuous integration.
4
5This is an alternative API for the clang-format command line.
6It runs over multiple files and directories in parallel.
7A diff output is produced and a sensible exit code is returned.
8
9"""
10
11from __future__ import print_function, unicode_literals
12
13import argparse
14import codecs
15import difflib
16import fnmatch
17import io
18import multiprocessing
19import os
20import signal
21import subprocess
22import sys
23import traceback
24
25from functools import partial
26
27DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx'
28
29
30class ExitStatus:
31    SUCCESS = 0
32    DIFF = 1
33    TROUBLE = 2
34
35
36def list_files(files, recursive=False, extensions=None, exclude=None):
37    if extensions is None:
38        extensions = []
39    if exclude is None:
40        exclude = []
41
42    out = []
43    for file in files:
44        if recursive and os.path.isdir(file):
45            for dirpath, dnames, fnames in os.walk(file):
46                fpaths = [os.path.join(dirpath, fname) for fname in fnames]
47                for pattern in exclude:
48                    # os.walk() supports trimming down the dnames list
49                    # by modifying it in-place,
50                    # to avoid unnecessary directory listings.
51                    dnames[:] = [
52                        x for x in dnames
53                        if
54                        not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)
55                    ]
56                    fpaths = [
57                        x for x in fpaths if not fnmatch.fnmatch(x, pattern)
58                    ]
59                for f in fpaths:
60                    ext = os.path.splitext(f)[1][1:]
61                    if ext in extensions:
62                        out.append(f)
63        else:
64            out.append(file)
65    return out
66
67
68def make_diff(file, original, reformatted):
69    return list(
70        difflib.unified_diff(
71            original,
72            reformatted,
73            fromfile='{}\t(original)'.format(file),
74            tofile='{}\t(reformatted)'.format(file),
75            n=3))
76
77
78class DiffError(Exception):
79    def __init__(self, message, errs=None):
80        super(DiffError, self).__init__(message)
81        self.errs = errs or []
82
83
84class UnexpectedError(Exception):
85    def __init__(self, message, exc=None):
86        super(UnexpectedError, self).__init__(message)
87        self.formatted_traceback = traceback.format_exc()
88        self.exc = exc
89
90
91def run_clang_format_diff_wrapper(args, file):
92    try:
93        ret = run_clang_format_diff(args, file)
94        return ret
95    except DiffError:
96        raise
97    except Exception as e:
98        raise UnexpectedError('{}: {}: {}'.format(file, e.__class__.__name__,
99                                                  e), e)
100
101
102def run_clang_format_diff(args, file):
103    try:
104        with io.open(file, 'r', encoding='utf-8') as f:
105            original = f.readlines()
106    except IOError as exc:
107        raise DiffError(str(exc))
108    invocation = [args.clang_format_executable, file]
109
110    # Use of utf-8 to decode the process output.
111    #
112    # Hopefully, this is the correct thing to do.
113    #
114    # It's done due to the following assumptions (which may be incorrect):
115    # - clang-format will returns the bytes read from the files as-is,
116    #   without conversion, and it is already assumed that the files use utf-8.
117    # - if the diagnostics were internationalized, they would use utf-8:
118    #   > Adding Translations to Clang
119    #   >
120    #   > Not possible yet!
121    #   > Diagnostic strings should be written in UTF-8,
122    #   > the client can translate to the relevant code page if needed.
123    #   > Each translation completely replaces the format string
124    #   > for the diagnostic.
125    #   > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation
126    #
127    # It's not pretty, due to Python 2 & 3 compatibility.
128    encoding_py3 = {}
129    if sys.version_info[0] >= 3:
130        encoding_py3['encoding'] = 'utf-8'
131
132    try:
133        proc = subprocess.Popen(
134            invocation,
135            stdout=subprocess.PIPE,
136            stderr=subprocess.PIPE,
137            universal_newlines=True,
138            **encoding_py3)
139    except OSError as exc:
140        raise DiffError(str(exc))
141    proc_stdout = proc.stdout
142    proc_stderr = proc.stderr
143    if sys.version_info[0] < 3:
144        # make the pipes compatible with Python 3,
145        # reading lines should output unicode
146        encoding = 'utf-8'
147        proc_stdout = codecs.getreader(encoding)(proc_stdout)
148        proc_stderr = codecs.getreader(encoding)(proc_stderr)
149    # hopefully the stderr pipe won't get full and block the process
150    outs = list(proc_stdout.readlines())
151    errs = list(proc_stderr.readlines())
152    proc.wait()
153    if proc.returncode:
154        raise DiffError("clang-format exited with status {}: '{}'".format(
155            proc.returncode, file), errs)
156    return make_diff(file, original, outs), errs
157
158
159def bold_red(s):
160    return '\x1b[1m\x1b[31m' + s + '\x1b[0m'
161
162
163def colorize(diff_lines):
164    def bold(s):
165        return '\x1b[1m' + s + '\x1b[0m'
166
167    def cyan(s):
168        return '\x1b[36m' + s + '\x1b[0m'
169
170    def green(s):
171        return '\x1b[32m' + s + '\x1b[0m'
172
173    def red(s):
174        return '\x1b[31m' + s + '\x1b[0m'
175
176    for line in diff_lines:
177        if line[:4] in ['--- ', '+++ ']:
178            yield bold(line)
179        elif line.startswith('@@ '):
180            yield cyan(line)
181        elif line.startswith('+'):
182            yield green(line)
183        elif line.startswith('-'):
184            yield red(line)
185        else:
186            yield line
187
188
189def print_diff(diff_lines, use_color):
190    if use_color:
191        diff_lines = colorize(diff_lines)
192    if sys.version_info[0] < 3:
193        sys.stdout.writelines((l.encode('utf-8') for l in diff_lines))
194    else:
195        sys.stdout.writelines(diff_lines)
196
197
198def print_trouble(prog, message, use_colors):
199    error_text = 'error:'
200    if use_colors:
201        error_text = bold_red(error_text)
202    print("{}: {} {}".format(prog, error_text, message), file=sys.stderr)
203
204
205def main():
206    parser = argparse.ArgumentParser(description=__doc__)
207    parser.add_argument(
208        '--clang-format-executable',
209        metavar='EXECUTABLE',
210        help='path to the clang-format executable',
211        default='clang-format')
212    parser.add_argument(
213        '--extensions',
214        help='comma separated list of file extensions (default: {})'.format(
215            DEFAULT_EXTENSIONS),
216        default=DEFAULT_EXTENSIONS)
217    parser.add_argument(
218        '-r',
219        '--recursive',
220        action='store_true',
221        help='run recursively over directories')
222    parser.add_argument('files', metavar='file', nargs='+')
223    parser.add_argument(
224        '-q',
225        '--quiet',
226        action='store_true')
227    parser.add_argument(
228        '-j',
229        metavar='N',
230        type=int,
231        default=0,
232        help='run N clang-format jobs in parallel'
233        ' (default number of cpus + 1)')
234    parser.add_argument(
235        '--color',
236        default='auto',
237        choices=['auto', 'always', 'never'],
238        help='show colored diff (default: auto)')
239    parser.add_argument(
240        '-e',
241        '--exclude',
242        metavar='PATTERN',
243        action='append',
244        default=[],
245        help='exclude paths matching the given glob-like pattern(s)'
246        ' from recursive search')
247
248    args = parser.parse_args()
249
250    # use default signal handling, like diff return SIGINT value on ^C
251    # https://bugs.python.org/issue14229#msg156446
252    signal.signal(signal.SIGINT, signal.SIG_DFL)
253    try:
254        signal.SIGPIPE
255    except AttributeError:
256        # compatibility, SIGPIPE does not exist on Windows
257        pass
258    else:
259        signal.signal(signal.SIGPIPE, signal.SIG_DFL)
260
261    colored_stdout = False
262    colored_stderr = False
263    if args.color == 'always':
264        colored_stdout = True
265        colored_stderr = True
266    elif args.color == 'auto':
267        colored_stdout = sys.stdout.isatty()
268        colored_stderr = sys.stderr.isatty()
269
270    retcode = ExitStatus.SUCCESS
271    files = list_files(
272        args.files,
273        recursive=args.recursive,
274        exclude=args.exclude,
275        extensions=args.extensions.split(','))
276
277    if not files:
278        return
279
280    njobs = args.j
281    if njobs == 0:
282        njobs = multiprocessing.cpu_count() + 1
283    njobs = min(len(files), njobs)
284
285    if njobs == 1:
286        # execute directly instead of in a pool,
287        # less overhead, simpler stacktraces
288        it = (run_clang_format_diff_wrapper(args, file) for file in files)
289        pool = None
290    else:
291        pool = multiprocessing.Pool(njobs)
292        it = pool.imap_unordered(
293            partial(run_clang_format_diff_wrapper, args), files)
294    while True:
295        try:
296            outs, errs = next(it)
297        except StopIteration:
298            break
299        except DiffError as e:
300            print_trouble(parser.prog, str(e), use_colors=colored_stderr)
301            retcode = ExitStatus.TROUBLE
302            sys.stderr.writelines(e.errs)
303        except UnexpectedError as e:
304            print_trouble(parser.prog, str(e), use_colors=colored_stderr)
305            sys.stderr.write(e.formatted_traceback)
306            retcode = ExitStatus.TROUBLE
307            # stop at the first unexpected error,
308            # something could be very wrong,
309            # don't process all files unnecessarily
310            if pool:
311                pool.terminate()
312            break
313        else:
314            sys.stderr.writelines(errs)
315            if outs == []:
316                continue
317            if not args.quiet:
318                print_diff(outs, use_color=colored_stdout)
319            if retcode == ExitStatus.SUCCESS:
320                retcode = ExitStatus.DIFF
321    return retcode
322
323
324if __name__ == '__main__':
325    sys.exit(main())
326