1#!/usr/bin/env python
2"""A wrapper script around clang-format, suitable for linting multiple files
3and to use for continuous integration.
4
5This is an alternative API for the clang-format command line.
6It runs over multiple files and directories in parallel.
7A diff output is produced and a sensible exit code is returned.
8
9"""
10
11from __future__ import print_function, unicode_literals
12
13import argparse
14import codecs
15import difflib
16import fnmatch
17import io
18import errno
19import multiprocessing
20import os
21import signal
22import subprocess
23import sys
24import traceback
25
26from functools import partial
27
28try:
29    from subprocess import DEVNULL  # py3k
30except ImportError:
31    DEVNULL = open(os.devnull, "wb")
32
33
34DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx'
35DEFAULT_CLANG_FORMAT_IGNORE = '.clang-format-ignore'
36
37
38class ExitStatus:
39    SUCCESS = 0
40    DIFF = 1
41    TROUBLE = 2
42
43def excludes_from_file(ignore_file):
44    excludes = []
45    try:
46        with io.open(ignore_file, 'r', encoding='utf-8') as f:
47            for line in f:
48                if line.startswith('#'):
49                    # ignore comments
50                    continue
51                pattern = line.rstrip()
52                if not pattern:
53                    # allow empty lines
54                    continue
55                excludes.append(pattern)
56    except EnvironmentError as e:
57        if e.errno != errno.ENOENT:
58            raise
59    return excludes;
60
61def list_files(files, recursive=False, extensions=None, exclude=None):
62    if extensions is None:
63        extensions = []
64    if exclude is None:
65        exclude = []
66
67    out = []
68    for file in files:
69        if recursive and os.path.isdir(file):
70            for dirpath, dnames, fnames in os.walk(file):
71                fpaths = [os.path.join(dirpath, fname) for fname in fnames]
72                for pattern in exclude:
73                    # os.walk() supports trimming down the dnames list
74                    # by modifying it in-place,
75                    # to avoid unnecessary directory listings.
76                    dnames[:] = [
77                        x for x in dnames
78                        if
79                        not fnmatch.fnmatch(os.path.join(dirpath, x), pattern)
80                    ]
81                    fpaths = [
82                        x for x in fpaths if not fnmatch.fnmatch(x, pattern)
83                    ]
84                for f in fpaths:
85                    ext = os.path.splitext(f)[1][1:]
86                    if ext in extensions:
87                        out.append(f)
88        else:
89            out.append(file)
90    return out
91
92
93def make_diff(file, original, reformatted):
94    return list(
95        difflib.unified_diff(
96            original,
97            reformatted,
98            fromfile='{}\t(original)'.format(file),
99            tofile='{}\t(reformatted)'.format(file),
100            n=3))
101
102
103class DiffError(Exception):
104    def __init__(self, message, errs=None):
105        super(DiffError, self).__init__(message)
106        self.errs = errs or []
107
108
109class UnexpectedError(Exception):
110    def __init__(self, message, exc=None):
111        super(UnexpectedError, self).__init__(message)
112        self.formatted_traceback = traceback.format_exc()
113        self.exc = exc
114
115
116def run_clang_format_diff_wrapper(args, file):
117    try:
118        ret = run_clang_format_diff(args, file)
119        return ret
120    except DiffError:
121        raise
122    except Exception as e:
123        raise UnexpectedError('{}: {}: {}'.format(file, e.__class__.__name__,
124                                                  e), e)
125
126
127def run_clang_format_diff(args, file):
128    try:
129        with io.open(file, 'r', encoding='utf-8') as f:
130            original = f.readlines()
131    except IOError as exc:
132        raise DiffError(str(exc))
133    invocation = [args.clang_format_executable, file]
134
135    # Use of utf-8 to decode the process output.
136    #
137    # Hopefully, this is the correct thing to do.
138    #
139    # It's done due to the following assumptions (which may be incorrect):
140    # - clang-format will returns the bytes read from the files as-is,
141    #   without conversion, and it is already assumed that the files use utf-8.
142    # - if the diagnostics were internationalized, they would use utf-8:
143    #   > Adding Translations to Clang
144    #   >
145    #   > Not possible yet!
146    #   > Diagnostic strings should be written in UTF-8,
147    #   > the client can translate to the relevant code page if needed.
148    #   > Each translation completely replaces the format string
149    #   > for the diagnostic.
150    #   > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation
151    #
152    # It's not pretty, due to Python 2 & 3 compatibility.
153    encoding_py3 = {}
154    if sys.version_info[0] >= 3:
155        encoding_py3['encoding'] = 'utf-8'
156
157    try:
158        proc = subprocess.Popen(
159            invocation,
160            stdout=subprocess.PIPE,
161            stderr=subprocess.PIPE,
162            universal_newlines=True,
163            **encoding_py3)
164    except OSError as exc:
165        raise DiffError(
166            "Command '{}' failed to start: {}".format(
167                subprocess.list2cmdline(invocation), exc
168            )
169        )
170    proc_stdout = proc.stdout
171    proc_stderr = proc.stderr
172    if sys.version_info[0] < 3:
173        # make the pipes compatible with Python 3,
174        # reading lines should output unicode
175        encoding = 'utf-8'
176        proc_stdout = codecs.getreader(encoding)(proc_stdout)
177        proc_stderr = codecs.getreader(encoding)(proc_stderr)
178    # hopefully the stderr pipe won't get full and block the process
179    outs = list(proc_stdout.readlines())
180    errs = list(proc_stderr.readlines())
181    proc.wait()
182    if proc.returncode:
183        raise DiffError(
184            "Command '{}' returned non-zero exit status {}".format(
185                subprocess.list2cmdline(invocation), proc.returncode
186            ),
187            errs,
188        )
189    return make_diff(file, original, outs), errs
190
191
192def bold_red(s):
193    return '\x1b[1m\x1b[31m' + s + '\x1b[0m'
194
195
196def colorize(diff_lines):
197    def bold(s):
198        return '\x1b[1m' + s + '\x1b[0m'
199
200    def cyan(s):
201        return '\x1b[36m' + s + '\x1b[0m'
202
203    def green(s):
204        return '\x1b[32m' + s + '\x1b[0m'
205
206    def red(s):
207        return '\x1b[31m' + s + '\x1b[0m'
208
209    for line in diff_lines:
210        if line[:4] in ['--- ', '+++ ']:
211            yield bold(line)
212        elif line.startswith('@@ '):
213            yield cyan(line)
214        elif line.startswith('+'):
215            yield green(line)
216        elif line.startswith('-'):
217            yield red(line)
218        else:
219            yield line
220
221
222def print_diff(diff_lines, use_color):
223    if use_color:
224        diff_lines = colorize(diff_lines)
225    if sys.version_info[0] < 3:
226        sys.stdout.writelines((l.encode('utf-8') for l in diff_lines))
227    else:
228        sys.stdout.writelines(diff_lines)
229
230
231def print_trouble(prog, message, use_colors):
232    error_text = 'error:'
233    if use_colors:
234        error_text = bold_red(error_text)
235    print("{}: {} {}".format(prog, error_text, message), file=sys.stderr)
236
237
238def main():
239    parser = argparse.ArgumentParser(description=__doc__)
240    parser.add_argument(
241        '--clang-format-executable',
242        metavar='EXECUTABLE',
243        help='path to the clang-format executable',
244        default='clang-format')
245    parser.add_argument(
246        '--extensions',
247        help='comma separated list of file extensions (default: {})'.format(
248            DEFAULT_EXTENSIONS),
249        default=DEFAULT_EXTENSIONS)
250    parser.add_argument(
251        '-r',
252        '--recursive',
253        action='store_true',
254        help='run recursively over directories')
255    parser.add_argument('files', metavar='file', nargs='+')
256    parser.add_argument(
257        '-q',
258        '--quiet',
259        action='store_true',
260        help="disable output, useful for the exit code")
261    parser.add_argument(
262        '-j',
263        metavar='N',
264        type=int,
265        default=0,
266        help='run N clang-format jobs in parallel'
267        ' (default number of cpus + 1)')
268    parser.add_argument(
269        '--color',
270        default='auto',
271        choices=['auto', 'always', 'never'],
272        help='show colored diff (default: auto)')
273    parser.add_argument(
274        '-e',
275        '--exclude',
276        metavar='PATTERN',
277        action='append',
278        default=[],
279        help='exclude paths matching the given glob-like pattern(s)'
280        ' from recursive search')
281
282    args = parser.parse_args()
283
284    # use default signal handling, like diff return SIGINT value on ^C
285    # https://bugs.python.org/issue14229#msg156446
286    signal.signal(signal.SIGINT, signal.SIG_DFL)
287    try:
288        signal.SIGPIPE
289    except AttributeError:
290        # compatibility, SIGPIPE does not exist on Windows
291        pass
292    else:
293        signal.signal(signal.SIGPIPE, signal.SIG_DFL)
294
295    colored_stdout = False
296    colored_stderr = False
297    if args.color == 'always':
298        colored_stdout = True
299        colored_stderr = True
300    elif args.color == 'auto':
301        colored_stdout = sys.stdout.isatty()
302        colored_stderr = sys.stderr.isatty()
303
304    version_invocation = [args.clang_format_executable, str("--version")]
305    try:
306        subprocess.check_call(version_invocation, stdout=DEVNULL)
307    except subprocess.CalledProcessError as e:
308        print_trouble(parser.prog, str(e), use_colors=colored_stderr)
309        return ExitStatus.TROUBLE
310    except OSError as e:
311        print_trouble(
312            parser.prog,
313            "Command '{}' failed to start: {}".format(
314                subprocess.list2cmdline(version_invocation), e
315            ),
316            use_colors=colored_stderr,
317        )
318        return ExitStatus.TROUBLE
319
320    retcode = ExitStatus.SUCCESS
321
322    excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE)
323    excludes.extend(args.exclude)
324
325    files = list_files(
326        args.files,
327        recursive=args.recursive,
328        exclude=excludes,
329        extensions=args.extensions.split(','))
330
331    if not files:
332        return
333
334    njobs = args.j
335    if njobs == 0:
336        njobs = multiprocessing.cpu_count() + 1
337    njobs = min(len(files), njobs)
338
339    if njobs == 1:
340        # execute directly instead of in a pool,
341        # less overhead, simpler stacktraces
342        it = (run_clang_format_diff_wrapper(args, file) for file in files)
343        pool = None
344    else:
345        pool = multiprocessing.Pool(njobs)
346        it = pool.imap_unordered(
347            partial(run_clang_format_diff_wrapper, args), files)
348    while True:
349        try:
350            outs, errs = next(it)
351        except StopIteration:
352            break
353        except DiffError as e:
354            print_trouble(parser.prog, str(e), use_colors=colored_stderr)
355            retcode = ExitStatus.TROUBLE
356            sys.stderr.writelines(e.errs)
357        except UnexpectedError as e:
358            print_trouble(parser.prog, str(e), use_colors=colored_stderr)
359            sys.stderr.write(e.formatted_traceback)
360            retcode = ExitStatus.TROUBLE
361            # stop at the first unexpected error,
362            # something could be very wrong,
363            # don't process all files unnecessarily
364            if pool:
365                pool.terminate()
366            break
367        else:
368            sys.stderr.writelines(errs)
369            if outs == []:
370                continue
371            if not args.quiet:
372                print_diff(outs, use_color=colored_stdout)
373            if retcode == ExitStatus.SUCCESS:
374                retcode = ExitStatus.DIFF
375    return retcode
376
377
378if __name__ == '__main__':
379    sys.exit(main())
380