1#!/usr/bin/env python3
2# pylint: disable=protected-access, unused-variable, locally-disabled, len-as-condition
3"""Lint helper to generate lint summary of source.
4
5Copyright by Contributors
6"""
7from __future__ import print_function
8import argparse
9import codecs
10import sys
11import re
12import os
13import cpplint
14from cpplint import _cpplint_state
15from pylint import epylint
16
17CXX_SUFFIX = set(['cc', 'c', 'cpp', 'h', 'cu', 'hpp'])
18PYTHON_SUFFIX = set(['py'])
19
20def filepath_enumerate(paths):
21    """Enumerate the file paths of all subfiles of the list of paths"""
22    out = []
23    for path in paths:
24        if os.path.isfile(path):
25            out.append(path)
26        else:
27            for root, dirs, files in os.walk(path):
28                for name in files:
29                    out.append(os.path.normpath(os.path.join(root, name)))
30    return out
31
32# pylint: disable=useless-object-inheritance
33class LintHelper(object):
34    """Class to help runing the lint and records summary"""
35
36    @staticmethod
37    def _print_summary_map(strm, result_map, ftype):
38        """Print summary of certain result map."""
39        if len(result_map) == 0:
40            return 0
41        npass = len([x for k, x in result_map.items() if len(x) == 0])
42        strm.write('=====%d/%d %s files passed check=====\n' % (npass, len(result_map), ftype))
43        for fname, emap in result_map.items():
44            if len(emap) == 0:
45                continue
46            strm.write('%s: %d Errors of %d Categories map=%s\n' % (
47                fname, sum(emap.values()), len(emap), str(emap)))
48        return len(result_map) - npass
49
50    def __init__(self):
51        self.project_name = None
52        self.cpp_header_map = {}
53        self.cpp_src_map = {}
54        self.python_map = {}
55        pylint_disable = ['superfluous-parens',
56                          'too-many-instance-attributes',
57                          'too-few-public-methods']
58        # setup pylint
59        self.pylint_opts = ['--extension-pkg-whitelist=numpy',
60                            '--disable=' + ','.join(pylint_disable)]
61
62        self.pylint_cats = set(['error', 'warning', 'convention', 'refactor'])
63        # setup cpp lint
64        cpplint_args = ['.', '--extensions=' + (','.join(CXX_SUFFIX))]
65        _ = cpplint.ParseArguments(cpplint_args)
66        cpplint._SetFilters(','.join(['-build/c++11',
67                                      '-build/namespaces',
68                                      '-build/include,',
69                                      '+build/include_what_you_use',
70                                      '+build/include_order']))
71        cpplint._SetCountingStyle('toplevel')
72        cpplint._line_length = 100
73
74    def process_cpp(self, path, suffix):
75        """Process a cpp file."""
76        _cpplint_state.ResetErrorCounts()
77        cpplint.ProcessFile(str(path), _cpplint_state.verbose_level)
78        _cpplint_state.PrintErrorCounts()
79        errors = _cpplint_state.errors_by_category.copy()
80
81        if suffix == 'h':
82            self.cpp_header_map[str(path)] = errors
83        else:
84            self.cpp_src_map[str(path)] = errors
85
86    def process_python(self, path):
87        """Process a python file."""
88        (pylint_stdout, pylint_stderr) = epylint.py_run(
89            ' '.join([str(path)] + self.pylint_opts), return_std=True)
90        emap = {}
91        err = pylint_stderr.read()
92        if len(err):
93            print(err)
94        for line in pylint_stdout:
95            sys.stderr.write(line)
96            key = line.split(':')[-1].split('(')[0].strip()
97            if key not in self.pylint_cats:
98                continue
99            if key not in emap:
100                emap[key] = 1
101            else:
102                emap[key] += 1
103        self.python_map[str(path)] = emap
104
105    def print_summary(self, strm):
106        """Print summary of lint."""
107        nerr = 0
108        nerr += LintHelper._print_summary_map(strm, self.cpp_header_map, 'cpp-header')
109        nerr += LintHelper._print_summary_map(strm, self.cpp_src_map, 'cpp-source')
110        nerr += LintHelper._print_summary_map(strm, self.python_map, 'python')
111        if nerr == 0:
112            strm.write('All passed!\n')
113        else:
114            strm.write('%d files failed lint\n' % nerr)
115        return nerr
116
117# singleton helper for lint check
118_HELPER = LintHelper()
119
120def get_header_guard_dmlc(filename):
121    """Get Header Guard Convention for DMLC Projects.
122
123    For headers in include, directly use the path
124    For headers in src, use project name plus path
125
126    Examples: with project-name = dmlc
127        include/dmlc/timer.h -> DMLC_TIMTER_H_
128        src/io/libsvm_parser.h -> DMLC_IO_LIBSVM_PARSER_H_
129    """
130    fileinfo = cpplint.FileInfo(filename)
131    file_path_from_root = fileinfo.RepositoryName()
132    inc_list = ['include', 'api', 'wrapper', 'contrib']
133    if os.name == 'nt':
134        inc_list.append("mshadow")
135
136    if file_path_from_root.find('src/') != -1 and _HELPER.project_name is not None:
137        idx = file_path_from_root.find('src/')
138        file_path_from_root = _HELPER.project_name +  file_path_from_root[idx + 3:]
139    else:
140        idx = file_path_from_root.find("include/")
141        if idx != -1:
142            file_path_from_root = file_path_from_root[idx + 8:]
143        for spath in inc_list:
144            prefix = spath + '/'
145            if file_path_from_root.startswith(prefix):
146                file_path_from_root = re.sub('^' + prefix, '', file_path_from_root)
147                break
148    return re.sub(r'[-./\s]', '_', file_path_from_root).upper() + '_'
149
150cpplint.GetHeaderGuardCPPVariable = get_header_guard_dmlc
151
152def process(fname, allow_type):
153    """Process a file."""
154    fname = str(fname)
155    arr = fname.rsplit('.', 1)
156    if fname.find('#') != -1 or arr[-1] not in allow_type:
157        return
158    if arr[-1] in CXX_SUFFIX:
159        _HELPER.process_cpp(fname, arr[-1])
160    if arr[-1] in PYTHON_SUFFIX:
161        _HELPER.process_python(fname)
162
163def main():
164    """Main entry function."""
165    parser = argparse.ArgumentParser(description="lint source codes")
166    parser.add_argument('project', help='project name')
167    parser.add_argument('filetype', choices=['python', 'cpp', 'all'],
168                        help='source code type')
169    parser.add_argument('path', nargs='+', help='path to traverse')
170    parser.add_argument('--exclude_path', nargs='+', default=[],
171                        help='exclude this path, and all subfolders if path is a folder')
172    parser.add_argument('--pylint-rc', default=None,
173                        help='pylint rc file')
174    args = parser.parse_args()
175
176    _HELPER.project_name = args.project
177    if args.pylint_rc is not None:
178        _HELPER.pylint_opts = ['--rcfile='+args.pylint_rc,]
179    file_type = args.filetype
180    allow_type = []
181    if file_type in ('python', 'all'):
182        allow_type += PYTHON_SUFFIX
183    if file_type in ('cpp', 'all'):
184        allow_type += CXX_SUFFIX
185    allow_type = set(allow_type)
186    if sys.version_info.major == 2 and os.name != 'nt':
187        sys.stderr = codecs.StreamReaderWriter(sys.stderr,
188                                               codecs.getreader('utf8'),
189                                               codecs.getwriter('utf8'),
190                                               'replace')
191    # get excluded files
192    excluded_paths = filepath_enumerate(args.exclude_path)
193    for path in args.path:
194        if os.path.isfile(path):
195            normpath = os.path.normpath(path)
196            if normpath not in excluded_paths:
197                process(path, allow_type)
198        else:
199            for root, dirs, files in os.walk(path):
200                for name in files:
201                    file_path = os.path.normpath(os.path.join(root, name))
202                    if file_path not in excluded_paths:
203                        process(file_path, allow_type)
204    nerr = _HELPER.print_summary(sys.stderr)
205    sys.exit(nerr > 0)
206
207if __name__ == '__main__':
208    main()
209