1#!/usr/bin/env python3 2# pylint: disable=protected-access, unused-variable, locally-disabled, len-as-condition 3"""Lint helper to generate lint summary of source. 4 5Copyright by Contributors 6""" 7from __future__ import print_function 8import argparse 9import codecs 10import sys 11import re 12import os 13import cpplint 14from cpplint import _cpplint_state 15from pylint import epylint 16 17CXX_SUFFIX = set(['cc', 'c', 'cpp', 'h', 'cu', 'hpp']) 18PYTHON_SUFFIX = set(['py']) 19 20def filepath_enumerate(paths): 21 """Enumerate the file paths of all subfiles of the list of paths""" 22 out = [] 23 for path in paths: 24 if os.path.isfile(path): 25 out.append(path) 26 else: 27 for root, dirs, files in os.walk(path): 28 for name in files: 29 out.append(os.path.normpath(os.path.join(root, name))) 30 return out 31 32# pylint: disable=useless-object-inheritance 33class LintHelper(object): 34 """Class to help runing the lint and records summary""" 35 36 @staticmethod 37 def _print_summary_map(strm, result_map, ftype): 38 """Print summary of certain result map.""" 39 if len(result_map) == 0: 40 return 0 41 npass = len([x for k, x in result_map.items() if len(x) == 0]) 42 strm.write('=====%d/%d %s files passed check=====\n' % (npass, len(result_map), ftype)) 43 for fname, emap in result_map.items(): 44 if len(emap) == 0: 45 continue 46 strm.write('%s: %d Errors of %d Categories map=%s\n' % ( 47 fname, sum(emap.values()), len(emap), str(emap))) 48 return len(result_map) - npass 49 50 def __init__(self): 51 self.project_name = None 52 self.cpp_header_map = {} 53 self.cpp_src_map = {} 54 self.python_map = {} 55 pylint_disable = ['superfluous-parens', 56 'too-many-instance-attributes', 57 'too-few-public-methods'] 58 # setup pylint 59 self.pylint_opts = ['--extension-pkg-whitelist=numpy', 60 '--disable=' + ','.join(pylint_disable)] 61 62 self.pylint_cats = set(['error', 'warning', 'convention', 'refactor']) 63 # setup cpp lint 64 cpplint_args = ['.', '--extensions=' + (','.join(CXX_SUFFIX))] 65 _ = cpplint.ParseArguments(cpplint_args) 66 cpplint._SetFilters(','.join(['-build/c++11', 67 '-build/namespaces', 68 '-build/include,', 69 '+build/include_what_you_use', 70 '+build/include_order'])) 71 cpplint._SetCountingStyle('toplevel') 72 cpplint._line_length = 100 73 74 def process_cpp(self, path, suffix): 75 """Process a cpp file.""" 76 _cpplint_state.ResetErrorCounts() 77 cpplint.ProcessFile(str(path), _cpplint_state.verbose_level) 78 _cpplint_state.PrintErrorCounts() 79 errors = _cpplint_state.errors_by_category.copy() 80 81 if suffix == 'h': 82 self.cpp_header_map[str(path)] = errors 83 else: 84 self.cpp_src_map[str(path)] = errors 85 86 def process_python(self, path): 87 """Process a python file.""" 88 (pylint_stdout, pylint_stderr) = epylint.py_run( 89 ' '.join([str(path)] + self.pylint_opts), return_std=True) 90 emap = {} 91 err = pylint_stderr.read() 92 if len(err): 93 print(err) 94 for line in pylint_stdout: 95 sys.stderr.write(line) 96 key = line.split(':')[-1].split('(')[0].strip() 97 if key not in self.pylint_cats: 98 continue 99 if key not in emap: 100 emap[key] = 1 101 else: 102 emap[key] += 1 103 self.python_map[str(path)] = emap 104 105 def print_summary(self, strm): 106 """Print summary of lint.""" 107 nerr = 0 108 nerr += LintHelper._print_summary_map(strm, self.cpp_header_map, 'cpp-header') 109 nerr += LintHelper._print_summary_map(strm, self.cpp_src_map, 'cpp-source') 110 nerr += LintHelper._print_summary_map(strm, self.python_map, 'python') 111 if nerr == 0: 112 strm.write('All passed!\n') 113 else: 114 strm.write('%d files failed lint\n' % nerr) 115 return nerr 116 117# singleton helper for lint check 118_HELPER = LintHelper() 119 120def get_header_guard_dmlc(filename): 121 """Get Header Guard Convention for DMLC Projects. 122 123 For headers in include, directly use the path 124 For headers in src, use project name plus path 125 126 Examples: with project-name = dmlc 127 include/dmlc/timer.h -> DMLC_TIMTER_H_ 128 src/io/libsvm_parser.h -> DMLC_IO_LIBSVM_PARSER_H_ 129 """ 130 fileinfo = cpplint.FileInfo(filename) 131 file_path_from_root = fileinfo.RepositoryName() 132 inc_list = ['include', 'api', 'wrapper', 'contrib'] 133 if os.name == 'nt': 134 inc_list.append("mshadow") 135 136 if file_path_from_root.find('src/') != -1 and _HELPER.project_name is not None: 137 idx = file_path_from_root.find('src/') 138 file_path_from_root = _HELPER.project_name + file_path_from_root[idx + 3:] 139 else: 140 idx = file_path_from_root.find("include/") 141 if idx != -1: 142 file_path_from_root = file_path_from_root[idx + 8:] 143 for spath in inc_list: 144 prefix = spath + '/' 145 if file_path_from_root.startswith(prefix): 146 file_path_from_root = re.sub('^' + prefix, '', file_path_from_root) 147 break 148 return re.sub(r'[-./\s]', '_', file_path_from_root).upper() + '_' 149 150cpplint.GetHeaderGuardCPPVariable = get_header_guard_dmlc 151 152def process(fname, allow_type): 153 """Process a file.""" 154 fname = str(fname) 155 arr = fname.rsplit('.', 1) 156 if fname.find('#') != -1 or arr[-1] not in allow_type: 157 return 158 if arr[-1] in CXX_SUFFIX: 159 _HELPER.process_cpp(fname, arr[-1]) 160 if arr[-1] in PYTHON_SUFFIX: 161 _HELPER.process_python(fname) 162 163def main(): 164 """Main entry function.""" 165 parser = argparse.ArgumentParser(description="lint source codes") 166 parser.add_argument('project', help='project name') 167 parser.add_argument('filetype', choices=['python', 'cpp', 'all'], 168 help='source code type') 169 parser.add_argument('path', nargs='+', help='path to traverse') 170 parser.add_argument('--exclude_path', nargs='+', default=[], 171 help='exclude this path, and all subfolders if path is a folder') 172 parser.add_argument('--pylint-rc', default=None, 173 help='pylint rc file') 174 args = parser.parse_args() 175 176 _HELPER.project_name = args.project 177 if args.pylint_rc is not None: 178 _HELPER.pylint_opts = ['--rcfile='+args.pylint_rc,] 179 file_type = args.filetype 180 allow_type = [] 181 if file_type in ('python', 'all'): 182 allow_type += PYTHON_SUFFIX 183 if file_type in ('cpp', 'all'): 184 allow_type += CXX_SUFFIX 185 allow_type = set(allow_type) 186 if sys.version_info.major == 2 and os.name != 'nt': 187 sys.stderr = codecs.StreamReaderWriter(sys.stderr, 188 codecs.getreader('utf8'), 189 codecs.getwriter('utf8'), 190 'replace') 191 # get excluded files 192 excluded_paths = filepath_enumerate(args.exclude_path) 193 for path in args.path: 194 if os.path.isfile(path): 195 normpath = os.path.normpath(path) 196 if normpath not in excluded_paths: 197 process(path, allow_type) 198 else: 199 for root, dirs, files in os.walk(path): 200 for name in files: 201 file_path = os.path.normpath(os.path.join(root, name)) 202 if file_path not in excluded_paths: 203 process(file_path, allow_type) 204 nerr = _HELPER.print_summary(sys.stderr) 205 sys.exit(nerr > 0) 206 207if __name__ == '__main__': 208 main() 209