1#!/usr/bin/env python 2# Copyright (c) 2013, AT&T Labs, Yun Mao <yunmao@gmail.com> 3# All Rights Reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); you may 6# not use this file except in compliance with the License. You may obtain 7# a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14# License for the specific language governing permissions and limitations 15# under the License. 16 17"""pylint error checking.""" 18 19import json 20import re 21import sys 22 23from pylint import lint 24from pylint.reporters import text 25from six.moves import cStringIO as StringIO 26 27ignore_codes = [ 28 # Note(maoy): E1103 is error code related to partial type inference 29 "E1103" 30] 31 32ignore_messages = [ 33 # Note(fengqian): this message is the pattern of [E0611]. 34 "No name 'urllib' in module '_MovedItems'", 35 36 # Note(xyang): these error messages are for the code [E1101]. 37 # They should be ignored because 'sha256' and 'sha224' are functions in 38 # 'hashlib'. 39 "Module 'hashlib' has no 'sha256' member", 40 "Module 'hashlib' has no 'sha224' member", 41 42 # six.moves 43 "Instance of '_MovedItems' has no 'builtins' member", 44 45 # This error message is for code [E1101] 46 "Instance of 'ResourceFilterManager' has no '_list' member", 47] 48 49ignore_modules = ["cinderclient/tests/"] 50 51KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions" 52 53 54class LintOutput(object): 55 56 _cached_filename = None 57 _cached_content = None 58 59 def __init__(self, filename, lineno, line_content, code, message, 60 lintoutput): 61 self.filename = filename 62 self.lineno = lineno 63 self.line_content = line_content 64 self.code = code 65 self.message = message 66 self.lintoutput = lintoutput 67 68 @classmethod 69 def from_line(cls, line): 70 m = re.search(r"(\S+):(\d+): \[(\S+)(, \S+)?] (.*)", line) 71 if m is None: 72 return None 73 matched = m.groups() 74 filename, lineno, code, message = (matched[0], int(matched[1]), 75 matched[2], matched[-1]) 76 if cls._cached_filename != filename: 77 with open(filename) as f: 78 cls._cached_content = list(f.readlines()) 79 cls._cached_filename = filename 80 line_content = cls._cached_content[lineno - 1].rstrip() 81 return cls(filename, lineno, line_content, code, message, 82 line.rstrip()) 83 84 @classmethod 85 def from_msg_to_dict(cls, msg): 86 """Convert pylint output to a dict. 87 88 From the output of pylint msg, to a dict, where each key 89 is a unique error identifier, value is a list of LintOutput 90 """ 91 result = {} 92 for line in msg.splitlines(): 93 obj = cls.from_line(line) 94 if obj is None or obj.is_ignored(): 95 continue 96 key = obj.key() 97 if key not in result: 98 result[key] = [] 99 result[key].append(obj) 100 return result 101 102 def is_ignored(self): 103 if self.code in ignore_codes: 104 return True 105 if any(self.filename.startswith(name) for name in ignore_modules): 106 return True 107 if any(msg in self.message for msg in ignore_messages): 108 return True 109 return False 110 111 def key(self): 112 if self.code in ["E1101", "E1103"]: 113 # These two types of errors are like Foo class has no member bar. 114 # We discard the source code so that the error will be ignored 115 # next time another Foo.bar is encountered. 116 return self.message, "" 117 return self.message, self.line_content.strip() 118 119 def json(self): 120 return json.dumps(self.__dict__) 121 122 def review_str(self): 123 return ("File %(filename)s\nLine %(lineno)d:%(line_content)s\n" 124 "%(code)s: %(message)s" % 125 {'filename': self.filename, 126 'lineno': self.lineno, 127 'line_content': self.line_content, 128 'code': self.code, 129 'message': self.message}) 130 131 132class ErrorKeys(object): 133 134 @classmethod 135 def print_json(cls, errors, output=sys.stdout): 136 print("# automatically generated by tools/lintstack.py", file=output) 137 for i in sorted(errors.keys()): 138 print(json.dumps(i), file=output) 139 140 @classmethod 141 def from_file(cls, filename): 142 keys = set() 143 for line in open(filename): 144 if line and line[0] != "#": 145 d = json.loads(line) 146 keys.add(tuple(d)) 147 return keys 148 149 150def run_pylint(): 151 buff = StringIO() 152 reporter = text.TextReporter(output=buff) 153 args = [ 154 "--msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}'", 155 "-E", "cinderclient"] 156 lint.Run(args, reporter=reporter, exit=False) 157 val = buff.getvalue() 158 buff.close() 159 return val 160 161 162def generate_error_keys(msg=None): 163 print("Generating", KNOWN_PYLINT_EXCEPTIONS_FILE) 164 if msg is None: 165 msg = run_pylint() 166 errors = LintOutput.from_msg_to_dict(msg) 167 with open(KNOWN_PYLINT_EXCEPTIONS_FILE, "w") as f: 168 ErrorKeys.print_json(errors, output=f) 169 170 171def validate(newmsg=None): 172 print("Loading", KNOWN_PYLINT_EXCEPTIONS_FILE) 173 known = ErrorKeys.from_file(KNOWN_PYLINT_EXCEPTIONS_FILE) 174 if newmsg is None: 175 print("Running pylint. Be patient...") 176 newmsg = run_pylint() 177 errors = LintOutput.from_msg_to_dict(newmsg) 178 179 print("Unique errors reported by pylint: was %d, now %d." 180 % (len(known), len(errors))) 181 passed = True 182 for err_key, err_list in errors.items(): 183 for err in err_list: 184 if err_key not in known: 185 print(err.lintoutput) 186 print() 187 passed = False 188 if passed: 189 print("Congrats! pylint check passed.") 190 redundant = known - set(errors.keys()) 191 if redundant: 192 print("Extra credit: some known pylint exceptions disappeared.") 193 for i in sorted(redundant): 194 print(json.dumps(i)) 195 print("Consider regenerating the exception file if you will.") 196 else: 197 print("Please fix the errors above. If you believe they are false " 198 "positives, run 'tools/lintstack.py generate' to overwrite.") 199 sys.exit(1) 200 201 202def usage(): 203 print("""Usage: tools/lintstack.py [generate|validate] 204 To generate pylint_exceptions file: tools/lintstack.py generate 205 To validate the current commit: tools/lintstack.py 206 """) 207 208 209def main(): 210 option = "validate" 211 if len(sys.argv) > 1: 212 option = sys.argv[1] 213 if option == "generate": 214 generate_error_keys() 215 elif option == "validate": 216 validate() 217 else: 218 usage() 219 220 221if __name__ == "__main__": 222 main() 223