1#!/usr/bin/env python
2# Copyright (c) 2013, AT&T Labs, Yun Mao <yunmao@gmail.com>
3# All Rights Reserved.
4#
5#    Licensed under the Apache License, Version 2.0 (the "License"); you may
6#    not use this file except in compliance with the License. You may obtain
7#    a copy of the License at
8#
9#         http://www.apache.org/licenses/LICENSE-2.0
10#
11#    Unless required by applicable law or agreed to in writing, software
12#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14#    License for the specific language governing permissions and limitations
15#    under the License.
16
17"""pylint error checking."""
18
19import json
20import re
21import sys
22
23from pylint import lint
24from pylint.reporters import text
25from six.moves import cStringIO as StringIO
26
27ignore_codes = [
28    # Note(maoy): E1103 is error code related to partial type inference
29    "E1103"
30]
31
32ignore_messages = [
33    # Note(fengqian): this message is the pattern of [E0611].
34    "No name 'urllib' in module '_MovedItems'",
35
36    # Note(xyang): these error messages are for the code [E1101].
37    # They should be ignored because 'sha256' and 'sha224' are functions in
38    # 'hashlib'.
39    "Module 'hashlib' has no 'sha256' member",
40    "Module 'hashlib' has no 'sha224' member",
41
42    # six.moves
43    "Instance of '_MovedItems' has no 'builtins' member",
44
45    # This error message is for code [E1101]
46    "Instance of 'ResourceFilterManager' has no '_list' member",
47]
48
49ignore_modules = ["cinderclient/tests/"]
50
51KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions"
52
53
54class LintOutput(object):
55
56    _cached_filename = None
57    _cached_content = None
58
59    def __init__(self, filename, lineno, line_content, code, message,
60                 lintoutput):
61        self.filename = filename
62        self.lineno = lineno
63        self.line_content = line_content
64        self.code = code
65        self.message = message
66        self.lintoutput = lintoutput
67
68    @classmethod
69    def from_line(cls, line):
70        m = re.search(r"(\S+):(\d+): \[(\S+)(, \S+)?] (.*)", line)
71        if m is None:
72            return None
73        matched = m.groups()
74        filename, lineno, code, message = (matched[0], int(matched[1]),
75                                           matched[2], matched[-1])
76        if cls._cached_filename != filename:
77            with open(filename) as f:
78                cls._cached_content = list(f.readlines())
79                cls._cached_filename = filename
80        line_content = cls._cached_content[lineno - 1].rstrip()
81        return cls(filename, lineno, line_content, code, message,
82                   line.rstrip())
83
84    @classmethod
85    def from_msg_to_dict(cls, msg):
86        """Convert pylint output to a dict.
87
88        From the output of pylint msg, to a dict, where each key
89        is a unique error identifier, value is a list of LintOutput
90        """
91        result = {}
92        for line in msg.splitlines():
93            obj = cls.from_line(line)
94            if obj is None or obj.is_ignored():
95                continue
96            key = obj.key()
97            if key not in result:
98                result[key] = []
99            result[key].append(obj)
100        return result
101
102    def is_ignored(self):
103        if self.code in ignore_codes:
104            return True
105        if any(self.filename.startswith(name) for name in ignore_modules):
106            return True
107        if any(msg in self.message for msg in ignore_messages):
108            return True
109        return False
110
111    def key(self):
112        if self.code in ["E1101", "E1103"]:
113            # These two types of errors are like Foo class has no member bar.
114            # We discard the source code so that the error will be ignored
115            # next time another Foo.bar is encountered.
116            return self.message, ""
117        return self.message, self.line_content.strip()
118
119    def json(self):
120        return json.dumps(self.__dict__)
121
122    def review_str(self):
123        return ("File %(filename)s\nLine %(lineno)d:%(line_content)s\n"
124                "%(code)s: %(message)s" %
125                {'filename': self.filename,
126                 'lineno': self.lineno,
127                 'line_content': self.line_content,
128                 'code': self.code,
129                 'message': self.message})
130
131
132class ErrorKeys(object):
133
134    @classmethod
135    def print_json(cls, errors, output=sys.stdout):
136        print("# automatically generated by tools/lintstack.py", file=output)
137        for i in sorted(errors.keys()):
138            print(json.dumps(i), file=output)
139
140    @classmethod
141    def from_file(cls, filename):
142        keys = set()
143        for line in open(filename):
144            if line and line[0] != "#":
145                d = json.loads(line)
146                keys.add(tuple(d))
147        return keys
148
149
150def run_pylint():
151    buff = StringIO()
152    reporter = text.TextReporter(output=buff)
153    args = [
154        "--msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}'",
155        "-E", "cinderclient"]
156    lint.Run(args, reporter=reporter, exit=False)
157    val = buff.getvalue()
158    buff.close()
159    return val
160
161
162def generate_error_keys(msg=None):
163    print("Generating", KNOWN_PYLINT_EXCEPTIONS_FILE)
164    if msg is None:
165        msg = run_pylint()
166    errors = LintOutput.from_msg_to_dict(msg)
167    with open(KNOWN_PYLINT_EXCEPTIONS_FILE, "w") as f:
168        ErrorKeys.print_json(errors, output=f)
169
170
171def validate(newmsg=None):
172    print("Loading", KNOWN_PYLINT_EXCEPTIONS_FILE)
173    known = ErrorKeys.from_file(KNOWN_PYLINT_EXCEPTIONS_FILE)
174    if newmsg is None:
175        print("Running pylint. Be patient...")
176        newmsg = run_pylint()
177    errors = LintOutput.from_msg_to_dict(newmsg)
178
179    print("Unique errors reported by pylint: was %d, now %d."
180          % (len(known), len(errors)))
181    passed = True
182    for err_key, err_list in errors.items():
183        for err in err_list:
184            if err_key not in known:
185                print(err.lintoutput)
186                print()
187                passed = False
188    if passed:
189        print("Congrats! pylint check passed.")
190        redundant = known - set(errors.keys())
191        if redundant:
192            print("Extra credit: some known pylint exceptions disappeared.")
193            for i in sorted(redundant):
194                print(json.dumps(i))
195            print("Consider regenerating the exception file if you will.")
196    else:
197        print("Please fix the errors above. If you believe they are false "
198              "positives, run 'tools/lintstack.py generate' to overwrite.")
199        sys.exit(1)
200
201
202def usage():
203    print("""Usage: tools/lintstack.py [generate|validate]
204    To generate pylint_exceptions file: tools/lintstack.py generate
205    To validate the current commit: tools/lintstack.py
206    """)
207
208
209def main():
210    option = "validate"
211    if len(sys.argv) > 1:
212        option = sys.argv[1]
213    if option == "generate":
214        generate_error_keys()
215    elif option == "validate":
216        validate()
217    else:
218        usage()
219
220
221if __name__ == "__main__":
222    main()
223