1#!/usr/local/bin/python3.8
2# Eclipse SUMO, Simulation of Urban MObility; see https://eclipse.org/sumo
3# Copyright (C) 2010-2019 German Aerospace Center (DLR) and others.
4# This program and the accompanying materials
5# are made available under the terms of the Eclipse Public License v2.0
6# which accompanies this distribution, and is available at
7# http://www.eclipse.org/legal/epl-v20.html
8# SPDX-License-Identifier: EPL-2.0
9
10# @file    checkStyle.py
11# @author  Michael Behrisch
12# @date    2010-08-29
13# @version $Id$
14
15"""
16Checks svn property settings for all files and pep8 for python
17as well as file headers.
18"""
19from __future__ import absolute_import
20from __future__ import print_function
21
22import os
23import subprocess
24import xml.sax
25import codecs
26from optparse import OptionParser
27try:
28    import flake8  # noqa
29    HAVE_FLAKE = True
30except ImportError:
31    HAVE_FLAKE = False
32try:
33    import autopep8  # noqa
34    HAVE_AUTOPEP = True
35except ImportError:
36    HAVE_AUTOPEP = False
37
38_SOURCE_EXT = [".h", ".cpp", ".py", ".pyw", ".pl", ".java", ".am", ".cs"]
39_TESTDATA_EXT = [".xml", ".prog", ".csv",
40                 ".complex", ".dfrouter", ".duarouter", ".jtrrouter", ".marouter",
41                 ".astar", ".chrouter", ".internal", ".tcl", ".txt",
42                 ".netconvert", ".netedit", ".netgen",
43                 ".od2trips", ".polyconvert", ".sumo",
44                 ".meso", ".tools", ".traci", ".activitygen",
45                 ".scenario", ".tapasVEU",
46                 ".sumocfg", ".netccfg", ".netgcfg"]
47_VS_EXT = [".vsprops", ".sln", ".vcproj",
48           ".bat", ".props", ".vcxproj", ".filters"]
49_IGNORE = set(["binstate.sumo", "binstate.sumo.meso", "image.tools"])
50_KEYWORDS = "HeadURL Id LastChangedBy LastChangedDate LastChangedRevision"
51
52SEPARATOR = "/****************************************************************************/\n"
53EPL_HEADER = """/****************************************************************************/
54// Eclipse SUMO, Simulation of Urban MObility; see https://eclipse.org/sumo
55// Copyright (C) 2001-2019 German Aerospace Center (DLR) and others.
56// This program and the accompanying materials
57// are made available under the terms of the Eclipse Public License v2.0
58// which accompanies this distribution, and is available at
59// http://www.eclipse.org/legal/epl-v20.html
60// SPDX-License-Identifier: EPL-2.0
61/****************************************************************************/
62"""
63
64
65class PropertyReader(xml.sax.handler.ContentHandler):
66
67    """Reads the svn properties of files as written by svn pl -v --xml"""
68
69    def __init__(self, doFix, doPep):
70        self._fix = doFix
71        self._pep = doPep
72        self._file = ""
73        self._property = None
74        self._value = ""
75        self._hadEOL = False
76        self._hadKeywords = False
77        self._haveFixed = False
78
79    def checkDoxyLines(self, lines, idx, comment="///"):
80        fileRef = "%s @file    %s\n" % (comment, os.path.basename(self._file))
81        try:
82            s = lines[idx].split()
83            if s != fileRef.split():
84                print(self._file, "broken @file reference", lines[idx].rstrip())
85                if self._fix and lines[idx].startswith("%s @file" % comment):
86                    lines[idx] = fileRef
87                    self._haveFixed = True
88            idx += 1
89            s = lines[idx].split()
90            if s[:2] != [comment, "@author"]:
91                print(self._file, "broken @author reference", s)
92            idx += 1
93            while lines[idx].split()[:2] == [comment, "@author"]:
94                idx += 1
95            s = lines[idx].split()
96            if s[:2] != [comment, "@date"]:
97                print(self._file, "broken @date reference", s)
98            idx += 1
99            s = lines[idx].split()
100            if s[:2] != [comment, "@version"]:
101                print(self._file, "broken @version reference", s)
102            idx += 1
103            if lines[idx] not in (comment + "\n", "\n"):
104                print(self._file, "missing empty line", idx, lines[idx].rstrip())
105        except IndexError:
106            print(self._file, "seems to be empty")
107        idx += 1
108        while idx < len(lines) and lines[idx].split()[:1] == ["//"]:
109            idx += 1
110        return idx
111
112    def checkFileHeader(self, ext):
113        lines = open(self._file).readlines()
114        if len(lines) == 0:
115            print(self._file, "is empty")
116            return
117        self._haveFixed = False
118        idx = 0
119        if ext in (".cpp", ".h"):
120            if lines[idx] == SEPARATOR:
121                year = lines[idx + 2][17:21]
122                end = idx + 9
123                license = EPL_HEADER.replace("2001", year)
124                if "module" in lines[idx + 3]:
125                    end += 2
126                    fileLicense = "".join(lines[idx:idx + 3]) + "".join(lines[idx + 5:end])
127                else:
128                    fileLicense = "".join(lines[idx:end])
129                if fileLicense != license:
130                    print(self._file, "invalid license")
131                    if options.verbose:
132                        print(fileLicense)
133                        print(license)
134                self.checkDoxyLines(lines, end)
135            else:
136                print(self._file, "header does not start")
137        if ext in (".py", ".pyw"):
138            if lines[0][:2] == '#!':
139                idx += 1
140                if lines[0] != '#!/usr/bin/env python\n':
141                    print(self._file, "wrong shebang")
142                    if self._fix:
143                        lines[0] = '#!/usr/bin/env python\n'
144                        self._haveFixed = True
145            if lines[idx][:5] == '# -*-':
146                idx += 1
147            license = EPL_HEADER.replace("//   ", "# ").replace("// ", "# ").replace("\n//", "")
148            end = lines.index("\n", idx)
149            if len(lines) < 13:
150                print(self._file, "is too short (%s lines, at least 13 required for valid header)" % len(lines))
151                return
152            year = lines[idx + 1][16:20]
153            license = license.replace("2001", year).replace(SEPARATOR, "")
154            if "module" in lines[idx + 2]:
155                fileLicense = "".join(lines[idx:idx + 2]) + "".join(lines[idx + 4:end])
156            else:
157                fileLicense = "".join(lines[idx:end])
158            if fileLicense != license:
159                print(self._file, "different license:")
160                print(fileLicense)
161                if options.verbose:
162                    print("!!%s!!" % os.path.commonprefix([fileLicense, license]))
163                    print(license)
164            self.checkDoxyLines(lines, end + 1, "#")
165        if self._haveFixed:
166            open(self._file, "w").write("".join(lines))
167
168    def startElement(self, name, attrs):
169        if name == 'target':
170            self._file = attrs['path']
171            seen.add(os.path.join(repoRoot, self._file))
172        if name == 'property':
173            self._property = attrs['name']
174
175    def characters(self, content):
176        if self._property:
177            self._value += content
178
179    def endElement(self, name):
180        ext = os.path.splitext(self._file)[1]
181        if name == 'property' and self._property == "svn:eol-style":
182            self._hadEOL = True
183        if name == 'property' and self._property == "svn:keywords":
184            self._hadKeywords = True
185        if os.path.basename(self._file) not in _IGNORE:
186            if ext in _SOURCE_EXT or ext in _TESTDATA_EXT or ext in _VS_EXT:
187                if (name == 'property' and self._property == "svn:executable" and
188                        ext not in (".py", ".pyw", ".pl", ".bat")):
189                    print(self._file, self._property, self._value)
190                    if self._fix:
191                        subprocess.call(
192                            ["svn", "pd", "svn:executable", self._file])
193                if name == 'property' and self._property == "svn:mime-type":
194                    print(self._file, self._property, self._value)
195                    if self._fix:
196                        subprocess.call(
197                            ["svn", "pd", "svn:mime-type", self._file])
198            if ext in _SOURCE_EXT or ext in _TESTDATA_EXT:
199                if ((name == 'property' and self._property == "svn:eol-style" and self._value != "LF") or
200                        (name == "target" and not self._hadEOL)):
201                    print(self._file, "svn:eol-style", self._value)
202                    if self._fix:
203                        if os.name == "posix":
204                            subprocess.call(
205                                ["sed", "-i", r's/\r$//', self._file])
206                            subprocess.call(
207                                ["sed", "-i", r's/\r/\n/g', self._file])
208                        subprocess.call(
209                            ["svn", "ps", "svn:eol-style", "LF", self._file])
210            if ext in _SOURCE_EXT:
211                if ((name == 'property' and self._property == "svn:keywords" and self._value != _KEYWORDS) or
212                        (name == "target" and not self._hadKeywords)):
213                    print(self._file, "svn:keywords", self._value)
214                    if self._fix:
215                        subprocess.call(
216                            ["svn", "ps", "svn:keywords", _KEYWORDS, self._file])
217                if name == 'target':
218                    self.checkFile()
219            if ext in _VS_EXT:
220                if ((name == 'property' and self._property == "svn:eol-style" and self._value != "CRLF") or
221                        (name == "target" and not self._hadEOL)):
222                    print(self._file, "svn:eol-style", self._value)
223                    if self._fix:
224                        subprocess.call(
225                            ["svn", "ps", "svn:eol-style", "CRLF", self._file])
226        if name == 'property':
227            self._value = ""
228            self._property = None
229        if name == 'target':
230            self._hadEOL = False
231            self._hadKeywords = False
232
233    def checkFile(self, fileName=None, exclude=None):
234        if fileName is not None:
235            self._file = fileName
236        ext = os.path.splitext(self._file)[1]
237        try:
238            codecs.open(self._file, 'r', 'utf8').read()
239        except UnicodeDecodeError as e:
240            print(self._file, e)
241        self.checkFileHeader(ext)
242        if exclude:
243            for x in exclude:
244                if x + "/" in self._file:
245                    return
246        if self._pep and ext == ".py":
247            if HAVE_FLAKE and os.path.getsize(self._file) < 1000000:  # flake hangs on very large files
248                subprocess.call(["flake8", "--max-line-length", "120", self._file])
249            if HAVE_AUTOPEP and self._fix:
250                subprocess.call(["autopep8", "--max-line-length", "120", "--in-place", self._file])
251
252
253optParser = OptionParser()
254optParser.add_option("-v", "--verbose", action="store_true",
255                     default=False, help="tell me what you are doing")
256optParser.add_option("-f", "--fix", action="store_true",
257                     default=False, help="fix invalid svn properties")
258optParser.add_option("-s", "--skip-pep", action="store_true",
259                     default=False, help="skip autopep8 and flake8 tests")
260optParser.add_option("-d", "--directory", help="check given subdirectory of sumo tree")
261optParser.add_option("-x", "--exclude", default="contributed",
262                     help="comma-separated list of (sub-)paths to exclude from pep checks")
263(options, args) = optParser.parse_args()
264seen = set()
265sumoRoot = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
266if len(args) > 0:
267    repoRoots = [os.path.abspath(a) for a in args]
268elif options.directory:
269    repoRoots = [os.path.join(sumoRoot, options.directory)]
270else:
271    repoRoots = [sumoRoot]
272for repoRoot in repoRoots:
273    if options.verbose:
274        print("checking", repoRoot)
275    propRead = PropertyReader(options.fix, not options.skip_pep)
276    try:
277        oldDir = os.getcwd()
278        os.chdir(repoRoot)
279        exclude = options.exclude.split(",")
280        for name in subprocess.check_output(["git", "ls-files"]).splitlines():
281            ext = os.path.splitext(name)[1]
282            if ext in _SOURCE_EXT:
283                propRead.checkFile(name, exclude)
284        os.chdir(oldDir)
285        continue
286    except (OSError, subprocess.CalledProcessError) as e:
287        print("This seems to be no valid git repository", repoRoot, e)
288        if options.verbose:
289            print("trying svn at", repoRoot)
290        output = subprocess.check_output(["svn", "pl", "-v", "-R", "--xml", repoRoot])
291        xml.sax.parseString(output, propRead)
292    if options.verbose:
293        print("re-checking tree at", repoRoot)
294    for root, dirs, files in os.walk(repoRoot):
295        for name in files:
296            ext = os.path.splitext(name)[1]
297            if name not in _IGNORE:
298                fullName = os.path.join(root, name)
299                if ext in _SOURCE_EXT or ext in _TESTDATA_EXT or ext in _VS_EXT:
300                    if fullName in seen or subprocess.call(["svn", "ls", fullName],
301                                                           stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT):
302                        continue
303                    print(fullName, "svn:eol-style")
304                    if options.fix:
305                        if ext in _VS_EXT:
306                            subprocess.call(
307                                ["svn", "ps", "svn:eol-style", "CRLF", fullName])
308                        else:
309                            if os.name == "posix":
310                                subprocess.call(["sed", "-i", 's/\r$//', fullName])
311                            subprocess.call(
312                                ["svn", "ps", "svn:eol-style", "LF", fullName])
313                if ext in _SOURCE_EXT:
314                    print(fullName, "svn:keywords")
315                    if options.fix:
316                        subprocess.call(
317                            ["svn", "ps", "svn:keywords", _KEYWORDS, fullName])
318        for ignoreDir in ['.svn', 'foreign', 'contributed', 'texttesttmp']:
319            if ignoreDir in dirs:
320                dirs.remove(ignoreDir)
321