1#!/usr/local/bin/python3.8 2# Eclipse SUMO, Simulation of Urban MObility; see https://eclipse.org/sumo 3# Copyright (C) 2010-2019 German Aerospace Center (DLR) and others. 4# This program and the accompanying materials 5# are made available under the terms of the Eclipse Public License v2.0 6# which accompanies this distribution, and is available at 7# http://www.eclipse.org/legal/epl-v20.html 8# SPDX-License-Identifier: EPL-2.0 9 10# @file checkStyle.py 11# @author Michael Behrisch 12# @date 2010-08-29 13# @version $Id$ 14 15""" 16Checks svn property settings for all files and pep8 for python 17as well as file headers. 18""" 19from __future__ import absolute_import 20from __future__ import print_function 21 22import os 23import subprocess 24import xml.sax 25import codecs 26from optparse import OptionParser 27try: 28 import flake8 # noqa 29 HAVE_FLAKE = True 30except ImportError: 31 HAVE_FLAKE = False 32try: 33 import autopep8 # noqa 34 HAVE_AUTOPEP = True 35except ImportError: 36 HAVE_AUTOPEP = False 37 38_SOURCE_EXT = [".h", ".cpp", ".py", ".pyw", ".pl", ".java", ".am", ".cs"] 39_TESTDATA_EXT = [".xml", ".prog", ".csv", 40 ".complex", ".dfrouter", ".duarouter", ".jtrrouter", ".marouter", 41 ".astar", ".chrouter", ".internal", ".tcl", ".txt", 42 ".netconvert", ".netedit", ".netgen", 43 ".od2trips", ".polyconvert", ".sumo", 44 ".meso", ".tools", ".traci", ".activitygen", 45 ".scenario", ".tapasVEU", 46 ".sumocfg", ".netccfg", ".netgcfg"] 47_VS_EXT = [".vsprops", ".sln", ".vcproj", 48 ".bat", ".props", ".vcxproj", ".filters"] 49_IGNORE = set(["binstate.sumo", "binstate.sumo.meso", "image.tools"]) 50_KEYWORDS = "HeadURL Id LastChangedBy LastChangedDate LastChangedRevision" 51 52SEPARATOR = "/****************************************************************************/\n" 53EPL_HEADER = """/****************************************************************************/ 54// Eclipse SUMO, Simulation of Urban MObility; see https://eclipse.org/sumo 55// Copyright (C) 2001-2019 German Aerospace Center (DLR) and others. 56// This program and the accompanying materials 57// are made available under the terms of the Eclipse Public License v2.0 58// which accompanies this distribution, and is available at 59// http://www.eclipse.org/legal/epl-v20.html 60// SPDX-License-Identifier: EPL-2.0 61/****************************************************************************/ 62""" 63 64 65class PropertyReader(xml.sax.handler.ContentHandler): 66 67 """Reads the svn properties of files as written by svn pl -v --xml""" 68 69 def __init__(self, doFix, doPep): 70 self._fix = doFix 71 self._pep = doPep 72 self._file = "" 73 self._property = None 74 self._value = "" 75 self._hadEOL = False 76 self._hadKeywords = False 77 self._haveFixed = False 78 79 def checkDoxyLines(self, lines, idx, comment="///"): 80 fileRef = "%s @file %s\n" % (comment, os.path.basename(self._file)) 81 try: 82 s = lines[idx].split() 83 if s != fileRef.split(): 84 print(self._file, "broken @file reference", lines[idx].rstrip()) 85 if self._fix and lines[idx].startswith("%s @file" % comment): 86 lines[idx] = fileRef 87 self._haveFixed = True 88 idx += 1 89 s = lines[idx].split() 90 if s[:2] != [comment, "@author"]: 91 print(self._file, "broken @author reference", s) 92 idx += 1 93 while lines[idx].split()[:2] == [comment, "@author"]: 94 idx += 1 95 s = lines[idx].split() 96 if s[:2] != [comment, "@date"]: 97 print(self._file, "broken @date reference", s) 98 idx += 1 99 s = lines[idx].split() 100 if s[:2] != [comment, "@version"]: 101 print(self._file, "broken @version reference", s) 102 idx += 1 103 if lines[idx] not in (comment + "\n", "\n"): 104 print(self._file, "missing empty line", idx, lines[idx].rstrip()) 105 except IndexError: 106 print(self._file, "seems to be empty") 107 idx += 1 108 while idx < len(lines) and lines[idx].split()[:1] == ["//"]: 109 idx += 1 110 return idx 111 112 def checkFileHeader(self, ext): 113 lines = open(self._file).readlines() 114 if len(lines) == 0: 115 print(self._file, "is empty") 116 return 117 self._haveFixed = False 118 idx = 0 119 if ext in (".cpp", ".h"): 120 if lines[idx] == SEPARATOR: 121 year = lines[idx + 2][17:21] 122 end = idx + 9 123 license = EPL_HEADER.replace("2001", year) 124 if "module" in lines[idx + 3]: 125 end += 2 126 fileLicense = "".join(lines[idx:idx + 3]) + "".join(lines[idx + 5:end]) 127 else: 128 fileLicense = "".join(lines[idx:end]) 129 if fileLicense != license: 130 print(self._file, "invalid license") 131 if options.verbose: 132 print(fileLicense) 133 print(license) 134 self.checkDoxyLines(lines, end) 135 else: 136 print(self._file, "header does not start") 137 if ext in (".py", ".pyw"): 138 if lines[0][:2] == '#!': 139 idx += 1 140 if lines[0] != '#!/usr/bin/env python\n': 141 print(self._file, "wrong shebang") 142 if self._fix: 143 lines[0] = '#!/usr/bin/env python\n' 144 self._haveFixed = True 145 if lines[idx][:5] == '# -*-': 146 idx += 1 147 license = EPL_HEADER.replace("// ", "# ").replace("// ", "# ").replace("\n//", "") 148 end = lines.index("\n", idx) 149 if len(lines) < 13: 150 print(self._file, "is too short (%s lines, at least 13 required for valid header)" % len(lines)) 151 return 152 year = lines[idx + 1][16:20] 153 license = license.replace("2001", year).replace(SEPARATOR, "") 154 if "module" in lines[idx + 2]: 155 fileLicense = "".join(lines[idx:idx + 2]) + "".join(lines[idx + 4:end]) 156 else: 157 fileLicense = "".join(lines[idx:end]) 158 if fileLicense != license: 159 print(self._file, "different license:") 160 print(fileLicense) 161 if options.verbose: 162 print("!!%s!!" % os.path.commonprefix([fileLicense, license])) 163 print(license) 164 self.checkDoxyLines(lines, end + 1, "#") 165 if self._haveFixed: 166 open(self._file, "w").write("".join(lines)) 167 168 def startElement(self, name, attrs): 169 if name == 'target': 170 self._file = attrs['path'] 171 seen.add(os.path.join(repoRoot, self._file)) 172 if name == 'property': 173 self._property = attrs['name'] 174 175 def characters(self, content): 176 if self._property: 177 self._value += content 178 179 def endElement(self, name): 180 ext = os.path.splitext(self._file)[1] 181 if name == 'property' and self._property == "svn:eol-style": 182 self._hadEOL = True 183 if name == 'property' and self._property == "svn:keywords": 184 self._hadKeywords = True 185 if os.path.basename(self._file) not in _IGNORE: 186 if ext in _SOURCE_EXT or ext in _TESTDATA_EXT or ext in _VS_EXT: 187 if (name == 'property' and self._property == "svn:executable" and 188 ext not in (".py", ".pyw", ".pl", ".bat")): 189 print(self._file, self._property, self._value) 190 if self._fix: 191 subprocess.call( 192 ["svn", "pd", "svn:executable", self._file]) 193 if name == 'property' and self._property == "svn:mime-type": 194 print(self._file, self._property, self._value) 195 if self._fix: 196 subprocess.call( 197 ["svn", "pd", "svn:mime-type", self._file]) 198 if ext in _SOURCE_EXT or ext in _TESTDATA_EXT: 199 if ((name == 'property' and self._property == "svn:eol-style" and self._value != "LF") or 200 (name == "target" and not self._hadEOL)): 201 print(self._file, "svn:eol-style", self._value) 202 if self._fix: 203 if os.name == "posix": 204 subprocess.call( 205 ["sed", "-i", r's/\r$//', self._file]) 206 subprocess.call( 207 ["sed", "-i", r's/\r/\n/g', self._file]) 208 subprocess.call( 209 ["svn", "ps", "svn:eol-style", "LF", self._file]) 210 if ext in _SOURCE_EXT: 211 if ((name == 'property' and self._property == "svn:keywords" and self._value != _KEYWORDS) or 212 (name == "target" and not self._hadKeywords)): 213 print(self._file, "svn:keywords", self._value) 214 if self._fix: 215 subprocess.call( 216 ["svn", "ps", "svn:keywords", _KEYWORDS, self._file]) 217 if name == 'target': 218 self.checkFile() 219 if ext in _VS_EXT: 220 if ((name == 'property' and self._property == "svn:eol-style" and self._value != "CRLF") or 221 (name == "target" and not self._hadEOL)): 222 print(self._file, "svn:eol-style", self._value) 223 if self._fix: 224 subprocess.call( 225 ["svn", "ps", "svn:eol-style", "CRLF", self._file]) 226 if name == 'property': 227 self._value = "" 228 self._property = None 229 if name == 'target': 230 self._hadEOL = False 231 self._hadKeywords = False 232 233 def checkFile(self, fileName=None, exclude=None): 234 if fileName is not None: 235 self._file = fileName 236 ext = os.path.splitext(self._file)[1] 237 try: 238 codecs.open(self._file, 'r', 'utf8').read() 239 except UnicodeDecodeError as e: 240 print(self._file, e) 241 self.checkFileHeader(ext) 242 if exclude: 243 for x in exclude: 244 if x + "/" in self._file: 245 return 246 if self._pep and ext == ".py": 247 if HAVE_FLAKE and os.path.getsize(self._file) < 1000000: # flake hangs on very large files 248 subprocess.call(["flake8", "--max-line-length", "120", self._file]) 249 if HAVE_AUTOPEP and self._fix: 250 subprocess.call(["autopep8", "--max-line-length", "120", "--in-place", self._file]) 251 252 253optParser = OptionParser() 254optParser.add_option("-v", "--verbose", action="store_true", 255 default=False, help="tell me what you are doing") 256optParser.add_option("-f", "--fix", action="store_true", 257 default=False, help="fix invalid svn properties") 258optParser.add_option("-s", "--skip-pep", action="store_true", 259 default=False, help="skip autopep8 and flake8 tests") 260optParser.add_option("-d", "--directory", help="check given subdirectory of sumo tree") 261optParser.add_option("-x", "--exclude", default="contributed", 262 help="comma-separated list of (sub-)paths to exclude from pep checks") 263(options, args) = optParser.parse_args() 264seen = set() 265sumoRoot = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 266if len(args) > 0: 267 repoRoots = [os.path.abspath(a) for a in args] 268elif options.directory: 269 repoRoots = [os.path.join(sumoRoot, options.directory)] 270else: 271 repoRoots = [sumoRoot] 272for repoRoot in repoRoots: 273 if options.verbose: 274 print("checking", repoRoot) 275 propRead = PropertyReader(options.fix, not options.skip_pep) 276 try: 277 oldDir = os.getcwd() 278 os.chdir(repoRoot) 279 exclude = options.exclude.split(",") 280 for name in subprocess.check_output(["git", "ls-files"]).splitlines(): 281 ext = os.path.splitext(name)[1] 282 if ext in _SOURCE_EXT: 283 propRead.checkFile(name, exclude) 284 os.chdir(oldDir) 285 continue 286 except (OSError, subprocess.CalledProcessError) as e: 287 print("This seems to be no valid git repository", repoRoot, e) 288 if options.verbose: 289 print("trying svn at", repoRoot) 290 output = subprocess.check_output(["svn", "pl", "-v", "-R", "--xml", repoRoot]) 291 xml.sax.parseString(output, propRead) 292 if options.verbose: 293 print("re-checking tree at", repoRoot) 294 for root, dirs, files in os.walk(repoRoot): 295 for name in files: 296 ext = os.path.splitext(name)[1] 297 if name not in _IGNORE: 298 fullName = os.path.join(root, name) 299 if ext in _SOURCE_EXT or ext in _TESTDATA_EXT or ext in _VS_EXT: 300 if fullName in seen or subprocess.call(["svn", "ls", fullName], 301 stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT): 302 continue 303 print(fullName, "svn:eol-style") 304 if options.fix: 305 if ext in _VS_EXT: 306 subprocess.call( 307 ["svn", "ps", "svn:eol-style", "CRLF", fullName]) 308 else: 309 if os.name == "posix": 310 subprocess.call(["sed", "-i", 's/\r$//', fullName]) 311 subprocess.call( 312 ["svn", "ps", "svn:eol-style", "LF", fullName]) 313 if ext in _SOURCE_EXT: 314 print(fullName, "svn:keywords") 315 if options.fix: 316 subprocess.call( 317 ["svn", "ps", "svn:keywords", _KEYWORDS, fullName]) 318 for ignoreDir in ['.svn', 'foreign', 'contributed', 'texttesttmp']: 319 if ignoreDir in dirs: 320 dirs.remove(ignoreDir) 321