1# This file is part of Buildbot. Buildbot is free software: you can 2# redistribute it and/or modify it under the terms of the GNU General Public 3# License as published by the Free Software Foundation, version 2. 4# 5# This program is distributed in the hope that it will be useful, but WITHOUT 6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8# details. 9# 10# You should have received a copy of the GNU General Public License along with 11# this program; if not, write to the Free Software Foundation, Inc., 51 12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13# 14# Copyright Buildbot Team Members 15 16import re 17 18from twisted.internet import defer 19 20from buildbot import config 21from buildbot.process import buildstep 22from buildbot.process import logobserver 23from buildbot.process.results import FAILURE 24from buildbot.process.results import SUCCESS 25from buildbot.process.results import WARNINGS 26from buildbot.process.results import Results 27 28 29class BuildEPYDoc(buildstep.ShellMixin, buildstep.BuildStep): 30 name = "epydoc" 31 command = ["make", "epydocs"] 32 description = "building epydocs" 33 descriptionDone = "epydoc" 34 35 def __init__(self, **kwargs): 36 kwargs = self.setupShellMixin(kwargs) 37 super().__init__(**kwargs) 38 self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self._log_consumer)) 39 40 def _log_consumer(self): 41 self.import_errors = 0 42 self.warnings = 0 43 self.errors = 0 44 45 while True: 46 stream, line = yield 47 if line.startswith("Error importing "): 48 self.import_errors += 1 49 if line.find("Warning: ") != -1: 50 self.warnings += 1 51 if line.find("Error: ") != -1: 52 self.errors += 1 53 54 def getResultSummary(self): 55 summary = ' '.join(self.descriptionDone) 56 if self.import_errors: 57 summary += " ierr={}".format(self.import_errors) 58 if self.warnings: 59 summary += " warn={}".format(self.warnings) 60 if self.errors: 61 summary += " err={}".format(self.errors) 62 if self.results != SUCCESS: 63 summary += ' ({})'.format(Results[self.results]) 64 return {'step': summary} 65 66 @defer.inlineCallbacks 67 def run(self): 68 cmd = yield self.makeRemoteShellCommand() 69 yield self.runCommand(cmd) 70 71 stdio_log = yield self.getLog('stdio') 72 yield stdio_log.finish() 73 74 if cmd.didFail(): 75 return FAILURE 76 if self.warnings or self.errors: 77 return WARNINGS 78 return SUCCESS 79 80 81class PyFlakes(buildstep.ShellMixin, buildstep.BuildStep): 82 name = "pyflakes" 83 command = ["make", "pyflakes"] 84 description = "running pyflakes" 85 descriptionDone = "pyflakes" 86 flunkOnFailure = False 87 88 # any pyflakes lines like this cause FAILURE 89 _flunkingIssues = ("undefined",) 90 91 _MESSAGES = ("unused", "undefined", "redefs", "import*", "misc") 92 93 def __init__(self, *args, **kwargs): 94 # PyFlakes return 1 for both warnings and errors. We 95 # categorize this initially as WARNINGS so that 96 # evaluateCommand below can inspect the results more closely. 97 kwargs['decodeRC'] = {0: SUCCESS, 1: WARNINGS} 98 99 kwargs = self.setupShellMixin(kwargs) 100 super().__init__(*args, **kwargs) 101 102 self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self._log_consumer)) 103 104 counts = self.counts = {} 105 summaries = self.summaries = {} 106 for m in self._MESSAGES: 107 counts[m] = 0 108 summaries[m] = [] 109 110 # we need a separate variable for syntax errors 111 self._hasSyntaxError = False 112 113 def _log_consumer(self): 114 counts = self.counts 115 summaries = self.summaries 116 first = True 117 while True: 118 stream, line = yield 119 if stream == 'h': 120 continue 121 # the first few lines might contain echoed commands from a 'make 122 # pyflakes' step, so don't count these as warnings. Stop ignoring 123 # the initial lines as soon as we see one with a colon. 124 if first: 125 if ':' in line: 126 # there's the colon, this is the first real line 127 first = False 128 # fall through and parse the line 129 else: 130 # skip this line, keep skipping non-colon lines 131 continue 132 133 if line.find("imported but unused") != -1: 134 m = "unused" 135 elif line.find("*' used; unable to detect undefined names") != -1: 136 m = "import*" 137 elif line.find("undefined name") != -1: 138 m = "undefined" 139 elif line.find("redefinition of unused") != -1: 140 m = "redefs" 141 elif line.find("invalid syntax") != -1: 142 self._hasSyntaxError = True 143 # we can do this, because if a syntax error occurs 144 # the output will only contain the info about it, nothing else 145 m = "misc" 146 else: 147 m = "misc" 148 149 summaries[m].append(line) 150 counts[m] += 1 151 152 def getResultSummary(self): 153 summary = ' '.join(self.descriptionDone) 154 for m in self._MESSAGES: 155 if self.counts[m]: 156 summary += " {}={}".format(m, self.counts[m]) 157 158 if self.results != SUCCESS: 159 summary += ' ({})'.format(Results[self.results]) 160 161 return {'step': summary} 162 163 @defer.inlineCallbacks 164 def run(self): 165 cmd = yield self.makeRemoteShellCommand() 166 yield self.runCommand(cmd) 167 168 stdio_log = yield self.getLog('stdio') 169 yield stdio_log.finish() 170 171 # we log 'misc' as syntax-error 172 if self._hasSyntaxError: 173 yield self.addCompleteLog("syntax-error", "\n".join(self.summaries['misc'])) 174 else: 175 for m in self._MESSAGES: 176 if self.counts[m]: 177 yield self.addCompleteLog(m, "\n".join(self.summaries[m])) 178 self.setProperty("pyflakes-{}".format(m), self.counts[m], "pyflakes") 179 self.setProperty("pyflakes-total", sum(self.counts.values()), "pyflakes") 180 181 if cmd.didFail() or self._hasSyntaxError: 182 return FAILURE 183 for m in self._flunkingIssues: 184 if m in self.counts and self.counts[m] > 0: 185 return FAILURE 186 if sum(self.counts.values()) > 0: 187 return WARNINGS 188 return SUCCESS 189 190 191class PyLint(buildstep.ShellMixin, buildstep.BuildStep): 192 193 '''A command that knows about pylint output. 194 It is a good idea to add --output-format=parseable to your 195 command, since it includes the filename in the message. 196 ''' 197 name = "pylint" 198 description = "running pylint" 199 descriptionDone = "pylint" 200 201 # pylint's return codes (see pylint(1) for details) 202 # 1 - 16 will be bit-ORed 203 204 RC_OK = 0 205 RC_FATAL = 1 206 RC_ERROR = 2 207 RC_WARNING = 4 208 RC_REFACTOR = 8 209 RC_CONVENTION = 16 210 RC_USAGE = 32 211 212 # Using the default text output, the message format is : 213 # MESSAGE_TYPE: LINE_NUM:[OBJECT:] MESSAGE 214 # with --output-format=parseable it is: (the outer brackets are literal) 215 # FILE_NAME:LINE_NUM: [MESSAGE_TYPE[, OBJECT]] MESSAGE 216 # message type consists of the type char and 4 digits 217 # The message types: 218 219 _MESSAGES = { 220 'C': "convention", # for programming standard violation 221 'R': "refactor", # for bad code smell 222 'W': "warning", # for python specific problems 223 'E': "error", # for much probably bugs in the code 224 'F': "fatal", # error prevented pylint from further processing. 225 'I': "info", 226 } 227 228 _flunkingIssues = ("F", "E") # msg categories that cause FAILURE 229 230 _msgtypes_re_str = '(?P<errtype>[{}])'.format(''.join(list(_MESSAGES))) 231 _default_line_re = re.compile(r'^{}(\d+)?: *\d+(, *\d+)?:.+'.format(_msgtypes_re_str)) 232 _default_2_0_0_line_re = \ 233 re.compile(r'^(?P<path>[^:]+):(?P<line>\d+):\d+: *{}(\d+)?:.+'.format(_msgtypes_re_str)) 234 _parseable_line_re = re.compile( 235 r'(?P<path>[^:]+):(?P<line>\d+): \[{}(\d+)?(\([a-z-]+\))?[,\]] .+'.format(_msgtypes_re_str)) 236 237 def __init__(self, store_results=True, **kwargs): 238 kwargs = self.setupShellMixin(kwargs) 239 super().__init__(**kwargs) 240 self._store_results = store_results 241 self.counts = {} 242 self.summaries = {} 243 self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self._log_consumer)) 244 245 # returns (message type, path, line) tuple if line has been matched, or None otherwise 246 def _match_line(self, line): 247 m = self._default_2_0_0_line_re.match(line) 248 if m: 249 try: 250 line_int = int(m.group('line')) 251 except ValueError: 252 line_int = None 253 return (m.group('errtype'), m.group('path'), line_int) 254 255 m = self._parseable_line_re.match(line) 256 if m: 257 try: 258 line_int = int(m.group('line')) 259 except ValueError: 260 line_int = None 261 return (m.group('errtype'), m.group('path'), line_int) 262 263 m = self._default_line_re.match(line) 264 if m: 265 return (m.group('errtype'), None, None) 266 267 return None 268 269 def _log_consumer(self): 270 for m in self._MESSAGES: 271 self.counts[m] = 0 272 self.summaries[m] = [] 273 274 while True: 275 stream, line = yield 276 if stream == 'h': 277 continue 278 279 ret = self._match_line(line) 280 if not ret: 281 continue 282 283 msgtype, path, line_number = ret 284 285 assert msgtype in self._MESSAGES 286 self.summaries[msgtype].append(line) 287 self.counts[msgtype] += 1 288 289 if self._store_results and path is not None: 290 self.addTestResult(self._result_setid, line, test_name=None, test_code_path=path, 291 line=line_number) 292 293 def getResultSummary(self): 294 summary = ' '.join(self.descriptionDone) 295 for msg, fullmsg in sorted(self._MESSAGES.items()): 296 if self.counts[msg]: 297 summary += " {}={}".format(fullmsg, self.counts[msg]) 298 299 if self.results != SUCCESS: 300 summary += ' ({})'.format(Results[self.results]) 301 302 return {'step': summary} 303 304 @defer.inlineCallbacks 305 def run(self): 306 cmd = yield self.makeRemoteShellCommand() 307 yield self.runCommand(cmd) 308 309 stdio_log = yield self.getLog('stdio') 310 yield stdio_log.finish() 311 312 for msg, fullmsg in sorted(self._MESSAGES.items()): 313 if self.counts[msg]: 314 yield self.addCompleteLog(fullmsg, "\n".join(self.summaries[msg])) 315 self.setProperty("pylint-{}".format(fullmsg), self.counts[msg], 'Pylint') 316 self.setProperty("pylint-total", sum(self.counts.values()), 'Pylint') 317 318 if cmd.rc & (self.RC_FATAL | self.RC_ERROR | self.RC_USAGE): 319 return FAILURE 320 321 for msg in self._flunkingIssues: 322 if msg in self.counts and self.counts[msg] > 0: 323 return FAILURE 324 if sum(self.counts.values()) > 0: 325 return WARNINGS 326 return SUCCESS 327 328 @defer.inlineCallbacks 329 def addTestResultSets(self): 330 if not self._store_results: 331 return 332 self._result_setid = yield self.addTestResultSet('Pylint warnings', 'code_issue', 'message') 333 334 335class Sphinx(buildstep.ShellMixin, buildstep.BuildStep): 336 337 ''' A Step to build sphinx documentation ''' 338 339 name = "sphinx" 340 description = "running sphinx" 341 descriptionDone = "sphinx" 342 343 haltOnFailure = True 344 345 def __init__(self, sphinx_sourcedir='.', sphinx_builddir=None, 346 sphinx_builder=None, sphinx='sphinx-build', tags=None, 347 defines=None, strict_warnings=False, mode='incremental', **kwargs): 348 349 if tags is None: 350 tags = [] 351 352 if defines is None: 353 defines = {} 354 355 if sphinx_builddir is None: 356 # Who the heck is not interested in the built doc ? 357 config.error("Sphinx argument sphinx_builddir is required") 358 359 if mode not in ('incremental', 'full'): 360 config.error("Sphinx argument mode has to be 'incremental' or" + 361 "'full' is required") 362 363 self.success = False 364 365 kwargs = self.setupShellMixin(kwargs) 366 367 super().__init__(**kwargs) 368 369 # build the command 370 command = [sphinx] 371 if sphinx_builder is not None: 372 command.extend(['-b', sphinx_builder]) 373 374 for tag in tags: 375 command.extend(['-t', tag]) 376 377 for key in sorted(defines): 378 if defines[key] is None: 379 command.extend(['-D', key]) 380 elif isinstance(defines[key], bool): 381 command.extend(['-D', 382 '{}={}'.format(key, defines[key] and 1 or 0)]) 383 else: 384 command.extend(['-D', '{}={}'.format(key, defines[key])]) 385 386 if mode == 'full': 387 command.extend(['-E']) # Don't use a saved environment 388 389 if strict_warnings: 390 command.extend(['-W']) # Convert warnings to errors 391 392 command.extend([sphinx_sourcedir, sphinx_builddir]) 393 self.command = command 394 395 self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self._log_consumer)) 396 397 _msgs = ('WARNING', 'ERROR', 'SEVERE') 398 399 def _log_consumer(self): 400 self.warnings = [] 401 next_is_warning = False 402 403 while True: 404 stream, line = yield 405 if line.startswith('build succeeded') or \ 406 line.startswith('no targets are out of date.'): 407 self.success = True 408 elif line.startswith('Warning, treated as error:'): 409 next_is_warning = True 410 else: 411 if next_is_warning: 412 self.warnings.append(line) 413 next_is_warning = False 414 else: 415 for msg in self._msgs: 416 if msg in line: 417 self.warnings.append(line) 418 419 def getResultSummary(self): 420 summary = '{} {} warnings'.format(self.name, len(self.warnings)) 421 422 if self.results != SUCCESS: 423 summary += ' ({})'.format(Results[self.results]) 424 425 return {'step': summary} 426 427 @defer.inlineCallbacks 428 def run(self): 429 cmd = yield self.makeRemoteShellCommand() 430 yield self.runCommand(cmd) 431 432 stdio_log = yield self.getLog('stdio') 433 yield stdio_log.finish() 434 435 if self.warnings: 436 yield self.addCompleteLog('warnings', "\n".join(self.warnings)) 437 438 self.setStatistic('warnings', len(self.warnings)) 439 440 if self.success: 441 if not self.warnings: 442 return SUCCESS 443 return WARNINGS 444 return FAILURE 445