1# This file is part of Buildbot.  Buildbot is free software: you can
2# redistribute it and/or modify it under the terms of the GNU General Public
3# License as published by the Free Software Foundation, version 2.
4#
5# This program is distributed in the hope that it will be useful, but WITHOUT
6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8# details.
9#
10# You should have received a copy of the GNU General Public License along with
11# this program; if not, write to the Free Software Foundation, Inc., 51
12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13#
14# Copyright Buildbot Team Members
15
16import re
17
18from twisted.internet import defer
19
20from buildbot import config
21from buildbot.process import buildstep
22from buildbot.process import logobserver
23from buildbot.process.results import FAILURE
24from buildbot.process.results import SUCCESS
25from buildbot.process.results import WARNINGS
26from buildbot.process.results import Results
27
28
29class BuildEPYDoc(buildstep.ShellMixin, buildstep.BuildStep):
30    name = "epydoc"
31    command = ["make", "epydocs"]
32    description = "building epydocs"
33    descriptionDone = "epydoc"
34
35    def __init__(self, **kwargs):
36        kwargs = self.setupShellMixin(kwargs)
37        super().__init__(**kwargs)
38        self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self._log_consumer))
39
40    def _log_consumer(self):
41        self.import_errors = 0
42        self.warnings = 0
43        self.errors = 0
44
45        while True:
46            stream, line = yield
47            if line.startswith("Error importing "):
48                self.import_errors += 1
49            if line.find("Warning: ") != -1:
50                self.warnings += 1
51            if line.find("Error: ") != -1:
52                self.errors += 1
53
54    def getResultSummary(self):
55        summary = ' '.join(self.descriptionDone)
56        if self.import_errors:
57            summary += " ierr={}".format(self.import_errors)
58        if self.warnings:
59            summary += " warn={}".format(self.warnings)
60        if self.errors:
61            summary += " err={}".format(self.errors)
62        if self.results != SUCCESS:
63            summary += ' ({})'.format(Results[self.results])
64        return {'step': summary}
65
66    @defer.inlineCallbacks
67    def run(self):
68        cmd = yield self.makeRemoteShellCommand()
69        yield self.runCommand(cmd)
70
71        stdio_log = yield self.getLog('stdio')
72        yield stdio_log.finish()
73
74        if cmd.didFail():
75            return FAILURE
76        if self.warnings or self.errors:
77            return WARNINGS
78        return SUCCESS
79
80
81class PyFlakes(buildstep.ShellMixin, buildstep.BuildStep):
82    name = "pyflakes"
83    command = ["make", "pyflakes"]
84    description = "running pyflakes"
85    descriptionDone = "pyflakes"
86    flunkOnFailure = False
87
88    # any pyflakes lines like this cause FAILURE
89    _flunkingIssues = ("undefined",)
90
91    _MESSAGES = ("unused", "undefined", "redefs", "import*", "misc")
92
93    def __init__(self, *args, **kwargs):
94        # PyFlakes return 1 for both warnings and errors. We
95        # categorize this initially as WARNINGS so that
96        # evaluateCommand below can inspect the results more closely.
97        kwargs['decodeRC'] = {0: SUCCESS, 1: WARNINGS}
98
99        kwargs = self.setupShellMixin(kwargs)
100        super().__init__(*args, **kwargs)
101
102        self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self._log_consumer))
103
104        counts = self.counts = {}
105        summaries = self.summaries = {}
106        for m in self._MESSAGES:
107            counts[m] = 0
108            summaries[m] = []
109
110        # we need a separate variable for syntax errors
111        self._hasSyntaxError = False
112
113    def _log_consumer(self):
114        counts = self.counts
115        summaries = self.summaries
116        first = True
117        while True:
118            stream, line = yield
119            if stream == 'h':
120                continue
121            # the first few lines might contain echoed commands from a 'make
122            # pyflakes' step, so don't count these as warnings. Stop ignoring
123            # the initial lines as soon as we see one with a colon.
124            if first:
125                if ':' in line:
126                    # there's the colon, this is the first real line
127                    first = False
128                    # fall through and parse the line
129                else:
130                    # skip this line, keep skipping non-colon lines
131                    continue
132
133            if line.find("imported but unused") != -1:
134                m = "unused"
135            elif line.find("*' used; unable to detect undefined names") != -1:
136                m = "import*"
137            elif line.find("undefined name") != -1:
138                m = "undefined"
139            elif line.find("redefinition of unused") != -1:
140                m = "redefs"
141            elif line.find("invalid syntax") != -1:
142                self._hasSyntaxError = True
143                # we can do this, because if a syntax error occurs
144                # the output will only contain the info about it, nothing else
145                m = "misc"
146            else:
147                m = "misc"
148
149            summaries[m].append(line)
150            counts[m] += 1
151
152    def getResultSummary(self):
153        summary = ' '.join(self.descriptionDone)
154        for m in self._MESSAGES:
155            if self.counts[m]:
156                summary += " {}={}".format(m, self.counts[m])
157
158        if self.results != SUCCESS:
159            summary += ' ({})'.format(Results[self.results])
160
161        return {'step': summary}
162
163    @defer.inlineCallbacks
164    def run(self):
165        cmd = yield self.makeRemoteShellCommand()
166        yield self.runCommand(cmd)
167
168        stdio_log = yield self.getLog('stdio')
169        yield stdio_log.finish()
170
171        # we log 'misc' as syntax-error
172        if self._hasSyntaxError:
173            yield self.addCompleteLog("syntax-error", "\n".join(self.summaries['misc']))
174        else:
175            for m in self._MESSAGES:
176                if self.counts[m]:
177                    yield self.addCompleteLog(m, "\n".join(self.summaries[m]))
178                self.setProperty("pyflakes-{}".format(m), self.counts[m], "pyflakes")
179            self.setProperty("pyflakes-total", sum(self.counts.values()), "pyflakes")
180
181        if cmd.didFail() or self._hasSyntaxError:
182            return FAILURE
183        for m in self._flunkingIssues:
184            if m in self.counts and self.counts[m] > 0:
185                return FAILURE
186        if sum(self.counts.values()) > 0:
187            return WARNINGS
188        return SUCCESS
189
190
191class PyLint(buildstep.ShellMixin, buildstep.BuildStep):
192
193    '''A command that knows about pylint output.
194    It is a good idea to add --output-format=parseable to your
195    command, since it includes the filename in the message.
196    '''
197    name = "pylint"
198    description = "running pylint"
199    descriptionDone = "pylint"
200
201    # pylint's return codes (see pylint(1) for details)
202    # 1 - 16 will be bit-ORed
203
204    RC_OK = 0
205    RC_FATAL = 1
206    RC_ERROR = 2
207    RC_WARNING = 4
208    RC_REFACTOR = 8
209    RC_CONVENTION = 16
210    RC_USAGE = 32
211
212    # Using the default text output, the message format is :
213    # MESSAGE_TYPE: LINE_NUM:[OBJECT:] MESSAGE
214    # with --output-format=parseable it is: (the outer brackets are literal)
215    # FILE_NAME:LINE_NUM: [MESSAGE_TYPE[, OBJECT]] MESSAGE
216    # message type consists of the type char and 4 digits
217    # The message types:
218
219    _MESSAGES = {
220        'C': "convention",  # for programming standard violation
221        'R': "refactor",  # for bad code smell
222        'W': "warning",  # for python specific problems
223        'E': "error",  # for much probably bugs in the code
224        'F': "fatal",  # error prevented pylint from further processing.
225        'I': "info",
226    }
227
228    _flunkingIssues = ("F", "E")  # msg categories that cause FAILURE
229
230    _msgtypes_re_str = '(?P<errtype>[{}])'.format(''.join(list(_MESSAGES)))
231    _default_line_re = re.compile(r'^{}(\d+)?: *\d+(, *\d+)?:.+'.format(_msgtypes_re_str))
232    _default_2_0_0_line_re = \
233        re.compile(r'^(?P<path>[^:]+):(?P<line>\d+):\d+: *{}(\d+)?:.+'.format(_msgtypes_re_str))
234    _parseable_line_re = re.compile(
235        r'(?P<path>[^:]+):(?P<line>\d+): \[{}(\d+)?(\([a-z-]+\))?[,\]] .+'.format(_msgtypes_re_str))
236
237    def __init__(self, store_results=True, **kwargs):
238        kwargs = self.setupShellMixin(kwargs)
239        super().__init__(**kwargs)
240        self._store_results = store_results
241        self.counts = {}
242        self.summaries = {}
243        self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self._log_consumer))
244
245    # returns (message type, path, line) tuple if line has been matched, or None otherwise
246    def _match_line(self, line):
247        m = self._default_2_0_0_line_re.match(line)
248        if m:
249            try:
250                line_int = int(m.group('line'))
251            except ValueError:
252                line_int = None
253            return (m.group('errtype'), m.group('path'), line_int)
254
255        m = self._parseable_line_re.match(line)
256        if m:
257            try:
258                line_int = int(m.group('line'))
259            except ValueError:
260                line_int = None
261            return (m.group('errtype'), m.group('path'), line_int)
262
263        m = self._default_line_re.match(line)
264        if m:
265            return (m.group('errtype'), None, None)
266
267        return None
268
269    def _log_consumer(self):
270        for m in self._MESSAGES:
271            self.counts[m] = 0
272            self.summaries[m] = []
273
274        while True:
275            stream, line = yield
276            if stream == 'h':
277                continue
278
279            ret = self._match_line(line)
280            if not ret:
281                continue
282
283            msgtype, path, line_number = ret
284
285            assert msgtype in self._MESSAGES
286            self.summaries[msgtype].append(line)
287            self.counts[msgtype] += 1
288
289            if self._store_results and path is not None:
290                self.addTestResult(self._result_setid, line, test_name=None, test_code_path=path,
291                                   line=line_number)
292
293    def getResultSummary(self):
294        summary = ' '.join(self.descriptionDone)
295        for msg, fullmsg in sorted(self._MESSAGES.items()):
296            if self.counts[msg]:
297                summary += " {}={}".format(fullmsg, self.counts[msg])
298
299        if self.results != SUCCESS:
300            summary += ' ({})'.format(Results[self.results])
301
302        return {'step': summary}
303
304    @defer.inlineCallbacks
305    def run(self):
306        cmd = yield self.makeRemoteShellCommand()
307        yield self.runCommand(cmd)
308
309        stdio_log = yield self.getLog('stdio')
310        yield stdio_log.finish()
311
312        for msg, fullmsg in sorted(self._MESSAGES.items()):
313            if self.counts[msg]:
314                yield self.addCompleteLog(fullmsg, "\n".join(self.summaries[msg]))
315            self.setProperty("pylint-{}".format(fullmsg), self.counts[msg], 'Pylint')
316        self.setProperty("pylint-total", sum(self.counts.values()), 'Pylint')
317
318        if cmd.rc & (self.RC_FATAL | self.RC_ERROR | self.RC_USAGE):
319            return FAILURE
320
321        for msg in self._flunkingIssues:
322            if msg in self.counts and self.counts[msg] > 0:
323                return FAILURE
324        if sum(self.counts.values()) > 0:
325            return WARNINGS
326        return SUCCESS
327
328    @defer.inlineCallbacks
329    def addTestResultSets(self):
330        if not self._store_results:
331            return
332        self._result_setid = yield self.addTestResultSet('Pylint warnings', 'code_issue', 'message')
333
334
335class Sphinx(buildstep.ShellMixin, buildstep.BuildStep):
336
337    ''' A Step to build sphinx documentation '''
338
339    name = "sphinx"
340    description = "running sphinx"
341    descriptionDone = "sphinx"
342
343    haltOnFailure = True
344
345    def __init__(self, sphinx_sourcedir='.', sphinx_builddir=None,
346                 sphinx_builder=None, sphinx='sphinx-build', tags=None,
347                 defines=None, strict_warnings=False, mode='incremental', **kwargs):
348
349        if tags is None:
350            tags = []
351
352        if defines is None:
353            defines = {}
354
355        if sphinx_builddir is None:
356            # Who the heck is not interested in the built doc ?
357            config.error("Sphinx argument sphinx_builddir is required")
358
359        if mode not in ('incremental', 'full'):
360            config.error("Sphinx argument mode has to be 'incremental' or" +
361                         "'full' is required")
362
363        self.success = False
364
365        kwargs = self.setupShellMixin(kwargs)
366
367        super().__init__(**kwargs)
368
369        # build the command
370        command = [sphinx]
371        if sphinx_builder is not None:
372            command.extend(['-b', sphinx_builder])
373
374        for tag in tags:
375            command.extend(['-t', tag])
376
377        for key in sorted(defines):
378            if defines[key] is None:
379                command.extend(['-D', key])
380            elif isinstance(defines[key], bool):
381                command.extend(['-D',
382                                '{}={}'.format(key, defines[key] and 1 or 0)])
383            else:
384                command.extend(['-D', '{}={}'.format(key, defines[key])])
385
386        if mode == 'full':
387            command.extend(['-E'])  # Don't use a saved environment
388
389        if strict_warnings:
390            command.extend(['-W'])  # Convert warnings to errors
391
392        command.extend([sphinx_sourcedir, sphinx_builddir])
393        self.command = command
394
395        self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self._log_consumer))
396
397    _msgs = ('WARNING', 'ERROR', 'SEVERE')
398
399    def _log_consumer(self):
400        self.warnings = []
401        next_is_warning = False
402
403        while True:
404            stream, line = yield
405            if line.startswith('build succeeded') or \
406               line.startswith('no targets are out of date.'):
407                self.success = True
408            elif line.startswith('Warning, treated as error:'):
409                next_is_warning = True
410            else:
411                if next_is_warning:
412                    self.warnings.append(line)
413                    next_is_warning = False
414                else:
415                    for msg in self._msgs:
416                        if msg in line:
417                            self.warnings.append(line)
418
419    def getResultSummary(self):
420        summary = '{} {} warnings'.format(self.name, len(self.warnings))
421
422        if self.results != SUCCESS:
423            summary += ' ({})'.format(Results[self.results])
424
425        return {'step': summary}
426
427    @defer.inlineCallbacks
428    def run(self):
429        cmd = yield self.makeRemoteShellCommand()
430        yield self.runCommand(cmd)
431
432        stdio_log = yield self.getLog('stdio')
433        yield stdio_log.finish()
434
435        if self.warnings:
436            yield self.addCompleteLog('warnings', "\n".join(self.warnings))
437
438        self.setStatistic('warnings', len(self.warnings))
439
440        if self.success:
441            if not self.warnings:
442                return SUCCESS
443            return WARNINGS
444        return FAILURE
445