1# This file is part of Buildbot.  Buildbot is free software: you can
2# redistribute it and/or modify it under the terms of the GNU General Public
3# License as published by the Free Software Foundation, version 2.
4#
5# This program is distributed in the hope that it will be useful, but WITHOUT
6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8# details.
9#
10# You should have received a copy of the GNU General Public License along with
11# this program; if not, write to the Free Software Foundation, Inc., 51
12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13#
14# Copyright Buildbot Team Members
15"""
16Push events to Gerrit
17"""
18
19import time
20import warnings
21from pkg_resources import parse_version
22
23from twisted.internet import defer
24from twisted.internet import reactor
25from twisted.internet.protocol import ProcessProtocol
26from twisted.python import log
27
28from buildbot.process.results import EXCEPTION
29from buildbot.process.results import FAILURE
30from buildbot.process.results import RETRY
31from buildbot.process.results import SUCCESS
32from buildbot.process.results import WARNINGS
33from buildbot.process.results import Results
34from buildbot.reporters import utils
35from buildbot.util import bytes2unicode
36from buildbot.util import service
37
38# Cache the version that the gerrit server is running for this many seconds
39GERRIT_VERSION_CACHE_TIMEOUT = 600
40
41GERRIT_LABEL_VERIFIED = 'Verified'
42GERRIT_LABEL_REVIEWED = 'Code-Review'
43
44
45def makeReviewResult(message, *labels):
46    """
47    helper to produce a review result
48    """
49    return dict(message=message, labels=dict(labels))
50
51
52def _handleLegacyResult(result):
53    """
54    make sure the result is backward compatible
55    """
56    if not isinstance(result, dict):
57        warnings.warn('The Gerrit status callback uses the old way to '
58                      'communicate results.  The outcome might be not what is '
59                      'expected.')
60        message, verified, reviewed = result
61        result = makeReviewResult(message,
62                                  (GERRIT_LABEL_VERIFIED, verified),
63                                  (GERRIT_LABEL_REVIEWED, reviewed))
64    return result
65
66
67def _old_add_label(label, value):
68    if label == GERRIT_LABEL_VERIFIED:
69        return ["--verified %d" % int(value)]
70    elif label == GERRIT_LABEL_REVIEWED:
71        return ["--code-review %d" % int(value)]
72    warnings.warn(('Gerrit older than 2.6 does not support custom labels. '
73                   'Setting {} is ignored.').format(label))
74    return []
75
76
77def _new_add_label(label, value):
78    return ["--label {}={}".format(label, int(value))]
79
80
81def defaultReviewCB(builderName, build, result, master, arg):
82    if result == RETRY:
83        return makeReviewResult(None)
84
85    message = "Buildbot finished compiling your patchset\n"
86    message += "on configuration: {}\n".format(builderName)
87    message += "The result is: {}\n".format(Results[result].upper())
88
89    return makeReviewResult(message,
90                            (GERRIT_LABEL_VERIFIED, result == SUCCESS or -1))
91
92
93def defaultSummaryCB(buildInfoList, results, master, arg):
94    success = False
95    failure = False
96
97    msgs = []
98
99    for buildInfo in buildInfoList:
100        msg = "Builder %(name)s %(resultText)s (%(text)s)" % buildInfo
101        link = buildInfo.get('url', None)
102        if link:
103            msg += " - " + link
104        else:
105            msg += "."
106        msgs.append(msg)
107
108        if buildInfo['result'] == SUCCESS:  # pylint: disable=simplifiable-if-statement
109            success = True
110        else:
111            failure = True
112
113    if success and not failure:
114        verified = 1
115    else:
116        verified = -1
117
118    return makeReviewResult('\n\n'.join(msgs), (GERRIT_LABEL_VERIFIED, verified))
119
120
121# These are just sentinel values for GerritStatusPush.__init__ args
122class DEFAULT_REVIEW:
123    pass
124
125
126class DEFAULT_SUMMARY:
127    pass
128
129
130class GerritStatusPush(service.BuildbotService):
131
132    """Event streamer to a gerrit ssh server."""
133    name = "GerritStatusPush"
134    gerrit_server = None
135    gerrit_username = None
136    gerrit_port = None
137    gerrit_version_time = None
138    gerrit_version = None
139    gerrit_identity_file = None
140    reviewCB = None
141    reviewArg = None
142    startCB = None
143    startArg = None
144    summaryCB = None
145    summaryArg = None
146    wantSteps = False
147    wantLogs = False
148    _gerrit_notify = None
149
150    def reconfigService(self, server, username, reviewCB=DEFAULT_REVIEW,
151                        startCB=None, port=29418, reviewArg=None,
152                        startArg=None, summaryCB=DEFAULT_SUMMARY, summaryArg=None,
153                        identity_file=None, builders=None, notify=None,
154                        wantSteps=False, wantLogs=False):
155
156        # If neither reviewCB nor summaryCB were specified, default to sending
157        # out "summary" reviews. But if we were given a reviewCB and only a
158        # reviewCB, disable the "summary" reviews, so we don't send out both
159        # by default.
160        if reviewCB is DEFAULT_REVIEW and summaryCB is DEFAULT_SUMMARY:
161            reviewCB = None
162            summaryCB = defaultSummaryCB
163        if reviewCB is DEFAULT_REVIEW:
164            reviewCB = None
165        if summaryCB is DEFAULT_SUMMARY:
166            summaryCB = None
167        # Parameters.
168        self.gerrit_server = server
169        self.gerrit_username = username
170        self.gerrit_port = port
171        self.gerrit_version = None
172        self.gerrit_version_time = 0
173        self.gerrit_identity_file = identity_file
174        self.reviewCB = reviewCB
175        self.reviewArg = reviewArg
176        self.startCB = startCB
177        self.startArg = startArg
178        self.summaryCB = summaryCB
179        self.summaryArg = summaryArg
180        self.builders = builders
181        self._gerrit_notify = notify
182        self.wantSteps = wantSteps
183        self.wantLogs = wantLogs
184
185    def _gerritCmd(self, *args):
186        '''Construct a command as a list of strings suitable for
187        :func:`subprocess.call`.
188        '''
189        if self.gerrit_identity_file is not None:
190            options = ['-i', self.gerrit_identity_file]
191        else:
192            options = []
193        return ['ssh', '-o', 'BatchMode=yes'] + options + [
194            '@'.join((self.gerrit_username, self.gerrit_server)),
195            '-p', str(self.gerrit_port),
196            'gerrit'
197        ] + list(args)
198
199    class VersionPP(ProcessProtocol):
200
201        def __init__(self, func):
202            self.func = func
203            self.gerrit_version = None
204
205        def outReceived(self, data):
206            vstr = b"gerrit version "
207            if not data.startswith(vstr):
208                log.msg(b"Error: Cannot interpret gerrit version info: " + data)
209                return
210            vers = data[len(vstr):].strip()
211            log.msg(b"gerrit version: " + vers)
212            self.gerrit_version = parse_version(bytes2unicode(vers))
213
214        def errReceived(self, data):
215            log.msg(b"gerriterr: " + data)
216
217        def processEnded(self, status_object):
218            if status_object.value.exitCode:
219                log.msg("gerrit version status: ERROR:", status_object)
220                return
221            if self.gerrit_version:
222                self.func(self.gerrit_version)
223
224    def getCachedVersion(self):
225        if self.gerrit_version is None:
226            return None
227        if time.time() - self.gerrit_version_time > GERRIT_VERSION_CACHE_TIMEOUT:
228            # cached version has expired
229            self.gerrit_version = None
230        return self.gerrit_version
231
232    def processVersion(self, gerrit_version, func):
233        self.gerrit_version = gerrit_version
234        self.gerrit_version_time = time.time()
235        func()
236
237    def callWithVersion(self, func):
238        command = self._gerritCmd("version")
239
240        def callback(gerrit_version):
241            return self.processVersion(gerrit_version, func)
242
243        self.spawnProcess(self.VersionPP(callback), command[0], command, env=None)
244
245    class LocalPP(ProcessProtocol):
246
247        def __init__(self, status):
248            self.status = status
249
250        def outReceived(self, data):
251            log.msg("gerritout:", data)
252
253        def errReceived(self, data):
254            log.msg("gerriterr:", data)
255
256        def processEnded(self, status_object):
257            if status_object.value.exitCode:
258                log.msg("gerrit status: ERROR:", status_object)
259            else:
260                log.msg("gerrit status: OK")
261
262    @defer.inlineCallbacks
263    def startService(self):
264        yield super().startService()
265        startConsuming = self.master.mq.startConsuming
266        self._buildsetCompleteConsumer = yield startConsuming(
267            self.buildsetComplete,
268            ('buildsets', None, 'complete'))
269
270        self._buildCompleteConsumer = yield startConsuming(
271            self.buildComplete,
272            ('builds', None, 'finished'))
273
274        self._buildStartedConsumer = yield startConsuming(
275            self.buildStarted,
276            ('builds', None, 'new'))
277
278    def stopService(self):
279        self._buildsetCompleteConsumer.stopConsuming()
280        self._buildCompleteConsumer.stopConsuming()
281        self._buildStartedConsumer.stopConsuming()
282
283    @defer.inlineCallbacks
284    def _got_event(self, key, msg):
285        # This function is used only from tests
286        if key[0] == 'builds':
287            if key[2] == 'new':
288                yield self.buildStarted(key, msg)
289                return
290            elif key[2] == 'finished':
291                yield self.buildComplete(key, msg)
292                return
293        if key[0] == 'buildsets' and key[2] == 'complete':  # pragma: no cover
294            yield self.buildsetComplete(key, msg)
295            return
296        raise Exception('Invalid key for _got_event: {}'.format(key))  # pragma: no cover
297
298    @defer.inlineCallbacks
299    def buildStarted(self, key, build):
300        if self.startCB is None:
301            return
302        yield self.getBuildDetails(build)
303        if self.isBuildReported(build):
304            result = yield self.startCB(build['builder']['name'], build, self.startArg)
305            self.sendCodeReviews(build, result)
306
307    @defer.inlineCallbacks
308    def buildComplete(self, key, build):
309        if self.reviewCB is None:
310            return
311        yield self.getBuildDetails(build)
312        if self.isBuildReported(build):
313            result = yield self.reviewCB(build['builder']['name'], build, build['results'],
314                                         self.master, self.reviewArg)
315            result = _handleLegacyResult(result)
316            self.sendCodeReviews(build, result)
317
318    @defer.inlineCallbacks
319    def getBuildDetails(self, build):
320        br = yield self.master.data.get(("buildrequests", build['buildrequestid']))
321        buildset = yield self.master.data.get(("buildsets", br['buildsetid']))
322        yield utils.getDetailsForBuilds(self.master,
323                                        buildset,
324                                        [build],
325                                        want_properties=True,
326                                        want_steps=self.wantSteps)
327
328    def isBuildReported(self, build):
329        return self.builders is None or build['builder']['name'] in self.builders
330
331    @defer.inlineCallbacks
332    def buildsetComplete(self, key, msg):
333        if not self.summaryCB:
334            return
335        bsid = msg['bsid']
336        res = yield utils.getDetailsForBuildset(self.master, bsid, want_properties=True,
337                                                want_steps=self.wantSteps, want_logs=self.wantLogs,
338                                                want_logs_content=self.wantLogs)
339        builds = res['builds']
340        buildset = res['buildset']
341        self.sendBuildSetSummary(buildset, builds)
342
343    @defer.inlineCallbacks
344    def sendBuildSetSummary(self, buildset, builds):
345        builds = [build for build in builds if self.isBuildReported(build)]
346        if builds and self.summaryCB:
347            def getBuildInfo(build):
348                result = build['results']
349                resultText = {
350                    SUCCESS: "succeeded",
351                    FAILURE: "failed",
352                    WARNINGS: "completed with warnings",
353                    EXCEPTION: "encountered an exception",
354                }.get(result, "completed with unknown result %d" % result)
355
356                return {'name': build['builder']['name'],
357                        'result': result,
358                        'resultText': resultText,
359                        'text': build['state_string'],
360                        'url': utils.getURLForBuild(self.master, build['builder']['builderid'],
361                                                    build['number']),
362                        'build': build
363                        }
364            buildInfoList = sorted(
365                [getBuildInfo(build) for build in builds], key=lambda bi: bi['name'])
366
367            result = yield self.summaryCB(buildInfoList,
368                                          Results[buildset['results']],
369                                          self.master,
370                                          self.summaryArg)
371
372            result = _handleLegacyResult(result)
373            self.sendCodeReviews(builds[0], result)
374
375    def sendCodeReviews(self, build, result):
376        message = result.get('message', None)
377        if message is None:
378            return
379
380        def getProperty(build, name):
381            return build['properties'].get(name, [None])[0]
382        # Gerrit + Repo
383        downloads = getProperty(build, "repo_downloads")
384        downloaded = getProperty(build, "repo_downloaded")
385        if downloads is not None and downloaded is not None:
386            downloaded = downloaded.split(" ")
387            if downloads and 2 * len(downloads) == len(downloaded):
388                for i, download in enumerate(downloads):
389                    try:
390                        project, change1 = download.split(" ")
391                    except ValueError:
392                        return  # something is wrong, abort
393                    change2 = downloaded[2 * i]
394                    revision = downloaded[2 * i + 1]
395                    if change1 == change2:
396                        self.sendCodeReview(project, revision, result)
397                    else:
398                        return  # something is wrong, abort
399            return
400
401        # Gerrit + Git
402        # used only to verify Gerrit source
403        if getProperty(build, "event.change.id") is not None:
404            project = getProperty(build, "event.change.project")
405            codebase = getProperty(build, "codebase")
406            revision = (getProperty(build, "event.patchSet.revision") or
407                        getProperty(build, "got_revision") or
408                        getProperty(build, "revision"))
409
410            if isinstance(revision, dict):
411                # in case of the revision is a codebase revision, we just take
412                # the revisionfor current codebase
413                if codebase is not None:
414                    revision = revision[codebase]
415                else:
416                    revision = None
417
418            if project is not None and revision is not None:
419                self.sendCodeReview(project, revision, result)
420                return
421
422    def sendCodeReview(self, project, revision, result):
423        gerrit_version = self.getCachedVersion()
424        if gerrit_version is None:
425            self.callWithVersion(
426                lambda: self.sendCodeReview(project, revision, result))
427            return
428
429        assert gerrit_version
430        command = self._gerritCmd("review", "--project {}".format(project))
431
432        if gerrit_version >= parse_version("2.13"):
433            command.append('--tag autogenerated:buildbot')
434
435        if self._gerrit_notify is not None:
436            command.append('--notify {}'.format(str(self._gerrit_notify)))
437
438        message = result.get('message', None)
439        if message:
440            command.append("--message '{}'".format(message.replace("'", "\"")))
441
442        labels = result.get('labels', None)
443        if labels:
444            if gerrit_version < parse_version("2.6"):
445                add_label = _old_add_label
446            else:
447                add_label = _new_add_label
448
449            for label, value in labels.items():
450                command.extend(add_label(label, value))
451
452        command.append(revision)
453        command = [str(s) for s in command]
454        self.spawnProcess(self.LocalPP(self), command[0], command, env=None)
455
456    def spawnProcess(self, *arg, **kw):
457        reactor.spawnProcess(*arg, **kw)
458