1# This file is part of Buildbot. Buildbot is free software: you can 2# redistribute it and/or modify it under the terms of the GNU General Public 3# License as published by the Free Software Foundation, version 2. 4# 5# This program is distributed in the hope that it will be useful, but WITHOUT 6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8# details. 9# 10# You should have received a copy of the GNU General Public License along with 11# this program; if not, write to the Free Software Foundation, Inc., 51 12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13# 14# Copyright Buildbot Team Members 15""" 16Push events to Gerrit 17""" 18 19import time 20import warnings 21from pkg_resources import parse_version 22 23from twisted.internet import defer 24from twisted.internet import reactor 25from twisted.internet.protocol import ProcessProtocol 26from twisted.python import log 27 28from buildbot.process.results import EXCEPTION 29from buildbot.process.results import FAILURE 30from buildbot.process.results import RETRY 31from buildbot.process.results import SUCCESS 32from buildbot.process.results import WARNINGS 33from buildbot.process.results import Results 34from buildbot.reporters import utils 35from buildbot.util import bytes2unicode 36from buildbot.util import service 37 38# Cache the version that the gerrit server is running for this many seconds 39GERRIT_VERSION_CACHE_TIMEOUT = 600 40 41GERRIT_LABEL_VERIFIED = 'Verified' 42GERRIT_LABEL_REVIEWED = 'Code-Review' 43 44 45def makeReviewResult(message, *labels): 46 """ 47 helper to produce a review result 48 """ 49 return dict(message=message, labels=dict(labels)) 50 51 52def _handleLegacyResult(result): 53 """ 54 make sure the result is backward compatible 55 """ 56 if not isinstance(result, dict): 57 warnings.warn('The Gerrit status callback uses the old way to ' 58 'communicate results. The outcome might be not what is ' 59 'expected.') 60 message, verified, reviewed = result 61 result = makeReviewResult(message, 62 (GERRIT_LABEL_VERIFIED, verified), 63 (GERRIT_LABEL_REVIEWED, reviewed)) 64 return result 65 66 67def _old_add_label(label, value): 68 if label == GERRIT_LABEL_VERIFIED: 69 return ["--verified %d" % int(value)] 70 elif label == GERRIT_LABEL_REVIEWED: 71 return ["--code-review %d" % int(value)] 72 warnings.warn(('Gerrit older than 2.6 does not support custom labels. ' 73 'Setting {} is ignored.').format(label)) 74 return [] 75 76 77def _new_add_label(label, value): 78 return ["--label {}={}".format(label, int(value))] 79 80 81def defaultReviewCB(builderName, build, result, master, arg): 82 if result == RETRY: 83 return makeReviewResult(None) 84 85 message = "Buildbot finished compiling your patchset\n" 86 message += "on configuration: {}\n".format(builderName) 87 message += "The result is: {}\n".format(Results[result].upper()) 88 89 return makeReviewResult(message, 90 (GERRIT_LABEL_VERIFIED, result == SUCCESS or -1)) 91 92 93def defaultSummaryCB(buildInfoList, results, master, arg): 94 success = False 95 failure = False 96 97 msgs = [] 98 99 for buildInfo in buildInfoList: 100 msg = "Builder %(name)s %(resultText)s (%(text)s)" % buildInfo 101 link = buildInfo.get('url', None) 102 if link: 103 msg += " - " + link 104 else: 105 msg += "." 106 msgs.append(msg) 107 108 if buildInfo['result'] == SUCCESS: # pylint: disable=simplifiable-if-statement 109 success = True 110 else: 111 failure = True 112 113 if success and not failure: 114 verified = 1 115 else: 116 verified = -1 117 118 return makeReviewResult('\n\n'.join(msgs), (GERRIT_LABEL_VERIFIED, verified)) 119 120 121# These are just sentinel values for GerritStatusPush.__init__ args 122class DEFAULT_REVIEW: 123 pass 124 125 126class DEFAULT_SUMMARY: 127 pass 128 129 130class GerritStatusPush(service.BuildbotService): 131 132 """Event streamer to a gerrit ssh server.""" 133 name = "GerritStatusPush" 134 gerrit_server = None 135 gerrit_username = None 136 gerrit_port = None 137 gerrit_version_time = None 138 gerrit_version = None 139 gerrit_identity_file = None 140 reviewCB = None 141 reviewArg = None 142 startCB = None 143 startArg = None 144 summaryCB = None 145 summaryArg = None 146 wantSteps = False 147 wantLogs = False 148 _gerrit_notify = None 149 150 def reconfigService(self, server, username, reviewCB=DEFAULT_REVIEW, 151 startCB=None, port=29418, reviewArg=None, 152 startArg=None, summaryCB=DEFAULT_SUMMARY, summaryArg=None, 153 identity_file=None, builders=None, notify=None, 154 wantSteps=False, wantLogs=False): 155 156 # If neither reviewCB nor summaryCB were specified, default to sending 157 # out "summary" reviews. But if we were given a reviewCB and only a 158 # reviewCB, disable the "summary" reviews, so we don't send out both 159 # by default. 160 if reviewCB is DEFAULT_REVIEW and summaryCB is DEFAULT_SUMMARY: 161 reviewCB = None 162 summaryCB = defaultSummaryCB 163 if reviewCB is DEFAULT_REVIEW: 164 reviewCB = None 165 if summaryCB is DEFAULT_SUMMARY: 166 summaryCB = None 167 # Parameters. 168 self.gerrit_server = server 169 self.gerrit_username = username 170 self.gerrit_port = port 171 self.gerrit_version = None 172 self.gerrit_version_time = 0 173 self.gerrit_identity_file = identity_file 174 self.reviewCB = reviewCB 175 self.reviewArg = reviewArg 176 self.startCB = startCB 177 self.startArg = startArg 178 self.summaryCB = summaryCB 179 self.summaryArg = summaryArg 180 self.builders = builders 181 self._gerrit_notify = notify 182 self.wantSteps = wantSteps 183 self.wantLogs = wantLogs 184 185 def _gerritCmd(self, *args): 186 '''Construct a command as a list of strings suitable for 187 :func:`subprocess.call`. 188 ''' 189 if self.gerrit_identity_file is not None: 190 options = ['-i', self.gerrit_identity_file] 191 else: 192 options = [] 193 return ['ssh', '-o', 'BatchMode=yes'] + options + [ 194 '@'.join((self.gerrit_username, self.gerrit_server)), 195 '-p', str(self.gerrit_port), 196 'gerrit' 197 ] + list(args) 198 199 class VersionPP(ProcessProtocol): 200 201 def __init__(self, func): 202 self.func = func 203 self.gerrit_version = None 204 205 def outReceived(self, data): 206 vstr = b"gerrit version " 207 if not data.startswith(vstr): 208 log.msg(b"Error: Cannot interpret gerrit version info: " + data) 209 return 210 vers = data[len(vstr):].strip() 211 log.msg(b"gerrit version: " + vers) 212 self.gerrit_version = parse_version(bytes2unicode(vers)) 213 214 def errReceived(self, data): 215 log.msg(b"gerriterr: " + data) 216 217 def processEnded(self, status_object): 218 if status_object.value.exitCode: 219 log.msg("gerrit version status: ERROR:", status_object) 220 return 221 if self.gerrit_version: 222 self.func(self.gerrit_version) 223 224 def getCachedVersion(self): 225 if self.gerrit_version is None: 226 return None 227 if time.time() - self.gerrit_version_time > GERRIT_VERSION_CACHE_TIMEOUT: 228 # cached version has expired 229 self.gerrit_version = None 230 return self.gerrit_version 231 232 def processVersion(self, gerrit_version, func): 233 self.gerrit_version = gerrit_version 234 self.gerrit_version_time = time.time() 235 func() 236 237 def callWithVersion(self, func): 238 command = self._gerritCmd("version") 239 240 def callback(gerrit_version): 241 return self.processVersion(gerrit_version, func) 242 243 self.spawnProcess(self.VersionPP(callback), command[0], command, env=None) 244 245 class LocalPP(ProcessProtocol): 246 247 def __init__(self, status): 248 self.status = status 249 250 def outReceived(self, data): 251 log.msg("gerritout:", data) 252 253 def errReceived(self, data): 254 log.msg("gerriterr:", data) 255 256 def processEnded(self, status_object): 257 if status_object.value.exitCode: 258 log.msg("gerrit status: ERROR:", status_object) 259 else: 260 log.msg("gerrit status: OK") 261 262 @defer.inlineCallbacks 263 def startService(self): 264 yield super().startService() 265 startConsuming = self.master.mq.startConsuming 266 self._buildsetCompleteConsumer = yield startConsuming( 267 self.buildsetComplete, 268 ('buildsets', None, 'complete')) 269 270 self._buildCompleteConsumer = yield startConsuming( 271 self.buildComplete, 272 ('builds', None, 'finished')) 273 274 self._buildStartedConsumer = yield startConsuming( 275 self.buildStarted, 276 ('builds', None, 'new')) 277 278 def stopService(self): 279 self._buildsetCompleteConsumer.stopConsuming() 280 self._buildCompleteConsumer.stopConsuming() 281 self._buildStartedConsumer.stopConsuming() 282 283 @defer.inlineCallbacks 284 def _got_event(self, key, msg): 285 # This function is used only from tests 286 if key[0] == 'builds': 287 if key[2] == 'new': 288 yield self.buildStarted(key, msg) 289 return 290 elif key[2] == 'finished': 291 yield self.buildComplete(key, msg) 292 return 293 if key[0] == 'buildsets' and key[2] == 'complete': # pragma: no cover 294 yield self.buildsetComplete(key, msg) 295 return 296 raise Exception('Invalid key for _got_event: {}'.format(key)) # pragma: no cover 297 298 @defer.inlineCallbacks 299 def buildStarted(self, key, build): 300 if self.startCB is None: 301 return 302 yield self.getBuildDetails(build) 303 if self.isBuildReported(build): 304 result = yield self.startCB(build['builder']['name'], build, self.startArg) 305 self.sendCodeReviews(build, result) 306 307 @defer.inlineCallbacks 308 def buildComplete(self, key, build): 309 if self.reviewCB is None: 310 return 311 yield self.getBuildDetails(build) 312 if self.isBuildReported(build): 313 result = yield self.reviewCB(build['builder']['name'], build, build['results'], 314 self.master, self.reviewArg) 315 result = _handleLegacyResult(result) 316 self.sendCodeReviews(build, result) 317 318 @defer.inlineCallbacks 319 def getBuildDetails(self, build): 320 br = yield self.master.data.get(("buildrequests", build['buildrequestid'])) 321 buildset = yield self.master.data.get(("buildsets", br['buildsetid'])) 322 yield utils.getDetailsForBuilds(self.master, 323 buildset, 324 [build], 325 want_properties=True, 326 want_steps=self.wantSteps) 327 328 def isBuildReported(self, build): 329 return self.builders is None or build['builder']['name'] in self.builders 330 331 @defer.inlineCallbacks 332 def buildsetComplete(self, key, msg): 333 if not self.summaryCB: 334 return 335 bsid = msg['bsid'] 336 res = yield utils.getDetailsForBuildset(self.master, bsid, want_properties=True, 337 want_steps=self.wantSteps, want_logs=self.wantLogs, 338 want_logs_content=self.wantLogs) 339 builds = res['builds'] 340 buildset = res['buildset'] 341 self.sendBuildSetSummary(buildset, builds) 342 343 @defer.inlineCallbacks 344 def sendBuildSetSummary(self, buildset, builds): 345 builds = [build for build in builds if self.isBuildReported(build)] 346 if builds and self.summaryCB: 347 def getBuildInfo(build): 348 result = build['results'] 349 resultText = { 350 SUCCESS: "succeeded", 351 FAILURE: "failed", 352 WARNINGS: "completed with warnings", 353 EXCEPTION: "encountered an exception", 354 }.get(result, "completed with unknown result %d" % result) 355 356 return {'name': build['builder']['name'], 357 'result': result, 358 'resultText': resultText, 359 'text': build['state_string'], 360 'url': utils.getURLForBuild(self.master, build['builder']['builderid'], 361 build['number']), 362 'build': build 363 } 364 buildInfoList = sorted( 365 [getBuildInfo(build) for build in builds], key=lambda bi: bi['name']) 366 367 result = yield self.summaryCB(buildInfoList, 368 Results[buildset['results']], 369 self.master, 370 self.summaryArg) 371 372 result = _handleLegacyResult(result) 373 self.sendCodeReviews(builds[0], result) 374 375 def sendCodeReviews(self, build, result): 376 message = result.get('message', None) 377 if message is None: 378 return 379 380 def getProperty(build, name): 381 return build['properties'].get(name, [None])[0] 382 # Gerrit + Repo 383 downloads = getProperty(build, "repo_downloads") 384 downloaded = getProperty(build, "repo_downloaded") 385 if downloads is not None and downloaded is not None: 386 downloaded = downloaded.split(" ") 387 if downloads and 2 * len(downloads) == len(downloaded): 388 for i, download in enumerate(downloads): 389 try: 390 project, change1 = download.split(" ") 391 except ValueError: 392 return # something is wrong, abort 393 change2 = downloaded[2 * i] 394 revision = downloaded[2 * i + 1] 395 if change1 == change2: 396 self.sendCodeReview(project, revision, result) 397 else: 398 return # something is wrong, abort 399 return 400 401 # Gerrit + Git 402 # used only to verify Gerrit source 403 if getProperty(build, "event.change.id") is not None: 404 project = getProperty(build, "event.change.project") 405 codebase = getProperty(build, "codebase") 406 revision = (getProperty(build, "event.patchSet.revision") or 407 getProperty(build, "got_revision") or 408 getProperty(build, "revision")) 409 410 if isinstance(revision, dict): 411 # in case of the revision is a codebase revision, we just take 412 # the revisionfor current codebase 413 if codebase is not None: 414 revision = revision[codebase] 415 else: 416 revision = None 417 418 if project is not None and revision is not None: 419 self.sendCodeReview(project, revision, result) 420 return 421 422 def sendCodeReview(self, project, revision, result): 423 gerrit_version = self.getCachedVersion() 424 if gerrit_version is None: 425 self.callWithVersion( 426 lambda: self.sendCodeReview(project, revision, result)) 427 return 428 429 assert gerrit_version 430 command = self._gerritCmd("review", "--project {}".format(project)) 431 432 if gerrit_version >= parse_version("2.13"): 433 command.append('--tag autogenerated:buildbot') 434 435 if self._gerrit_notify is not None: 436 command.append('--notify {}'.format(str(self._gerrit_notify))) 437 438 message = result.get('message', None) 439 if message: 440 command.append("--message '{}'".format(message.replace("'", "\""))) 441 442 labels = result.get('labels', None) 443 if labels: 444 if gerrit_version < parse_version("2.6"): 445 add_label = _old_add_label 446 else: 447 add_label = _new_add_label 448 449 for label, value in labels.items(): 450 command.extend(add_label(label, value)) 451 452 command.append(revision) 453 command = [str(s) for s in command] 454 self.spawnProcess(self.LocalPP(self), command[0], command, env=None) 455 456 def spawnProcess(self, *arg, **kw): 457 reactor.spawnProcess(*arg, **kw) 458