1# This file is part of Buildbot. Buildbot is free software: you can 2# redistribute it and/or modify it under the terms of the GNU General Public 3# License as published by the Free Software Foundation, version 2. 4# 5# This program is distributed in the hope that it will be useful, but WITHOUT 6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8# details. 9# 10# You should have received a copy of the GNU General Public License along with 11# this program; if not, write to the Free Software Foundation, Inc., 51 12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13# 14# Copyright Buildbot Team Members 15 16import base64 17import json 18import os 19import random 20import re 21import shlex 22import string 23import sys 24import time 25 26from twisted.cred import credentials 27from twisted.internet import defer 28from twisted.internet import protocol 29from twisted.internet import reactor 30from twisted.internet import task 31from twisted.internet import utils 32from twisted.python import log 33from twisted.python import runtime 34from twisted.python.procutils import which 35from twisted.spread import pb 36 37from buildbot.process.results import SUCCESS 38from buildbot.process.results import Results 39from buildbot.util import bytes2unicode 40from buildbot.util import now 41from buildbot.util import unicode2bytes 42from buildbot.util.eventual import fireEventually 43 44 45class SourceStamp: 46 47 def __init__(self, branch, revision, patch, repository=''): 48 self.branch = branch 49 self.revision = revision 50 self.patch = patch 51 self.repository = repository 52 53 54def output(*msg): 55 print(' '.join([str(m)for m in msg])) 56 57 58class SourceStampExtractor: 59 60 def __init__(self, treetop, branch, repository): 61 self.treetop = treetop 62 self.repository = repository 63 self.branch = branch 64 exes = which(self.vcexe) 65 if not exes: 66 output("Could not find executable '{}'.".format(self.vcexe)) 67 sys.exit(1) 68 self.exe = exes[0] 69 70 def dovc(self, cmd): 71 """This accepts the arguments of a command, without the actual 72 command itself.""" 73 env = os.environ.copy() 74 env['LC_ALL'] = "C" 75 d = utils.getProcessOutputAndValue(self.exe, cmd, env=env, 76 path=self.treetop) 77 d.addCallback(self._didvc, cmd) 78 return d 79 80 def _didvc(self, res, cmd): 81 (stdout, stderr, code) = res 82 # 'bzr diff' sets rc=1 if there were any differences. 83 # cvs does something similar, so don't bother requiring rc=0. 84 return stdout 85 86 def get(self): 87 """Return a Deferred that fires with a SourceStamp instance.""" 88 d = self.getBaseRevision() 89 d.addCallback(self.getPatch) 90 d.addCallback(self.done) 91 return d 92 93 def readPatch(self, diff, patchlevel): 94 if not diff: 95 diff = None 96 self.patch = (patchlevel, diff) 97 98 def done(self, res): 99 if not self.repository: 100 self.repository = self.treetop 101 # TODO: figure out the branch and project too 102 ss = SourceStamp(bytes2unicode(self.branch), self.baserev, self.patch, 103 repository=self.repository) 104 return ss 105 106 107class CVSExtractor(SourceStampExtractor): 108 patchlevel = 0 109 vcexe = "cvs" 110 111 def getBaseRevision(self): 112 # this depends upon our local clock and the repository's clock being 113 # reasonably synchronized with each other. We express everything in 114 # UTC because the '%z' format specifier for strftime doesn't always 115 # work. 116 self.baserev = time.strftime("%Y-%m-%d %H:%M:%S +0000", 117 time.gmtime(now())) 118 return defer.succeed(None) 119 120 def getPatch(self, res): 121 # the -q tells CVS to not announce each directory as it works 122 if self.branch is not None: 123 # 'cvs diff' won't take both -r and -D at the same time (it 124 # ignores the -r). As best I can tell, there is no way to make 125 # cvs give you a diff relative to a timestamp on the non-trunk 126 # branch. A bare 'cvs diff' will tell you about the changes 127 # relative to your checked-out versions, but I know of no way to 128 # find out what those checked-out versions are. 129 output("Sorry, CVS 'try' builds don't work with branches") 130 sys.exit(1) 131 args = ['-q', 'diff', '-u', '-D', self.baserev] 132 d = self.dovc(args) 133 d.addCallback(self.readPatch, self.patchlevel) 134 return d 135 136 137class SVNExtractor(SourceStampExtractor): 138 patchlevel = 0 139 vcexe = "svn" 140 141 def getBaseRevision(self): 142 d = self.dovc(["status", "-u"]) 143 d.addCallback(self.parseStatus) 144 return d 145 146 def parseStatus(self, res): 147 # svn shows the base revision for each file that has been modified or 148 # which needs an update. You can update each file to a different 149 # version, so each file is displayed with its individual base 150 # revision. It also shows the repository-wide latest revision number 151 # on the last line ("Status against revision: \d+"). 152 153 # for our purposes, we use the latest revision number as the "base" 154 # revision, and get a diff against that. This means we will get 155 # reverse-diffs for local files that need updating, but the resulting 156 # tree will still be correct. The only weirdness is that the baserev 157 # that we emit may be different than the version of the tree that we 158 # first checked out. 159 160 # to do this differently would probably involve scanning the revision 161 # numbers to find the max (or perhaps the min) revision, and then 162 # using that as a base. 163 164 for line in res.split(b"\n"): 165 m = re.search(br'^Status against revision:\s+(\d+)', line) 166 if m: 167 self.baserev = m.group(1) 168 return 169 output( 170 b"Could not find 'Status against revision' in SVN output: " + res) 171 sys.exit(1) 172 173 def getPatch(self, res): 174 d = self.dovc(["diff", "-r{}".format(self.baserev)]) 175 d.addCallback(self.readPatch, self.patchlevel) 176 return d 177 178 179class BzrExtractor(SourceStampExtractor): 180 patchlevel = 0 181 vcexe = "bzr" 182 183 def getBaseRevision(self): 184 d = self.dovc(["revision-info", "-rsubmit:"]) 185 d.addCallback(self.get_revision_number) 186 return d 187 188 def get_revision_number(self, out): 189 revno, revid = out.split() 190 self.baserev = 'revid:' + revid 191 return 192 193 def getPatch(self, res): 194 d = self.dovc(["diff", "-r{}..".format(self.baserev)]) 195 d.addCallback(self.readPatch, self.patchlevel) 196 return d 197 198 199class MercurialExtractor(SourceStampExtractor): 200 patchlevel = 1 201 vcexe = "hg" 202 203 def _didvc(self, res, cmd): 204 (stdout, stderr, code) = res 205 206 if code: 207 cs = ' '.join(['hg'] + cmd) 208 if stderr: 209 stderr = '\n' + stderr.rstrip() 210 raise RuntimeError("{} returned {} {}".format(cs, code, stderr)) 211 212 return stdout 213 214 @defer.inlineCallbacks 215 def getBaseRevision(self): 216 upstream = "" 217 if self.repository: 218 upstream = "r'{}'".format(self.repository) 219 output = '' 220 try: 221 output = yield self.dovc(["log", "--template", "{node}\\n", "-r", 222 "max(::. - outgoing({}))".format(upstream)]) 223 except RuntimeError: 224 # outgoing() will abort if no default-push/default path is 225 # configured 226 if upstream: 227 raise 228 # fall back to current working directory parent 229 output = yield self.dovc(["log", "--template", "{node}\\n", "-r", "p1()"]) 230 m = re.search(br'^(\w+)', output) 231 if not m: 232 raise RuntimeError( 233 "Revision {!r} is not in the right format".format(output)) 234 self.baserev = m.group(0) 235 236 def getPatch(self, res): 237 d = self.dovc(["diff", "-r", self.baserev]) 238 d.addCallback(self.readPatch, self.patchlevel) 239 return d 240 241 242class PerforceExtractor(SourceStampExtractor): 243 patchlevel = 0 244 vcexe = "p4" 245 246 def getBaseRevision(self): 247 d = self.dovc(["changes", "-m1", "..."]) 248 d.addCallback(self.parseStatus) 249 return d 250 251 def parseStatus(self, res): 252 # 253 # extract the base change number 254 # 255 m = re.search(br'Change (\d+)', res) 256 if m: 257 self.baserev = m.group(1) 258 return 259 260 output(b"Could not find change number in output: " + res) 261 sys.exit(1) 262 263 def readPatch(self, res, patchlevel): 264 # 265 # extract the actual patch from "res" 266 # 267 if not self.branch: 268 output("you must specify a branch") 269 sys.exit(1) 270 mpatch = "" 271 found = False 272 for line in res.split("\n"): 273 m = re.search('==== //depot/' + self.branch 274 + r'/([\w/\.\d\-_]+)#(\d+) -', line) 275 if m: 276 mpatch += "--- {}#{}\n".format(m.group(1), m.group(2)) 277 mpatch += "+++ {}\n".format(m.group(1)) 278 found = True 279 else: 280 mpatch += line 281 mpatch += "\n" 282 if not found: 283 output(b"could not parse patch file") 284 sys.exit(1) 285 self.patch = (patchlevel, unicode2bytes(mpatch)) 286 287 def getPatch(self, res): 288 d = self.dovc(["diff"]) 289 d.addCallback(self.readPatch, self.patchlevel) 290 return d 291 292 293class DarcsExtractor(SourceStampExtractor): 294 patchlevel = 1 295 vcexe = "darcs" 296 297 def getBaseRevision(self): 298 d = self.dovc(["changes", "--context"]) 299 d.addCallback(self.parseStatus) 300 return d 301 302 def parseStatus(self, res): 303 self.baserev = res # the whole context file 304 305 def getPatch(self, res): 306 d = self.dovc(["diff", "-u"]) 307 d.addCallback(self.readPatch, self.patchlevel) 308 return d 309 310 311class GitExtractor(SourceStampExtractor): 312 patchlevel = 1 313 vcexe = "git" 314 config = None 315 316 def getBaseRevision(self): 317 # If a branch is specified, parse out the rev it points to 318 # and extract the local name. 319 if self.branch: 320 d = self.dovc(["rev-parse", self.branch]) 321 d.addCallback(self.override_baserev) 322 d.addCallback(self.extractLocalBranch) 323 return d 324 d = self.dovc(["branch", "--no-color", "-v", "--no-abbrev"]) 325 d.addCallback(self.parseStatus) 326 return d 327 328 # remove remote-prefix from self.branch (assumes format <prefix>/<branch>) 329 # this uses "git remote" to retrieve all configured remote names 330 def extractLocalBranch(self, res): 331 if '/' in self.branch: 332 d = self.dovc(["remote"]) 333 d.addCallback(self.fixBranch) 334 return d 335 return None 336 337 # strip remote prefix from self.branch 338 def fixBranch(self, remotes): 339 for l in bytes2unicode(remotes).split("\n"): 340 r = l.strip() 341 if r and self.branch.startswith(r + "/"): 342 self.branch = self.branch[len(r) + 1:] 343 break 344 345 def readConfig(self): 346 if self.config: 347 return defer.succeed(self.config) 348 d = self.dovc(["config", "-l"]) 349 d.addCallback(self.parseConfig) 350 return d 351 352 def parseConfig(self, res): 353 self.config = {} 354 for l in res.split(b"\n"): 355 if l.strip(): 356 parts = l.strip().split(b"=", 2) 357 if len(parts) < 2: 358 parts.append('true') 359 self.config[parts[0]] = parts[1] 360 return self.config 361 362 def parseTrackingBranch(self, res): 363 # If we're tracking a remote, consider that the base. 364 remote = self.config.get(b"branch." + self.branch + b".remote") 365 ref = self.config.get(b"branch." + self.branch + b".merge") 366 if remote and ref: 367 remote_branch = ref.split(b"/", 2)[-1] 368 baserev = remote + b"/" + remote_branch 369 else: 370 baserev = b"master" 371 372 d = self.dovc(["rev-parse", baserev]) 373 d.addCallback(self.override_baserev) 374 return d 375 376 def override_baserev(self, res): 377 self.baserev = bytes2unicode(res).strip() 378 379 def parseStatus(self, res): 380 # The current branch is marked by '*' at the start of the 381 # line, followed by the branch name and the SHA1. 382 # 383 # Branch names may contain pretty much anything but whitespace. 384 m = re.search(br'^\* (\S+)\s+([0-9a-f]{40})', res, re.MULTILINE) 385 if m: 386 self.baserev = m.group(2) 387 self.branch = m.group(1) 388 d = self.readConfig() 389 d.addCallback(self.parseTrackingBranch) 390 return d 391 output(b"Could not find current GIT branch: " + res) 392 sys.exit(1) 393 394 def getPatch(self, res): 395 d = self.dovc(["diff", "--src-prefix=a/", "--dst-prefix=b/", 396 "--no-textconv", "--no-ext-diff", self.baserev]) 397 d.addCallback(self.readPatch, self.patchlevel) 398 return d 399 400 401class MonotoneExtractor(SourceStampExtractor): 402 patchlevel = 0 403 vcexe = "mtn" 404 405 def getBaseRevision(self): 406 d = self.dovc(["automate", "get_base_revision_id"]) 407 d.addCallback(self.parseStatus) 408 return d 409 410 def parseStatus(self, output): 411 hash = output.strip() 412 if len(hash) != 40: 413 self.baserev = None 414 self.baserev = hash 415 416 def getPatch(self, res): 417 d = self.dovc(["diff"]) 418 d.addCallback(self.readPatch, self.patchlevel) 419 return d 420 421 422def getSourceStamp(vctype, treetop, branch=None, repository=None): 423 if vctype == "cvs": 424 cls = CVSExtractor 425 elif vctype == "svn": 426 cls = SVNExtractor 427 elif vctype == "bzr": 428 cls = BzrExtractor 429 elif vctype == "hg": 430 cls = MercurialExtractor 431 elif vctype == "p4": 432 cls = PerforceExtractor 433 elif vctype == "darcs": 434 cls = DarcsExtractor 435 elif vctype == "git": 436 cls = GitExtractor 437 elif vctype == "mtn": 438 cls = MonotoneExtractor 439 elif vctype == "none": 440 return defer.succeed(SourceStamp("", "", (1, ""), "")) 441 else: 442 output("unknown vctype '{}'".format(vctype)) 443 sys.exit(1) 444 return cls(treetop, branch, repository).get() 445 446 447def ns(s): 448 return "{}:{},".format(len(s), s) 449 450 451def createJobfile(jobid, branch, baserev, patch_level, patch_body, repository, 452 project, who, comment, builderNames, properties): 453 # Determine job file version from provided arguments 454 try: 455 bytes2unicode(patch_body) 456 version = 5 457 except UnicodeDecodeError: 458 version = 6 459 460 job = "" 461 job += ns(str(version)) 462 job_dict = { 463 'jobid': jobid, 464 'branch': branch, 465 'baserev': str(baserev), 466 'patch_level': patch_level, 467 'repository': repository, 468 'project': project, 469 'who': who, 470 'comment': comment, 471 'builderNames': builderNames, 472 'properties': properties, 473 } 474 if version > 5: 475 job_dict['patch_body_base64'] = bytes2unicode(base64.b64encode(patch_body)) 476 else: 477 job_dict['patch_body'] = bytes2unicode(patch_body) 478 479 job += ns(json.dumps(job_dict)) 480 return job 481 482 483def getTopdir(topfile, start=None): 484 """walk upwards from the current directory until we find this topfile""" 485 if not start: 486 start = os.getcwd() 487 here = start 488 toomany = 20 489 while toomany > 0: 490 if os.path.exists(os.path.join(here, topfile)): 491 return here 492 next = os.path.dirname(here) 493 if next == here: 494 break # we've hit the root 495 here = next 496 toomany -= 1 497 output("Unable to find topfile '{}' anywhere " 498 "from {} upwards".format(topfile, start)) 499 sys.exit(1) 500 501 502class RemoteTryPP(protocol.ProcessProtocol): 503 504 def __init__(self, job): 505 self.job = job 506 self.d = defer.Deferred() 507 508 def connectionMade(self): 509 self.transport.write(unicode2bytes(self.job)) 510 self.transport.closeStdin() 511 512 def outReceived(self, data): 513 sys.stdout.write(bytes2unicode(data)) 514 515 def errReceived(self, data): 516 sys.stderr.write(bytes2unicode(data)) 517 518 def processEnded(self, status_object): 519 sig = status_object.value.signal 520 rc = status_object.value.exitCode 521 if sig is not None or rc != 0: 522 self.d.errback(RuntimeError("remote 'buildbot tryserver' failed" 523 ": sig={}, rc={}".format(sig, rc))) 524 return 525 self.d.callback((sig, rc)) 526 527 528class FakeBuildSetStatus: 529 def callRemote(self, name): 530 if name == "getBuildRequests": 531 return defer.succeed([]) 532 raise NotImplementedError() 533 534 535class Try(pb.Referenceable): 536 buildsetStatus = None 537 quiet = False 538 printloop = False 539 540 def __init__(self, config): 541 self.config = config 542 self.connect = self.getopt('connect') 543 if self.connect not in ['ssh', 'pb']: 544 output("you must specify a connect style: ssh or pb") 545 sys.exit(1) 546 self.builderNames = self.getopt('builders') 547 self.project = self.getopt('project', '') 548 self.who = self.getopt('who') 549 self.comment = self.getopt('comment') 550 551 def getopt(self, config_name, default=None): 552 value = self.config.get(config_name) 553 if value is None or value == []: 554 value = default 555 return value 556 557 def createJob(self): 558 # returns a Deferred which fires when the job parameters have been 559 # created 560 561 # generate a random (unique) string. It would make sense to add a 562 # hostname and process ID here, but a) I suspect that would cause 563 # windows portability problems, and b) really this is good enough 564 self.bsid = "{}-{}".format(time.time(), random.randint(0, 1000000)) 565 566 # common options 567 branch = self.getopt("branch") 568 569 difffile = self.config.get("diff") 570 if difffile: 571 baserev = self.config.get("baserev") 572 if difffile == "-": 573 diff = sys.stdin.read() 574 else: 575 with open(difffile, "rb") as f: 576 diff = f.read() 577 if not diff: 578 diff = None 579 patch = (self.config['patchlevel'], diff) 580 ss = SourceStamp( 581 branch, baserev, patch, repository=self.getopt("repository")) 582 d = defer.succeed(ss) 583 else: 584 vc = self.getopt("vc") 585 if vc in ("cvs", "svn"): 586 # we need to find the tree-top 587 topdir = self.getopt("topdir") 588 if topdir: 589 treedir = os.path.expanduser(topdir) 590 else: 591 topfile = self.getopt("topfile") 592 if topfile: 593 treedir = getTopdir(topfile) 594 else: 595 output("Must specify topdir or topfile.") 596 sys.exit(1) 597 else: 598 treedir = os.getcwd() 599 d = getSourceStamp(vc, treedir, branch, self.getopt("repository")) 600 d.addCallback(self._createJob_1) 601 return d 602 603 def _createJob_1(self, ss): 604 self.sourcestamp = ss 605 patchlevel, diff = ss.patch 606 if diff is None: 607 raise RuntimeError("There is no patch to try, diff is empty.") 608 609 if self.connect == "ssh": 610 revspec = ss.revision 611 if revspec is None: 612 revspec = "" 613 self.jobfile = createJobfile( 614 self.bsid, ss.branch or "", revspec, patchlevel, diff, 615 ss.repository, self.project, self.who, self.comment, 616 self.builderNames, self.config.get('properties', {})) 617 618 def fakeDeliverJob(self): 619 # Display the job to be delivered, but don't perform delivery. 620 ss = self.sourcestamp 621 output("Job:\n\tRepository: {}\n\tProject: {}\n\tBranch: {}\n\t" 622 "Revision: {}\n\tBuilders: {}\n{}".format( 623 ss.repository, self.project, ss.branch, 624 ss.revision, 625 self.builderNames, 626 ss.patch[1])) 627 self.buildsetStatus = FakeBuildSetStatus() 628 d = defer.Deferred() 629 d.callback(True) 630 return d 631 632 def deliver_job_ssh(self): 633 tryhost = self.getopt("host") 634 tryport = self.getopt("port") 635 tryuser = self.getopt("username") 636 trydir = self.getopt("jobdir") 637 buildbotbin = self.getopt("buildbotbin") 638 ssh_command = self.getopt("ssh") 639 if not ssh_command: 640 ssh_commands = which("ssh") 641 if not ssh_commands: 642 raise RuntimeError("couldn't find ssh executable, make sure " 643 "it is available in the PATH") 644 645 argv = [ssh_commands[0]] 646 else: 647 # Split the string on whitespace to allow passing options in 648 # ssh command too, but preserving whitespace inside quotes to 649 # allow using paths with spaces in them which is common under 650 # Windows. And because Windows uses backslashes in paths, we 651 # can't just use shlex.split there as it would interpret them 652 # specially, so do it by hand. 653 if runtime.platformType == 'win32': 654 # Note that regex here matches the arguments, not the 655 # separators, as it's simpler to do it like this. And then we 656 # just need to get all of them together using the slice and 657 # also remove the quotes from those that were quoted. 658 argv = [string.strip(a, '"') for a in 659 re.split(r'''([^" ]+|"[^"]+")''', ssh_command)[1::2]] 660 else: 661 # Do use standard tokenization logic under POSIX. 662 argv = shlex.split(ssh_command) 663 664 if tryuser: 665 argv += ["-l", tryuser] 666 667 if tryport: 668 argv += ["-p", tryport] 669 670 argv += [tryhost, buildbotbin, "tryserver", "--jobdir", trydir] 671 pp = RemoteTryPP(self.jobfile) 672 reactor.spawnProcess(pp, argv[0], argv, os.environ) 673 d = pp.d 674 return d 675 676 @defer.inlineCallbacks 677 def deliver_job_pb(self): 678 user = self.getopt("username") 679 passwd = self.getopt("passwd") 680 master = self.getopt("master") 681 tryhost, tryport = master.split(":") 682 tryport = int(tryport) 683 f = pb.PBClientFactory() 684 d = f.login(credentials.UsernamePassword(unicode2bytes(user), unicode2bytes(passwd))) 685 reactor.connectTCP(tryhost, tryport, f) 686 remote = yield d 687 688 ss = self.sourcestamp 689 output("Delivering job; comment=", self.comment) 690 691 self.buildsetStatus = \ 692 yield remote.callRemote("try", ss.branch, ss.revision, ss.patch, ss.repository, 693 self.project, self.builderNames, self.who, self.comment, 694 self.config.get('properties', {})) 695 696 def deliverJob(self): 697 # returns a Deferred that fires when the job has been delivered 698 if self.connect == "ssh": 699 return self.deliver_job_ssh() 700 if self.connect == "pb": 701 return self.deliver_job_pb() 702 raise RuntimeError("unknown connecttype '{}', " 703 "should be 'ssh' or 'pb'".format(self.connect)) 704 705 def getStatus(self): 706 # returns a Deferred that fires when the builds have finished, and 707 # may emit status messages while we wait 708 wait = bool(self.getopt("wait")) 709 if not wait: 710 output("not waiting for builds to finish") 711 elif self.connect == "ssh": 712 output("waiting for builds with ssh is not supported") 713 else: 714 self.running = defer.Deferred() 715 if not self.buildsetStatus: 716 output("try scheduler on the master does not have the builder configured") 717 return None 718 719 self._getStatus_1() # note that we don't wait for the returned Deferred 720 if bool(self.config.get("dryrun")): 721 self.statusDone() 722 return self.running 723 return None 724 725 @defer.inlineCallbacks 726 def _getStatus_1(self): 727 # gather the set of BuildRequests 728 brs = yield self.buildsetStatus.callRemote("getBuildRequests") 729 730 self.builderNames = [] 731 self.buildRequests = {} 732 733 # self.builds holds the current BuildStatus object for each one 734 self.builds = {} 735 736 # self.outstanding holds the list of builderNames which haven't 737 # finished yet 738 self.outstanding = [] 739 740 # self.results holds the list of build results. It holds a tuple of 741 # (result, text) 742 self.results = {} 743 744 # self.currentStep holds the name of the Step that each build is 745 # currently running 746 self.currentStep = {} 747 748 # self.ETA holds the expected finishing time (absolute time since 749 # epoch) 750 self.ETA = {} 751 752 for n, br in brs: 753 self.builderNames.append(n) 754 self.buildRequests[n] = br 755 self.builds[n] = None 756 self.outstanding.append(n) 757 self.results[n] = [None, None] 758 self.currentStep[n] = None 759 self.ETA[n] = None 760 # get new Builds for this buildrequest. We follow each one until 761 # it finishes or is interrupted. 762 br.callRemote("subscribe", self) 763 764 # now that those queries are in transit, we can start the 765 # display-status-every-30-seconds loop 766 if not self.getopt("quiet"): 767 self.printloop = task.LoopingCall(self.printStatus) 768 self.printloop.start(3, now=False) 769 770 # these methods are invoked by the status objects we've subscribed to 771 772 def remote_newbuild(self, bs, builderName): 773 if self.builds[builderName]: 774 self.builds[builderName].callRemote("unsubscribe", self) 775 self.builds[builderName] = bs 776 bs.callRemote("subscribe", self, 20) 777 d = bs.callRemote("waitUntilFinished") 778 d.addCallback(self._build_finished, builderName) 779 780 def remote_stepStarted(self, buildername, build, stepname, step): 781 self.currentStep[buildername] = stepname 782 783 def remote_stepFinished(self, buildername, build, stepname, step, results): 784 pass 785 786 def remote_buildETAUpdate(self, buildername, build, eta): 787 self.ETA[buildername] = now() + eta 788 789 @defer.inlineCallbacks 790 def _build_finished(self, bs, builderName): 791 # we need to collect status from the newly-finished build. We don't 792 # remove the build from self.outstanding until we've collected 793 # everything we want. 794 self.builds[builderName] = None 795 self.ETA[builderName] = None 796 self.currentStep[builderName] = "finished" 797 798 self.results[builderName][0] = yield bs.callRemote("getResults") 799 self.results[builderName][1] = yield bs.callRemote("getText") 800 801 self.outstanding.remove(builderName) 802 if not self.outstanding: 803 self.statusDone() 804 805 def printStatus(self): 806 try: 807 names = sorted(self.buildRequests.keys()) 808 for n in names: 809 if n not in self.outstanding: 810 # the build is finished, and we have results 811 code, text = self.results[n] 812 t = Results[code] 813 if text: 814 t += " ({})".format(" ".join(text)) 815 elif self.builds[n]: 816 t = self.currentStep[n] or "building" 817 if self.ETA[n]: 818 t += " [ETA {}s]".format(self.ETA[n] - now()) 819 else: 820 t = "no build" 821 self.announce("{}: {}".format(n, t)) 822 self.announce("") 823 except Exception: 824 log.err(None, "printing status") 825 826 def statusDone(self): 827 if self.printloop: 828 self.printloop.stop() 829 self.printloop = None 830 output("All Builds Complete") 831 # TODO: include a URL for all failing builds 832 names = sorted(self.buildRequests.keys()) 833 happy = True 834 for n in names: 835 code, text = self.results[n] 836 t = "{}: {}".format(n, Results[code]) 837 if text: 838 t += " ({})".format(" ".join(text)) 839 output(t) 840 if code != SUCCESS: 841 happy = False 842 843 if happy: 844 self.exitcode = 0 845 else: 846 self.exitcode = 1 847 self.running.callback(self.exitcode) 848 849 @defer.inlineCallbacks 850 def getAvailableBuilderNames(self): 851 # This logs into the master using the PB protocol to 852 # get the names of the configured builders that can 853 # be used for the --builder argument 854 if self.connect == "pb": 855 user = self.getopt("username") 856 passwd = self.getopt("passwd") 857 master = self.getopt("master") 858 tryhost, tryport = master.split(":") 859 tryport = int(tryport) 860 f = pb.PBClientFactory() 861 d = f.login(credentials.UsernamePassword(unicode2bytes(user), unicode2bytes(passwd))) 862 reactor.connectTCP(tryhost, tryport, f) 863 remote = yield d 864 buildernames = yield remote.callRemote("getAvailableBuilderNames") 865 866 output("The following builders are available for the try scheduler: ") 867 for buildername in buildernames: 868 output(buildername) 869 870 yield remote.broker.transport.loseConnection() 871 return 872 if self.connect == "ssh": 873 output("Cannot get available builders over ssh.") 874 sys.exit(1) 875 raise RuntimeError( 876 "unknown connecttype '{}', should be 'pb'".format(self.connect)) 877 878 def announce(self, message): 879 if not self.quiet: 880 output(message) 881 882 @defer.inlineCallbacks 883 def run_impl(self): 884 output("using '{}' connect method".format(self.connect)) 885 self.exitcode = 0 886 887 # we can't do spawnProcess until we're inside reactor.run(), so force asynchronous execution 888 yield fireEventually(None) 889 890 try: 891 if bool(self.config.get("get-builder-names")): 892 yield self.getAvailableBuilderNames() 893 else: 894 yield self.createJob() 895 yield self.announce("job created") 896 if bool(self.config.get("dryrun")): 897 yield self.fakeDeliverJob() 898 else: 899 yield self.deliverJob() 900 yield self.announce("job has been delivered") 901 yield self.getStatus() 902 903 if not bool(self.config.get("dryrun")): 904 yield self.cleanup() 905 except SystemExit as e: 906 self.exitcode = e.code 907 except Exception as e: 908 log.err(e) 909 raise 910 911 def run(self): 912 d = self.run_impl() 913 d.addCallback(lambda res: reactor.stop()) 914 915 reactor.run() 916 sys.exit(self.exitcode) 917 918 def trapSystemExit(self, why): 919 why.trap(SystemExit) 920 self.exitcode = why.value.code 921 922 def cleanup(self, res=None): 923 if self.buildsetStatus: 924 self.buildsetStatus.broker.transport.loseConnection() 925