1# This file is part of Buildbot. Buildbot is free software: you can 2# redistribute it and/or modify it under the terms of the GNU General Public 3# License as published by the Free Software Foundation, version 2. 4# 5# This program is distributed in the hope that it will be useful, but WITHOUT 6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8# details. 9# 10# You should have received a copy of the GNU General Public License along with 11# this program; if not, write to the Free Software Foundation, Inc., 51 12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13# 14# Copyright Buildbot Team Members 15 16import inspect 17import sys 18 19from twisted.internet import defer 20from twisted.internet import error 21from twisted.python import deprecate 22from twisted.python import log 23from twisted.python import versions 24from twisted.python.deprecate import deprecatedModuleAttribute 25from twisted.python.failure import Failure 26from twisted.python.reflect import accumulateClassList 27from twisted.python.versions import Version 28from twisted.web.util import formatFailure 29from zope.interface import implementer 30 31from buildbot import config 32from buildbot import interfaces 33from buildbot import util 34from buildbot.interfaces import IRenderable 35from buildbot.interfaces import WorkerSetupError 36from buildbot.process import log as plog 37from buildbot.process import logobserver 38from buildbot.process import properties 39from buildbot.process import remotecommand 40from buildbot.process import results 41# (WithProperties used to be available in this module) 42from buildbot.process.properties import WithProperties 43from buildbot.process.results import ALL_RESULTS 44from buildbot.process.results import CANCELLED 45from buildbot.process.results import EXCEPTION 46from buildbot.process.results import FAILURE 47from buildbot.process.results import RETRY 48from buildbot.process.results import SKIPPED 49from buildbot.process.results import SUCCESS 50from buildbot.process.results import WARNINGS 51from buildbot.process.results import Results 52from buildbot.util import bytes2unicode 53from buildbot.util import debounce 54from buildbot.util import flatten 55from buildbot.util.test_result_submitter import TestResultSubmitter 56from buildbot.warnings import warn_deprecated 57 58 59class BuildStepFailed(Exception): 60 pass 61 62 63class BuildStepCancelled(Exception): 64 # used internally for signalling 65 pass 66 67 68class CallableAttributeError(Exception): 69 # attribute error raised from a callable run inside a property 70 pass 71 72 73# old import paths for these classes 74RemoteCommand = remotecommand.RemoteCommand 75deprecatedModuleAttribute( 76 Version("buildbot", 2, 10, 1), 77 message="Use buildbot.process.remotecommand.RemoteCommand instead.", 78 moduleName="buildbot.process.buildstep", 79 name="RemoteCommand", 80) 81 82LoggedRemoteCommand = remotecommand.LoggedRemoteCommand 83deprecatedModuleAttribute( 84 Version("buildbot", 2, 10, 1), 85 message="Use buildbot.process.remotecommand.LoggedRemoteCommand instead.", 86 moduleName="buildbot.process.buildstep", 87 name="LoggedRemoteCommand", 88) 89 90RemoteShellCommand = remotecommand.RemoteShellCommand 91deprecatedModuleAttribute( 92 Version("buildbot", 2, 10, 1), 93 message="Use buildbot.process.remotecommand.RemoteShellCommand instead.", 94 moduleName="buildbot.process.buildstep", 95 name="RemoteShellCommand", 96) 97 98LogObserver = logobserver.LogObserver 99deprecatedModuleAttribute( 100 Version("buildbot", 2, 10, 1), 101 message="Use buildbot.process.logobserver.LogObserver instead.", 102 moduleName="buildbot.process.buildstep", 103 name="LogObserver", 104) 105 106LogLineObserver = logobserver.LogLineObserver 107deprecatedModuleAttribute( 108 Version("buildbot", 2, 10, 1), 109 message="Use buildbot.util.LogLineObserver instead.", 110 moduleName="buildbot.process.buildstep", 111 name="LogLineObserver", 112) 113 114OutputProgressObserver = logobserver.OutputProgressObserver 115deprecatedModuleAttribute( 116 Version("buildbot", 2, 10, 1), 117 message="Use buildbot.process.logobserver.OutputProgressObserver instead.", 118 moduleName="buildbot.process.buildstep", 119 name="OutputProgressObserver", 120) 121 122 123@implementer(interfaces.IBuildStepFactory) 124class _BuildStepFactory(util.ComparableMixin): 125 126 """ 127 This is a wrapper to record the arguments passed to as BuildStep subclass. 128 We use an instance of this class, rather than a closure mostly to make it 129 easier to test that the right factories are getting created. 130 """ 131 compare_attrs = ('factory', 'args', 'kwargs') 132 133 def __init__(self, factory, *args, **kwargs): 134 self.factory = factory 135 self.args = args 136 self.kwargs = kwargs 137 138 def buildStep(self): 139 try: 140 return self.factory(*self.args, **self.kwargs) 141 except Exception: 142 log.msg("error while creating step, factory={}, args={}, kwargs={}".format(self.factory, 143 self.args, self.kwargs)) 144 raise 145 146 147class BuildStepStatus: 148 # used only for old-style steps 149 pass 150 151 152def get_factory_from_step_or_factory(step_or_factory): 153 if hasattr(step_or_factory, 'get_step_factory'): 154 factory = step_or_factory.get_step_factory() 155 else: 156 factory = step_or_factory 157 # make sure the returned value actually implements IBuildStepFactory 158 return interfaces.IBuildStepFactory(factory) 159 160 161def create_step_from_step_or_factory(step_or_factory): 162 return get_factory_from_step_or_factory(step_or_factory).buildStep() 163 164 165@implementer(interfaces.IBuildStep) 166class BuildStep(results.ResultComputingConfigMixin, 167 properties.PropertiesMixin, 168 util.ComparableMixin): 169 # Note that the BuildStep is at the same time a template from which per-build steps are 170 # constructed. This works by creating a new IBuildStepFactory in __new__, retrieving it via 171 # get_step_factory() and then calling buildStep() on that factory. 172 173 alwaysRun = False 174 doStepIf = True 175 hideStepIf = False 176 compare_attrs = ("_factory",) 177 # properties set on a build step are, by nature, always runtime properties 178 set_runtime_properties = True 179 180 renderables = results.ResultComputingConfigMixin.resultConfig + [ 181 'alwaysRun', 182 'description', 183 'descriptionDone', 184 'descriptionSuffix', 185 'doStepIf', 186 'hideStepIf', 187 'workdir', 188 ] 189 190 # 'parms' holds a list of all the parameters we care about, to allow 191 # users to instantiate a subclass of BuildStep with a mixture of 192 # arguments, some of which are for us, some of which are for the subclass 193 # (or a delegate of the subclass, like how ShellCommand delivers many 194 # arguments to the RemoteShellCommand that it creates). Such delegating 195 # subclasses will use this list to figure out which arguments are meant 196 # for us and which should be given to someone else. 197 parms = [ 198 'alwaysRun', 199 'description', 200 'descriptionDone', 201 'descriptionSuffix', 202 'doStepIf', 203 'flunkOnFailure', 204 'flunkOnWarnings', 205 'haltOnFailure', 206 'updateBuildSummaryPolicy', 207 'hideStepIf', 208 'locks', 209 'logEncoding', 210 'name', 211 'progressMetrics', 212 'useProgress', 213 'warnOnFailure', 214 'warnOnWarnings', 215 'workdir', 216 ] 217 218 name = "generic" 219 description = None # set this to a list of short strings to override 220 descriptionDone = None # alternate description when the step is complete 221 descriptionSuffix = None # extra information to append to suffix 222 updateBuildSummaryPolicy = None 223 locks = [] 224 progressMetrics = () # 'time' is implicit 225 useProgress = True # set to False if step is really unpredictable 226 build = None 227 step_status = None 228 progress = None 229 logEncoding = None 230 cmd = None 231 rendered = False # true if attributes are rendered 232 _workdir = None 233 _waitingForLocks = False 234 235 def __init__(self, **kwargs): 236 self.worker = None 237 238 for p in self.__class__.parms: 239 if p in kwargs: 240 setattr(self, p, kwargs.pop(p)) 241 242 if kwargs: 243 config.error("{}.__init__ got unexpected keyword argument(s) {}".format(self.__class__, 244 list(kwargs))) 245 self._pendingLogObservers = [] 246 247 if not isinstance(self.name, str) and not IRenderable.providedBy(self.name): 248 config.error("BuildStep name must be a string or a renderable object: " 249 "%r" % (self.name,)) 250 251 if isinstance(self.description, str): 252 self.description = [self.description] 253 if isinstance(self.descriptionDone, str): 254 self.descriptionDone = [self.descriptionDone] 255 if isinstance(self.descriptionSuffix, str): 256 self.descriptionSuffix = [self.descriptionSuffix] 257 258 if self.updateBuildSummaryPolicy is None: 259 # compute default value for updateBuildSummaryPolicy 260 self.updateBuildSummaryPolicy = [EXCEPTION, RETRY, CANCELLED] 261 if self.flunkOnFailure or self.haltOnFailure or self.warnOnFailure: 262 self.updateBuildSummaryPolicy.append(FAILURE) 263 if self.warnOnWarnings or self.flunkOnWarnings: 264 self.updateBuildSummaryPolicy.append(WARNINGS) 265 if self.updateBuildSummaryPolicy is False: 266 self.updateBuildSummaryPolicy = [] 267 if self.updateBuildSummaryPolicy is True: 268 self.updateBuildSummaryPolicy = ALL_RESULTS 269 if not isinstance(self.updateBuildSummaryPolicy, list): 270 config.error("BuildStep updateBuildSummaryPolicy must be " 271 "a list of result ids or boolean but it is %r" % 272 (self.updateBuildSummaryPolicy,)) 273 self._acquiringLocks = [] 274 self.stopped = False 275 self.master = None 276 self.statistics = {} 277 self.logs = {} 278 self._running = False 279 self.stepid = None 280 self.results = None 281 self._start_unhandled_deferreds = None 282 self._test_result_submitters = {} 283 284 def __new__(klass, *args, **kwargs): 285 self = object.__new__(klass) 286 self._factory = _BuildStepFactory(klass, *args, **kwargs) 287 return self 288 289 def __str__(self): 290 args = [repr(x) for x in self._factory.args] 291 args.extend([str(k) + "=" + repr(v) 292 for k, v in self._factory.kwargs.items()]) 293 return "{}({})".format( 294 self.__class__.__name__, ", ".join(args)) 295 __repr__ = __str__ 296 297 def setBuild(self, build): 298 self.build = build 299 self.master = self.build.master 300 301 def setWorker(self, worker): 302 self.worker = worker 303 304 @deprecate.deprecated(versions.Version("buildbot", 0, 9, 0)) 305 def setDefaultWorkdir(self, workdir): 306 if self._workdir is None: 307 self._workdir = workdir 308 309 @property 310 def workdir(self): 311 # default the workdir appropriately 312 if self._workdir is not None or self.build is None: 313 return self._workdir 314 else: 315 # see :ref:`Factory-Workdir-Functions` for details on how to 316 # customize this 317 if callable(self.build.workdir): 318 try: 319 return self.build.workdir(self.build.sources) 320 except AttributeError as e: 321 # if the callable raises an AttributeError 322 # python thinks it is actually workdir that is not existing. 323 # python will then swallow the attribute error and call 324 # __getattr__ from worker_transition 325 _, _, traceback = sys.exc_info() 326 raise CallableAttributeError(e).with_traceback(traceback) 327 # we re-raise the original exception by changing its type, 328 # but keeping its stacktrace 329 else: 330 return self.build.workdir 331 332 @workdir.setter 333 def workdir(self, workdir): 334 self._workdir = workdir 335 336 def getProperties(self): 337 return self.build.getProperties() 338 339 def get_step_factory(self): 340 return self._factory 341 342 def setupProgress(self): 343 # this function temporarily does nothing 344 pass 345 346 def setProgress(self, metric, value): 347 # this function temporarily does nothing 348 pass 349 350 def getCurrentSummary(self): 351 if self.description is not None: 352 stepsumm = util.join_list(self.description) 353 if self.descriptionSuffix: 354 stepsumm += ' ' + util.join_list(self.descriptionSuffix) 355 else: 356 stepsumm = 'running' 357 return {'step': stepsumm} 358 359 def getResultSummary(self): 360 if self.descriptionDone is not None or self.description is not None: 361 stepsumm = util.join_list(self.descriptionDone or self.description) 362 if self.descriptionSuffix: 363 stepsumm += ' ' + util.join_list(self.descriptionSuffix) 364 else: 365 stepsumm = 'finished' 366 367 if self.results != SUCCESS: 368 stepsumm += ' ({})'.format(Results[self.results]) 369 370 return {'step': stepsumm} 371 372 @defer.inlineCallbacks 373 def getBuildResultSummary(self): 374 summary = yield self.getResultSummary() 375 if self.results in self.updateBuildSummaryPolicy and \ 376 'build' not in summary and 'step' in summary: 377 summary['build'] = summary['step'] 378 return summary 379 380 @debounce.method(wait=1) 381 @defer.inlineCallbacks 382 def updateSummary(self): 383 def methodInfo(m): 384 lines = inspect.getsourcelines(m) 385 return "\nat {}:{}:\n {}".format(inspect.getsourcefile(m), lines[1], 386 "\n".join(lines[0])) 387 if not self._running: 388 summary = yield self.getResultSummary() 389 if not isinstance(summary, dict): 390 raise TypeError('getResultSummary must return a dictionary: ' + 391 methodInfo(self.getResultSummary)) 392 else: 393 summary = yield self.getCurrentSummary() 394 if not isinstance(summary, dict): 395 raise TypeError('getCurrentSummary must return a dictionary: ' + 396 methodInfo(self.getCurrentSummary)) 397 398 stepResult = summary.get('step', 'finished') 399 if not isinstance(stepResult, str): 400 raise TypeError("step result string must be unicode (got %r)" 401 % (stepResult,)) 402 if self.stepid is not None: 403 stepResult = self.build.properties.cleanupTextFromSecrets( 404 stepResult) 405 yield self.master.data.updates.setStepStateString(self.stepid, 406 stepResult) 407 408 if not self._running: 409 buildResult = summary.get('build', None) 410 if buildResult and not isinstance(buildResult, str): 411 raise TypeError("build result string must be unicode") 412 413 @defer.inlineCallbacks 414 def addStep(self): 415 # create and start the step, noting that the name may be altered to 416 # ensure uniqueness 417 self.name = yield self.build.render(self.name) 418 self.build.setUniqueStepName(self) 419 self.stepid, self.number, self.name = yield self.master.data.updates.addStep( 420 buildid=self.build.buildid, 421 name=util.bytes2unicode(self.name)) 422 yield self.master.data.updates.startStep(self.stepid) 423 424 @defer.inlineCallbacks 425 def startStep(self, remote): 426 self.remote = remote 427 428 yield self.addStep() 429 self.locks = yield self.build.render(self.locks) 430 431 # convert all locks into their real form 432 botmaster = self.build.builder.botmaster 433 self.locks = yield botmaster.getLockFromLockAccesses(self.locks, self.build.config_version) 434 435 # then narrow WorkerLocks down to the worker that this build is being 436 # run on 437 self.locks = [(l.getLockForWorker(self.build.workerforbuilder.worker.workername), 438 la) 439 for l, la in self.locks] 440 441 for l, la in self.locks: 442 if l in self.build.locks: 443 log.msg(("Hey, lock {} is claimed by both a Step ({}) and the" 444 " parent Build ({})").format(l, self, self.build)) 445 raise RuntimeError("lock claimed by both Step and Build") 446 447 try: 448 # set up locks 449 yield self.acquireLocks() 450 451 if self.stopped: 452 raise BuildStepCancelled 453 454 # render renderables in parallel 455 renderables = [] 456 accumulateClassList(self.__class__, 'renderables', renderables) 457 458 def setRenderable(res, attr): 459 setattr(self, attr, res) 460 461 dl = [] 462 for renderable in renderables: 463 d = self.build.render(getattr(self, renderable)) 464 d.addCallback(setRenderable, renderable) 465 dl.append(d) 466 yield defer.gatherResults(dl) 467 self.rendered = True 468 # we describe ourselves only when renderables are interpolated 469 self.updateSummary() 470 471 # check doStepIf (after rendering) 472 if isinstance(self.doStepIf, bool): 473 doStep = self.doStepIf 474 else: 475 doStep = yield self.doStepIf(self) 476 477 # run -- or skip -- the step 478 if doStep: 479 yield self.addTestResultSets() 480 try: 481 self._running = True 482 self.results = yield self.run() 483 finally: 484 self._running = False 485 else: 486 self.results = SKIPPED 487 488 # NOTE: all of these `except` blocks must set self.results immediately! 489 except BuildStepCancelled: 490 self.results = CANCELLED 491 492 except BuildStepFailed: 493 self.results = FAILURE 494 495 except error.ConnectionLost: 496 self.results = RETRY 497 498 except Exception: 499 self.results = EXCEPTION 500 why = Failure() 501 log.err(why, "BuildStep.failed; traceback follows") 502 yield self.addLogWithFailure(why) 503 504 if self.stopped and self.results != RETRY: 505 # We handle this specially because we don't care about 506 # the return code of an interrupted command; we know 507 # that this should just be exception due to interrupt 508 # At the same time we must respect RETRY status because it's used 509 # to retry interrupted build due to some other issues for example 510 # due to worker lost 511 if self.results != CANCELLED: 512 self.results = EXCEPTION 513 514 # determine whether we should hide this step 515 hidden = self.hideStepIf 516 if callable(hidden): 517 try: 518 hidden = hidden(self.results, self) 519 except Exception: 520 why = Failure() 521 log.err(why, "hidden callback failed; traceback follows") 522 yield self.addLogWithFailure(why) 523 self.results = EXCEPTION 524 hidden = False 525 526 # perform final clean ups 527 success = yield self._cleanup_logs() 528 if not success: 529 self.results = EXCEPTION 530 531 # update the summary one last time, make sure that completes, 532 # and then don't update it any more. 533 self.updateSummary() 534 yield self.updateSummary.stop() 535 536 for sub in self._test_result_submitters.values(): 537 yield sub.finish() 538 539 self.releaseLocks() 540 541 yield self.master.data.updates.finishStep(self.stepid, self.results, 542 hidden) 543 544 return self.results 545 546 def setBuildData(self, name, value, source): 547 # returns a Deferred that yields nothing 548 return self.master.data.updates.setBuildData(self.build.buildid, name, value, source) 549 550 @defer.inlineCallbacks 551 def _cleanup_logs(self): 552 all_success = True 553 not_finished_logs = [v for (k, v) in self.logs.items() if not v.finished] 554 finish_logs = yield defer.DeferredList([v.finish() for v in not_finished_logs], 555 consumeErrors=True) 556 for success, res in finish_logs: 557 if not success: 558 log.err(res, "when trying to finish a log") 559 all_success = False 560 561 for log_ in self.logs.values(): 562 if log_.had_errors(): 563 all_success = False 564 565 return all_success 566 567 def addTestResultSets(self): 568 return defer.succeed(None) 569 570 @defer.inlineCallbacks 571 def addTestResultSet(self, description, category, value_unit): 572 sub = TestResultSubmitter() 573 yield sub.setup(self, description, category, value_unit) 574 setid = sub.get_test_result_set_id() 575 self._test_result_submitters[setid] = sub 576 return setid 577 578 def addTestResult(self, setid, value, test_name=None, test_code_path=None, line=None, 579 duration_ns=None): 580 self._test_result_submitters[setid].add_test_result(value, test_name=test_name, 581 test_code_path=test_code_path, 582 line=line, duration_ns=duration_ns) 583 584 def acquireLocks(self, res=None): 585 if not self.locks: 586 return defer.succeed(None) 587 if self.stopped: 588 return defer.succeed(None) 589 log.msg("acquireLocks(step {}, locks {})".format(self, self.locks)) 590 for lock, access in self.locks: 591 for waited_lock, _, _ in self._acquiringLocks: 592 if lock is waited_lock: 593 continue 594 595 if not lock.isAvailable(self, access): 596 self._waitingForLocks = True 597 log.msg("step {} waiting for lock {}".format(self, lock)) 598 d = lock.waitUntilMaybeAvailable(self, access) 599 self._acquiringLocks.append((lock, access, d)) 600 d.addCallback(self.acquireLocks) 601 return d 602 # all locks are available, claim them all 603 for lock, access in self.locks: 604 lock.claim(self, access) 605 self._acquiringLocks = [] 606 self._waitingForLocks = False 607 return defer.succeed(None) 608 609 def run(self): 610 raise NotImplementedError("A custom build step must implement run()") 611 612 def isNewStyle(self): 613 warn_deprecated('3.0.0', 'BuildStep.isNewStyle() always returns True') 614 return True 615 616 @defer.inlineCallbacks 617 def interrupt(self, reason): 618 if self.stopped: 619 return 620 self.stopped = True 621 if self._acquiringLocks: 622 for (lock, access, d) in self._acquiringLocks: 623 lock.stopWaitingUntilAvailable(self, access, d) 624 self._acquiringLocks = [] 625 626 if self._waitingForLocks: 627 yield self.addCompleteLog( 628 'cancelled while waiting for locks', str(reason)) 629 else: 630 yield self.addCompleteLog('cancelled', str(reason)) 631 632 if self.cmd: 633 d = self.cmd.interrupt(reason) 634 d.addErrback(log.err, 'while cancelling command') 635 yield d 636 637 def releaseLocks(self): 638 log.msg("releaseLocks({}): {}".format(self, self.locks)) 639 for lock, access in self.locks: 640 if lock.isOwner(self, access): 641 lock.release(self, access) 642 else: 643 # This should only happen if we've been interrupted 644 assert self.stopped 645 646 # utility methods that BuildSteps may find useful 647 648 def workerVersion(self, command, oldversion=None): 649 return self.build.getWorkerCommandVersion(command, oldversion) 650 651 def workerVersionIsOlderThan(self, command, minversion): 652 sv = self.build.getWorkerCommandVersion(command, None) 653 if sv is None: 654 return True 655 if [int(s) for s in sv.split(".")] < [int(m) for m in minversion.split(".")]: 656 return True 657 return False 658 659 def checkWorkerHasCommand(self, command): 660 if not self.workerVersion(command): 661 message = "worker is too old, does not know about {}".format(command) 662 raise WorkerSetupError(message) 663 664 def getWorkerName(self): 665 return self.build.getWorkerName() 666 667 def addLog(self, name, type='s', logEncoding=None): 668 if self.stepid is None: 669 raise BuildStepCancelled 670 d = self.master.data.updates.addLog(self.stepid, 671 util.bytes2unicode(name), 672 str(type)) 673 674 @d.addCallback 675 def newLog(logid): 676 return self._newLog(name, type, logid, logEncoding) 677 return d 678 679 def getLog(self, name): 680 return self.logs[name] 681 682 @defer.inlineCallbacks 683 def addCompleteLog(self, name, text): 684 if self.stepid is None: 685 raise BuildStepCancelled 686 logid = yield self.master.data.updates.addLog(self.stepid, 687 util.bytes2unicode(name), 't') 688 _log = self._newLog(name, 't', logid) 689 yield _log.addContent(text) 690 yield _log.finish() 691 692 @defer.inlineCallbacks 693 def addHTMLLog(self, name, html): 694 if self.stepid is None: 695 raise BuildStepCancelled 696 logid = yield self.master.data.updates.addLog(self.stepid, 697 util.bytes2unicode(name), 'h') 698 _log = self._newLog(name, 'h', logid) 699 html = bytes2unicode(html) 700 yield _log.addContent(html) 701 yield _log.finish() 702 703 @defer.inlineCallbacks 704 def addLogWithFailure(self, why, logprefix=""): 705 # helper for showing exceptions to the users 706 try: 707 yield self.addCompleteLog(logprefix + "err.text", why.getTraceback()) 708 yield self.addHTMLLog(logprefix + "err.html", formatFailure(why)) 709 except Exception: 710 log.err(Failure(), "error while formatting exceptions") 711 712 def addLogWithException(self, why, logprefix=""): 713 return self.addLogWithFailure(Failure(why), logprefix) 714 715 def addLogObserver(self, logname, observer): 716 assert interfaces.ILogObserver.providedBy(observer) 717 observer.setStep(self) 718 self._pendingLogObservers.append((logname, observer)) 719 self._connectPendingLogObservers() 720 721 def _newLog(self, name, type, logid, logEncoding=None): 722 if not logEncoding: 723 logEncoding = self.logEncoding 724 if not logEncoding: 725 logEncoding = self.master.config.logEncoding 726 log = plog.Log.new(self.master, name, type, logid, logEncoding) 727 self.logs[name] = log 728 self._connectPendingLogObservers() 729 return log 730 731 def _connectPendingLogObservers(self): 732 for logname, observer in self._pendingLogObservers[:]: 733 if logname in self.logs: 734 observer.setLog(self.logs[logname]) 735 self._pendingLogObservers.remove((logname, observer)) 736 737 @defer.inlineCallbacks 738 def addURL(self, name, url): 739 yield self.master.data.updates.addStepURL(self.stepid, str(name), str(url)) 740 return None 741 742 @defer.inlineCallbacks 743 def runCommand(self, command): 744 if self.stopped: 745 return CANCELLED 746 747 self.cmd = command 748 command.worker = self.worker 749 try: 750 res = yield command.run(self, self.remote, self.build.builder.name) 751 finally: 752 self.cmd = None 753 return res 754 755 def hasStatistic(self, name): 756 return name in self.statistics 757 758 def getStatistic(self, name, default=None): 759 return self.statistics.get(name, default) 760 761 def getStatistics(self): 762 return self.statistics.copy() 763 764 def setStatistic(self, name, value): 765 self.statistics[name] = value 766 767 768class CommandMixin: 769 770 @defer.inlineCallbacks 771 def _runRemoteCommand(self, cmd, abandonOnFailure, args, makeResult=None): 772 cmd = remotecommand.RemoteCommand(cmd, args) 773 try: 774 log = self.getLog('stdio') 775 except Exception: 776 log = yield self.addLog('stdio') 777 cmd.useLog(log, False) 778 yield self.runCommand(cmd) 779 if abandonOnFailure and cmd.didFail(): 780 raise BuildStepFailed() 781 if makeResult: 782 return makeResult(cmd) 783 else: 784 return not cmd.didFail() 785 786 def runRmdir(self, dir, log=None, abandonOnFailure=True): 787 return self._runRemoteCommand('rmdir', abandonOnFailure, 788 {'dir': dir, 'logEnviron': False}) 789 790 def pathExists(self, path, log=None): 791 return self._runRemoteCommand('stat', False, 792 {'file': path, 'logEnviron': False}) 793 794 def runMkdir(self, dir, log=None, abandonOnFailure=True): 795 return self._runRemoteCommand('mkdir', abandonOnFailure, 796 {'dir': dir, 'logEnviron': False}) 797 798 def runGlob(self, path): 799 return self._runRemoteCommand( 800 'glob', True, {'path': path, 'logEnviron': False}, 801 makeResult=lambda cmd: cmd.updates['files'][0]) 802 803 804class ShellMixin: 805 806 command = None 807 env = {} 808 want_stdout = True 809 want_stderr = True 810 usePTY = None 811 logfiles = {} 812 lazylogfiles = {} 813 timeout = 1200 814 maxTime = None 815 logEnviron = True 816 interruptSignal = 'KILL' 817 sigtermTime = None 818 initialStdin = None 819 decodeRC = {0: SUCCESS} 820 821 _shellMixinArgs = [ 822 'command', 823 'workdir', 824 'env', 825 'want_stdout', 826 'want_stderr', 827 'usePTY', 828 'logfiles', 829 'lazylogfiles', 830 'timeout', 831 'maxTime', 832 'logEnviron', 833 'interruptSignal', 834 'sigtermTime', 835 'initialStdin', 836 'decodeRC', 837 ] 838 renderables = _shellMixinArgs 839 840 def setupShellMixin(self, constructorArgs, prohibitArgs=None): 841 constructorArgs = constructorArgs.copy() 842 843 if prohibitArgs is None: 844 prohibitArgs = [] 845 846 def bad(arg): 847 config.error("invalid {} argument {}".format(self.__class__.__name__, arg)) 848 for arg in self._shellMixinArgs: 849 if arg not in constructorArgs: 850 continue 851 if arg in prohibitArgs: 852 bad(arg) 853 else: 854 setattr(self, arg, constructorArgs[arg]) 855 del constructorArgs[arg] 856 for arg in list(constructorArgs): 857 if arg not in BuildStep.parms: 858 bad(arg) 859 del constructorArgs[arg] 860 return constructorArgs 861 862 @defer.inlineCallbacks 863 def makeRemoteShellCommand(self, collectStdout=False, collectStderr=False, 864 stdioLogName='stdio', 865 **overrides): 866 kwargs = {arg: getattr(self, arg) 867 for arg in self._shellMixinArgs} 868 kwargs.update(overrides) 869 stdio = None 870 if stdioLogName is not None: 871 # Reuse an existing log if possible; otherwise, create one. 872 try: 873 stdio = yield self.getLog(stdioLogName) 874 except KeyError: 875 stdio = yield self.addLog(stdioLogName) 876 877 kwargs['command'] = flatten(kwargs['command'], (list, tuple)) 878 879 # store command away for display 880 self.command = kwargs['command'] 881 882 # check for the usePTY flag 883 if kwargs['usePTY'] is not None: 884 if self.workerVersionIsOlderThan("shell", "2.7"): 885 if stdio is not None: 886 yield stdio.addHeader( 887 "NOTE: worker does not allow master to override usePTY\n") 888 del kwargs['usePTY'] 889 890 # check for the interruptSignal flag 891 if kwargs["interruptSignal"] and self.workerVersionIsOlderThan("shell", "2.15"): 892 if stdio is not None: 893 yield stdio.addHeader( 894 "NOTE: worker does not allow master to specify interruptSignal\n") 895 del kwargs['interruptSignal'] 896 897 # lazylogfiles are handled below 898 del kwargs['lazylogfiles'] 899 900 # merge the builder's environment with that supplied here 901 builderEnv = self.build.builder.config.env 902 kwargs['env'] = yield self.build.render(builderEnv) 903 kwargs['env'].update(self.env) 904 kwargs['stdioLogName'] = stdioLogName 905 906 # default the workdir appropriately 907 if not kwargs.get('workdir') and not self.workdir: 908 if callable(self.build.workdir): 909 kwargs['workdir'] = self.build.workdir(self.build.sources) 910 else: 911 kwargs['workdir'] = self.build.workdir 912 913 # the rest of the args go to RemoteShellCommand 914 cmd = remotecommand.RemoteShellCommand( 915 collectStdout=collectStdout, 916 collectStderr=collectStderr, 917 **kwargs 918 ) 919 920 # set up logging 921 if stdio is not None: 922 cmd.useLog(stdio, False) 923 for logname, remotefilename in self.logfiles.items(): 924 if self.lazylogfiles: 925 # it's OK if this does, or does not, return a Deferred 926 def callback(cmd_arg, local_logname=logname): 927 return self.addLog(local_logname) 928 cmd.useLogDelayed(logname, callback, True) 929 else: 930 # add a LogFile 931 newlog = yield self.addLog(logname) 932 # and tell the RemoteCommand to feed it 933 cmd.useLog(newlog, False) 934 935 return cmd 936 937 def getResultSummary(self): 938 if self.descriptionDone is not None: 939 return super().getResultSummary() 940 summary = util.command_to_string(self.command) 941 if summary: 942 if self.results != SUCCESS: 943 summary += ' ({})'.format(Results[self.results]) 944 return {'step': summary} 945 return super().getResultSummary() 946 947 948_hush_pyflakes = [WithProperties] 949del _hush_pyflakes 950