1# This file is part of Buildbot.  Buildbot is free software: you can
2# redistribute it and/or modify it under the terms of the GNU General Public
3# License as published by the Free Software Foundation, version 2.
4#
5# This program is distributed in the hope that it will be useful, but WITHOUT
6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8# details.
9#
10# You should have received a copy of the GNU General Public License along with
11# this program; if not, write to the Free Software Foundation, Inc., 51
12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13#
14# Copyright Buildbot Team Members
15
16import inspect
17import sys
18
19from twisted.internet import defer
20from twisted.internet import error
21from twisted.python import deprecate
22from twisted.python import log
23from twisted.python import versions
24from twisted.python.deprecate import deprecatedModuleAttribute
25from twisted.python.failure import Failure
26from twisted.python.reflect import accumulateClassList
27from twisted.python.versions import Version
28from twisted.web.util import formatFailure
29from zope.interface import implementer
30
31from buildbot import config
32from buildbot import interfaces
33from buildbot import util
34from buildbot.interfaces import IRenderable
35from buildbot.interfaces import WorkerSetupError
36from buildbot.process import log as plog
37from buildbot.process import logobserver
38from buildbot.process import properties
39from buildbot.process import remotecommand
40from buildbot.process import results
41# (WithProperties used to be available in this module)
42from buildbot.process.properties import WithProperties
43from buildbot.process.results import ALL_RESULTS
44from buildbot.process.results import CANCELLED
45from buildbot.process.results import EXCEPTION
46from buildbot.process.results import FAILURE
47from buildbot.process.results import RETRY
48from buildbot.process.results import SKIPPED
49from buildbot.process.results import SUCCESS
50from buildbot.process.results import WARNINGS
51from buildbot.process.results import Results
52from buildbot.util import bytes2unicode
53from buildbot.util import debounce
54from buildbot.util import flatten
55from buildbot.util.test_result_submitter import TestResultSubmitter
56from buildbot.warnings import warn_deprecated
57
58
59class BuildStepFailed(Exception):
60    pass
61
62
63class BuildStepCancelled(Exception):
64    # used internally for signalling
65    pass
66
67
68class CallableAttributeError(Exception):
69    # attribute error raised from a callable run inside a property
70    pass
71
72
73# old import paths for these classes
74RemoteCommand = remotecommand.RemoteCommand
75deprecatedModuleAttribute(
76    Version("buildbot", 2, 10, 1),
77    message="Use buildbot.process.remotecommand.RemoteCommand instead.",
78    moduleName="buildbot.process.buildstep",
79    name="RemoteCommand",
80)
81
82LoggedRemoteCommand = remotecommand.LoggedRemoteCommand
83deprecatedModuleAttribute(
84    Version("buildbot", 2, 10, 1),
85    message="Use buildbot.process.remotecommand.LoggedRemoteCommand instead.",
86    moduleName="buildbot.process.buildstep",
87    name="LoggedRemoteCommand",
88)
89
90RemoteShellCommand = remotecommand.RemoteShellCommand
91deprecatedModuleAttribute(
92    Version("buildbot", 2, 10, 1),
93    message="Use buildbot.process.remotecommand.RemoteShellCommand instead.",
94    moduleName="buildbot.process.buildstep",
95    name="RemoteShellCommand",
96)
97
98LogObserver = logobserver.LogObserver
99deprecatedModuleAttribute(
100    Version("buildbot", 2, 10, 1),
101    message="Use buildbot.process.logobserver.LogObserver instead.",
102    moduleName="buildbot.process.buildstep",
103    name="LogObserver",
104)
105
106LogLineObserver = logobserver.LogLineObserver
107deprecatedModuleAttribute(
108    Version("buildbot", 2, 10, 1),
109    message="Use buildbot.util.LogLineObserver instead.",
110    moduleName="buildbot.process.buildstep",
111    name="LogLineObserver",
112)
113
114OutputProgressObserver = logobserver.OutputProgressObserver
115deprecatedModuleAttribute(
116    Version("buildbot", 2, 10, 1),
117    message="Use buildbot.process.logobserver.OutputProgressObserver instead.",
118    moduleName="buildbot.process.buildstep",
119    name="OutputProgressObserver",
120)
121
122
123@implementer(interfaces.IBuildStepFactory)
124class _BuildStepFactory(util.ComparableMixin):
125
126    """
127    This is a wrapper to record the arguments passed to as BuildStep subclass.
128    We use an instance of this class, rather than a closure mostly to make it
129    easier to test that the right factories are getting created.
130    """
131    compare_attrs = ('factory', 'args', 'kwargs')
132
133    def __init__(self, factory, *args, **kwargs):
134        self.factory = factory
135        self.args = args
136        self.kwargs = kwargs
137
138    def buildStep(self):
139        try:
140            return self.factory(*self.args, **self.kwargs)
141        except Exception:
142            log.msg("error while creating step, factory={}, args={}, kwargs={}".format(self.factory,
143                    self.args, self.kwargs))
144            raise
145
146
147class BuildStepStatus:
148    # used only for old-style steps
149    pass
150
151
152def get_factory_from_step_or_factory(step_or_factory):
153    if hasattr(step_or_factory, 'get_step_factory'):
154        factory = step_or_factory.get_step_factory()
155    else:
156        factory = step_or_factory
157    # make sure the returned value actually implements IBuildStepFactory
158    return interfaces.IBuildStepFactory(factory)
159
160
161def create_step_from_step_or_factory(step_or_factory):
162    return get_factory_from_step_or_factory(step_or_factory).buildStep()
163
164
165@implementer(interfaces.IBuildStep)
166class BuildStep(results.ResultComputingConfigMixin,
167                properties.PropertiesMixin,
168                util.ComparableMixin):
169    # Note that the BuildStep is at the same time a template from which per-build steps are
170    # constructed. This works by creating a new IBuildStepFactory in __new__, retrieving it via
171    # get_step_factory() and then calling buildStep() on that factory.
172
173    alwaysRun = False
174    doStepIf = True
175    hideStepIf = False
176    compare_attrs = ("_factory",)
177    # properties set on a build step are, by nature, always runtime properties
178    set_runtime_properties = True
179
180    renderables = results.ResultComputingConfigMixin.resultConfig + [
181        'alwaysRun',
182        'description',
183        'descriptionDone',
184        'descriptionSuffix',
185        'doStepIf',
186        'hideStepIf',
187        'workdir',
188    ]
189
190    # 'parms' holds a list of all the parameters we care about, to allow
191    # users to instantiate a subclass of BuildStep with a mixture of
192    # arguments, some of which are for us, some of which are for the subclass
193    # (or a delegate of the subclass, like how ShellCommand delivers many
194    # arguments to the RemoteShellCommand that it creates). Such delegating
195    # subclasses will use this list to figure out which arguments are meant
196    # for us and which should be given to someone else.
197    parms = [
198        'alwaysRun',
199        'description',
200        'descriptionDone',
201        'descriptionSuffix',
202        'doStepIf',
203        'flunkOnFailure',
204        'flunkOnWarnings',
205        'haltOnFailure',
206        'updateBuildSummaryPolicy',
207        'hideStepIf',
208        'locks',
209        'logEncoding',
210        'name',
211        'progressMetrics',
212        'useProgress',
213        'warnOnFailure',
214        'warnOnWarnings',
215        'workdir',
216    ]
217
218    name = "generic"
219    description = None  # set this to a list of short strings to override
220    descriptionDone = None  # alternate description when the step is complete
221    descriptionSuffix = None  # extra information to append to suffix
222    updateBuildSummaryPolicy = None
223    locks = []
224    progressMetrics = ()  # 'time' is implicit
225    useProgress = True  # set to False if step is really unpredictable
226    build = None
227    step_status = None
228    progress = None
229    logEncoding = None
230    cmd = None
231    rendered = False  # true if attributes are rendered
232    _workdir = None
233    _waitingForLocks = False
234
235    def __init__(self, **kwargs):
236        self.worker = None
237
238        for p in self.__class__.parms:
239            if p in kwargs:
240                setattr(self, p, kwargs.pop(p))
241
242        if kwargs:
243            config.error("{}.__init__ got unexpected keyword argument(s) {}".format(self.__class__,
244                                                                                    list(kwargs)))
245        self._pendingLogObservers = []
246
247        if not isinstance(self.name, str) and not IRenderable.providedBy(self.name):
248            config.error("BuildStep name must be a string or a renderable object: "
249                         "%r" % (self.name,))
250
251        if isinstance(self.description, str):
252            self.description = [self.description]
253        if isinstance(self.descriptionDone, str):
254            self.descriptionDone = [self.descriptionDone]
255        if isinstance(self.descriptionSuffix, str):
256            self.descriptionSuffix = [self.descriptionSuffix]
257
258        if self.updateBuildSummaryPolicy is None:
259            # compute default value for updateBuildSummaryPolicy
260            self.updateBuildSummaryPolicy = [EXCEPTION, RETRY, CANCELLED]
261            if self.flunkOnFailure or self.haltOnFailure or self.warnOnFailure:
262                self.updateBuildSummaryPolicy.append(FAILURE)
263            if self.warnOnWarnings or self.flunkOnWarnings:
264                self.updateBuildSummaryPolicy.append(WARNINGS)
265        if self.updateBuildSummaryPolicy is False:
266            self.updateBuildSummaryPolicy = []
267        if self.updateBuildSummaryPolicy is True:
268            self.updateBuildSummaryPolicy = ALL_RESULTS
269        if not isinstance(self.updateBuildSummaryPolicy, list):
270            config.error("BuildStep updateBuildSummaryPolicy must be "
271                         "a list of result ids or boolean but it is %r" %
272                         (self.updateBuildSummaryPolicy,))
273        self._acquiringLocks = []
274        self.stopped = False
275        self.master = None
276        self.statistics = {}
277        self.logs = {}
278        self._running = False
279        self.stepid = None
280        self.results = None
281        self._start_unhandled_deferreds = None
282        self._test_result_submitters = {}
283
284    def __new__(klass, *args, **kwargs):
285        self = object.__new__(klass)
286        self._factory = _BuildStepFactory(klass, *args, **kwargs)
287        return self
288
289    def __str__(self):
290        args = [repr(x) for x in self._factory.args]
291        args.extend([str(k) + "=" + repr(v)
292                     for k, v in self._factory.kwargs.items()])
293        return "{}({})".format(
294            self.__class__.__name__, ", ".join(args))
295    __repr__ = __str__
296
297    def setBuild(self, build):
298        self.build = build
299        self.master = self.build.master
300
301    def setWorker(self, worker):
302        self.worker = worker
303
304    @deprecate.deprecated(versions.Version("buildbot", 0, 9, 0))
305    def setDefaultWorkdir(self, workdir):
306        if self._workdir is None:
307            self._workdir = workdir
308
309    @property
310    def workdir(self):
311        # default the workdir appropriately
312        if self._workdir is not None or self.build is None:
313            return self._workdir
314        else:
315            # see :ref:`Factory-Workdir-Functions` for details on how to
316            # customize this
317            if callable(self.build.workdir):
318                try:
319                    return self.build.workdir(self.build.sources)
320                except AttributeError as e:
321                    # if the callable raises an AttributeError
322                    # python thinks it is actually workdir that is not existing.
323                    # python will then swallow the attribute error and call
324                    # __getattr__ from worker_transition
325                    _, _, traceback = sys.exc_info()
326                    raise CallableAttributeError(e).with_traceback(traceback)
327                    # we re-raise the original exception by changing its type,
328                    # but keeping its stacktrace
329            else:
330                return self.build.workdir
331
332    @workdir.setter
333    def workdir(self, workdir):
334        self._workdir = workdir
335
336    def getProperties(self):
337        return self.build.getProperties()
338
339    def get_step_factory(self):
340        return self._factory
341
342    def setupProgress(self):
343        # this function temporarily does nothing
344        pass
345
346    def setProgress(self, metric, value):
347        # this function temporarily does nothing
348        pass
349
350    def getCurrentSummary(self):
351        if self.description is not None:
352            stepsumm = util.join_list(self.description)
353            if self.descriptionSuffix:
354                stepsumm += ' ' + util.join_list(self.descriptionSuffix)
355        else:
356            stepsumm = 'running'
357        return {'step': stepsumm}
358
359    def getResultSummary(self):
360        if self.descriptionDone is not None or self.description is not None:
361            stepsumm = util.join_list(self.descriptionDone or self.description)
362            if self.descriptionSuffix:
363                stepsumm += ' ' + util.join_list(self.descriptionSuffix)
364        else:
365            stepsumm = 'finished'
366
367        if self.results != SUCCESS:
368            stepsumm += ' ({})'.format(Results[self.results])
369
370        return {'step': stepsumm}
371
372    @defer.inlineCallbacks
373    def getBuildResultSummary(self):
374        summary = yield self.getResultSummary()
375        if self.results in self.updateBuildSummaryPolicy and \
376                'build' not in summary and 'step' in summary:
377            summary['build'] = summary['step']
378        return summary
379
380    @debounce.method(wait=1)
381    @defer.inlineCallbacks
382    def updateSummary(self):
383        def methodInfo(m):
384            lines = inspect.getsourcelines(m)
385            return "\nat {}:{}:\n {}".format(inspect.getsourcefile(m), lines[1],
386                                             "\n".join(lines[0]))
387        if not self._running:
388            summary = yield self.getResultSummary()
389            if not isinstance(summary, dict):
390                raise TypeError('getResultSummary must return a dictionary: ' +
391                                methodInfo(self.getResultSummary))
392        else:
393            summary = yield self.getCurrentSummary()
394            if not isinstance(summary, dict):
395                raise TypeError('getCurrentSummary must return a dictionary: ' +
396                                methodInfo(self.getCurrentSummary))
397
398        stepResult = summary.get('step', 'finished')
399        if not isinstance(stepResult, str):
400            raise TypeError("step result string must be unicode (got %r)"
401                            % (stepResult,))
402        if self.stepid is not None:
403            stepResult = self.build.properties.cleanupTextFromSecrets(
404                stepResult)
405            yield self.master.data.updates.setStepStateString(self.stepid,
406                                                              stepResult)
407
408        if not self._running:
409            buildResult = summary.get('build', None)
410            if buildResult and not isinstance(buildResult, str):
411                raise TypeError("build result string must be unicode")
412
413    @defer.inlineCallbacks
414    def addStep(self):
415        # create and start the step, noting that the name may be altered to
416        # ensure uniqueness
417        self.name = yield self.build.render(self.name)
418        self.build.setUniqueStepName(self)
419        self.stepid, self.number, self.name = yield self.master.data.updates.addStep(
420            buildid=self.build.buildid,
421            name=util.bytes2unicode(self.name))
422        yield self.master.data.updates.startStep(self.stepid)
423
424    @defer.inlineCallbacks
425    def startStep(self, remote):
426        self.remote = remote
427
428        yield self.addStep()
429        self.locks = yield self.build.render(self.locks)
430
431        # convert all locks into their real form
432        botmaster = self.build.builder.botmaster
433        self.locks = yield botmaster.getLockFromLockAccesses(self.locks, self.build.config_version)
434
435        # then narrow WorkerLocks down to the worker that this build is being
436        # run on
437        self.locks = [(l.getLockForWorker(self.build.workerforbuilder.worker.workername),
438                       la)
439                      for l, la in self.locks]
440
441        for l, la in self.locks:
442            if l in self.build.locks:
443                log.msg(("Hey, lock {} is claimed by both a Step ({}) and the"
444                         " parent Build ({})").format(l, self, self.build))
445                raise RuntimeError("lock claimed by both Step and Build")
446
447        try:
448            # set up locks
449            yield self.acquireLocks()
450
451            if self.stopped:
452                raise BuildStepCancelled
453
454            # render renderables in parallel
455            renderables = []
456            accumulateClassList(self.__class__, 'renderables', renderables)
457
458            def setRenderable(res, attr):
459                setattr(self, attr, res)
460
461            dl = []
462            for renderable in renderables:
463                d = self.build.render(getattr(self, renderable))
464                d.addCallback(setRenderable, renderable)
465                dl.append(d)
466            yield defer.gatherResults(dl)
467            self.rendered = True
468            # we describe ourselves only when renderables are interpolated
469            self.updateSummary()
470
471            # check doStepIf (after rendering)
472            if isinstance(self.doStepIf, bool):
473                doStep = self.doStepIf
474            else:
475                doStep = yield self.doStepIf(self)
476
477            # run -- or skip -- the step
478            if doStep:
479                yield self.addTestResultSets()
480                try:
481                    self._running = True
482                    self.results = yield self.run()
483                finally:
484                    self._running = False
485            else:
486                self.results = SKIPPED
487
488        # NOTE: all of these `except` blocks must set self.results immediately!
489        except BuildStepCancelled:
490            self.results = CANCELLED
491
492        except BuildStepFailed:
493            self.results = FAILURE
494
495        except error.ConnectionLost:
496            self.results = RETRY
497
498        except Exception:
499            self.results = EXCEPTION
500            why = Failure()
501            log.err(why, "BuildStep.failed; traceback follows")
502            yield self.addLogWithFailure(why)
503
504        if self.stopped and self.results != RETRY:
505            # We handle this specially because we don't care about
506            # the return code of an interrupted command; we know
507            # that this should just be exception due to interrupt
508            # At the same time we must respect RETRY status because it's used
509            # to retry interrupted build due to some other issues for example
510            # due to worker lost
511            if self.results != CANCELLED:
512                self.results = EXCEPTION
513
514        # determine whether we should hide this step
515        hidden = self.hideStepIf
516        if callable(hidden):
517            try:
518                hidden = hidden(self.results, self)
519            except Exception:
520                why = Failure()
521                log.err(why, "hidden callback failed; traceback follows")
522                yield self.addLogWithFailure(why)
523                self.results = EXCEPTION
524                hidden = False
525
526        # perform final clean ups
527        success = yield self._cleanup_logs()
528        if not success:
529            self.results = EXCEPTION
530
531        # update the summary one last time, make sure that completes,
532        # and then don't update it any more.
533        self.updateSummary()
534        yield self.updateSummary.stop()
535
536        for sub in self._test_result_submitters.values():
537            yield sub.finish()
538
539        self.releaseLocks()
540
541        yield self.master.data.updates.finishStep(self.stepid, self.results,
542                                                  hidden)
543
544        return self.results
545
546    def setBuildData(self, name, value, source):
547        # returns a Deferred that yields nothing
548        return self.master.data.updates.setBuildData(self.build.buildid, name, value, source)
549
550    @defer.inlineCallbacks
551    def _cleanup_logs(self):
552        all_success = True
553        not_finished_logs = [v for (k, v) in self.logs.items() if not v.finished]
554        finish_logs = yield defer.DeferredList([v.finish() for v in not_finished_logs],
555                                               consumeErrors=True)
556        for success, res in finish_logs:
557            if not success:
558                log.err(res, "when trying to finish a log")
559                all_success = False
560
561        for log_ in self.logs.values():
562            if log_.had_errors():
563                all_success = False
564
565        return all_success
566
567    def addTestResultSets(self):
568        return defer.succeed(None)
569
570    @defer.inlineCallbacks
571    def addTestResultSet(self, description, category, value_unit):
572        sub = TestResultSubmitter()
573        yield sub.setup(self, description, category, value_unit)
574        setid = sub.get_test_result_set_id()
575        self._test_result_submitters[setid] = sub
576        return setid
577
578    def addTestResult(self, setid, value, test_name=None, test_code_path=None, line=None,
579                      duration_ns=None):
580        self._test_result_submitters[setid].add_test_result(value, test_name=test_name,
581                                                            test_code_path=test_code_path,
582                                                            line=line, duration_ns=duration_ns)
583
584    def acquireLocks(self, res=None):
585        if not self.locks:
586            return defer.succeed(None)
587        if self.stopped:
588            return defer.succeed(None)
589        log.msg("acquireLocks(step {}, locks {})".format(self, self.locks))
590        for lock, access in self.locks:
591            for waited_lock, _, _ in self._acquiringLocks:
592                if lock is waited_lock:
593                    continue
594
595            if not lock.isAvailable(self, access):
596                self._waitingForLocks = True
597                log.msg("step {} waiting for lock {}".format(self, lock))
598                d = lock.waitUntilMaybeAvailable(self, access)
599                self._acquiringLocks.append((lock, access, d))
600                d.addCallback(self.acquireLocks)
601                return d
602        # all locks are available, claim them all
603        for lock, access in self.locks:
604            lock.claim(self, access)
605        self._acquiringLocks = []
606        self._waitingForLocks = False
607        return defer.succeed(None)
608
609    def run(self):
610        raise NotImplementedError("A custom build step must implement run()")
611
612    def isNewStyle(self):
613        warn_deprecated('3.0.0', 'BuildStep.isNewStyle() always returns True')
614        return True
615
616    @defer.inlineCallbacks
617    def interrupt(self, reason):
618        if self.stopped:
619            return
620        self.stopped = True
621        if self._acquiringLocks:
622            for (lock, access, d) in self._acquiringLocks:
623                lock.stopWaitingUntilAvailable(self, access, d)
624            self._acquiringLocks = []
625
626        if self._waitingForLocks:
627            yield self.addCompleteLog(
628                'cancelled while waiting for locks', str(reason))
629        else:
630            yield self.addCompleteLog('cancelled', str(reason))
631
632        if self.cmd:
633            d = self.cmd.interrupt(reason)
634            d.addErrback(log.err, 'while cancelling command')
635            yield d
636
637    def releaseLocks(self):
638        log.msg("releaseLocks({}): {}".format(self, self.locks))
639        for lock, access in self.locks:
640            if lock.isOwner(self, access):
641                lock.release(self, access)
642            else:
643                # This should only happen if we've been interrupted
644                assert self.stopped
645
646    # utility methods that BuildSteps may find useful
647
648    def workerVersion(self, command, oldversion=None):
649        return self.build.getWorkerCommandVersion(command, oldversion)
650
651    def workerVersionIsOlderThan(self, command, minversion):
652        sv = self.build.getWorkerCommandVersion(command, None)
653        if sv is None:
654            return True
655        if [int(s) for s in sv.split(".")] < [int(m) for m in minversion.split(".")]:
656            return True
657        return False
658
659    def checkWorkerHasCommand(self, command):
660        if not self.workerVersion(command):
661            message = "worker is too old, does not know about {}".format(command)
662            raise WorkerSetupError(message)
663
664    def getWorkerName(self):
665        return self.build.getWorkerName()
666
667    def addLog(self, name, type='s', logEncoding=None):
668        if self.stepid is None:
669            raise BuildStepCancelled
670        d = self.master.data.updates.addLog(self.stepid,
671                                            util.bytes2unicode(name),
672                                            str(type))
673
674        @d.addCallback
675        def newLog(logid):
676            return self._newLog(name, type, logid, logEncoding)
677        return d
678
679    def getLog(self, name):
680        return self.logs[name]
681
682    @defer.inlineCallbacks
683    def addCompleteLog(self, name, text):
684        if self.stepid is None:
685            raise BuildStepCancelled
686        logid = yield self.master.data.updates.addLog(self.stepid,
687                                                      util.bytes2unicode(name), 't')
688        _log = self._newLog(name, 't', logid)
689        yield _log.addContent(text)
690        yield _log.finish()
691
692    @defer.inlineCallbacks
693    def addHTMLLog(self, name, html):
694        if self.stepid is None:
695            raise BuildStepCancelled
696        logid = yield self.master.data.updates.addLog(self.stepid,
697                                                      util.bytes2unicode(name), 'h')
698        _log = self._newLog(name, 'h', logid)
699        html = bytes2unicode(html)
700        yield _log.addContent(html)
701        yield _log.finish()
702
703    @defer.inlineCallbacks
704    def addLogWithFailure(self, why, logprefix=""):
705        # helper for showing exceptions to the users
706        try:
707            yield self.addCompleteLog(logprefix + "err.text", why.getTraceback())
708            yield self.addHTMLLog(logprefix + "err.html", formatFailure(why))
709        except Exception:
710            log.err(Failure(), "error while formatting exceptions")
711
712    def addLogWithException(self, why, logprefix=""):
713        return self.addLogWithFailure(Failure(why), logprefix)
714
715    def addLogObserver(self, logname, observer):
716        assert interfaces.ILogObserver.providedBy(observer)
717        observer.setStep(self)
718        self._pendingLogObservers.append((logname, observer))
719        self._connectPendingLogObservers()
720
721    def _newLog(self, name, type, logid, logEncoding=None):
722        if not logEncoding:
723            logEncoding = self.logEncoding
724        if not logEncoding:
725            logEncoding = self.master.config.logEncoding
726        log = plog.Log.new(self.master, name, type, logid, logEncoding)
727        self.logs[name] = log
728        self._connectPendingLogObservers()
729        return log
730
731    def _connectPendingLogObservers(self):
732        for logname, observer in self._pendingLogObservers[:]:
733            if logname in self.logs:
734                observer.setLog(self.logs[logname])
735                self._pendingLogObservers.remove((logname, observer))
736
737    @defer.inlineCallbacks
738    def addURL(self, name, url):
739        yield self.master.data.updates.addStepURL(self.stepid, str(name), str(url))
740        return None
741
742    @defer.inlineCallbacks
743    def runCommand(self, command):
744        if self.stopped:
745            return CANCELLED
746
747        self.cmd = command
748        command.worker = self.worker
749        try:
750            res = yield command.run(self, self.remote, self.build.builder.name)
751        finally:
752            self.cmd = None
753        return res
754
755    def hasStatistic(self, name):
756        return name in self.statistics
757
758    def getStatistic(self, name, default=None):
759        return self.statistics.get(name, default)
760
761    def getStatistics(self):
762        return self.statistics.copy()
763
764    def setStatistic(self, name, value):
765        self.statistics[name] = value
766
767
768class CommandMixin:
769
770    @defer.inlineCallbacks
771    def _runRemoteCommand(self, cmd, abandonOnFailure, args, makeResult=None):
772        cmd = remotecommand.RemoteCommand(cmd, args)
773        try:
774            log = self.getLog('stdio')
775        except Exception:
776            log = yield self.addLog('stdio')
777        cmd.useLog(log, False)
778        yield self.runCommand(cmd)
779        if abandonOnFailure and cmd.didFail():
780            raise BuildStepFailed()
781        if makeResult:
782            return makeResult(cmd)
783        else:
784            return not cmd.didFail()
785
786    def runRmdir(self, dir, log=None, abandonOnFailure=True):
787        return self._runRemoteCommand('rmdir', abandonOnFailure,
788                                      {'dir': dir, 'logEnviron': False})
789
790    def pathExists(self, path, log=None):
791        return self._runRemoteCommand('stat', False,
792                                      {'file': path, 'logEnviron': False})
793
794    def runMkdir(self, dir, log=None, abandonOnFailure=True):
795        return self._runRemoteCommand('mkdir', abandonOnFailure,
796                                      {'dir': dir, 'logEnviron': False})
797
798    def runGlob(self, path):
799        return self._runRemoteCommand(
800            'glob', True, {'path': path, 'logEnviron': False},
801            makeResult=lambda cmd: cmd.updates['files'][0])
802
803
804class ShellMixin:
805
806    command = None
807    env = {}
808    want_stdout = True
809    want_stderr = True
810    usePTY = None
811    logfiles = {}
812    lazylogfiles = {}
813    timeout = 1200
814    maxTime = None
815    logEnviron = True
816    interruptSignal = 'KILL'
817    sigtermTime = None
818    initialStdin = None
819    decodeRC = {0: SUCCESS}
820
821    _shellMixinArgs = [
822        'command',
823        'workdir',
824        'env',
825        'want_stdout',
826        'want_stderr',
827        'usePTY',
828        'logfiles',
829        'lazylogfiles',
830        'timeout',
831        'maxTime',
832        'logEnviron',
833        'interruptSignal',
834        'sigtermTime',
835        'initialStdin',
836        'decodeRC',
837    ]
838    renderables = _shellMixinArgs
839
840    def setupShellMixin(self, constructorArgs, prohibitArgs=None):
841        constructorArgs = constructorArgs.copy()
842
843        if prohibitArgs is None:
844            prohibitArgs = []
845
846        def bad(arg):
847            config.error("invalid {} argument {}".format(self.__class__.__name__, arg))
848        for arg in self._shellMixinArgs:
849            if arg not in constructorArgs:
850                continue
851            if arg in prohibitArgs:
852                bad(arg)
853            else:
854                setattr(self, arg, constructorArgs[arg])
855            del constructorArgs[arg]
856        for arg in list(constructorArgs):
857            if arg not in BuildStep.parms:
858                bad(arg)
859                del constructorArgs[arg]
860        return constructorArgs
861
862    @defer.inlineCallbacks
863    def makeRemoteShellCommand(self, collectStdout=False, collectStderr=False,
864                               stdioLogName='stdio',
865                               **overrides):
866        kwargs = {arg: getattr(self, arg)
867                  for arg in self._shellMixinArgs}
868        kwargs.update(overrides)
869        stdio = None
870        if stdioLogName is not None:
871            # Reuse an existing log if possible; otherwise, create one.
872            try:
873                stdio = yield self.getLog(stdioLogName)
874            except KeyError:
875                stdio = yield self.addLog(stdioLogName)
876
877        kwargs['command'] = flatten(kwargs['command'], (list, tuple))
878
879        # store command away for display
880        self.command = kwargs['command']
881
882        # check for the usePTY flag
883        if kwargs['usePTY'] is not None:
884            if self.workerVersionIsOlderThan("shell", "2.7"):
885                if stdio is not None:
886                    yield stdio.addHeader(
887                        "NOTE: worker does not allow master to override usePTY\n")
888                del kwargs['usePTY']
889
890        # check for the interruptSignal flag
891        if kwargs["interruptSignal"] and self.workerVersionIsOlderThan("shell", "2.15"):
892            if stdio is not None:
893                yield stdio.addHeader(
894                    "NOTE: worker does not allow master to specify interruptSignal\n")
895            del kwargs['interruptSignal']
896
897        # lazylogfiles are handled below
898        del kwargs['lazylogfiles']
899
900        # merge the builder's environment with that supplied here
901        builderEnv = self.build.builder.config.env
902        kwargs['env'] = yield self.build.render(builderEnv)
903        kwargs['env'].update(self.env)
904        kwargs['stdioLogName'] = stdioLogName
905
906        # default the workdir appropriately
907        if not kwargs.get('workdir') and not self.workdir:
908            if callable(self.build.workdir):
909                kwargs['workdir'] = self.build.workdir(self.build.sources)
910            else:
911                kwargs['workdir'] = self.build.workdir
912
913        # the rest of the args go to RemoteShellCommand
914        cmd = remotecommand.RemoteShellCommand(
915            collectStdout=collectStdout,
916            collectStderr=collectStderr,
917            **kwargs
918        )
919
920        # set up logging
921        if stdio is not None:
922            cmd.useLog(stdio, False)
923        for logname, remotefilename in self.logfiles.items():
924            if self.lazylogfiles:
925                # it's OK if this does, or does not, return a Deferred
926                def callback(cmd_arg, local_logname=logname):
927                    return self.addLog(local_logname)
928                cmd.useLogDelayed(logname, callback, True)
929            else:
930                # add a LogFile
931                newlog = yield self.addLog(logname)
932                # and tell the RemoteCommand to feed it
933                cmd.useLog(newlog, False)
934
935        return cmd
936
937    def getResultSummary(self):
938        if self.descriptionDone is not None:
939            return super().getResultSummary()
940        summary = util.command_to_string(self.command)
941        if summary:
942            if self.results != SUCCESS:
943                summary += ' ({})'.format(Results[self.results])
944            return {'step': summary}
945        return super().getResultSummary()
946
947
948_hush_pyflakes = [WithProperties]
949del _hush_pyflakes
950