1# This file is part of Buildbot. Buildbot is free software: you can 2# redistribute it and/or modify it under the terms of the GNU General Public 3# License as published by the Free Software Foundation, version 2. 4# 5# This program is distributed in the hope that it will be useful, but WITHOUT 6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8# details. 9# 10# You should have received a copy of the GNU General Public License along with 11# this program; if not, write to the Free Software Foundation, Inc., 51 12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13# 14# Copyright Buildbot Team Members 15 16import io 17import os 18import subprocess 19 20from twisted.internet import defer 21from twisted.internet import error 22from twisted.internet import protocol 23from twisted.python import failure 24from twisted.python import log 25from twisted.python import runtime 26 27from buildbot.util import unicode2bytes 28 29 30class RunProcessPP(protocol.ProcessProtocol): 31 def __init__(self, run_process, initial_stdin=None): 32 self.run_process = run_process 33 self.initial_stdin = initial_stdin 34 35 def connectionMade(self): 36 if self.initial_stdin: 37 self.transport.write(self.initial_stdin) 38 self.transport.closeStdin() 39 40 def outReceived(self, data): 41 self.run_process.add_stdout(data) 42 43 def errReceived(self, data): 44 self.run_process.add_stderr(data) 45 46 def processEnded(self, reason): 47 self.run_process.process_ended(reason.value.signal, reason.value.exitCode) 48 49 50class RunProcess: 51 52 TIMEOUT_KILL = 5 53 interrupt_signal = "KILL" 54 55 def __init__(self, reactor, command, workdir=None, env=None, 56 collect_stdout=True, collect_stderr=True, stderr_is_error=False, 57 io_timeout=300, runtime_timeout=3600, sigterm_timeout=5, initial_stdin=None): 58 59 self._reactor = reactor 60 self.command = command 61 62 self.workdir = workdir 63 self.process = None 64 65 self.environ = env 66 67 self.initial_stdin = initial_stdin 68 69 self.output_stdout = io.BytesIO() if collect_stdout else None 70 self.output_stderr = io.BytesIO() if collect_stderr else None 71 self.stderr_is_error = stderr_is_error 72 73 self.io_timeout = io_timeout 74 self.io_timer = None 75 76 self.sigterm_timeout = sigterm_timeout 77 self.sigterm_timer = None 78 79 self.runtime_timeout = runtime_timeout 80 self.runtime_timer = None 81 82 self.killed = False 83 self.kill_timer = None 84 85 def __repr__(self): 86 return "<{0} '{1}'>".format(self.__class__.__name__, self.command) 87 88 def get_os_env(self): 89 return os.environ 90 91 def resolve_environment(self, env): 92 os_env = self.get_os_env() 93 if env is None: 94 return os_env.copy() 95 96 new_env = {} 97 for key in os_env: 98 if key not in env or env[key] is not None: 99 new_env[key] = os_env[key] 100 for key, value in env.items(): 101 if value is not None: 102 new_env[key] = value 103 return new_env 104 105 def start(self): 106 self.deferred = defer.Deferred() 107 try: 108 self._start_command() 109 except Exception as e: 110 self.deferred.errback(failure.Failure(e)) 111 return self.deferred 112 113 def _start_command(self): 114 self.pp = RunProcessPP(self, initial_stdin=self.initial_stdin) 115 116 environ = self.resolve_environment(self.environ) 117 118 # $PWD usually indicates the current directory; spawnProcess may not 119 # update this value, though, so we set it explicitly here. This causes 120 # weird problems (bug #456) on msys 121 if not environ.get('MACHTYPE', None) == 'i686-pc-msys' and self.workdir is not None: 122 environ['PWD'] = os.path.abspath(self.workdir) 123 124 argv = unicode2bytes(self.command) 125 self.process = self._reactor.spawnProcess(self.pp, argv[0], argv, environ, self.workdir) 126 127 if self.io_timeout: 128 self.io_timer = self._reactor.callLater(self.io_timeout, self.io_timed_out) 129 130 if self.runtime_timeout: 131 self.runtime_timer = self._reactor.callLater(self.runtime_timeout, 132 self.runtime_timed_out) 133 134 def add_stdout(self, data): 135 if self.output_stdout is not None: 136 self.output_stdout.write(data) 137 138 if self.io_timer: 139 self.io_timer.reset(self.io_timeout) 140 141 def add_stderr(self, data): 142 if self.output_stderr is not None: 143 self.output_stderr.write(data) 144 elif self.stderr_is_error: 145 self.kill('command produced stderr which is interpreted as error') 146 147 if self.io_timer: 148 self.io_timer.reset(self.io_timeout) 149 150 def _build_result(self, rc): 151 if self.output_stdout is not None and self.output_stderr is not None: 152 return (rc, self.output_stdout.getvalue(), self.output_stderr.getvalue()) 153 if self.output_stdout is not None: 154 return (rc, self.output_stdout.getvalue()) 155 if self.output_stderr is not None: 156 return (rc, self.output_stderr.getvalue()) 157 return rc 158 159 def process_ended(self, sig, rc): 160 if self.killed and rc == 0: 161 log.msg("process was killed, but exited with status 0; faking a failure") 162 163 # windows returns '1' even for signalled failures, while POSIX returns -1 164 if runtime.platformType == 'win32': 165 rc = 1 166 else: 167 rc = -1 168 169 if sig is not None: 170 rc = -1 171 172 self._cancel_timers() 173 d = self.deferred 174 self.deferred = None 175 if d: 176 d.callback(self._build_result(rc)) 177 else: 178 log.err("{}: command finished twice".format(self)) 179 180 def failed(self, why): 181 self._cancel_timers() 182 d = self.deferred 183 self.deferred = None 184 if d: 185 d.errback(why) 186 else: 187 log.err("{}: command finished twice".format(self)) 188 189 def io_timed_out(self): 190 self.io_timer = None 191 msg = "{}: command timed out: {} seconds without output".format(self, self.io_timeout) 192 self.kill(msg) 193 194 def runtime_timed_out(self): 195 self.runtime_timer = None 196 msg = "{}: command timed out: {} seconds elapsed".format(self, self.runtime_timeout) 197 self.kill(msg) 198 199 def is_dead(self): 200 if self.process.pid is None: 201 return True 202 pid = int(self.process.pid) 203 try: 204 os.kill(pid, 0) 205 except OSError: 206 return True 207 return False 208 209 def check_process_was_killed(self): 210 211 self.sigterm_timer = None 212 if not self.is_dead(): 213 if not self.send_signal(self.interrupt_signal): 214 log.msg("{}: failed to kill process again".format(self)) 215 216 self.cleanup_killed_process() 217 218 def cleanup_killed_process(self): 219 if runtime.platformType == "posix": 220 # we only do this under posix because the win32eventreactor 221 # blocks here until the process has terminated, while closing 222 # stderr. This is weird. 223 self.pp.transport.loseConnection() 224 225 if self.deferred: 226 # finished ought to be called momentarily. Just in case it doesn't, 227 # set a timer which will abandon the command. 228 self.kill_timer = self._reactor.callLater(self.TIMEOUT_KILL, self.kill_timed_out) 229 230 def send_signal(self, interrupt_signal): 231 success = False 232 233 log.msg('{}: killing process using {}'.format(self, interrupt_signal)) 234 235 if runtime.platformType == "win32": 236 if interrupt_signal is not None and self.process.pid is not None: 237 if interrupt_signal == "TERM": 238 # TODO: blocks 239 subprocess.check_call("TASKKILL /PID {0} /T".format(self.process.pid)) 240 success = True 241 elif interrupt_signal == "KILL": 242 # TODO: blocks 243 subprocess.check_call("TASKKILL /F /PID {0} /T".format(self.process.pid)) 244 success = True 245 246 # try signalling the process itself (works on Windows too, sorta) 247 if not success: 248 try: 249 self.process.signalProcess(interrupt_signal) 250 success = True 251 except OSError as e: 252 log.err("{}: from process.signalProcess: {}".format(self, e)) 253 # could be no-such-process, because they finished very recently 254 except error.ProcessExitedAlready: 255 log.msg("{}: process exited already - can't kill".format(self)) 256 257 # the process has already exited, and likely finished() has 258 # been called already or will be called shortly 259 260 return success 261 262 def kill(self, msg): 263 log.msg('{}: killing process because {}'.format(self, msg)) 264 self._cancel_timers() 265 266 self.killed = True 267 268 if self.sigterm_timeout is not None: 269 self.send_signal("TERM") 270 self.sigterm_timer = self._reactor.callLater(self.sigterm_timeout, 271 self.check_process_was_killed) 272 else: 273 if not self.send_signal(self.interrupt_signal): 274 log.msg("{}: failed to kill process".format(self)) 275 276 self.cleanup_killed_process() 277 278 def kill_timed_out(self): 279 self.kill_timer = None 280 log.msg("{}: attempted to kill process, but it wouldn't die".format(self)) 281 282 self.failed(RuntimeError("SIG{} failed to kill process".format(self.interrupt_signal))) 283 284 def _cancel_timers(self): 285 for name in ('io_timer', 'kill_timer', 'runtime_timer', 'sigterm_timer'): 286 timer = getattr(self, name, None) 287 if timer: 288 timer.cancel() 289 setattr(self, name, None) 290 291 292def run_process(*args, **kwargs): 293 process = RunProcess(*args, **kwargs) 294 return process.start() 295