1# This file is part of Buildbot. Buildbot is free software: you can 2# redistribute it and/or modify it under the terms of the GNU General Public 3# License as published by the Free Software Foundation, version 2. 4# 5# This program is distributed in the hope that it will be useful, but WITHOUT 6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8# details. 9# 10# You should have received a copy of the GNU General Public License along with 11# this program; if not, write to the Free Software Foundation, Inc., 51 12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13# 14# Copyright Buildbot Team Members 15""" 16Source step code for Monotone 17""" 18 19 20from twisted.internet import defer 21from twisted.internet import reactor 22from twisted.python import log 23 24from buildbot.config import ConfigErrors 25from buildbot.interfaces import WorkerSetupError 26from buildbot.process import buildstep 27from buildbot.process import remotecommand 28from buildbot.process.results import SUCCESS 29from buildbot.steps.source.base import Source 30 31 32class Monotone(Source): 33 34 """ Class for Monotone with all smarts """ 35 36 name = 'monotone' 37 38 renderables = ['repourl'] 39 possible_methods = ('clobber', 'copy', 'fresh', 'clean') 40 41 def __init__(self, repourl=None, branch=None, progress=False, 42 mode='incremental', method=None, **kwargs): 43 44 self.repourl = repourl 45 self.method = method 46 self.mode = mode 47 self.branch = branch 48 self.sourcedata = "{}?{}".format(self.repourl, self.branch) 49 self.database = 'db.mtn' 50 self.progress = progress 51 super().__init__(**kwargs) 52 errors = [] 53 54 if not self._hasAttrGroupMember('mode', self.mode): 55 errors.append("mode {} is not one of {}".format(self.mode, 56 self._listAttrGroupMembers('mode'))) 57 if self.mode == 'incremental' and self.method: 58 errors.append("Incremental mode does not require method") 59 60 if self.mode == 'full': 61 if self.method is None: 62 self.method = 'copy' 63 elif self.method not in self.possible_methods: 64 errors.append("Invalid method for mode == {}".format(self.mode)) 65 66 if repourl is None: 67 errors.append("you must provide repourl") 68 69 if branch is None: 70 errors.append("you must provide branch") 71 72 if errors: 73 raise ConfigErrors(errors) 74 75 @defer.inlineCallbacks 76 def run_vc(self, branch, revision, patch): 77 self.revision = revision 78 self.stdio_log = yield self.addLogForRemoteCommands("stdio") 79 80 try: 81 monotoneInstalled = yield self.checkMonotone() 82 if not monotoneInstalled: 83 raise WorkerSetupError("Monotone is not installed on worker") 84 85 yield self._checkDb() 86 yield self._retryPull() 87 88 # If we're not throwing away the workdir, check if it's 89 # somehow patched or modified and revert. 90 if self.mode != 'full' or self.method not in ('clobber', 'copy'): 91 patched = yield self.sourcedirIsPatched() 92 if patched: 93 yield self.clean() 94 95 # Call a mode specific method 96 fn = self._getAttrGroupMember('mode', self.mode) 97 yield fn() 98 99 if patch: 100 yield self.patch(patch) 101 yield self.parseGotRevision() 102 return SUCCESS 103 finally: 104 pass # FIXME: remove this try:raise block 105 106 @defer.inlineCallbacks 107 def mode_full(self): 108 if self.method == 'clobber': 109 yield self.clobber() 110 return 111 elif self.method == 'copy': 112 yield self.copy() 113 return 114 115 updatable = yield self._sourcedirIsUpdatable() 116 if not updatable: 117 yield self.clobber() 118 elif self.method == 'clean': 119 yield self.clean() 120 yield self._update() 121 elif self.method == 'fresh': 122 yield self.clean(False) 123 yield self._update() 124 else: 125 raise ValueError("Unknown method, check your configuration") 126 127 @defer.inlineCallbacks 128 def mode_incremental(self): 129 updatable = yield self._sourcedirIsUpdatable() 130 if not updatable: 131 yield self.clobber() 132 else: 133 yield self._update() 134 135 @defer.inlineCallbacks 136 def clobber(self): 137 yield self.runRmdir(self.workdir) 138 yield self._checkout() 139 140 @defer.inlineCallbacks 141 def copy(self): 142 cmd = remotecommand.RemoteCommand('rmdir', { 143 'dir': self.workdir, 144 'logEnviron': self.logEnviron, 145 'timeout': self.timeout, }) 146 cmd.useLog(self.stdio_log, False) 147 yield self.runCommand(cmd) 148 149 self.workdir = 'source' 150 yield self.mode_incremental() 151 cmd = remotecommand.RemoteCommand('cpdir', 152 {'fromdir': 'source', 153 'todir': 'build', 154 'logEnviron': self.logEnviron, 155 'timeout': self.timeout, }) 156 cmd.useLog(self.stdio_log, False) 157 yield self.runCommand(cmd) 158 159 self.workdir = 'build' 160 return 0 161 162 @defer.inlineCallbacks 163 def checkMonotone(self): 164 cmd = remotecommand.RemoteShellCommand(self.workdir, 165 ['mtn', '--version'], 166 env=self.env, 167 logEnviron=self.logEnviron, 168 timeout=self.timeout) 169 cmd.useLog(self.stdio_log, False) 170 yield self.runCommand(cmd) 171 return cmd.rc == 0 172 173 @defer.inlineCallbacks 174 def clean(self, ignore_ignored=True): 175 files = [] 176 commands = [['mtn', 'ls', 'unknown']] 177 if not ignore_ignored: 178 commands.append(['mtn', 'ls', 'ignored']) 179 for cmd in commands: 180 stdout = yield self._dovccmd(cmd, workdir=self.workdir, 181 collectStdout=True) 182 if not stdout: 183 continue 184 for filename in stdout.strip().split('\n'): 185 filename = self.workdir + '/' + str(filename) 186 files.append(filename) 187 188 if not files: 189 rc = 0 190 else: 191 if self.workerVersionIsOlderThan('rmdir', '2.14'): 192 rc = yield self.removeFiles(files) 193 else: 194 rc = yield self.runRmdir(files, abandonOnFailure=False) 195 196 if rc != 0: 197 log.msg("Failed removing files") 198 raise buildstep.BuildStepFailed() 199 200 @defer.inlineCallbacks 201 def removeFiles(self, files): 202 for filename in files: 203 res = yield self.runRmdir(filename, abandonOnFailure=False) 204 if res: 205 return res 206 return 0 207 208 def _checkout(self, abandonOnFailure=False): 209 command = ['mtn', 'checkout', self.workdir, '--db', self.database] 210 if self.revision: 211 command.extend(['--revision', self.revision]) 212 command.extend(['--branch', self.branch]) 213 return self._dovccmd(command, workdir='.', 214 abandonOnFailure=abandonOnFailure) 215 216 def _update(self, abandonOnFailure=False): 217 command = ['mtn', 'update'] 218 if self.revision: 219 command.extend(['--revision', self.revision]) 220 else: 221 command.extend(['--revision', 'h:' + self.branch]) 222 command.extend(['--branch', self.branch]) 223 return self._dovccmd(command, workdir=self.workdir, 224 abandonOnFailure=abandonOnFailure) 225 226 def _pull(self, abandonOnFailure=False): 227 command = ['mtn', 'pull', self.sourcedata, '--db', self.database] 228 if self.progress: 229 command.extend(['--ticker=dot']) 230 else: 231 command.extend(['--ticker=none']) 232 d = self._dovccmd(command, workdir='.', 233 abandonOnFailure=abandonOnFailure) 234 return d 235 236 @defer.inlineCallbacks 237 def _retryPull(self): 238 if self.retry: 239 abandonOnFailure = (self.retry[1] <= 0) 240 else: 241 abandonOnFailure = True 242 243 res = yield self._pull(abandonOnFailure) 244 if self.retry: 245 delay, repeats = self.retry 246 if self.stopped or res == 0 or repeats <= 0: 247 return res 248 else: 249 log.msg("Checkout failed, trying %d more times after %d seconds" 250 % (repeats, delay)) 251 self.retry = (delay, repeats - 1) 252 df = defer.Deferred() 253 df.addCallback(lambda _: self._retryPull()) 254 reactor.callLater(delay, df.callback, None) 255 yield df 256 return None 257 258 @defer.inlineCallbacks 259 def parseGotRevision(self): 260 stdout = yield self._dovccmd(['mtn', 'automate', 'select', 'w:'], 261 workdir=self.workdir, 262 collectStdout=True) 263 revision = stdout.strip() 264 if len(revision) != 40: 265 raise buildstep.BuildStepFailed() 266 log.msg("Got Monotone revision {}".format(revision)) 267 self.updateSourceProperty('got_revision', revision) 268 return 0 269 270 @defer.inlineCallbacks 271 def _dovccmd(self, command, workdir, 272 collectStdout=False, initialStdin=None, decodeRC=None, 273 abandonOnFailure=True): 274 if not command: 275 raise ValueError("No command specified") 276 277 if decodeRC is None: 278 decodeRC = {0: SUCCESS} 279 cmd = remotecommand.RemoteShellCommand(workdir, command, 280 env=self.env, 281 logEnviron=self.logEnviron, 282 timeout=self.timeout, 283 collectStdout=collectStdout, 284 initialStdin=initialStdin, 285 decodeRC=decodeRC) 286 cmd.useLog(self.stdio_log, False) 287 yield self.runCommand(cmd) 288 289 if abandonOnFailure and cmd.didFail(): 290 log.msg("Source step failed while running command {}".format(cmd)) 291 raise buildstep.BuildStepFailed() 292 if collectStdout: 293 return cmd.stdout 294 else: 295 return cmd.rc 296 297 @defer.inlineCallbacks 298 def _checkDb(self): 299 db_exists = yield self.pathExists(self.database) 300 db_needs_init = False 301 if db_exists: 302 stdout = yield self._dovccmd( 303 ['mtn', 'db', 'info', '--db', self.database], 304 workdir='.', 305 collectStdout=True) 306 if stdout.find("migration needed") >= 0: 307 log.msg("Older format database found, migrating it") 308 yield self._dovccmd(['mtn', 'db', 'migrate', '--db', 309 self.database], 310 workdir='.') 311 elif stdout.find("too new, cannot use") >= 0 or \ 312 stdout.find("database has no tables") >= 0: 313 # The database is of a newer format which the worker's 314 # mtn version can not handle. Drop it and pull again 315 # with that monotone version installed on the 316 # worker. Do the same if it's an empty file. 317 yield self.runRmdir(self.database) 318 db_needs_init = True 319 elif stdout.find("not a monotone database") >= 0: 320 # There exists a database file, but it's not a valid 321 # monotone database. Do not delete it, but fail with 322 # an error. 323 raise buildstep.BuildStepFailed() 324 else: 325 log.msg("Database exists and compatible") 326 else: 327 db_needs_init = True 328 log.msg("Database does not exist") 329 330 if db_needs_init: 331 command = ['mtn', 'db', 'init', '--db', self.database] 332 yield self._dovccmd(command, workdir='.') 333 334 @defer.inlineCallbacks 335 def _sourcedirIsUpdatable(self): 336 workdir_path = self.build.path_module.join(self.workdir, '_MTN') 337 workdir_exists = yield self.pathExists(workdir_path) 338 339 if not workdir_exists: 340 log.msg("Workdir does not exist, falling back to a fresh clone") 341 342 return workdir_exists 343