1# This file is part of Buildbot.  Buildbot is free software: you can
2# redistribute it and/or modify it under the terms of the GNU General Public
3# License as published by the Free Software Foundation, version 2.
4#
5# This program is distributed in the hope that it will be useful, but WITHOUT
6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
8# details.
9#
10# You should have received a copy of the GNU General Public License along with
11# this program; if not, write to the Free Software Foundation, Inc., 51
12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13#
14# Copyright Buildbot Team Members
15"""
16Source step code for Monotone
17"""
18
19
20from twisted.internet import defer
21from twisted.internet import reactor
22from twisted.python import log
23
24from buildbot.config import ConfigErrors
25from buildbot.interfaces import WorkerSetupError
26from buildbot.process import buildstep
27from buildbot.process import remotecommand
28from buildbot.process.results import SUCCESS
29from buildbot.steps.source.base import Source
30
31
32class Monotone(Source):
33
34    """ Class for Monotone with all smarts """
35
36    name = 'monotone'
37
38    renderables = ['repourl']
39    possible_methods = ('clobber', 'copy', 'fresh', 'clean')
40
41    def __init__(self, repourl=None, branch=None, progress=False,
42                 mode='incremental', method=None, **kwargs):
43
44        self.repourl = repourl
45        self.method = method
46        self.mode = mode
47        self.branch = branch
48        self.sourcedata = "{}?{}".format(self.repourl, self.branch)
49        self.database = 'db.mtn'
50        self.progress = progress
51        super().__init__(**kwargs)
52        errors = []
53
54        if not self._hasAttrGroupMember('mode', self.mode):
55            errors.append("mode {} is not one of {}".format(self.mode,
56                                                            self._listAttrGroupMembers('mode')))
57        if self.mode == 'incremental' and self.method:
58            errors.append("Incremental mode does not require method")
59
60        if self.mode == 'full':
61            if self.method is None:
62                self.method = 'copy'
63            elif self.method not in self.possible_methods:
64                errors.append("Invalid method for mode == {}".format(self.mode))
65
66        if repourl is None:
67            errors.append("you must provide repourl")
68
69        if branch is None:
70            errors.append("you must provide branch")
71
72        if errors:
73            raise ConfigErrors(errors)
74
75    @defer.inlineCallbacks
76    def run_vc(self, branch, revision, patch):
77        self.revision = revision
78        self.stdio_log = yield self.addLogForRemoteCommands("stdio")
79
80        try:
81            monotoneInstalled = yield self.checkMonotone()
82            if not monotoneInstalled:
83                raise WorkerSetupError("Monotone is not installed on worker")
84
85            yield self._checkDb()
86            yield self._retryPull()
87
88            # If we're not throwing away the workdir, check if it's
89            # somehow patched or modified and revert.
90            if self.mode != 'full' or self.method not in ('clobber', 'copy'):
91                patched = yield self.sourcedirIsPatched()
92                if patched:
93                    yield self.clean()
94
95            # Call a mode specific method
96            fn = self._getAttrGroupMember('mode', self.mode)
97            yield fn()
98
99            if patch:
100                yield self.patch(patch)
101            yield self.parseGotRevision()
102            return SUCCESS
103        finally:
104            pass  # FIXME: remove this try:raise block
105
106    @defer.inlineCallbacks
107    def mode_full(self):
108        if self.method == 'clobber':
109            yield self.clobber()
110            return
111        elif self.method == 'copy':
112            yield self.copy()
113            return
114
115        updatable = yield self._sourcedirIsUpdatable()
116        if not updatable:
117            yield self.clobber()
118        elif self.method == 'clean':
119            yield self.clean()
120            yield self._update()
121        elif self.method == 'fresh':
122            yield self.clean(False)
123            yield self._update()
124        else:
125            raise ValueError("Unknown method, check your configuration")
126
127    @defer.inlineCallbacks
128    def mode_incremental(self):
129        updatable = yield self._sourcedirIsUpdatable()
130        if not updatable:
131            yield self.clobber()
132        else:
133            yield self._update()
134
135    @defer.inlineCallbacks
136    def clobber(self):
137        yield self.runRmdir(self.workdir)
138        yield self._checkout()
139
140    @defer.inlineCallbacks
141    def copy(self):
142        cmd = remotecommand.RemoteCommand('rmdir', {
143            'dir': self.workdir,
144            'logEnviron': self.logEnviron,
145            'timeout': self.timeout, })
146        cmd.useLog(self.stdio_log, False)
147        yield self.runCommand(cmd)
148
149        self.workdir = 'source'
150        yield self.mode_incremental()
151        cmd = remotecommand.RemoteCommand('cpdir',
152                                          {'fromdir': 'source',
153                                           'todir': 'build',
154                                           'logEnviron': self.logEnviron,
155                                           'timeout': self.timeout, })
156        cmd.useLog(self.stdio_log, False)
157        yield self.runCommand(cmd)
158
159        self.workdir = 'build'
160        return 0
161
162    @defer.inlineCallbacks
163    def checkMonotone(self):
164        cmd = remotecommand.RemoteShellCommand(self.workdir,
165                                               ['mtn', '--version'],
166                                               env=self.env,
167                                               logEnviron=self.logEnviron,
168                                               timeout=self.timeout)
169        cmd.useLog(self.stdio_log, False)
170        yield self.runCommand(cmd)
171        return cmd.rc == 0
172
173    @defer.inlineCallbacks
174    def clean(self, ignore_ignored=True):
175        files = []
176        commands = [['mtn', 'ls', 'unknown']]
177        if not ignore_ignored:
178            commands.append(['mtn', 'ls', 'ignored'])
179        for cmd in commands:
180            stdout = yield self._dovccmd(cmd, workdir=self.workdir,
181                                         collectStdout=True)
182            if not stdout:
183                continue
184            for filename in stdout.strip().split('\n'):
185                filename = self.workdir + '/' + str(filename)
186                files.append(filename)
187
188        if not files:
189            rc = 0
190        else:
191            if self.workerVersionIsOlderThan('rmdir', '2.14'):
192                rc = yield self.removeFiles(files)
193            else:
194                rc = yield self.runRmdir(files, abandonOnFailure=False)
195
196        if rc != 0:
197            log.msg("Failed removing files")
198            raise buildstep.BuildStepFailed()
199
200    @defer.inlineCallbacks
201    def removeFiles(self, files):
202        for filename in files:
203            res = yield self.runRmdir(filename, abandonOnFailure=False)
204            if res:
205                return res
206        return 0
207
208    def _checkout(self, abandonOnFailure=False):
209        command = ['mtn', 'checkout', self.workdir, '--db', self.database]
210        if self.revision:
211            command.extend(['--revision', self.revision])
212        command.extend(['--branch', self.branch])
213        return self._dovccmd(command, workdir='.',
214                             abandonOnFailure=abandonOnFailure)
215
216    def _update(self, abandonOnFailure=False):
217        command = ['mtn', 'update']
218        if self.revision:
219            command.extend(['--revision', self.revision])
220        else:
221            command.extend(['--revision', 'h:' + self.branch])
222        command.extend(['--branch', self.branch])
223        return self._dovccmd(command, workdir=self.workdir,
224                             abandonOnFailure=abandonOnFailure)
225
226    def _pull(self, abandonOnFailure=False):
227        command = ['mtn', 'pull', self.sourcedata, '--db', self.database]
228        if self.progress:
229            command.extend(['--ticker=dot'])
230        else:
231            command.extend(['--ticker=none'])
232        d = self._dovccmd(command, workdir='.',
233                          abandonOnFailure=abandonOnFailure)
234        return d
235
236    @defer.inlineCallbacks
237    def _retryPull(self):
238        if self.retry:
239            abandonOnFailure = (self.retry[1] <= 0)
240        else:
241            abandonOnFailure = True
242
243        res = yield self._pull(abandonOnFailure)
244        if self.retry:
245            delay, repeats = self.retry
246            if self.stopped or res == 0 or repeats <= 0:
247                return res
248            else:
249                log.msg("Checkout failed, trying %d more times after %d seconds"
250                        % (repeats, delay))
251                self.retry = (delay, repeats - 1)
252                df = defer.Deferred()
253                df.addCallback(lambda _: self._retryPull())
254                reactor.callLater(delay, df.callback, None)
255                yield df
256        return None
257
258    @defer.inlineCallbacks
259    def parseGotRevision(self):
260        stdout = yield self._dovccmd(['mtn', 'automate', 'select', 'w:'],
261                                     workdir=self.workdir,
262                                     collectStdout=True)
263        revision = stdout.strip()
264        if len(revision) != 40:
265            raise buildstep.BuildStepFailed()
266        log.msg("Got Monotone revision {}".format(revision))
267        self.updateSourceProperty('got_revision', revision)
268        return 0
269
270    @defer.inlineCallbacks
271    def _dovccmd(self, command, workdir,
272                 collectStdout=False, initialStdin=None, decodeRC=None,
273                 abandonOnFailure=True):
274        if not command:
275            raise ValueError("No command specified")
276
277        if decodeRC is None:
278            decodeRC = {0: SUCCESS}
279        cmd = remotecommand.RemoteShellCommand(workdir, command,
280                                               env=self.env,
281                                               logEnviron=self.logEnviron,
282                                               timeout=self.timeout,
283                                               collectStdout=collectStdout,
284                                               initialStdin=initialStdin,
285                                               decodeRC=decodeRC)
286        cmd.useLog(self.stdio_log, False)
287        yield self.runCommand(cmd)
288
289        if abandonOnFailure and cmd.didFail():
290            log.msg("Source step failed while running command {}".format(cmd))
291            raise buildstep.BuildStepFailed()
292        if collectStdout:
293            return cmd.stdout
294        else:
295            return cmd.rc
296
297    @defer.inlineCallbacks
298    def _checkDb(self):
299        db_exists = yield self.pathExists(self.database)
300        db_needs_init = False
301        if db_exists:
302            stdout = yield self._dovccmd(
303                ['mtn', 'db', 'info', '--db', self.database],
304                workdir='.',
305                collectStdout=True)
306            if stdout.find("migration needed") >= 0:
307                log.msg("Older format database found, migrating it")
308                yield self._dovccmd(['mtn', 'db', 'migrate', '--db',
309                                     self.database],
310                                    workdir='.')
311            elif stdout.find("too new, cannot use") >= 0 or \
312                    stdout.find("database has no tables") >= 0:
313                # The database is of a newer format which the worker's
314                # mtn version can not handle. Drop it and pull again
315                # with that monotone version installed on the
316                # worker. Do the same if it's an empty file.
317                yield self.runRmdir(self.database)
318                db_needs_init = True
319            elif stdout.find("not a monotone database") >= 0:
320                # There exists a database file, but it's not a valid
321                # monotone database. Do not delete it, but fail with
322                # an error.
323                raise buildstep.BuildStepFailed()
324            else:
325                log.msg("Database exists and compatible")
326        else:
327            db_needs_init = True
328            log.msg("Database does not exist")
329
330        if db_needs_init:
331            command = ['mtn', 'db', 'init', '--db', self.database]
332            yield self._dovccmd(command, workdir='.')
333
334    @defer.inlineCallbacks
335    def _sourcedirIsUpdatable(self):
336        workdir_path = self.build.path_module.join(self.workdir, '_MTN')
337        workdir_exists = yield self.pathExists(workdir_path)
338
339        if not workdir_exists:
340            log.msg("Workdir does not exist, falling back to a fresh clone")
341
342        return workdir_exists
343