1# -*- mode: python; coding: utf-8 -*-
2# :Progetto: vcpx -- Syncable targets
3# :Creato:   ven 04 giu 2004 00:27:07 CEST
4# :Autore:   Lele Gaifax <lele@nautilus.homeip.net>
5# :Licenza:  GNU General Public License
6#
7
8"""
9Syncronizable targets are the simplest abstract wrappers around a
10working directory under two different version control systems.
11"""
12from __future__ import absolute_import
13
14from builtins import str
15__docformat__ = 'reStructuredText'
16
17import socket
18from signal import signal, SIGINT, SIG_IGN
19from vcpx import TailorBug, TailorException
20from vcpx.workdir import WorkingDir
21
22
23HOST = socket.getfqdn()
24AUTHOR = "tailor"
25BOOTSTRAP_PATCHNAME = 'Tailorization'
26BOOTSTRAP_CHANGELOG = """\
27Import of the upstream sources from
28%(source_repository)s
29   Revision: %(revision)s
30"""
31
32
33class TargetInitializationFailure(TailorException):
34    "Failure initializing the target VCS"
35
36
37class ChangesetReplayFailure(TailorException):
38    "Failure replaying the changeset on the target system"
39
40
41class PostCommitCheckFailure(TailorException):
42    "Most probably a tailor bug, not everything has been committed."
43
44
45class SynchronizableTargetWorkingDir(WorkingDir):
46    """
47    This is an abstract working dir usable as a *shadow* of another
48    kind of VC, sharing the same working directory.
49
50    Most interesting entry points are:
51
52    replayChangeset
53        to replay an already applied changeset, to mimic the actions
54        performed by the upstream VC system on the tree such as
55        renames, deletions and adds.  This is an useful argument to
56        feed as ``replay`` to ``applyUpstreamChangesets``
57
58    importFirstRevision
59        to initialize a pristine working directory tree under this VC
60        system, possibly extracted under a different kind of VC
61
62    Subclasses MUST override at least the _underscoredMethods.
63    """
64
65    PATCH_NAME_FORMAT = '[%(project)s @ %(revision)s]'
66    """
67    The format string used to compute the patch name, used by underlying VCS.
68    """
69
70    REMOVE_FIRST_LOG_LINE = False
71    """
72    When true, remove the first line from the upstream changelog.
73    """
74
75    def __getPatchNameAndLog(self, changeset):
76        """
77        Return a tuple (patchname, changelog) interpolating changeset's
78        information with the template above.
79        """
80
81        if changeset.log == '':
82            firstlogline = 'Empty log message'
83            remaininglog = ''
84        else:
85            loglines = changeset.log.split('\n')
86            if len(loglines)>1:
87                firstlogline = loglines[0]
88                remaininglog = '\n'.join(loglines[1:])
89            else:
90                firstlogline = changeset.log
91                remaininglog = ''
92
93        patchname = self.PATCH_NAME_FORMAT % {
94            'project': self.repository.projectref().name,
95            'revision': changeset.revision,
96            'author': changeset.author,
97            'date': changeset.date,
98            'firstlogline': firstlogline,
99            'remaininglog': remaininglog}
100        if self.REMOVE_FIRST_LOG_LINE:
101            changelog = remaininglog
102        else:
103            changelog = changeset.log
104        return patchname, changelog
105
106    def _prepareToReplayChangeset(self, changeset):
107        """
108        This is called **before** fetching and applying the source
109        changeset. This implementation does nothing more than
110        returning True. Subclasses may override it, for example to
111        preexecute some entries such as moves.
112
113        Returning False the changeset won't be applied and the
114        process will stop.
115        """
116
117        return True
118
119    def replayChangeset(self, changeset):
120        """
121        Do whatever is needed to replay the changes under the target
122        VC, to register the already applied (under the other VC)
123        changeset.
124        """
125
126        try:
127            changeset = self._adaptChangeset(changeset)
128        except:
129            self.log.exception("Failure adapting: %s", str(changeset))
130            raise
131
132        if changeset is None:
133            return
134
135        try:
136            self._replayChangeset(changeset)
137        except:
138            self.log.exception("Failure replaying: %s", str(changeset))
139            raise
140        patchname, log = self.__getPatchNameAndLog(changeset)
141        entries = self._getCommitEntries(changeset)
142        previous = signal(SIGINT, SIG_IGN)
143        try:
144            self._commit(changeset.date, changeset.author, patchname, log,
145                         entries, tags = changeset.tags)
146            if changeset.tags:
147                for tag in changeset.tags:
148                    self._tag(tag, changeset.date, changeset.author)
149            if self.repository.post_commit_check:
150                self._postCommitCheck()
151        finally:
152            signal(SIGINT, previous)
153
154        try:
155            self._dismissChangeset(changeset)
156        except:
157            self.log.exception("Failure dismissing: %s", str(changeset))
158            raise
159
160    def __getPrefixToSource(self):
161        """
162        Compute and return the "offset" between source and target basedirs,
163        or None when not using shared directories, or there's no offset.
164        """
165
166        project = self.repository.projectref()
167        ssubdir = project.source.subdir
168        tsubdir = project.target.subdir
169        if self.shared_basedirs and ssubdir != tsubdir:
170            if tsubdir == '.':
171                prefix = ssubdir
172            else:
173                if not tsubdir.endswith('/'):
174                    tsubdir += '/'
175                prefix = ssubdir[len(tsubdir):]
176            return prefix
177        else:
178            return None
179
180    def _normalizeEntryPaths(self, entry):
181        """
182        Normalize the name and old_name of an entry.
183
184        The ``name`` and ``old_name`` of an entry are pathnames coming
185        from the upstream system, and is usually (although there is no
186        guarantee it actually is) a UNIX style path with forward
187        slashes "/" as separators.
188
189        This implementation uses normpath to adapt the path to the
190        actual OS convention, but subclasses may eventually override
191        this to use their own canonicalization of ``name`` and
192        ``old_name``.
193        """
194
195        from os.path import normpath
196
197        entry.name = normpath(entry.name)
198        if entry.old_name:
199            entry.old_name = normpath(entry.old_name)
200
201    def __adaptEntriesPath(self, changeset):
202        """
203        If the source basedir is a subdirectory of the target, adjust
204        all the pathnames adding the prefix computed by difference.
205        """
206
207        from copy import deepcopy
208        from os.path import join
209
210        if not changeset.entries:
211            return changeset
212
213        prefix = self.__getPrefixToSource()
214        adapted = deepcopy(changeset)
215        for e in adapted.entries:
216            if prefix:
217                e.name = join(prefix, e.name)
218                if e.old_name:
219                    e.old_name = join(prefix, e.old_name)
220            self._normalizeEntryPaths(e)
221        return adapted
222
223    def _adaptEntries(self, changeset):
224        """
225        Do whatever is needed to adapt entries to the target system.
226
227        This implementation adds a prefix to each path if needed, when
228        the target basedir *contains* the source basedir. Also, each
229        path is normalized thru ``normpath()`` or whatever equivalent
230        operation provided by the specific target. It operates on and
231        returns a copy of the given changeset.
232
233        Subclasses shall eventually extend this to exclude unwanted
234        entries, eventually returning None when all entries were
235        excluded, to avoid the commit on target of an empty changeset.
236        """
237
238        adapted = self.__adaptEntriesPath(changeset)
239        return adapted
240
241    def _adaptChangeset(self, changeset):
242        """
243        Do whatever needed before replay and return the adapted changeset.
244
245        This implementation calls ``self._adaptEntries()``, then
246        executes the adapters defined by before-commit on the project:
247        each adapter is run in turn, and may return False to indicate
248        that the changeset shouldn't be replayed at all. They are
249        otherwise free to alter the changeset in any meaningful way.
250        """
251
252        from copy import copy
253
254        adapted = self._adaptEntries(changeset)
255        if adapted:
256            project = self.repository.projectref()
257            if project.before_commit:
258                adapted = copy(adapted)
259
260                for adapter in project.before_commit:
261                    if not adapter(self, adapted):
262                        return None
263        return adapted
264
265    def _dismissChangeset(self, changeset):
266        """
267        Do whatever needed after commit.
268
269        This execute the adapters defined by after-commit on the project,
270        for example tagging in some way the target repository upon some
271        particular kind of changeset.
272        """
273
274        project = self.repository.projectref()
275        if project.after_commit:
276            for farewell in project.after_commit:
277                farewell(self, changeset)
278
279    def _getCommitEntries(self, changeset):
280        """
281        Extract the names of the entries for the commit phase.
282        """
283
284        # Since the commit may use cli tools to do its job, and the
285        # machinery may split the list into smaller chunks to avoid
286        # too long command lines, anticipates added stuff.  I think
287        # this is needed only when coming from CVS (or HG or in
288        # general from systems that don't handle directories): its
289        # _applyChangeset *appends* to the entries a fake ADD for
290        # each new subdir.
291
292        entries = []
293        added = 0
294        for e in changeset.entries:
295            if e.action_kind == e.ADDED:
296                entries.insert(added, e.name)
297                added += 1
298            else:
299                # Add also the name of the old file: for some systems
300                # it may not be strictly needed, but it is for most.
301                if e.action_kind == e.RENAMED:
302                    entries.append(e.old_name)
303                entries.append(e.name)
304        return entries
305
306    def _replayChangeset(self, changeset):
307        """
308        Replay each entry of the changeset, that is execute the action associated
309        to each kind of change for each entry, possibly grouping consecutive entries
310        of the same kind.
311        """
312
313        from .changes import ChangesetEntry
314
315        actions = { ChangesetEntry.ADDED: self._addEntries,
316                    ChangesetEntry.DELETED: self._removeEntries,
317                    ChangesetEntry.RENAMED: self._renameEntries,
318                    ChangesetEntry.UPDATED: self._editEntries
319                    }
320
321        # Group the changes by kind and perform the corresponding action
322
323        last = None
324        group = []
325        for e in changeset.entries:
326            if last is None or last.action_kind == e.action_kind:
327                last = e
328                group.append(e)
329            if last.action_kind != e.action_kind:
330                action = actions.get(last.action_kind)
331                if action is not None:
332                    action(group)
333                group = [e]
334                last = e
335        if group:
336            action = actions.get(group[0].action_kind)
337            if action is not None:
338                action(group)
339
340    def _addEntries(self, entries):
341        """
342        Add a sequence of entries
343        """
344
345        self._addPathnames([e.name for e in entries])
346
347    def _addPathnames(self, names):
348        """
349        Add some new filesystem objects.
350        """
351
352        raise TailorBug("%s should override this method!" % self.__class__)
353
354    def _addSubtree(self, subdir):
355        """
356        Add a whole subtree.
357
358        This implementation crawl down the whole subtree, adding
359        entries (subdirs, skipping the usual VC-specific control
360        directories such as ``.svn``, ``_darcs`` or ``CVS``, and
361        files).
362
363        Subclasses may use a better way, if the backend implements
364        a recursive add that skips the various metadata directories.
365        """
366
367        from os.path import join
368        from os import walk
369        from .dualwd import IGNORED_METADIRS
370
371        exclude = []
372
373        if self.state_file.filename.startswith(self.repository.basedir):
374            sfrelname = self.state_file.filename[len(self.repository.basedir)+1:]
375            exclude.append(sfrelname)
376            exclude.append(sfrelname+'.old')
377            exclude.append(sfrelname+'.journal')
378
379        if self.logfile.startswith(self.repository.basedir):
380            exclude.append(self.logfile[len(self.repository.basedir)+1:])
381
382        if subdir and subdir!='.':
383            self._addPathnames([subdir])
384
385        for dir, subdirs, files in walk(join(self.repository.basedir, subdir)):
386            for excd in IGNORED_METADIRS:
387                if excd in subdirs:
388                    subdirs.remove(excd)
389
390            for excf in exclude:
391                if excf in files:
392                    files.remove(excf)
393
394            if subdirs or files:
395                self._addPathnames([join(dir, df)[len(self.repository.basedir)+1:]
396                                    for df in subdirs + files])
397
398    def _commit(self, date, author, patchname, changelog=None, entries=None,
399                tags = [], isinitialcommit = False):
400        """
401        Commit the changeset.
402        """
403
404        raise TailorBug("%s should override this method!" % self.__class__)
405
406    def _postCommitCheck(self):
407        """
408        Perform any safety-belt check to assert everything's ok. This
409        implementation does nothing, subclasses should reimplement the
410        method.
411        """
412
413    def _removeEntries(self, entries):
414        """
415        Remove a sequence of entries.
416        """
417
418        self._removePathnames([e.name for e in entries])
419
420    def _removePathnames(self, names):
421        """
422        Remove some filesystem object.
423        """
424
425        raise TailorBug("%s should override this method!" % self.__class__)
426
427    def _editEntries(self, entries):
428        """
429        Records a sequence of entries as updated.
430        """
431
432        self._editPathnames([e.name for e in entries])
433
434    def _editPathnames(self, names):
435        """
436        Records a sequence of filesystem objects as updated.
437        """
438
439        pass
440
441    def _renameEntries(self, entries):
442        """
443        Rename a sequence of entries, adding all the parent directories
444        of each entry.
445        """
446
447        from os import rename, walk
448        from shutil import rmtree
449        from os.path import split, join, exists, isdir
450
451        added = []
452        for e in entries:
453            parents = []
454            parent = split(e.name)[0]
455            while parent:
456                if not parent in added:
457                    parents.append(parent)
458                    added.append(parent)
459                parent = split(parent)[0]
460            if parents:
461                parents.reverse()
462                self._addPathnames(parents)
463
464            other = False
465            if self.shared_basedirs:
466                # Check to see if the oldentry is still there. If it is,
467                # that probably means one thing: it's been moved and then
468                # replaced, see svn 'R' event. In this case, rename the
469                # existing old entry to something else to trick targets
470                # (that will assume the move was already done manually) and
471                # finally restore its name.
472
473                absold = join(self.repository.basedir, e.old_name)
474                renamed = exists(absold)
475                if renamed:
476                    rename(absold, absold + '-TAILOR-HACKED-TEMP-NAME')
477            else:
478                # With disjunct directories, old entries are *always*
479                # there because we dropped the --delete option to rsync.
480                # So, instead of renaming the old entry, we temporarily
481                # rename the new one, perform the target system rename
482                # and replace back the real content (it may be a
483                # renamed+edited event).
484
485                # Hide the real new file from rename
486                absnew = join(self.repository.basedir, e.name)
487                renamed = exists(absnew)
488                if renamed:
489                    rename(absnew, absnew + '-TAILOR-HACKED-TEMP-NAME')
490
491                # If 'absold' exist, then the file was moved and replaced
492                # with an other file. Hide the other file from rename.
493                absold = join(self.repository.basedir, e.old_name)
494                other = exists(absold)
495                if other:
496                    rename(absold, absold + '-TAILOR-HACKED-OTHER-NAME')
497
498                # Restore the old file from backup.
499                oldfile = exists(absold + '-TAILOR-HACKED-OLD-NAME')
500                if oldfile:
501                    rename(absold + '-TAILOR-HACKED-OLD-NAME', absold)
502
503            try:
504                self._renamePathname(e.old_name, e.name)
505            finally:
506
507                # Restore other NEW target
508                if other:
509                    rename(absold + '-TAILOR-HACKED-OTHER-NAME', absold)
510
511                if renamed:
512                    if self.shared_basedirs:
513                        # it's possible that the target already handled
514                        # this
515                        if exists(absold + '-TAILOR-HACKED-TEMP-NAME'):
516                            rename(absold + '-TAILOR-HACKED-TEMP-NAME', absold)
517                    else:
518
519                        # before rsync      after rsync      the HACK            after "svn mv"           result
520                        # /basedir          /basedir         /basedir            /basedir                 /basedir
521                        # |                 |                |                   |                        |
522                        # +- /dirold        +- /dirold       +- /dirold          +- /dirnew        move   |
523                        #    |              |  |             |  |                |  |~~~~~~        hack   |
524                        #    +- /.svn       |  +- /.svn      |  +- /.svn         |  +- /.svn  >-------+   |
525                        #    |              |  |             |  |                |  |                 |   |
526                        #    +- /subdir     |  +- /subdir    |  +- /subdir       |  +- /subdir        v   |
527                        #       |           |     |          |     |             |     |              |   |
528                        #       +- /.svn    |     +- /.svn   |     +- /.svn      |     +- /.svn  >--+ |   |
529                        #                   |                |                   |                  | |   |
530                        #                   +- /dirnew       +- /dirnew-HACKED   +- /dirnew-HACKED  v |   +- /dirnew
531                        #                      |~~~~~~          |      ~~~~~~~      |               | |      |
532                        #                      |                |                   |               +-|--->  +- /.svn
533                        #                      |                |                   |                 |      |   ~~~~
534                        #                      +- /subdir       +- /subdir          +- /subdir        |      +- /subdir
535                        #                          ~~~~~~                                             |        |
536                        #                                                                             +----->  +- /.svn
537                        #                                                                                          ~~~~
538
539                        # Ticket #65, #125
540                        # If the target reposity has files in subdirectory,
541                        # then remove the complete dir.
542                        # But keep the dir '.svn', '_CVS', or what ever
543                        if isdir(absnew):
544                            if self.repository.METADIR != None:
545                                for root, dirs, files in walk(absnew):
546                                    if self.repository.METADIR in dirs:
547                                        dirs.remove(self.repository.METADIR)  # don't visit SVN directories
548                                        svnnew = join(root, self.repository.METADIR)
549                                        hacked = join(absnew + '-TAILOR-HACKED-TEMP-NAME' + root[len(absnew):], self.repository.METADIR)
550                                        rename(svnnew, hacked)
551
552                            rmtree(absnew)
553
554                        rename(absnew + '-TAILOR-HACKED-TEMP-NAME', absnew)
555
556    def _renamePathname(self, oldname, newname):
557        """
558        Rename a filesystem object to some other name/location.
559        """
560
561        raise TailorBug("%s should override this method!" % self.__class__)
562
563    def prepareWorkingDirectory(self, source_repo):
564        """
565        Do anything required to setup the hosting working directory.
566        """
567
568        self._prepareWorkingDirectory(source_repo)
569
570    def _prepareWorkingDirectory(self, source_repo):
571        """
572        Possibly checkout a working copy of the target VC, that will host the
573        upstream source tree, when overriden by subclasses.
574        """
575
576    def prepareTargetRepository(self):
577        """
578        Do anything required to host the target repository.
579        """
580
581        from os import makedirs
582        from os.path import join, exists
583
584        if not exists(self.repository.basedir):
585            makedirs(self.repository.basedir)
586
587        self._prepareTargetRepository()
588
589        prefix = self.__getPrefixToSource()
590        if prefix:
591            if not exists(join(self.repository.basedir, prefix)):
592                # At bootstrap time, we assume that if the user
593                # extracted the source manually, she added
594                # the subdir, before doing that.
595                makedirs(join(self.repository.basedir, prefix))
596                self._addPathnames([prefix])
597
598    def _prepareTargetRepository(self):
599        """
600        Possibly create or connect to the repository, when overriden
601        by subclasses.
602        """
603
604    def importFirstRevision(self, source_repo, changeset, initial):
605        """
606        Initialize a new working directory, just extracted from
607        some other VC system, importing everything's there.
608        """
609
610        self._initializeWorkingDir()
611        # Execute the precommit hooks, but ignore None results
612        changeset = self._adaptChangeset(changeset) or changeset
613        revision = changeset.revision
614        source_repository = str(source_repo)
615        if initial:
616            author = changeset.author
617            patchname, log = self.__getPatchNameAndLog(changeset)
618        else:
619            author = "%s@%s" % (AUTHOR, HOST)
620            patchname = BOOTSTRAP_PATCHNAME
621            log = BOOTSTRAP_CHANGELOG % locals()
622        self._commit(changeset.date, author, patchname, log,
623                     isinitialcommit = True)
624
625        if changeset.tags:
626            for tag in changeset.tags:
627                self._tag(tag, changeset.date, author)
628
629        self._dismissChangeset(changeset)
630
631    def _initializeWorkingDir(self):
632        """
633        Assuming the ``basedir`` directory contains a working copy ``module``
634        extracted from some VC repository, add it and all its content
635        to the target repository.
636
637        This implementation recursively add every file in the subtree.
638        Subclasses should override this method doing whatever is
639        appropriate for the backend.
640        """
641
642        self._addSubtree('.')
643
644    def _tag(self, tagname, date, author):
645        """
646        Tag the current version, if the VC type supports it, otherwise
647        do nothing.
648        """
649        pass
650