1# -*- mode: python; coding: utf-8 -*- 2# :Progetto: vcpx -- Syncable targets 3# :Creato: ven 04 giu 2004 00:27:07 CEST 4# :Autore: Lele Gaifax <lele@nautilus.homeip.net> 5# :Licenza: GNU General Public License 6# 7 8""" 9Syncronizable targets are the simplest abstract wrappers around a 10working directory under two different version control systems. 11""" 12from __future__ import absolute_import 13 14from builtins import str 15__docformat__ = 'reStructuredText' 16 17import socket 18from signal import signal, SIGINT, SIG_IGN 19from vcpx import TailorBug, TailorException 20from vcpx.workdir import WorkingDir 21 22 23HOST = socket.getfqdn() 24AUTHOR = "tailor" 25BOOTSTRAP_PATCHNAME = 'Tailorization' 26BOOTSTRAP_CHANGELOG = """\ 27Import of the upstream sources from 28%(source_repository)s 29 Revision: %(revision)s 30""" 31 32 33class TargetInitializationFailure(TailorException): 34 "Failure initializing the target VCS" 35 36 37class ChangesetReplayFailure(TailorException): 38 "Failure replaying the changeset on the target system" 39 40 41class PostCommitCheckFailure(TailorException): 42 "Most probably a tailor bug, not everything has been committed." 43 44 45class SynchronizableTargetWorkingDir(WorkingDir): 46 """ 47 This is an abstract working dir usable as a *shadow* of another 48 kind of VC, sharing the same working directory. 49 50 Most interesting entry points are: 51 52 replayChangeset 53 to replay an already applied changeset, to mimic the actions 54 performed by the upstream VC system on the tree such as 55 renames, deletions and adds. This is an useful argument to 56 feed as ``replay`` to ``applyUpstreamChangesets`` 57 58 importFirstRevision 59 to initialize a pristine working directory tree under this VC 60 system, possibly extracted under a different kind of VC 61 62 Subclasses MUST override at least the _underscoredMethods. 63 """ 64 65 PATCH_NAME_FORMAT = '[%(project)s @ %(revision)s]' 66 """ 67 The format string used to compute the patch name, used by underlying VCS. 68 """ 69 70 REMOVE_FIRST_LOG_LINE = False 71 """ 72 When true, remove the first line from the upstream changelog. 73 """ 74 75 def __getPatchNameAndLog(self, changeset): 76 """ 77 Return a tuple (patchname, changelog) interpolating changeset's 78 information with the template above. 79 """ 80 81 if changeset.log == '': 82 firstlogline = 'Empty log message' 83 remaininglog = '' 84 else: 85 loglines = changeset.log.split('\n') 86 if len(loglines)>1: 87 firstlogline = loglines[0] 88 remaininglog = '\n'.join(loglines[1:]) 89 else: 90 firstlogline = changeset.log 91 remaininglog = '' 92 93 patchname = self.PATCH_NAME_FORMAT % { 94 'project': self.repository.projectref().name, 95 'revision': changeset.revision, 96 'author': changeset.author, 97 'date': changeset.date, 98 'firstlogline': firstlogline, 99 'remaininglog': remaininglog} 100 if self.REMOVE_FIRST_LOG_LINE: 101 changelog = remaininglog 102 else: 103 changelog = changeset.log 104 return patchname, changelog 105 106 def _prepareToReplayChangeset(self, changeset): 107 """ 108 This is called **before** fetching and applying the source 109 changeset. This implementation does nothing more than 110 returning True. Subclasses may override it, for example to 111 preexecute some entries such as moves. 112 113 Returning False the changeset won't be applied and the 114 process will stop. 115 """ 116 117 return True 118 119 def replayChangeset(self, changeset): 120 """ 121 Do whatever is needed to replay the changes under the target 122 VC, to register the already applied (under the other VC) 123 changeset. 124 """ 125 126 try: 127 changeset = self._adaptChangeset(changeset) 128 except: 129 self.log.exception("Failure adapting: %s", str(changeset)) 130 raise 131 132 if changeset is None: 133 return 134 135 try: 136 self._replayChangeset(changeset) 137 except: 138 self.log.exception("Failure replaying: %s", str(changeset)) 139 raise 140 patchname, log = self.__getPatchNameAndLog(changeset) 141 entries = self._getCommitEntries(changeset) 142 previous = signal(SIGINT, SIG_IGN) 143 try: 144 self._commit(changeset.date, changeset.author, patchname, log, 145 entries, tags = changeset.tags) 146 if changeset.tags: 147 for tag in changeset.tags: 148 self._tag(tag, changeset.date, changeset.author) 149 if self.repository.post_commit_check: 150 self._postCommitCheck() 151 finally: 152 signal(SIGINT, previous) 153 154 try: 155 self._dismissChangeset(changeset) 156 except: 157 self.log.exception("Failure dismissing: %s", str(changeset)) 158 raise 159 160 def __getPrefixToSource(self): 161 """ 162 Compute and return the "offset" between source and target basedirs, 163 or None when not using shared directories, or there's no offset. 164 """ 165 166 project = self.repository.projectref() 167 ssubdir = project.source.subdir 168 tsubdir = project.target.subdir 169 if self.shared_basedirs and ssubdir != tsubdir: 170 if tsubdir == '.': 171 prefix = ssubdir 172 else: 173 if not tsubdir.endswith('/'): 174 tsubdir += '/' 175 prefix = ssubdir[len(tsubdir):] 176 return prefix 177 else: 178 return None 179 180 def _normalizeEntryPaths(self, entry): 181 """ 182 Normalize the name and old_name of an entry. 183 184 The ``name`` and ``old_name`` of an entry are pathnames coming 185 from the upstream system, and is usually (although there is no 186 guarantee it actually is) a UNIX style path with forward 187 slashes "/" as separators. 188 189 This implementation uses normpath to adapt the path to the 190 actual OS convention, but subclasses may eventually override 191 this to use their own canonicalization of ``name`` and 192 ``old_name``. 193 """ 194 195 from os.path import normpath 196 197 entry.name = normpath(entry.name) 198 if entry.old_name: 199 entry.old_name = normpath(entry.old_name) 200 201 def __adaptEntriesPath(self, changeset): 202 """ 203 If the source basedir is a subdirectory of the target, adjust 204 all the pathnames adding the prefix computed by difference. 205 """ 206 207 from copy import deepcopy 208 from os.path import join 209 210 if not changeset.entries: 211 return changeset 212 213 prefix = self.__getPrefixToSource() 214 adapted = deepcopy(changeset) 215 for e in adapted.entries: 216 if prefix: 217 e.name = join(prefix, e.name) 218 if e.old_name: 219 e.old_name = join(prefix, e.old_name) 220 self._normalizeEntryPaths(e) 221 return adapted 222 223 def _adaptEntries(self, changeset): 224 """ 225 Do whatever is needed to adapt entries to the target system. 226 227 This implementation adds a prefix to each path if needed, when 228 the target basedir *contains* the source basedir. Also, each 229 path is normalized thru ``normpath()`` or whatever equivalent 230 operation provided by the specific target. It operates on and 231 returns a copy of the given changeset. 232 233 Subclasses shall eventually extend this to exclude unwanted 234 entries, eventually returning None when all entries were 235 excluded, to avoid the commit on target of an empty changeset. 236 """ 237 238 adapted = self.__adaptEntriesPath(changeset) 239 return adapted 240 241 def _adaptChangeset(self, changeset): 242 """ 243 Do whatever needed before replay and return the adapted changeset. 244 245 This implementation calls ``self._adaptEntries()``, then 246 executes the adapters defined by before-commit on the project: 247 each adapter is run in turn, and may return False to indicate 248 that the changeset shouldn't be replayed at all. They are 249 otherwise free to alter the changeset in any meaningful way. 250 """ 251 252 from copy import copy 253 254 adapted = self._adaptEntries(changeset) 255 if adapted: 256 project = self.repository.projectref() 257 if project.before_commit: 258 adapted = copy(adapted) 259 260 for adapter in project.before_commit: 261 if not adapter(self, adapted): 262 return None 263 return adapted 264 265 def _dismissChangeset(self, changeset): 266 """ 267 Do whatever needed after commit. 268 269 This execute the adapters defined by after-commit on the project, 270 for example tagging in some way the target repository upon some 271 particular kind of changeset. 272 """ 273 274 project = self.repository.projectref() 275 if project.after_commit: 276 for farewell in project.after_commit: 277 farewell(self, changeset) 278 279 def _getCommitEntries(self, changeset): 280 """ 281 Extract the names of the entries for the commit phase. 282 """ 283 284 # Since the commit may use cli tools to do its job, and the 285 # machinery may split the list into smaller chunks to avoid 286 # too long command lines, anticipates added stuff. I think 287 # this is needed only when coming from CVS (or HG or in 288 # general from systems that don't handle directories): its 289 # _applyChangeset *appends* to the entries a fake ADD for 290 # each new subdir. 291 292 entries = [] 293 added = 0 294 for e in changeset.entries: 295 if e.action_kind == e.ADDED: 296 entries.insert(added, e.name) 297 added += 1 298 else: 299 # Add also the name of the old file: for some systems 300 # it may not be strictly needed, but it is for most. 301 if e.action_kind == e.RENAMED: 302 entries.append(e.old_name) 303 entries.append(e.name) 304 return entries 305 306 def _replayChangeset(self, changeset): 307 """ 308 Replay each entry of the changeset, that is execute the action associated 309 to each kind of change for each entry, possibly grouping consecutive entries 310 of the same kind. 311 """ 312 313 from .changes import ChangesetEntry 314 315 actions = { ChangesetEntry.ADDED: self._addEntries, 316 ChangesetEntry.DELETED: self._removeEntries, 317 ChangesetEntry.RENAMED: self._renameEntries, 318 ChangesetEntry.UPDATED: self._editEntries 319 } 320 321 # Group the changes by kind and perform the corresponding action 322 323 last = None 324 group = [] 325 for e in changeset.entries: 326 if last is None or last.action_kind == e.action_kind: 327 last = e 328 group.append(e) 329 if last.action_kind != e.action_kind: 330 action = actions.get(last.action_kind) 331 if action is not None: 332 action(group) 333 group = [e] 334 last = e 335 if group: 336 action = actions.get(group[0].action_kind) 337 if action is not None: 338 action(group) 339 340 def _addEntries(self, entries): 341 """ 342 Add a sequence of entries 343 """ 344 345 self._addPathnames([e.name for e in entries]) 346 347 def _addPathnames(self, names): 348 """ 349 Add some new filesystem objects. 350 """ 351 352 raise TailorBug("%s should override this method!" % self.__class__) 353 354 def _addSubtree(self, subdir): 355 """ 356 Add a whole subtree. 357 358 This implementation crawl down the whole subtree, adding 359 entries (subdirs, skipping the usual VC-specific control 360 directories such as ``.svn``, ``_darcs`` or ``CVS``, and 361 files). 362 363 Subclasses may use a better way, if the backend implements 364 a recursive add that skips the various metadata directories. 365 """ 366 367 from os.path import join 368 from os import walk 369 from .dualwd import IGNORED_METADIRS 370 371 exclude = [] 372 373 if self.state_file.filename.startswith(self.repository.basedir): 374 sfrelname = self.state_file.filename[len(self.repository.basedir)+1:] 375 exclude.append(sfrelname) 376 exclude.append(sfrelname+'.old') 377 exclude.append(sfrelname+'.journal') 378 379 if self.logfile.startswith(self.repository.basedir): 380 exclude.append(self.logfile[len(self.repository.basedir)+1:]) 381 382 if subdir and subdir!='.': 383 self._addPathnames([subdir]) 384 385 for dir, subdirs, files in walk(join(self.repository.basedir, subdir)): 386 for excd in IGNORED_METADIRS: 387 if excd in subdirs: 388 subdirs.remove(excd) 389 390 for excf in exclude: 391 if excf in files: 392 files.remove(excf) 393 394 if subdirs or files: 395 self._addPathnames([join(dir, df)[len(self.repository.basedir)+1:] 396 for df in subdirs + files]) 397 398 def _commit(self, date, author, patchname, changelog=None, entries=None, 399 tags = [], isinitialcommit = False): 400 """ 401 Commit the changeset. 402 """ 403 404 raise TailorBug("%s should override this method!" % self.__class__) 405 406 def _postCommitCheck(self): 407 """ 408 Perform any safety-belt check to assert everything's ok. This 409 implementation does nothing, subclasses should reimplement the 410 method. 411 """ 412 413 def _removeEntries(self, entries): 414 """ 415 Remove a sequence of entries. 416 """ 417 418 self._removePathnames([e.name for e in entries]) 419 420 def _removePathnames(self, names): 421 """ 422 Remove some filesystem object. 423 """ 424 425 raise TailorBug("%s should override this method!" % self.__class__) 426 427 def _editEntries(self, entries): 428 """ 429 Records a sequence of entries as updated. 430 """ 431 432 self._editPathnames([e.name for e in entries]) 433 434 def _editPathnames(self, names): 435 """ 436 Records a sequence of filesystem objects as updated. 437 """ 438 439 pass 440 441 def _renameEntries(self, entries): 442 """ 443 Rename a sequence of entries, adding all the parent directories 444 of each entry. 445 """ 446 447 from os import rename, walk 448 from shutil import rmtree 449 from os.path import split, join, exists, isdir 450 451 added = [] 452 for e in entries: 453 parents = [] 454 parent = split(e.name)[0] 455 while parent: 456 if not parent in added: 457 parents.append(parent) 458 added.append(parent) 459 parent = split(parent)[0] 460 if parents: 461 parents.reverse() 462 self._addPathnames(parents) 463 464 other = False 465 if self.shared_basedirs: 466 # Check to see if the oldentry is still there. If it is, 467 # that probably means one thing: it's been moved and then 468 # replaced, see svn 'R' event. In this case, rename the 469 # existing old entry to something else to trick targets 470 # (that will assume the move was already done manually) and 471 # finally restore its name. 472 473 absold = join(self.repository.basedir, e.old_name) 474 renamed = exists(absold) 475 if renamed: 476 rename(absold, absold + '-TAILOR-HACKED-TEMP-NAME') 477 else: 478 # With disjunct directories, old entries are *always* 479 # there because we dropped the --delete option to rsync. 480 # So, instead of renaming the old entry, we temporarily 481 # rename the new one, perform the target system rename 482 # and replace back the real content (it may be a 483 # renamed+edited event). 484 485 # Hide the real new file from rename 486 absnew = join(self.repository.basedir, e.name) 487 renamed = exists(absnew) 488 if renamed: 489 rename(absnew, absnew + '-TAILOR-HACKED-TEMP-NAME') 490 491 # If 'absold' exist, then the file was moved and replaced 492 # with an other file. Hide the other file from rename. 493 absold = join(self.repository.basedir, e.old_name) 494 other = exists(absold) 495 if other: 496 rename(absold, absold + '-TAILOR-HACKED-OTHER-NAME') 497 498 # Restore the old file from backup. 499 oldfile = exists(absold + '-TAILOR-HACKED-OLD-NAME') 500 if oldfile: 501 rename(absold + '-TAILOR-HACKED-OLD-NAME', absold) 502 503 try: 504 self._renamePathname(e.old_name, e.name) 505 finally: 506 507 # Restore other NEW target 508 if other: 509 rename(absold + '-TAILOR-HACKED-OTHER-NAME', absold) 510 511 if renamed: 512 if self.shared_basedirs: 513 # it's possible that the target already handled 514 # this 515 if exists(absold + '-TAILOR-HACKED-TEMP-NAME'): 516 rename(absold + '-TAILOR-HACKED-TEMP-NAME', absold) 517 else: 518 519 # before rsync after rsync the HACK after "svn mv" result 520 # /basedir /basedir /basedir /basedir /basedir 521 # | | | | | 522 # +- /dirold +- /dirold +- /dirold +- /dirnew move | 523 # | | | | | | |~~~~~~ hack | 524 # +- /.svn | +- /.svn | +- /.svn | +- /.svn >-------+ | 525 # | | | | | | | | | 526 # +- /subdir | +- /subdir | +- /subdir | +- /subdir v | 527 # | | | | | | | | | 528 # +- /.svn | +- /.svn | +- /.svn | +- /.svn >--+ | | 529 # | | | | | | 530 # +- /dirnew +- /dirnew-HACKED +- /dirnew-HACKED v | +- /dirnew 531 # |~~~~~~ | ~~~~~~~ | | | | 532 # | | | +-|---> +- /.svn 533 # | | | | | ~~~~ 534 # +- /subdir +- /subdir +- /subdir | +- /subdir 535 # ~~~~~~ | | 536 # +-----> +- /.svn 537 # ~~~~ 538 539 # Ticket #65, #125 540 # If the target reposity has files in subdirectory, 541 # then remove the complete dir. 542 # But keep the dir '.svn', '_CVS', or what ever 543 if isdir(absnew): 544 if self.repository.METADIR != None: 545 for root, dirs, files in walk(absnew): 546 if self.repository.METADIR in dirs: 547 dirs.remove(self.repository.METADIR) # don't visit SVN directories 548 svnnew = join(root, self.repository.METADIR) 549 hacked = join(absnew + '-TAILOR-HACKED-TEMP-NAME' + root[len(absnew):], self.repository.METADIR) 550 rename(svnnew, hacked) 551 552 rmtree(absnew) 553 554 rename(absnew + '-TAILOR-HACKED-TEMP-NAME', absnew) 555 556 def _renamePathname(self, oldname, newname): 557 """ 558 Rename a filesystem object to some other name/location. 559 """ 560 561 raise TailorBug("%s should override this method!" % self.__class__) 562 563 def prepareWorkingDirectory(self, source_repo): 564 """ 565 Do anything required to setup the hosting working directory. 566 """ 567 568 self._prepareWorkingDirectory(source_repo) 569 570 def _prepareWorkingDirectory(self, source_repo): 571 """ 572 Possibly checkout a working copy of the target VC, that will host the 573 upstream source tree, when overriden by subclasses. 574 """ 575 576 def prepareTargetRepository(self): 577 """ 578 Do anything required to host the target repository. 579 """ 580 581 from os import makedirs 582 from os.path import join, exists 583 584 if not exists(self.repository.basedir): 585 makedirs(self.repository.basedir) 586 587 self._prepareTargetRepository() 588 589 prefix = self.__getPrefixToSource() 590 if prefix: 591 if not exists(join(self.repository.basedir, prefix)): 592 # At bootstrap time, we assume that if the user 593 # extracted the source manually, she added 594 # the subdir, before doing that. 595 makedirs(join(self.repository.basedir, prefix)) 596 self._addPathnames([prefix]) 597 598 def _prepareTargetRepository(self): 599 """ 600 Possibly create or connect to the repository, when overriden 601 by subclasses. 602 """ 603 604 def importFirstRevision(self, source_repo, changeset, initial): 605 """ 606 Initialize a new working directory, just extracted from 607 some other VC system, importing everything's there. 608 """ 609 610 self._initializeWorkingDir() 611 # Execute the precommit hooks, but ignore None results 612 changeset = self._adaptChangeset(changeset) or changeset 613 revision = changeset.revision 614 source_repository = str(source_repo) 615 if initial: 616 author = changeset.author 617 patchname, log = self.__getPatchNameAndLog(changeset) 618 else: 619 author = "%s@%s" % (AUTHOR, HOST) 620 patchname = BOOTSTRAP_PATCHNAME 621 log = BOOTSTRAP_CHANGELOG % locals() 622 self._commit(changeset.date, author, patchname, log, 623 isinitialcommit = True) 624 625 if changeset.tags: 626 for tag in changeset.tags: 627 self._tag(tag, changeset.date, author) 628 629 self._dismissChangeset(changeset) 630 631 def _initializeWorkingDir(self): 632 """ 633 Assuming the ``basedir`` directory contains a working copy ``module`` 634 extracted from some VC repository, add it and all its content 635 to the target repository. 636 637 This implementation recursively add every file in the subtree. 638 Subclasses should override this method doing whatever is 639 appropriate for the backend. 640 """ 641 642 self._addSubtree('.') 643 644 def _tag(self, tagname, date, author): 645 """ 646 Tag the current version, if the VC type supports it, otherwise 647 do nothing. 648 """ 649 pass 650