1# Copyright (C) 2006-2011 Canonical Ltd 2# Copyright (C) 2020 Breezy Developers 3# 4# This program is free software; you can redistribute it and/or modify 5# it under the terms of the GNU General Public License as published by 6# the Free Software Foundation; either version 2 of the License, or 7# (at your option) any later version. 8# 9# This program is distributed in the hope that it will be useful, 10# but WITHOUT ANY WARRANTY; without even the implied warranty of 11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12# GNU General Public License for more details. 13# 14# You should have received a copy of the GNU General Public License 15# along with this program; if not, write to the Free Software 16# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17 18from __future__ import absolute_import 19 20import errno 21import os 22import posixpath 23from stat import S_IEXEC, S_ISREG 24import time 25 26from .mapping import encode_git_path, mode_kind, mode_is_executable, object_mode 27from .tree import GitTree, GitTreeDirectory, GitTreeSymlink, GitTreeFile 28 29from .. import ( 30 annotate, 31 conflicts, 32 errors, 33 multiparent, 34 osutils, 35 revision as _mod_revision, 36 trace, 37 ui, 38 urlutils, 39 ) 40from ..i18n import gettext 41from ..mutabletree import MutableTree 42from ..tree import InterTree, TreeChange 43from ..transform import ( 44 PreviewTree, 45 TreeTransform, 46 _TransformResults, 47 _FileMover, 48 FinalPaths, 49 joinpath, 50 unique_add, 51 TransformRenameFailed, 52 ImmortalLimbo, 53 ROOT_PARENT, 54 ReusingTransform, 55 MalformedTransform, 56 ) 57 58from dulwich.index import commit_tree, blob_from_path_and_stat 59from dulwich.objects import Blob 60 61 62class TreeTransformBase(TreeTransform): 63 """The base class for TreeTransform and its kin.""" 64 65 def __init__(self, tree, pb=None, case_sensitive=True): 66 """Constructor. 67 68 :param tree: The tree that will be transformed, but not necessarily 69 the output tree. 70 :param pb: ignored 71 :param case_sensitive: If True, the target of the transform is 72 case sensitive, not just case preserving. 73 """ 74 super(TreeTransformBase, self).__init__(tree, pb=pb) 75 # mapping of trans_id => (sha1 of content, stat_value) 76 self._observed_sha1s = {} 77 # Set of versioned trans ids 78 self._versioned = set() 79 # The trans_id that will be used as the tree root 80 self.root = self.trans_id_tree_path('') 81 # Whether the target is case sensitive 82 self._case_sensitive_target = case_sensitive 83 self._symlink_target = {} 84 85 @property 86 def mapping(self): 87 return self._tree.mapping 88 89 def finalize(self): 90 """Release the working tree lock, if held. 91 92 This is required if apply has not been invoked, but can be invoked 93 even after apply. 94 """ 95 if self._tree is None: 96 return 97 for hook in MutableTree.hooks['post_transform']: 98 hook(self._tree, self) 99 self._tree.unlock() 100 self._tree = None 101 102 def create_path(self, name, parent): 103 """Assign a transaction id to a new path""" 104 trans_id = self.assign_id() 105 unique_add(self._new_name, trans_id, name) 106 unique_add(self._new_parent, trans_id, parent) 107 return trans_id 108 109 def adjust_root_path(self, name, parent): 110 """Emulate moving the root by moving all children, instead. 111 """ 112 113 def fixup_new_roots(self): 114 """Reinterpret requests to change the root directory 115 116 Instead of creating a root directory, or moving an existing directory, 117 all the attributes and children of the new root are applied to the 118 existing root directory. 119 120 This means that the old root trans-id becomes obsolete, so it is 121 recommended only to invoke this after the root trans-id has become 122 irrelevant. 123 124 """ 125 new_roots = [k for k, v in self._new_parent.items() 126 if v == ROOT_PARENT] 127 if len(new_roots) < 1: 128 return 129 if len(new_roots) != 1: 130 raise ValueError('A tree cannot have two roots!') 131 old_new_root = new_roots[0] 132 # unversion the new root's directory. 133 if old_new_root in self._versioned: 134 self.cancel_versioning(old_new_root) 135 else: 136 self.unversion_file(old_new_root) 137 138 # Now move children of new root into old root directory. 139 # Ensure all children are registered with the transaction, but don't 140 # use directly-- some tree children have new parents 141 list(self.iter_tree_children(old_new_root)) 142 # Move all children of new root into old root directory. 143 for child in self.by_parent().get(old_new_root, []): 144 self.adjust_path(self.final_name(child), self.root, child) 145 146 # Ensure old_new_root has no directory. 147 if old_new_root in self._new_contents: 148 self.cancel_creation(old_new_root) 149 else: 150 self.delete_contents(old_new_root) 151 152 # prevent deletion of root directory. 153 if self.root in self._removed_contents: 154 self.cancel_deletion(self.root) 155 156 # destroy path info for old_new_root. 157 del self._new_parent[old_new_root] 158 del self._new_name[old_new_root] 159 160 def trans_id_file_id(self, file_id): 161 """Determine or set the transaction id associated with a file ID. 162 A new id is only created for file_ids that were never present. If 163 a transaction has been unversioned, it is deliberately still returned. 164 (this will likely lead to an unversioned parent conflict.) 165 """ 166 if file_id is None: 167 raise ValueError('None is not a valid file id') 168 path = self.mapping.parse_file_id(file_id) 169 return self.trans_id_tree_path(path) 170 171 def version_file(self, trans_id, file_id=None): 172 """Schedule a file to become versioned.""" 173 if trans_id in self._versioned: 174 raise errors.DuplicateKey(key=trans_id) 175 self._versioned.add(trans_id) 176 177 def cancel_versioning(self, trans_id): 178 """Undo a previous versioning of a file""" 179 raise NotImplementedError(self.cancel_versioning) 180 181 def new_paths(self, filesystem_only=False): 182 """Determine the paths of all new and changed files. 183 184 :param filesystem_only: if True, only calculate values for files 185 that require renames or execute bit changes. 186 """ 187 new_ids = set() 188 if filesystem_only: 189 stale_ids = self._needs_rename.difference(self._new_name) 190 stale_ids.difference_update(self._new_parent) 191 stale_ids.difference_update(self._new_contents) 192 stale_ids.difference_update(self._versioned) 193 needs_rename = self._needs_rename.difference(stale_ids) 194 id_sets = (needs_rename, self._new_executability) 195 else: 196 id_sets = (self._new_name, self._new_parent, self._new_contents, 197 self._versioned, self._new_executability) 198 for id_set in id_sets: 199 new_ids.update(id_set) 200 return sorted(FinalPaths(self).get_paths(new_ids)) 201 202 def final_is_versioned(self, trans_id): 203 if trans_id in self._versioned: 204 return True 205 if trans_id in self._removed_id: 206 return False 207 orig_path = self.tree_path(trans_id) 208 if orig_path is None: 209 return False 210 return self._tree.is_versioned(orig_path) 211 212 def find_raw_conflicts(self): 213 """Find any violations of inventory or filesystem invariants""" 214 if self._done is True: 215 raise ReusingTransform() 216 conflicts = [] 217 # ensure all children of all existent parents are known 218 # all children of non-existent parents are known, by definition. 219 self._add_tree_children() 220 by_parent = self.by_parent() 221 conflicts.extend(self._parent_loops()) 222 conflicts.extend(self._duplicate_entries(by_parent)) 223 conflicts.extend(self._parent_type_conflicts(by_parent)) 224 conflicts.extend(self._improper_versioning()) 225 conflicts.extend(self._executability_conflicts()) 226 conflicts.extend(self._overwrite_conflicts()) 227 return conflicts 228 229 def _check_malformed(self): 230 conflicts = self.find_raw_conflicts() 231 if len(conflicts) != 0: 232 raise MalformedTransform(conflicts=conflicts) 233 234 def _add_tree_children(self): 235 """Add all the children of all active parents to the known paths. 236 237 Active parents are those which gain children, and those which are 238 removed. This is a necessary first step in detecting conflicts. 239 """ 240 parents = list(self.by_parent()) 241 parents.extend([t for t in self._removed_contents if 242 self.tree_kind(t) == 'directory']) 243 for trans_id in self._removed_id: 244 path = self.tree_path(trans_id) 245 if path is not None: 246 try: 247 if self._tree.stored_kind(path) == 'directory': 248 parents.append(trans_id) 249 except errors.NoSuchFile: 250 pass 251 elif self.tree_kind(trans_id) == 'directory': 252 parents.append(trans_id) 253 254 for parent_id in parents: 255 # ensure that all children are registered with the transaction 256 list(self.iter_tree_children(parent_id)) 257 258 def _has_named_child(self, name, parent_id, known_children): 259 """Does a parent already have a name child. 260 261 :param name: The searched for name. 262 263 :param parent_id: The parent for which the check is made. 264 265 :param known_children: The already known children. This should have 266 been recently obtained from `self.by_parent.get(parent_id)` 267 (or will be if None is passed). 268 """ 269 if known_children is None: 270 known_children = self.by_parent().get(parent_id, []) 271 for child in known_children: 272 if self.final_name(child) == name: 273 return True 274 parent_path = self._tree_id_paths.get(parent_id, None) 275 if parent_path is None: 276 # No parent... no children 277 return False 278 child_path = joinpath(parent_path, name) 279 child_id = self._tree_path_ids.get(child_path, None) 280 if child_id is None: 281 # Not known by the tree transform yet, check the filesystem 282 return osutils.lexists(self._tree.abspath(child_path)) 283 else: 284 raise AssertionError('child_id is missing: %s, %s, %s' 285 % (name, parent_id, child_id)) 286 287 def _available_backup_name(self, name, target_id): 288 """Find an available backup name. 289 290 :param name: The basename of the file. 291 292 :param target_id: The directory trans_id where the backup should 293 be placed. 294 """ 295 known_children = self.by_parent().get(target_id, []) 296 return osutils.available_backup_name( 297 name, 298 lambda base: self._has_named_child( 299 base, target_id, known_children)) 300 301 def _parent_loops(self): 302 """No entry should be its own ancestor""" 303 for trans_id in self._new_parent: 304 seen = set() 305 parent_id = trans_id 306 while parent_id != ROOT_PARENT: 307 seen.add(parent_id) 308 try: 309 parent_id = self.final_parent(parent_id) 310 except KeyError: 311 break 312 if parent_id == trans_id: 313 yield ('parent loop', trans_id) 314 if parent_id in seen: 315 break 316 317 def _improper_versioning(self): 318 """Cannot version a file with no contents, or a bad type. 319 320 However, existing entries with no contents are okay. 321 """ 322 for trans_id in self._versioned: 323 kind = self.final_kind(trans_id) 324 if kind == 'symlink' and not self._tree.supports_symlinks(): 325 # Ignore symlinks as they are not supported on this platform 326 continue 327 if kind is None: 328 yield ('versioning no contents', trans_id) 329 continue 330 if not self._tree.versionable_kind(kind): 331 yield ('versioning bad kind', trans_id, kind) 332 333 def _executability_conflicts(self): 334 """Check for bad executability changes. 335 336 Only versioned files may have their executability set, because 337 1. only versioned entries can have executability under windows 338 2. only files can be executable. (The execute bit on a directory 339 does not indicate searchability) 340 """ 341 for trans_id in self._new_executability: 342 if not self.final_is_versioned(trans_id): 343 yield ('unversioned executability', trans_id) 344 else: 345 if self.final_kind(trans_id) != "file": 346 yield ('non-file executability', trans_id) 347 348 def _overwrite_conflicts(self): 349 """Check for overwrites (not permitted on Win32)""" 350 for trans_id in self._new_contents: 351 if self.tree_kind(trans_id) is None: 352 continue 353 if trans_id not in self._removed_contents: 354 yield ('overwrite', trans_id, self.final_name(trans_id)) 355 356 def _duplicate_entries(self, by_parent): 357 """No directory may have two entries with the same name.""" 358 if (self._new_name, self._new_parent) == ({}, {}): 359 return 360 for children in by_parent.values(): 361 name_ids = [] 362 for child_tid in children: 363 name = self.final_name(child_tid) 364 if name is not None: 365 # Keep children only if they still exist in the end 366 if not self._case_sensitive_target: 367 name = name.lower() 368 name_ids.append((name, child_tid)) 369 name_ids.sort() 370 last_name = None 371 last_trans_id = None 372 for name, trans_id in name_ids: 373 kind = self.final_kind(trans_id) 374 if kind is None and not self.final_is_versioned(trans_id): 375 continue 376 if name == last_name: 377 yield ('duplicate', last_trans_id, trans_id, name) 378 last_name = name 379 last_trans_id = trans_id 380 381 def _parent_type_conflicts(self, by_parent): 382 """Children must have a directory parent""" 383 for parent_id, children in by_parent.items(): 384 if parent_id == ROOT_PARENT: 385 continue 386 no_children = True 387 for child_id in children: 388 if self.final_kind(child_id) is not None: 389 no_children = False 390 break 391 if no_children: 392 continue 393 # There is at least a child, so we need an existing directory to 394 # contain it. 395 kind = self.final_kind(parent_id) 396 if kind is None: 397 # The directory will be deleted 398 yield ('missing parent', parent_id) 399 elif kind != "directory": 400 # Meh, we need a *directory* to put something in it 401 yield ('non-directory parent', parent_id) 402 403 def _set_executability(self, path, trans_id): 404 """Set the executability of versioned files """ 405 if self._tree._supports_executable(): 406 new_executability = self._new_executability[trans_id] 407 abspath = self._tree.abspath(path) 408 current_mode = os.stat(abspath).st_mode 409 if new_executability: 410 umask = os.umask(0) 411 os.umask(umask) 412 to_mode = current_mode | (0o100 & ~umask) 413 # Enable x-bit for others only if they can read it. 414 if current_mode & 0o004: 415 to_mode |= 0o001 & ~umask 416 if current_mode & 0o040: 417 to_mode |= 0o010 & ~umask 418 else: 419 to_mode = current_mode & ~0o111 420 osutils.chmod_if_possible(abspath, to_mode) 421 422 def _new_entry(self, name, parent_id, file_id): 423 """Helper function to create a new filesystem entry.""" 424 trans_id = self.create_path(name, parent_id) 425 if file_id is not None: 426 self.version_file(trans_id, file_id=file_id) 427 return trans_id 428 429 def new_file(self, name, parent_id, contents, file_id=None, 430 executable=None, sha1=None): 431 """Convenience method to create files. 432 433 name is the name of the file to create. 434 parent_id is the transaction id of the parent directory of the file. 435 contents is an iterator of bytestrings, which will be used to produce 436 the file. 437 :param file_id: The inventory ID of the file, if it is to be versioned. 438 :param executable: Only valid when a file_id has been supplied. 439 """ 440 trans_id = self._new_entry(name, parent_id, file_id) 441 # TODO: rather than scheduling a set_executable call, 442 # have create_file create the file with the right mode. 443 self.create_file(contents, trans_id, sha1=sha1) 444 if executable is not None: 445 self.set_executability(executable, trans_id) 446 return trans_id 447 448 def new_directory(self, name, parent_id, file_id=None): 449 """Convenience method to create directories. 450 451 name is the name of the directory to create. 452 parent_id is the transaction id of the parent directory of the 453 directory. 454 file_id is the inventory ID of the directory, if it is to be versioned. 455 """ 456 trans_id = self._new_entry(name, parent_id, file_id) 457 self.create_directory(trans_id) 458 return trans_id 459 460 def new_symlink(self, name, parent_id, target, file_id=None): 461 """Convenience method to create symbolic link. 462 463 name is the name of the symlink to create. 464 parent_id is the transaction id of the parent directory of the symlink. 465 target is a bytestring of the target of the symlink. 466 file_id is the inventory ID of the file, if it is to be versioned. 467 """ 468 trans_id = self._new_entry(name, parent_id, file_id) 469 self.create_symlink(target, trans_id) 470 return trans_id 471 472 def new_orphan(self, trans_id, parent_id): 473 """Schedule an item to be orphaned. 474 475 When a directory is about to be removed, its children, if they are not 476 versioned are moved out of the way: they don't have a parent anymore. 477 478 :param trans_id: The trans_id of the existing item. 479 :param parent_id: The parent trans_id of the item. 480 """ 481 raise NotImplementedError(self.new_orphan) 482 483 def _get_potential_orphans(self, dir_id): 484 """Find the potential orphans in a directory. 485 486 A directory can't be safely deleted if there are versioned files in it. 487 If all the contained files are unversioned then they can be orphaned. 488 489 The 'None' return value means that the directory contains at least one 490 versioned file and should not be deleted. 491 492 :param dir_id: The directory trans id. 493 494 :return: A list of the orphan trans ids or None if at least one 495 versioned file is present. 496 """ 497 orphans = [] 498 # Find the potential orphans, stop if one item should be kept 499 for child_tid in self.by_parent()[dir_id]: 500 if child_tid in self._removed_contents: 501 # The child is removed as part of the transform. Since it was 502 # versioned before, it's not an orphan 503 continue 504 if not self.final_is_versioned(child_tid): 505 # The child is not versioned 506 orphans.append(child_tid) 507 else: 508 # We have a versioned file here, searching for orphans is 509 # meaningless. 510 orphans = None 511 break 512 return orphans 513 514 def _affected_ids(self): 515 """Return the set of transform ids affected by the transform""" 516 trans_ids = set(self._removed_id) 517 trans_ids.update(self._versioned) 518 trans_ids.update(self._removed_contents) 519 trans_ids.update(self._new_contents) 520 trans_ids.update(self._new_executability) 521 trans_ids.update(self._new_name) 522 trans_ids.update(self._new_parent) 523 return trans_ids 524 525 def iter_changes(self, want_unversioned=False): 526 """Produce output in the same format as Tree.iter_changes. 527 528 Will produce nonsensical results if invoked while inventory/filesystem 529 conflicts (as reported by TreeTransform.find_raw_conflicts()) are present. 530 """ 531 final_paths = FinalPaths(self) 532 trans_ids = self._affected_ids() 533 results = [] 534 # Now iterate through all active paths 535 for trans_id in trans_ids: 536 from_path = self.tree_path(trans_id) 537 modified = False 538 # find file ids, and determine versioning state 539 if from_path is None: 540 from_versioned = False 541 else: 542 from_versioned = self._tree.is_versioned(from_path) 543 if not want_unversioned and not from_versioned: 544 from_path = None 545 to_path = final_paths.get_path(trans_id) 546 if to_path is None: 547 to_versioned = False 548 else: 549 to_versioned = self.final_is_versioned(trans_id) 550 if not want_unversioned and not to_versioned: 551 to_path = None 552 553 if from_versioned: 554 # get data from working tree if versioned 555 from_entry = next(self._tree.iter_entries_by_dir( 556 specific_files=[from_path]))[1] 557 from_name = from_entry.name 558 else: 559 from_entry = None 560 if from_path is None: 561 # File does not exist in FROM state 562 from_name = None 563 else: 564 # File exists, but is not versioned. Have to use path- 565 # splitting stuff 566 from_name = os.path.basename(from_path) 567 if from_path is not None: 568 from_kind, from_executable, from_stats = \ 569 self._tree._comparison_data(from_entry, from_path) 570 else: 571 from_kind = None 572 from_executable = False 573 574 to_name = self.final_name(trans_id) 575 to_kind = self.final_kind(trans_id) 576 if trans_id in self._new_executability: 577 to_executable = self._new_executability[trans_id] 578 else: 579 to_executable = from_executable 580 581 if from_versioned and from_kind != to_kind: 582 modified = True 583 elif to_kind in ('file', 'symlink') and ( 584 trans_id in self._new_contents): 585 modified = True 586 if (not modified and from_versioned == to_versioned 587 and from_path == to_path 588 and from_name == to_name 589 and from_executable == to_executable): 590 continue 591 if (from_path, to_path) == (None, None): 592 continue 593 results.append( 594 TreeChange( 595 (from_path, to_path), modified, 596 (from_versioned, to_versioned), 597 (from_name, to_name), 598 (from_kind, to_kind), 599 (from_executable, to_executable))) 600 601 def path_key(c): 602 return (c.path[0] or '', c.path[1] or '') 603 return iter(sorted(results, key=path_key)) 604 605 def get_preview_tree(self): 606 """Return a tree representing the result of the transform. 607 608 The tree is a snapshot, and altering the TreeTransform will invalidate 609 it. 610 """ 611 return GitPreviewTree(self) 612 613 def commit(self, branch, message, merge_parents=None, strict=False, 614 timestamp=None, timezone=None, committer=None, authors=None, 615 revprops=None, revision_id=None): 616 """Commit the result of this TreeTransform to a branch. 617 618 :param branch: The branch to commit to. 619 :param message: The message to attach to the commit. 620 :param merge_parents: Additional parent revision-ids specified by 621 pending merges. 622 :param strict: If True, abort the commit if there are unversioned 623 files. 624 :param timestamp: if not None, seconds-since-epoch for the time and 625 date. (May be a float.) 626 :param timezone: Optional timezone for timestamp, as an offset in 627 seconds. 628 :param committer: Optional committer in email-id format. 629 (e.g. "J Random Hacker <jrandom@example.com>") 630 :param authors: Optional list of authors in email-id format. 631 :param revprops: Optional dictionary of revision properties. 632 :param revision_id: Optional revision id. (Specifying a revision-id 633 may reduce performance for some non-native formats.) 634 :return: The revision_id of the revision committed. 635 """ 636 self._check_malformed() 637 if strict: 638 unversioned = set(self._new_contents).difference(set(self._versioned)) 639 for trans_id in unversioned: 640 if not self.final_is_versioned(trans_id): 641 raise errors.StrictCommitFailed() 642 643 revno, last_rev_id = branch.last_revision_info() 644 if last_rev_id == _mod_revision.NULL_REVISION: 645 if merge_parents is not None: 646 raise ValueError('Cannot supply merge parents for first' 647 ' commit.') 648 parent_ids = [] 649 else: 650 parent_ids = [last_rev_id] 651 if merge_parents is not None: 652 parent_ids.extend(merge_parents) 653 if self._tree.get_revision_id() != last_rev_id: 654 raise ValueError('TreeTransform not based on branch basis: %s' % 655 self._tree.get_revision_id().decode('utf-8')) 656 from .. import commit 657 revprops = commit.Commit.update_revprops(revprops, branch, authors) 658 builder = branch.get_commit_builder(parent_ids, 659 timestamp=timestamp, 660 timezone=timezone, 661 committer=committer, 662 revprops=revprops, 663 revision_id=revision_id) 664 preview = self.get_preview_tree() 665 list(builder.record_iter_changes(preview, last_rev_id, 666 self.iter_changes())) 667 builder.finish_inventory() 668 revision_id = builder.commit(message) 669 branch.set_last_revision_info(revno + 1, revision_id) 670 return revision_id 671 672 def _text_parent(self, trans_id): 673 path = self.tree_path(trans_id) 674 try: 675 if path is None or self._tree.kind(path) != 'file': 676 return None 677 except errors.NoSuchFile: 678 return None 679 return path 680 681 def _get_parents_texts(self, trans_id): 682 """Get texts for compression parents of this file.""" 683 path = self._text_parent(trans_id) 684 if path is None: 685 return () 686 return (self._tree.get_file_text(path),) 687 688 def _get_parents_lines(self, trans_id): 689 """Get lines for compression parents of this file.""" 690 path = self._text_parent(trans_id) 691 if path is None: 692 return () 693 return (self._tree.get_file_lines(path),) 694 695 def create_file(self, contents, trans_id, mode_id=None, sha1=None): 696 """Schedule creation of a new file. 697 698 :seealso: new_file. 699 700 :param contents: an iterator of strings, all of which will be written 701 to the target destination. 702 :param trans_id: TreeTransform handle 703 :param mode_id: If not None, force the mode of the target file to match 704 the mode of the object referenced by mode_id. 705 Otherwise, we will try to preserve mode bits of an existing file. 706 :param sha1: If the sha1 of this content is already known, pass it in. 707 We can use it to prevent future sha1 computations. 708 """ 709 raise NotImplementedError(self.create_file) 710 711 def create_directory(self, trans_id): 712 """Schedule creation of a new directory. 713 714 See also new_directory. 715 """ 716 raise NotImplementedError(self.create_directory) 717 718 def create_symlink(self, target, trans_id): 719 """Schedule creation of a new symbolic link. 720 721 target is a bytestring. 722 See also new_symlink. 723 """ 724 raise NotImplementedError(self.create_symlink) 725 726 def create_hardlink(self, path, trans_id): 727 """Schedule creation of a hard link""" 728 raise NotImplementedError(self.create_hardlink) 729 730 def cancel_creation(self, trans_id): 731 """Cancel the creation of new file contents.""" 732 raise NotImplementedError(self.cancel_creation) 733 734 def apply(self, no_conflicts=False, _mover=None): 735 """Apply all changes to the inventory and filesystem. 736 737 If filesystem or inventory conflicts are present, MalformedTransform 738 will be thrown. 739 740 If apply succeeds, finalize is not necessary. 741 742 :param no_conflicts: if True, the caller guarantees there are no 743 conflicts, so no check is made. 744 :param _mover: Supply an alternate FileMover, for testing 745 """ 746 raise NotImplementedError(self.apply) 747 748 def cook_conflicts(self, raw_conflicts): 749 """Generate a list of cooked conflicts, sorted by file path""" 750 if not raw_conflicts: 751 return 752 fp = FinalPaths(self) 753 from .workingtree import TextConflict 754 for c in raw_conflicts: 755 if c[0] == 'text conflict': 756 yield TextConflict(fp.get_path(c[1])) 757 elif c[0] == 'duplicate': 758 yield TextConflict(fp.get_path(c[2])) 759 elif c[0] == 'contents conflict': 760 yield TextConflict(fp.get_path(c[1][0])) 761 elif c[0] == 'missing parent': 762 # TODO(jelmer): This should not make it to here 763 yield TextConflict(fp.get_path(c[2])) 764 elif c[0] == 'non-directory parent': 765 yield TextConflict(fp.get_path(c[2])) 766 elif c[0] == 'deleting parent': 767 # TODO(jelmer): This should not make it to here 768 yield TextConflict(fp.get_path(c[2])) 769 elif c[0] == 'parent loop': 770 # TODO(jelmer): This should not make it to here 771 yield TextConflict(fp.get_path(c[2])) 772 elif c[0] == 'path conflict': 773 yield TextConflict(fp.get_path(c[1])) 774 else: 775 raise AssertionError('unknown conflict %s' % c[0]) 776 777 778class DiskTreeTransform(TreeTransformBase): 779 """Tree transform storing its contents on disk.""" 780 781 def __init__(self, tree, limbodir, pb=None, case_sensitive=True): 782 """Constructor. 783 :param tree: The tree that will be transformed, but not necessarily 784 the output tree. 785 :param limbodir: A directory where new files can be stored until 786 they are installed in their proper places 787 :param pb: ignored 788 :param case_sensitive: If True, the target of the transform is 789 case sensitive, not just case preserving. 790 """ 791 TreeTransformBase.__init__(self, tree, pb, case_sensitive) 792 self._limbodir = limbodir 793 self._deletiondir = None 794 # A mapping of transform ids to their limbo filename 795 self._limbo_files = {} 796 self._possibly_stale_limbo_files = set() 797 # A mapping of transform ids to a set of the transform ids of children 798 # that their limbo directory has 799 self._limbo_children = {} 800 # Map transform ids to maps of child filename to child transform id 801 self._limbo_children_names = {} 802 # List of transform ids that need to be renamed from limbo into place 803 self._needs_rename = set() 804 self._creation_mtime = None 805 self._create_symlinks = osutils.supports_symlinks(self._limbodir) 806 807 def finalize(self): 808 """Release the working tree lock, if held, clean up limbo dir. 809 810 This is required if apply has not been invoked, but can be invoked 811 even after apply. 812 """ 813 if self._tree is None: 814 return 815 try: 816 limbo_paths = list(self._limbo_files.values()) 817 limbo_paths.extend(self._possibly_stale_limbo_files) 818 limbo_paths.sort(reverse=True) 819 for path in limbo_paths: 820 try: 821 osutils.delete_any(path) 822 except OSError as e: 823 if e.errno != errno.ENOENT: 824 raise 825 # XXX: warn? perhaps we just got interrupted at an 826 # inconvenient moment, but perhaps files are disappearing 827 # from under us? 828 try: 829 osutils.delete_any(self._limbodir) 830 except OSError: 831 # We don't especially care *why* the dir is immortal. 832 raise ImmortalLimbo(self._limbodir) 833 try: 834 if self._deletiondir is not None: 835 osutils.delete_any(self._deletiondir) 836 except OSError: 837 raise errors.ImmortalPendingDeletion(self._deletiondir) 838 finally: 839 TreeTransformBase.finalize(self) 840 841 def _limbo_supports_executable(self): 842 """Check if the limbo path supports the executable bit.""" 843 return osutils.supports_executable(self._limbodir) 844 845 def _limbo_name(self, trans_id): 846 """Generate the limbo name of a file""" 847 limbo_name = self._limbo_files.get(trans_id) 848 if limbo_name is None: 849 limbo_name = self._generate_limbo_path(trans_id) 850 self._limbo_files[trans_id] = limbo_name 851 return limbo_name 852 853 def _generate_limbo_path(self, trans_id): 854 """Generate a limbo path using the trans_id as the relative path. 855 856 This is suitable as a fallback, and when the transform should not be 857 sensitive to the path encoding of the limbo directory. 858 """ 859 self._needs_rename.add(trans_id) 860 return osutils.pathjoin(self._limbodir, trans_id) 861 862 def adjust_path(self, name, parent, trans_id): 863 previous_parent = self._new_parent.get(trans_id) 864 previous_name = self._new_name.get(trans_id) 865 super(DiskTreeTransform, self).adjust_path(name, parent, trans_id) 866 if (trans_id in self._limbo_files 867 and trans_id not in self._needs_rename): 868 self._rename_in_limbo([trans_id]) 869 if previous_parent != parent: 870 self._limbo_children[previous_parent].remove(trans_id) 871 if previous_parent != parent or previous_name != name: 872 del self._limbo_children_names[previous_parent][previous_name] 873 874 def _rename_in_limbo(self, trans_ids): 875 """Fix limbo names so that the right final path is produced. 876 877 This means we outsmarted ourselves-- we tried to avoid renaming 878 these files later by creating them with their final names in their 879 final parents. But now the previous name or parent is no longer 880 suitable, so we have to rename them. 881 882 Even for trans_ids that have no new contents, we must remove their 883 entries from _limbo_files, because they are now stale. 884 """ 885 for trans_id in trans_ids: 886 old_path = self._limbo_files[trans_id] 887 self._possibly_stale_limbo_files.add(old_path) 888 del self._limbo_files[trans_id] 889 if trans_id not in self._new_contents: 890 continue 891 new_path = self._limbo_name(trans_id) 892 os.rename(old_path, new_path) 893 self._possibly_stale_limbo_files.remove(old_path) 894 for descendant in self._limbo_descendants(trans_id): 895 desc_path = self._limbo_files[descendant] 896 desc_path = new_path + desc_path[len(old_path):] 897 self._limbo_files[descendant] = desc_path 898 899 def _limbo_descendants(self, trans_id): 900 """Return the set of trans_ids whose limbo paths descend from this.""" 901 descendants = set(self._limbo_children.get(trans_id, [])) 902 for descendant in list(descendants): 903 descendants.update(self._limbo_descendants(descendant)) 904 return descendants 905 906 def _set_mode(self, trans_id, mode_id, typefunc): 907 raise NotImplementedError(self._set_mode) 908 909 def create_file(self, contents, trans_id, mode_id=None, sha1=None): 910 """Schedule creation of a new file. 911 912 :seealso: new_file. 913 914 :param contents: an iterator of strings, all of which will be written 915 to the target destination. 916 :param trans_id: TreeTransform handle 917 :param mode_id: If not None, force the mode of the target file to match 918 the mode of the object referenced by mode_id. 919 Otherwise, we will try to preserve mode bits of an existing file. 920 :param sha1: If the sha1 of this content is already known, pass it in. 921 We can use it to prevent future sha1 computations. 922 """ 923 name = self._limbo_name(trans_id) 924 with open(name, 'wb') as f: 925 unique_add(self._new_contents, trans_id, 'file') 926 f.writelines(contents) 927 self._set_mtime(name) 928 self._set_mode(trans_id, mode_id, S_ISREG) 929 # It is unfortunate we have to use lstat instead of fstat, but we just 930 # used utime and chmod on the file, so we need the accurate final 931 # details. 932 if sha1 is not None: 933 self._observed_sha1s[trans_id] = (sha1, osutils.lstat(name)) 934 935 def _read_symlink_target(self, trans_id): 936 return os.readlink(self._limbo_name(trans_id)) 937 938 def _set_mtime(self, path): 939 """All files that are created get the same mtime. 940 941 This time is set by the first object to be created. 942 """ 943 if self._creation_mtime is None: 944 self._creation_mtime = time.time() 945 os.utime(path, (self._creation_mtime, self._creation_mtime)) 946 947 def create_hardlink(self, path, trans_id): 948 """Schedule creation of a hard link""" 949 name = self._limbo_name(trans_id) 950 try: 951 os.link(path, name) 952 except OSError as e: 953 if e.errno != errno.EPERM: 954 raise 955 raise errors.HardLinkNotSupported(path) 956 try: 957 unique_add(self._new_contents, trans_id, 'file') 958 except BaseException: 959 # Clean up the file, it never got registered so 960 # TreeTransform.finalize() won't clean it up. 961 os.unlink(name) 962 raise 963 964 def create_directory(self, trans_id): 965 """Schedule creation of a new directory. 966 967 See also new_directory. 968 """ 969 os.mkdir(self._limbo_name(trans_id)) 970 unique_add(self._new_contents, trans_id, 'directory') 971 972 def create_symlink(self, target, trans_id): 973 """Schedule creation of a new symbolic link. 974 975 target is a bytestring. 976 See also new_symlink. 977 """ 978 if self._create_symlinks: 979 os.symlink(target, self._limbo_name(trans_id)) 980 else: 981 try: 982 path = FinalPaths(self).get_path(trans_id) 983 except KeyError: 984 path = None 985 trace.warning( 986 'Unable to create symlink "%s" on this filesystem.' % (path,)) 987 self._symlink_target[trans_id] = target 988 # We add symlink to _new_contents even if they are unsupported 989 # and not created. These entries are subsequently used to avoid 990 # conflicts on platforms that don't support symlink 991 unique_add(self._new_contents, trans_id, 'symlink') 992 993 def cancel_creation(self, trans_id): 994 """Cancel the creation of new file contents.""" 995 del self._new_contents[trans_id] 996 if trans_id in self._observed_sha1s: 997 del self._observed_sha1s[trans_id] 998 children = self._limbo_children.get(trans_id) 999 # if this is a limbo directory with children, move them before removing 1000 # the directory 1001 if children is not None: 1002 self._rename_in_limbo(children) 1003 del self._limbo_children[trans_id] 1004 del self._limbo_children_names[trans_id] 1005 osutils.delete_any(self._limbo_name(trans_id)) 1006 1007 def new_orphan(self, trans_id, parent_id): 1008 conf = self._tree.get_config_stack() 1009 handle_orphan = conf.get('transform.orphan_policy') 1010 handle_orphan(self, trans_id, parent_id) 1011 1012 def final_entry(self, trans_id): 1013 is_versioned = self.final_is_versioned(trans_id) 1014 fp = FinalPaths(self) 1015 tree_path = fp.get_path(trans_id) 1016 if trans_id in self._new_contents: 1017 path = self._limbo_name(trans_id) 1018 st = os.lstat(path) 1019 kind = mode_kind(st.st_mode) 1020 name = self.final_name(trans_id) 1021 file_id = self._tree.mapping.generate_file_id(tree_path) 1022 parent_id = self._tree.mapping.generate_file_id(os.path.dirname(tree_path)) 1023 if kind == 'directory': 1024 return GitTreeDirectory( 1025 file_id, self.final_name(trans_id), parent_id=parent_id), is_versioned 1026 executable = mode_is_executable(st.st_mode) 1027 mode = object_mode(kind, executable) 1028 blob = blob_from_path_and_stat(encode_git_path(path), st) 1029 if kind == 'symlink': 1030 return GitTreeSymlink( 1031 file_id, name, parent_id, 1032 decode_git_path(blob.data)), is_versioned 1033 elif kind == 'file': 1034 return GitTreeFile( 1035 file_id, name, executable=executable, parent_id=parent_id, 1036 git_sha1=blob.id, text_size=len(blob.data)), is_versioned 1037 else: 1038 raise AssertionError(kind) 1039 elif trans_id in self._removed_contents: 1040 return None, None 1041 else: 1042 orig_path = self.tree_path(trans_id) 1043 if orig_path is None: 1044 return None, None 1045 file_id = self._tree.mapping.generate_file_id(tree_path) 1046 if tree_path == '': 1047 parent_id = None 1048 else: 1049 parent_id = self._tree.mapping.generate_file_id(os.path.dirname(tree_path)) 1050 try: 1051 ie = next(self._tree.iter_entries_by_dir( 1052 specific_files=[orig_path]))[1] 1053 ie.file_id = file_id 1054 ie.parent_id = parent_id 1055 return ie, is_versioned 1056 except StopIteration: 1057 try: 1058 if self.tree_kind(trans_id) == 'directory': 1059 return GitTreeDirectory( 1060 file_id, self.final_name(trans_id), parent_id=parent_id), is_versioned 1061 except OSError as e: 1062 if e.errno != errno.ENOTDIR: 1063 raise 1064 return None, None 1065 1066 def final_git_entry(self, trans_id): 1067 if trans_id in self._new_contents: 1068 path = self._limbo_name(trans_id) 1069 st = os.lstat(path) 1070 kind = mode_kind(st.st_mode) 1071 if kind == 'directory': 1072 return None, None 1073 executable = mode_is_executable(st.st_mode) 1074 mode = object_mode(kind, executable) 1075 blob = blob_from_path_and_stat(encode_git_path(path), st) 1076 elif trans_id in self._removed_contents: 1077 return None, None 1078 else: 1079 orig_path = self.tree_path(trans_id) 1080 kind = self._tree.kind(orig_path) 1081 executable = self._tree.is_executable(orig_path) 1082 mode = object_mode(kind, executable) 1083 if kind == 'symlink': 1084 contents = self._tree.get_symlink_target(orig_path) 1085 elif kind == 'file': 1086 contents = self._tree.get_file_text(orig_path) 1087 elif kind == 'directory': 1088 return None, None 1089 else: 1090 raise AssertionError(kind) 1091 blob = Blob.from_string(contents) 1092 return blob, mode 1093 1094 1095class GitTreeTransform(DiskTreeTransform): 1096 """Represent a tree transformation. 1097 1098 This object is designed to support incremental generation of the transform, 1099 in any order. 1100 1101 However, it gives optimum performance when parent directories are created 1102 before their contents. The transform is then able to put child files 1103 directly in their parent directory, avoiding later renames. 1104 1105 It is easy to produce malformed transforms, but they are generally 1106 harmless. Attempting to apply a malformed transform will cause an 1107 exception to be raised before any modifications are made to the tree. 1108 1109 Many kinds of malformed transforms can be corrected with the 1110 resolve_conflicts function. The remaining ones indicate programming error, 1111 such as trying to create a file with no path. 1112 1113 Two sets of file creation methods are supplied. Convenience methods are: 1114 * new_file 1115 * new_directory 1116 * new_symlink 1117 1118 These are composed of the low-level methods: 1119 * create_path 1120 * create_file or create_directory or create_symlink 1121 * version_file 1122 * set_executability 1123 1124 Transform/Transaction ids 1125 ------------------------- 1126 trans_ids are temporary ids assigned to all files involved in a transform. 1127 It's possible, even common, that not all files in the Tree have trans_ids. 1128 1129 trans_ids are used because filenames and file_ids are not good enough 1130 identifiers; filenames change. 1131 1132 trans_ids are only valid for the TreeTransform that generated them. 1133 1134 Limbo 1135 ----- 1136 Limbo is a temporary directory use to hold new versions of files. 1137 Files are added to limbo by create_file, create_directory, create_symlink, 1138 and their convenience variants (new_*). Files may be removed from limbo 1139 using cancel_creation. Files are renamed from limbo into their final 1140 location as part of TreeTransform.apply 1141 1142 Limbo must be cleaned up, by either calling TreeTransform.apply or 1143 calling TreeTransform.finalize. 1144 1145 Files are placed into limbo inside their parent directories, where 1146 possible. This reduces subsequent renames, and makes operations involving 1147 lots of files faster. This optimization is only possible if the parent 1148 directory is created *before* creating any of its children, so avoid 1149 creating children before parents, where possible. 1150 1151 Pending-deletion 1152 ---------------- 1153 This temporary directory is used by _FileMover for storing files that are 1154 about to be deleted. In case of rollback, the files will be restored. 1155 FileMover does not delete files until it is sure that a rollback will not 1156 happen. 1157 """ 1158 1159 def __init__(self, tree, pb=None): 1160 """Note: a tree_write lock is taken on the tree. 1161 1162 Use TreeTransform.finalize() to release the lock (can be omitted if 1163 TreeTransform.apply() called). 1164 """ 1165 tree.lock_tree_write() 1166 try: 1167 limbodir = urlutils.local_path_from_url( 1168 tree._transport.abspath('limbo')) 1169 osutils.ensure_empty_directory_exists( 1170 limbodir, 1171 errors.ExistingLimbo) 1172 deletiondir = urlutils.local_path_from_url( 1173 tree._transport.abspath('pending-deletion')) 1174 osutils.ensure_empty_directory_exists( 1175 deletiondir, 1176 errors.ExistingPendingDeletion) 1177 except BaseException: 1178 tree.unlock() 1179 raise 1180 1181 # Cache of realpath results, to speed up canonical_path 1182 self._realpaths = {} 1183 # Cache of relpath results, to speed up canonical_path 1184 self._relpaths = {} 1185 DiskTreeTransform.__init__(self, tree, limbodir, pb, 1186 tree.case_sensitive) 1187 self._deletiondir = deletiondir 1188 1189 def canonical_path(self, path): 1190 """Get the canonical tree-relative path""" 1191 # don't follow final symlinks 1192 abs = self._tree.abspath(path) 1193 if abs in self._relpaths: 1194 return self._relpaths[abs] 1195 dirname, basename = os.path.split(abs) 1196 if dirname not in self._realpaths: 1197 self._realpaths[dirname] = os.path.realpath(dirname) 1198 dirname = self._realpaths[dirname] 1199 abs = osutils.pathjoin(dirname, basename) 1200 if dirname in self._relpaths: 1201 relpath = osutils.pathjoin(self._relpaths[dirname], basename) 1202 relpath = relpath.rstrip('/\\') 1203 else: 1204 relpath = self._tree.relpath(abs) 1205 self._relpaths[abs] = relpath 1206 return relpath 1207 1208 def tree_kind(self, trans_id): 1209 """Determine the file kind in the working tree. 1210 1211 :returns: The file kind or None if the file does not exist 1212 """ 1213 path = self._tree_id_paths.get(trans_id) 1214 if path is None: 1215 return None 1216 try: 1217 return osutils.file_kind(self._tree.abspath(path)) 1218 except errors.NoSuchFile: 1219 return None 1220 1221 def _set_mode(self, trans_id, mode_id, typefunc): 1222 """Set the mode of new file contents. 1223 The mode_id is the existing file to get the mode from (often the same 1224 as trans_id). The operation is only performed if there's a mode match 1225 according to typefunc. 1226 """ 1227 if mode_id is None: 1228 mode_id = trans_id 1229 try: 1230 old_path = self._tree_id_paths[mode_id] 1231 except KeyError: 1232 return 1233 try: 1234 mode = os.stat(self._tree.abspath(old_path)).st_mode 1235 except OSError as e: 1236 if e.errno in (errno.ENOENT, errno.ENOTDIR): 1237 # Either old_path doesn't exist, or the parent of the 1238 # target is not a directory (but will be one eventually) 1239 # Either way, we know it doesn't exist *right now* 1240 # See also bug #248448 1241 return 1242 else: 1243 raise 1244 if typefunc(mode): 1245 osutils.chmod_if_possible(self._limbo_name(trans_id), mode) 1246 1247 def iter_tree_children(self, parent_id): 1248 """Iterate through the entry's tree children, if any""" 1249 try: 1250 path = self._tree_id_paths[parent_id] 1251 except KeyError: 1252 return 1253 try: 1254 children = os.listdir(self._tree.abspath(path)) 1255 except OSError as e: 1256 if not (osutils._is_error_enotdir(e) or 1257 e.errno in (errno.ENOENT, errno.ESRCH)): 1258 raise 1259 return 1260 1261 for child in children: 1262 childpath = joinpath(path, child) 1263 if self._tree.is_control_filename(childpath): 1264 continue 1265 yield self.trans_id_tree_path(childpath) 1266 1267 def _generate_limbo_path(self, trans_id): 1268 """Generate a limbo path using the final path if possible. 1269 1270 This optimizes the performance of applying the tree transform by 1271 avoiding renames. These renames can be avoided only when the parent 1272 directory is already scheduled for creation. 1273 1274 If the final path cannot be used, falls back to using the trans_id as 1275 the relpath. 1276 """ 1277 parent = self._new_parent.get(trans_id) 1278 # if the parent directory is already in limbo (e.g. when building a 1279 # tree), choose a limbo name inside the parent, to reduce further 1280 # renames. 1281 use_direct_path = False 1282 if self._new_contents.get(parent) == 'directory': 1283 filename = self._new_name.get(trans_id) 1284 if filename is not None: 1285 if parent not in self._limbo_children: 1286 self._limbo_children[parent] = set() 1287 self._limbo_children_names[parent] = {} 1288 use_direct_path = True 1289 # the direct path can only be used if no other file has 1290 # already taken this pathname, i.e. if the name is unused, or 1291 # if it is already associated with this trans_id. 1292 elif self._case_sensitive_target: 1293 if (self._limbo_children_names[parent].get(filename) 1294 in (trans_id, None)): 1295 use_direct_path = True 1296 else: 1297 for l_filename, l_trans_id in ( 1298 self._limbo_children_names[parent].items()): 1299 if l_trans_id == trans_id: 1300 continue 1301 if l_filename.lower() == filename.lower(): 1302 break 1303 else: 1304 use_direct_path = True 1305 1306 if not use_direct_path: 1307 return DiskTreeTransform._generate_limbo_path(self, trans_id) 1308 1309 limbo_name = osutils.pathjoin(self._limbo_files[parent], filename) 1310 self._limbo_children[parent].add(trans_id) 1311 self._limbo_children_names[parent][filename] = trans_id 1312 return limbo_name 1313 1314 def cancel_versioning(self, trans_id): 1315 """Undo a previous versioning of a file""" 1316 self._versioned.remove(trans_id) 1317 1318 def apply(self, no_conflicts=False, _mover=None): 1319 """Apply all changes to the inventory and filesystem. 1320 1321 If filesystem or inventory conflicts are present, MalformedTransform 1322 will be thrown. 1323 1324 If apply succeeds, finalize is not necessary. 1325 1326 :param no_conflicts: if True, the caller guarantees there are no 1327 conflicts, so no check is made. 1328 :param _mover: Supply an alternate FileMover, for testing 1329 """ 1330 for hook in MutableTree.hooks['pre_transform']: 1331 hook(self._tree, self) 1332 if not no_conflicts: 1333 self._check_malformed() 1334 self.rename_count = 0 1335 with ui.ui_factory.nested_progress_bar() as child_pb: 1336 child_pb.update(gettext('Apply phase'), 0, 2) 1337 index_changes = self._generate_index_changes() 1338 offset = 1 1339 if _mover is None: 1340 mover = _FileMover() 1341 else: 1342 mover = _mover 1343 try: 1344 child_pb.update(gettext('Apply phase'), 0 + offset, 2 + offset) 1345 self._apply_removals(mover) 1346 child_pb.update(gettext('Apply phase'), 1 + offset, 2 + offset) 1347 modified_paths = self._apply_insertions(mover) 1348 except BaseException: 1349 mover.rollback() 1350 raise 1351 else: 1352 mover.apply_deletions() 1353 self._tree._apply_index_changes(index_changes) 1354 self._done = True 1355 self.finalize() 1356 return _TransformResults(modified_paths, self.rename_count) 1357 1358 def _apply_removals(self, mover): 1359 """Perform tree operations that remove directory/inventory names. 1360 1361 That is, delete files that are to be deleted, and put any files that 1362 need renaming into limbo. This must be done in strict child-to-parent 1363 order. 1364 1365 If inventory_delta is None, no inventory delta generation is performed. 1366 """ 1367 tree_paths = sorted(self._tree_path_ids.items(), reverse=True) 1368 with ui.ui_factory.nested_progress_bar() as child_pb: 1369 for num, (path, trans_id) in enumerate(tree_paths): 1370 # do not attempt to move root into a subdirectory of itself. 1371 if path == '': 1372 continue 1373 child_pb.update(gettext('removing file'), num, len(tree_paths)) 1374 full_path = self._tree.abspath(path) 1375 if trans_id in self._removed_contents: 1376 delete_path = os.path.join(self._deletiondir, trans_id) 1377 mover.pre_delete(full_path, delete_path) 1378 elif (trans_id in self._new_name or 1379 trans_id in self._new_parent): 1380 try: 1381 mover.rename(full_path, self._limbo_name(trans_id)) 1382 except TransformRenameFailed as e: 1383 if e.errno != errno.ENOENT: 1384 raise 1385 else: 1386 self.rename_count += 1 1387 1388 def _apply_insertions(self, mover): 1389 """Perform tree operations that insert directory/inventory names. 1390 1391 That is, create any files that need to be created, and restore from 1392 limbo any files that needed renaming. This must be done in strict 1393 parent-to-child order. 1394 1395 If inventory_delta is None, no inventory delta is calculated, and 1396 no list of modified paths is returned. 1397 """ 1398 new_paths = self.new_paths(filesystem_only=True) 1399 modified_paths = [] 1400 with ui.ui_factory.nested_progress_bar() as child_pb: 1401 for num, (path, trans_id) in enumerate(new_paths): 1402 if (num % 10) == 0: 1403 child_pb.update(gettext('adding file'), 1404 num, len(new_paths)) 1405 full_path = self._tree.abspath(path) 1406 if trans_id in self._needs_rename: 1407 try: 1408 mover.rename(self._limbo_name(trans_id), full_path) 1409 except TransformRenameFailed as e: 1410 # We may be renaming a dangling inventory id 1411 if e.errno != errno.ENOENT: 1412 raise 1413 else: 1414 self.rename_count += 1 1415 # TODO: if trans_id in self._observed_sha1s, we should 1416 # re-stat the final target, since ctime will be 1417 # updated by the change. 1418 if (trans_id in self._new_contents 1419 or self.path_changed(trans_id)): 1420 if trans_id in self._new_contents: 1421 modified_paths.append(full_path) 1422 if trans_id in self._new_executability: 1423 self._set_executability(path, trans_id) 1424 if trans_id in self._observed_sha1s: 1425 o_sha1, o_st_val = self._observed_sha1s[trans_id] 1426 st = osutils.lstat(full_path) 1427 self._observed_sha1s[trans_id] = (o_sha1, st) 1428 for path, trans_id in new_paths: 1429 # new_paths includes stuff like workingtree conflicts. Only the 1430 # stuff in new_contents actually comes from limbo. 1431 if trans_id in self._limbo_files: 1432 del self._limbo_files[trans_id] 1433 self._new_contents.clear() 1434 return modified_paths 1435 1436 def _generate_index_changes(self): 1437 """Generate an inventory delta for the current transform.""" 1438 removed_id = set(self._removed_id) 1439 removed_id.update(self._removed_contents) 1440 changes = {} 1441 changed_ids = set() 1442 for id_set in [self._new_name, self._new_parent, 1443 self._new_executability, self._new_contents]: 1444 changed_ids.update(id_set) 1445 for id_set in [self._new_name, self._new_parent]: 1446 removed_id.update(id_set) 1447 # so does adding 1448 changed_kind = set(self._new_contents) 1449 # Ignore entries that are already known to have changed. 1450 changed_kind.difference_update(changed_ids) 1451 # to keep only the truly changed ones 1452 changed_kind = (t for t in changed_kind 1453 if self.tree_kind(t) != self.final_kind(t)) 1454 changed_ids.update(changed_kind) 1455 for t in changed_kind: 1456 if self.final_kind(t) == 'directory': 1457 removed_id.add(t) 1458 changed_ids.remove(t) 1459 new_paths = sorted(FinalPaths(self).get_paths(changed_ids)) 1460 total_entries = len(new_paths) + len(removed_id) 1461 with ui.ui_factory.nested_progress_bar() as child_pb: 1462 for num, trans_id in enumerate(removed_id): 1463 if (num % 10) == 0: 1464 child_pb.update(gettext('removing file'), 1465 num, total_entries) 1466 try: 1467 path = self._tree_id_paths[trans_id] 1468 except KeyError: 1469 continue 1470 changes[path] = (None, None, None, None) 1471 for num, (path, trans_id) in enumerate(new_paths): 1472 if (num % 10) == 0: 1473 child_pb.update(gettext('adding file'), 1474 num + len(removed_id), total_entries) 1475 1476 kind = self.final_kind(trans_id) 1477 if kind is None: 1478 continue 1479 versioned = self.final_is_versioned(trans_id) 1480 if not versioned: 1481 continue 1482 executability = self._new_executability.get(trans_id) 1483 reference_revision = self._new_reference_revision.get(trans_id) 1484 symlink_target = self._symlink_target.get(trans_id) 1485 changes[path] = ( 1486 kind, executability, reference_revision, symlink_target) 1487 return [(p, k, e, rr, st) for (p, (k, e, rr, st)) in changes.items()] 1488 1489 1490class GitTransformPreview(GitTreeTransform): 1491 """A TreeTransform for generating preview trees. 1492 1493 Unlike TreeTransform, this version works when the input tree is a 1494 RevisionTree, rather than a WorkingTree. As a result, it tends to ignore 1495 unversioned files in the input tree. 1496 """ 1497 1498 def __init__(self, tree, pb=None, case_sensitive=True): 1499 tree.lock_read() 1500 limbodir = osutils.mkdtemp(prefix='git-limbo-') 1501 DiskTreeTransform.__init__(self, tree, limbodir, pb, case_sensitive) 1502 1503 def canonical_path(self, path): 1504 return path 1505 1506 def tree_kind(self, trans_id): 1507 path = self.tree_path(trans_id) 1508 if path is None: 1509 return None 1510 kind = self._tree.path_content_summary(path)[0] 1511 if kind == 'missing': 1512 kind = None 1513 return kind 1514 1515 def _set_mode(self, trans_id, mode_id, typefunc): 1516 """Set the mode of new file contents. 1517 The mode_id is the existing file to get the mode from (often the same 1518 as trans_id). The operation is only performed if there's a mode match 1519 according to typefunc. 1520 """ 1521 # is it ok to ignore this? probably 1522 pass 1523 1524 def iter_tree_children(self, parent_id): 1525 """Iterate through the entry's tree children, if any""" 1526 try: 1527 path = self._tree_id_paths[parent_id] 1528 except KeyError: 1529 return 1530 try: 1531 for child in self._tree.iter_child_entries(path): 1532 childpath = joinpath(path, child.name) 1533 yield self.trans_id_tree_path(childpath) 1534 except errors.NoSuchFile: 1535 return 1536 1537 def new_orphan(self, trans_id, parent_id): 1538 raise NotImplementedError(self.new_orphan) 1539 1540 1541class GitPreviewTree(PreviewTree, GitTree): 1542 """Partial implementation of Tree to support show_diff_trees""" 1543 1544 def __init__(self, transform): 1545 PreviewTree.__init__(self, transform) 1546 self.store = transform._tree.store 1547 self.mapping = transform._tree.mapping 1548 self._final_paths = FinalPaths(transform) 1549 1550 def supports_setting_file_ids(self): 1551 return False 1552 1553 def _supports_executable(self): 1554 return self._transform._limbo_supports_executable() 1555 1556 def walkdirs(self, prefix=''): 1557 pending = [self._transform.root] 1558 while len(pending) > 0: 1559 parent_id = pending.pop() 1560 children = [] 1561 subdirs = [] 1562 prefix = prefix.rstrip('/') 1563 parent_path = self._final_paths.get_path(parent_id) 1564 for child_id in self._all_children(parent_id): 1565 path_from_root = self._final_paths.get_path(child_id) 1566 basename = self._transform.final_name(child_id) 1567 kind = self._transform.final_kind(child_id) 1568 if kind is not None: 1569 versioned_kind = kind 1570 else: 1571 kind = 'unknown' 1572 versioned_kind = self._transform._tree.stored_kind( 1573 path_from_root) 1574 if versioned_kind == 'directory': 1575 subdirs.append(child_id) 1576 children.append((path_from_root, basename, kind, None, 1577 versioned_kind)) 1578 children.sort() 1579 if parent_path.startswith(prefix): 1580 yield parent_path, children 1581 pending.extend(sorted(subdirs, key=self._final_paths.get_path, 1582 reverse=True)) 1583 1584 def iter_changes(self, from_tree, include_unchanged=False, 1585 specific_files=None, pb=None, extra_trees=None, 1586 require_versioned=True, want_unversioned=False): 1587 """See InterTree.iter_changes. 1588 1589 This has a fast path that is only used when the from_tree matches 1590 the transform tree, and no fancy options are supplied. 1591 """ 1592 return InterTree.get(from_tree, self).iter_changes( 1593 include_unchanged=include_unchanged, 1594 specific_files=specific_files, 1595 pb=pb, 1596 extra_trees=extra_trees, 1597 require_versioned=require_versioned, 1598 want_unversioned=want_unversioned) 1599 1600 def get_file(self, path): 1601 """See Tree.get_file""" 1602 trans_id = self._path2trans_id(path) 1603 if trans_id is None: 1604 raise errors.NoSuchFile(path) 1605 if trans_id in self._transform._new_contents: 1606 name = self._transform._limbo_name(trans_id) 1607 return open(name, 'rb') 1608 if trans_id in self._transform._removed_contents: 1609 raise errors.NoSuchFile(path) 1610 orig_path = self._transform.tree_path(trans_id) 1611 return self._transform._tree.get_file(orig_path) 1612 1613 def get_symlink_target(self, path): 1614 """See Tree.get_symlink_target""" 1615 trans_id = self._path2trans_id(path) 1616 if trans_id is None: 1617 raise errors.NoSuchFile(path) 1618 if trans_id not in self._transform._new_contents: 1619 orig_path = self._transform.tree_path(trans_id) 1620 return self._transform._tree.get_symlink_target(orig_path) 1621 name = self._transform._limbo_name(trans_id) 1622 return osutils.readlink(name) 1623 1624 def annotate_iter(self, path, default_revision=_mod_revision.CURRENT_REVISION): 1625 trans_id = self._path2trans_id(path) 1626 if trans_id is None: 1627 return None 1628 orig_path = self._transform.tree_path(trans_id) 1629 if orig_path is not None: 1630 old_annotation = self._transform._tree.annotate_iter( 1631 orig_path, default_revision=default_revision) 1632 else: 1633 old_annotation = [] 1634 try: 1635 lines = self.get_file_lines(path) 1636 except errors.NoSuchFile: 1637 return None 1638 return annotate.reannotate([old_annotation], lines, default_revision) 1639 1640 def get_file_text(self, path): 1641 """Return the byte content of a file. 1642 1643 :param path: The path of the file. 1644 1645 :returns: A single byte string for the whole file. 1646 """ 1647 with self.get_file(path) as my_file: 1648 return my_file.read() 1649 1650 def get_file_lines(self, path): 1651 """Return the content of a file, as lines. 1652 1653 :param path: The path of the file. 1654 """ 1655 return osutils.split_lines(self.get_file_text(path)) 1656 1657 def extras(self): 1658 possible_extras = set(self._transform.trans_id_tree_path(p) for p 1659 in self._transform._tree.extras()) 1660 possible_extras.update(self._transform._new_contents) 1661 possible_extras.update(self._transform._removed_id) 1662 for trans_id in possible_extras: 1663 if not self._transform.final_is_versioned(trans_id): 1664 yield self._final_paths._determine_path(trans_id) 1665 1666 def path_content_summary(self, path): 1667 trans_id = self._path2trans_id(path) 1668 tt = self._transform 1669 tree_path = tt.tree_path(trans_id) 1670 kind = tt._new_contents.get(trans_id) 1671 if kind is None: 1672 if tree_path is None or trans_id in tt._removed_contents: 1673 return 'missing', None, None, None 1674 summary = tt._tree.path_content_summary(tree_path) 1675 kind, size, executable, link_or_sha1 = summary 1676 else: 1677 link_or_sha1 = None 1678 limbo_name = tt._limbo_name(trans_id) 1679 if trans_id in tt._new_reference_revision: 1680 kind = 'tree-reference' 1681 if kind == 'file': 1682 statval = os.lstat(limbo_name) 1683 size = statval.st_size 1684 if not tt._limbo_supports_executable(): 1685 executable = False 1686 else: 1687 executable = statval.st_mode & S_IEXEC 1688 else: 1689 size = None 1690 executable = None 1691 if kind == 'symlink': 1692 link_or_sha1 = os.readlink(limbo_name) 1693 if not isinstance(link_or_sha1, str): 1694 link_or_sha1 = link_or_sha1.decode(osutils._fs_enc) 1695 executable = tt._new_executability.get(trans_id, executable) 1696 return kind, size, executable, link_or_sha1 1697 1698 def get_file_mtime(self, path): 1699 """See Tree.get_file_mtime""" 1700 trans_id = self._path2trans_id(path) 1701 if trans_id is None: 1702 raise errors.NoSuchFile(path) 1703 if trans_id not in self._transform._new_contents: 1704 return self._transform._tree.get_file_mtime( 1705 self._transform.tree_path(trans_id)) 1706 name = self._transform._limbo_name(trans_id) 1707 statval = os.lstat(name) 1708 return statval.st_mtime 1709 1710 def is_versioned(self, path): 1711 trans_id = self._path2trans_id(path) 1712 if trans_id is None: 1713 # It doesn't exist, so it's not versioned. 1714 return False 1715 if trans_id in self._transform._versioned: 1716 return True 1717 if trans_id in self._transform._removed_id: 1718 return False 1719 orig_path = self._transform.tree_path(trans_id) 1720 return self._transform._tree.is_versioned(orig_path) 1721 1722 def iter_entries_by_dir(self, specific_files=None, recurse_nested=False): 1723 if recurse_nested: 1724 raise NotImplementedError( 1725 'follow tree references not yet supported') 1726 1727 # This may not be a maximally efficient implementation, but it is 1728 # reasonably straightforward. An implementation that grafts the 1729 # TreeTransform changes onto the tree's iter_entries_by_dir results 1730 # might be more efficient, but requires tricky inferences about stack 1731 # position. 1732 for trans_id, path in self._list_files_by_dir(): 1733 entry, is_versioned = self._transform.final_entry(trans_id) 1734 if entry is None: 1735 continue 1736 if not is_versioned and entry.kind != 'directory': 1737 continue 1738 if specific_files is not None and path not in specific_files: 1739 continue 1740 if entry is not None: 1741 yield path, entry 1742 1743 def _list_files_by_dir(self): 1744 todo = [ROOT_PARENT] 1745 while len(todo) > 0: 1746 parent = todo.pop() 1747 children = list(self._all_children(parent)) 1748 paths = dict(zip(children, self._final_paths.get_paths(children))) 1749 children.sort(key=paths.get) 1750 todo.extend(reversed(children)) 1751 for trans_id in children: 1752 yield trans_id, paths[trans_id][0] 1753 1754 def revision_tree(self, revision_id): 1755 return self._transform._tree.revision_tree(revision_id) 1756 1757 def _stat_limbo_file(self, trans_id): 1758 name = self._transform._limbo_name(trans_id) 1759 return os.lstat(name) 1760 1761 def git_snapshot(self, want_unversioned=False): 1762 extra = set() 1763 os = [] 1764 for trans_id, path in self._list_files_by_dir(): 1765 if not self._transform.final_is_versioned(trans_id): 1766 if not want_unversioned: 1767 continue 1768 extra.add(path) 1769 o, mode = self._transform.final_git_entry(trans_id) 1770 if o is not None: 1771 self.store.add_object(o) 1772 os.append((encode_git_path(path), o.id, mode)) 1773 if not os: 1774 return None, extra 1775 return commit_tree(self.store, os), extra 1776 1777 def iter_child_entries(self, path): 1778 trans_id = self._path2trans_id(path) 1779 if trans_id is None: 1780 raise errors.NoSuchFile(path) 1781 for child_trans_id in self._all_children(trans_id): 1782 entry, is_versioned = self._transform.final_entry(trans_id) 1783 if not is_versioned: 1784 continue 1785 if entry is not None: 1786 yield entry 1787