1# Copyright 2002, 2003, 2004, 2005 Ben Escoto 2# 3# This file is part of rdiff-backup. 4# 5# rdiff-backup is free software; you can redistribute it and/or modify 6# under the terms of the GNU General Public License as published by the 7# Free Software Foundation; either version 2 of the License, or (at your 8# option) any later version. 9# 10# rdiff-backup is distributed in the hope that it will be useful, but 11# WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13# General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with rdiff-backup; if not, write to the Free Software 17# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 18# 02110-1301, USA 19"""Read increment files and restore to original""" 20 21import tempfile 22import io 23from . import rorpiter, FilenameMapping 24 25 26class RestoreError(Exception): 27 pass 28 29 30def Restore(mirror_rp, inc_rpath, target, restore_to_time): 31 """Recursively restore mirror and inc_rpath to target at restore_to_time 32 in epoch format""" 33 34 # Store references to classes over the connection 35 MirrorS = mirror_rp.conn.restore.MirrorStruct 36 TargetS = target.conn.restore.TargetStruct 37 38 MirrorS.set_mirror_and_rest_times(restore_to_time) 39 MirrorS.initialize_rf_cache(mirror_rp, inc_rpath) 40 target_iter = TargetS.get_initial_iter(target) 41 diff_iter = MirrorS.get_diffs(target_iter) 42 TargetS.patch(target, diff_iter) 43 MirrorS.close_rf_cache() 44 45 46def get_inclist(inc_rpath): 47 """Returns increments with given base""" 48 dirname, basename = inc_rpath.dirsplit() 49 if Globals.chars_to_quote: 50 basename = FilenameMapping.unquote(basename) 51 parent_dir = inc_rpath.__class__(inc_rpath.conn, dirname, ()) 52 if not parent_dir.isdir(): 53 return [] # inc directory not created yet 54 55 inc_list = [] 56 for filename in parent_dir.listdir(): 57 inc_info = rpath.get_incfile_info(filename) 58 if inc_info and inc_info[3] == basename: 59 inc = parent_dir.append(filename) 60 assert inc.isincfile() 61 inc_list.append(inc) 62 return inc_list 63 64 65def ListChangedSince(mirror_rp, inc_rp, restore_to_time): 66 """List the changed files under mirror_rp since rest time 67 68 Notice the output is an iterator of RORPs. We do this because we 69 want to give the remote connection the data in buffered 70 increments, and this is done automatically for rorp iterators. 71 Encode the lines in the first element of the rorp's index. 72 73 """ 74 assert mirror_rp.conn is Globals.local_connection, "Run locally only" 75 MirrorStruct.set_mirror_and_rest_times(restore_to_time) 76 MirrorStruct.initialize_rf_cache(mirror_rp, inc_rp) 77 78 old_iter = MirrorStruct.get_mirror_rorp_iter(MirrorStruct._rest_time, 1) 79 cur_iter = MirrorStruct.get_mirror_rorp_iter(MirrorStruct._mirror_time, 1) 80 collated = rorpiter.Collate2Iters(old_iter, cur_iter) 81 for old_rorp, cur_rorp in collated: 82 if not old_rorp: 83 change = "new" 84 elif not cur_rorp: 85 change = "deleted" 86 elif old_rorp == cur_rorp: 87 continue 88 else: 89 change = "changed" 90 path_desc = (old_rorp and old_rorp.get_safeindexpath() 91 or cur_rorp.get_safeindexpath()) 92 yield rpath.RORPath(("%-7s %s" % (change, path_desc), )) 93 MirrorStruct.close_rf_cache() 94 95 96def ListAtTime(mirror_rp, inc_rp, time): 97 """List the files in archive at the given time 98 99 Output is a RORP Iterator with info in index. See ListChangedSince. 100 101 """ 102 assert mirror_rp.conn is Globals.local_connection, "Run locally only" 103 MirrorStruct.set_mirror_and_rest_times(time) 104 MirrorStruct.initialize_rf_cache(mirror_rp, inc_rp) 105 old_iter = MirrorStruct.get_mirror_rorp_iter() 106 for rorp in old_iter: 107 yield rorp 108 109 110class MirrorStruct: 111 """Hold functions to be run on the mirror side""" 112 # If selection command line arguments given, use Select here 113 _select = None 114 # This will be set to the time of the current mirror 115 _mirror_time = None 116 # This will be set to the exact time to restore to (not restore_to_time) 117 _rest_time = None 118 119 @classmethod 120 def set_mirror_and_rest_times(cls, restore_to_time): 121 """Set class variables _mirror_time and _rest_time on mirror conn""" 122 MirrorStruct._mirror_time = cls.get_mirror_time() 123 MirrorStruct._rest_time = cls.get_rest_time(restore_to_time) 124 125 @classmethod 126 def get_mirror_time(cls): 127 """Return time (in seconds) of latest mirror""" 128 cur_mirror_incs = get_inclist(Globals.rbdir.append(b"current_mirror")) 129 if not cur_mirror_incs: 130 log.Log.FatalError("Could not get time of current mirror") 131 elif len(cur_mirror_incs) > 1: 132 log.Log("Warning, two different times for current mirror found", 2) 133 return cur_mirror_incs[0].getinctime() 134 135 @classmethod 136 def get_rest_time(cls, restore_to_time): 137 """Return older time, if restore_to_time is in between two inc times 138 139 There is a slightly tricky reason for doing this: The rest of the 140 code just ignores increments that are older than restore_to_time. 141 But sometimes we want to consider the very next increment older 142 than rest time, because rest_time will be between two increments, 143 and what was actually on the mirror side will correspond to the 144 older one. 145 146 So if restore_to_time is inbetween two increments, return the 147 older one. 148 149 """ 150 inctimes = cls.get_increment_times() 151 older_times = [time for time in inctimes if time <= restore_to_time] 152 if older_times: 153 return max(older_times) 154 else: # restore time older than oldest increment, just return that 155 return min(inctimes) 156 157 @classmethod 158 def get_increment_times(cls, rp=None): 159 """Return list of times of backups, including current mirror 160 161 Take the total list of times from the increments.<time>.dir 162 file and the mirror_metadata file. Sorted ascending. 163 164 """ 165 # use dictionary to remove dups 166 if not cls._mirror_time: 167 d = {cls.get_mirror_time(): None} 168 else: 169 d = {cls._mirror_time: None} 170 if not rp or not rp.index: 171 rp = Globals.rbdir.append(b"increments") 172 for inc in get_inclist(rp): 173 d[inc.getinctime()] = None 174 for inc in get_inclist(Globals.rbdir.append(b"mirror_metadata")): 175 d[inc.getinctime()] = None 176 return_list = list(d.keys()) 177 return_list.sort() 178 return return_list 179 180 @classmethod 181 def initialize_rf_cache(cls, mirror_base, inc_base): 182 """Set cls.rf_cache to CachedRF object""" 183 inc_list = get_inclist(inc_base) 184 rf = RestoreFile(mirror_base, inc_base, inc_list) 185 cls.mirror_base, cls.inc_base = mirror_base, inc_base 186 cls.root_rf = rf 187 cls.rf_cache = CachedRF(rf) 188 189 @classmethod 190 def close_rf_cache(cls): 191 """Run anything remaining on CachedRF object""" 192 cls.rf_cache.close() 193 194 @classmethod 195 def get_mirror_rorp_iter(cls, rest_time=None, require_metadata=None): 196 """Return iter of mirror rps at given restore time 197 198 Usually we can use the metadata file, but if this is 199 unavailable, we may have to build it from scratch. 200 201 If the cls._select object is set, use it to filter out the 202 unwanted files from the metadata_iter. 203 204 """ 205 if rest_time is None: 206 rest_time = cls._rest_time 207 208 metadata.SetManager() 209 rorp_iter = metadata.ManagerObj.GetAtTime(rest_time, 210 cls.mirror_base.index) 211 if not rorp_iter: 212 if require_metadata: 213 log.Log.FatalError("Mirror metadata not found") 214 log.Log( 215 "Warning: Mirror metadata not found, " 216 "reading from directory", 2) 217 rorp_iter = cls.get_rorp_iter_from_rf(cls.root_rf) 218 219 if cls._select: 220 rorp_iter = selection.FilterIter(cls._select, rorp_iter) 221 return rorp_iter 222 223 @classmethod 224 def set_mirror_select(cls, target_rp, select_opts, *filelists): 225 """Initialize the mirror selection object""" 226 assert select_opts, "If no selection options, don't use selector" 227 cls._select = selection.Select(target_rp) 228 cls._select.ParseArgs(select_opts, filelists) 229 230 @classmethod 231 def get_rorp_iter_from_rf(cls, rf): 232 """Recursively yield mirror rorps from rf""" 233 rorp = rf.get_attribs() 234 yield rorp 235 if rorp.isdir(): 236 for sub_rf in rf.yield_sub_rfs(): 237 for attribs in cls.get_rorp_iter_from_rf(sub_rf): 238 yield attribs 239 240 @classmethod 241 def subtract_indices(cls, index, rorp_iter): 242 """Subtract index from index of each rorp in rorp_iter 243 244 subtract_indices is necessary because we 245 may not be restoring from the root index. 246 247 """ 248 if index == (): 249 return rorp_iter 250 251 def get_iter(): 252 for rorp in rorp_iter: 253 assert rorp.index[:len(index)] == index, (rorp.index, index) 254 rorp.index = rorp.index[len(index):] 255 yield rorp 256 257 return get_iter() 258 259 @classmethod 260 def get_diffs(cls, target_iter): 261 """Given rorp iter of target files, return diffs 262 263 Here the target_iter doesn't contain any actual data, just 264 attribute listings. Thus any diffs we generate will be 265 snapshots. 266 267 """ 268 mir_iter = cls.subtract_indices(cls.mirror_base.index, 269 cls.get_mirror_rorp_iter()) 270 collated = rorpiter.Collate2Iters(mir_iter, target_iter) 271 return cls.get_diffs_from_collated(collated) 272 273 @classmethod 274 def get_diffs_from_collated(cls, collated): 275 """Get diff iterator from collated""" 276 for mir_rorp, target_rorp in collated: 277 if Globals.preserve_hardlinks and mir_rorp: 278 Hardlink.add_rorp(mir_rorp, target_rorp) 279 if (not target_rorp or not mir_rorp or not mir_rorp == target_rorp 280 or (Globals.preserve_hardlinks 281 and not Hardlink.rorp_eq(mir_rorp, target_rorp))): 282 diff = cls.get_diff(mir_rorp, target_rorp) 283 else: 284 diff = None 285 if Globals.preserve_hardlinks and mir_rorp: 286 Hardlink.del_rorp(mir_rorp) 287 if diff: 288 yield diff 289 290 @classmethod 291 def get_diff(cls, mir_rorp, target_rorp): 292 """Get a diff for mir_rorp at time""" 293 if not mir_rorp: 294 mir_rorp = rpath.RORPath(target_rorp.index) 295 elif Globals.preserve_hardlinks and Hardlink.islinked(mir_rorp): 296 mir_rorp.flaglinked(Hardlink.get_link_index(mir_rorp)) 297 elif mir_rorp.isreg(): 298 expanded_index = cls.mirror_base.index + mir_rorp.index 299 file_fp = cls.rf_cache.get_fp(expanded_index, mir_rorp) 300 mir_rorp.setfile(hash.FileWrapper(file_fp)) 301 mir_rorp.set_attached_filetype('snapshot') 302 return mir_rorp 303 304 305class TargetStruct: 306 """Hold functions to be run on the target side when restoring""" 307 _select = None 308 309 @classmethod 310 def set_target_select(cls, target, select_opts, *filelists): 311 """Return a selection object iterating the rorpaths in target""" 312 cls._select = selection.Select(target) 313 cls._select.ParseArgs(select_opts, filelists) 314 315 @classmethod 316 def get_initial_iter(cls, target): 317 """Return selector previously set with set_initial_iter""" 318 if cls._select: 319 return cls._select.set_iter() 320 else: 321 return selection.Select(target).set_iter() 322 323 @classmethod 324 def patch(cls, target, diff_iter): 325 """Patch target with the diffs from the mirror side 326 327 This function and the associated ITRB is similar to the 328 patching code in backup.py, but they have different error 329 correction requirements, so it seemed easier to just repeat it 330 all in this module. 331 332 """ 333 ITR = rorpiter.IterTreeReducer(PatchITRB, [target]) 334 for diff in rorpiter.FillInIter(diff_iter, target): 335 log.Log("Processing changed file %s" % diff.get_safeindexpath(), 5) 336 ITR(diff.index, diff) 337 ITR.Finish() 338 target.setdata() 339 340 341class CachedRF: 342 """Store RestoreFile objects until they are needed 343 344 The code above would like to pretend it has random access to RFs, 345 making one for a particular index at will. However, in general 346 this involves listing and filtering a directory, which can get 347 expensive. 348 349 Thus, when a CachedRF retrieves an RestoreFile, it creates all the 350 RFs of that directory at the same time, and doesn't have to 351 recalculate. It assumes the indices will be in order, so the 352 cache is deleted if a later index is requested. 353 354 """ 355 356 def __init__(self, root_rf): 357 """Initialize CachedRF, self.rf_list variable""" 358 self.root_rf = root_rf 359 self.rf_list = [] # list should filled in index order 360 if Globals.process_uid != 0: 361 self.perm_changer = PermissionChanger(root_rf.mirror_rp) 362 363 def list_rfs_in_cache(self, index): 364 """Used for debugging, return indices of cache rfs for printing""" 365 s1 = "-------- Cached RF for %s -------" % (index, ) 366 s2 = " ".join([str(rf.index) for rf in self.rf_list]) 367 s3 = "--------------------------" 368 return "\n".join((s1, s2, s3)) 369 370 def get_rf(self, index, mir_rorp=None): 371 """Get a RestoreFile for given index, or None""" 372 while 1: 373 if not self.rf_list: 374 if not self.add_rfs(index, mir_rorp): 375 return None 376 rf = self.rf_list[0] 377 if rf.index == index: 378 if Globals.process_uid != 0: 379 self.perm_changer(index, mir_rorp) 380 return rf 381 elif rf.index > index: 382 # Try to add earlier indices. But if first is 383 # already from same directory, or we can't find any 384 # from that directory, then we know it can't be added. 385 if (index[:-1] == rf.index[:-1] 386 or not self.add_rfs(index, mir_rorp)): 387 return None 388 else: 389 del self.rf_list[0] 390 391 def get_fp(self, index, mir_rorp): 392 """Return the file object (for reading) of given index""" 393 rf = longname.update_rf( 394 self.get_rf(index, mir_rorp), mir_rorp, self.root_rf.mirror_rp) 395 if not rf: 396 log.Log( 397 "Error: Unable to retrieve data for file %s!\nThe " 398 "cause is probably data loss from the backup repository." % 399 (index and "/".join(index) or '.', ), 2) 400 return io.BytesIO() 401 return rf.get_restore_fp() 402 403 def add_rfs(self, index, mir_rorp=None): 404 """Given index, add the rfs in that same directory 405 406 Returns false if no rfs are available, which usually indicates 407 an error. 408 409 """ 410 if not index: 411 return self.root_rf 412 if mir_rorp.has_alt_mirror_name(): 413 return # longname alias separate 414 parent_index = index[:-1] 415 if Globals.process_uid != 0: 416 self.perm_changer(parent_index) 417 temp_rf = RestoreFile( 418 self.root_rf.mirror_rp.new_index(parent_index), 419 self.root_rf.inc_rp.new_index(parent_index), []) 420 new_rfs = list(temp_rf.yield_sub_rfs()) 421 if not new_rfs: 422 return 0 423 self.rf_list[0:0] = new_rfs 424 return 1 425 426 def close(self): 427 """Finish remaining rps in PermissionChanger""" 428 if Globals.process_uid != 0: 429 self.perm_changer.finish() 430 431 432class RestoreFile: 433 """Hold data about a single mirror file and its related increments 434 435 self.relevant_incs will be set to a list of increments that matter 436 for restoring a regular file. If the patches are to mirror_rp, it 437 will be the first element in self.relevant.incs 438 439 """ 440 441 def __init__(self, mirror_rp, inc_rp, inc_list): 442 self.index = mirror_rp.index 443 self.mirror_rp = mirror_rp 444 self.inc_rp, self.inc_list = inc_rp, inc_list 445 self.set_relevant_incs() 446 447 def __str__(self): 448 return "Index: %s, Mirror: %s, Increment: %s\nIncList: %s\nIncRel: %s" % ( 449 self.index, self.mirror_rp, self.inc_rp, 450 list(map(str, self.inc_list)), list(map(str, self.relevant_incs))) 451 452 def relevant_incs_string(self): 453 """Return printable string of relevant incs, used for debugging""" 454 inc_header = ["---- Relevant incs for %s" % ("/".join(self.index), )] 455 inc_header.extend([ 456 "%s %s %s" % (inc.getinctype(), inc.lstat(), inc.path) 457 for inc in self.relevant_incs 458 ]) 459 inc_header.append("--------------------------------") 460 return "\n".join(inc_header) 461 462 def set_relevant_incs(self): 463 """Set self.relevant_incs to increments that matter for restoring 464 465 relevant_incs is sorted newest first. If mirror_rp matters, 466 it will be (first) in relevant_incs. 467 468 """ 469 self.mirror_rp.inc_type = b'snapshot' 470 self.mirror_rp.inc_compressed = 0 471 if (not self.inc_list 472 or MirrorStruct._rest_time >= MirrorStruct._mirror_time): 473 self.relevant_incs = [self.mirror_rp] 474 return 475 476 newer_incs = self.get_newer_incs() 477 i = 0 478 while (i < len(newer_incs)): 479 # Only diff type increments require later versions 480 if newer_incs[i].getinctype() != b"diff": 481 break 482 i = i + 1 483 self.relevant_incs = newer_incs[:i + 1] 484 if (not self.relevant_incs 485 or self.relevant_incs[-1].getinctype() == b"diff"): 486 self.relevant_incs.append(self.mirror_rp) 487 self.relevant_incs.reverse() # return in reversed order 488 489 def get_newer_incs(self): 490 """Return list of newer incs sorted by time (increasing) 491 492 Also discard increments older than rest_time (rest_time we are 493 assuming is the exact time rdiff-backup was run, so no need to 494 consider the next oldest increment or any of that) 495 496 """ 497 incpairs = [] 498 for inc in self.inc_list: 499 time = inc.getinctime() 500 if time >= MirrorStruct._rest_time: 501 incpairs.append((time, inc)) 502 incpairs.sort() 503 return [pair[1] for pair in incpairs] 504 505 def get_attribs(self): 506 """Return RORP with restored attributes, but no data 507 508 This should only be necessary if the metadata file is lost for 509 some reason. Otherwise the file provides all data. The size 510 will be wrong here, because the attribs may be taken from 511 diff. 512 513 """ 514 last_inc = self.relevant_incs[-1] 515 if last_inc.getinctype() == b'missing': 516 return rpath.RORPath(self.index) 517 518 rorp = last_inc.getRORPath() 519 rorp.index = self.index 520 if last_inc.getinctype() == b'dir': 521 rorp.data['type'] = 'dir' 522 return rorp 523 524 def get_restore_fp(self): 525 """Return file object of restored data""" 526 527 def get_fp(): 528 current_fp = self.get_first_fp() 529 for inc_diff in self.relevant_incs[1:]: 530 log.Log("Applying patch %s" % (inc_diff.get_safeindexpath(), ), 531 7) 532 assert inc_diff.getinctype() == b'diff' 533 delta_fp = inc_diff.open("rb", inc_diff.isinccompressed()) 534 new_fp = tempfile.TemporaryFile() 535 Rdiff.write_patched_fp(current_fp, delta_fp, new_fp) 536 new_fp.seek(0) 537 current_fp = new_fp 538 return current_fp 539 540 def error_handler(exc): 541 log.Log( 542 "Error reading %s, substituting empty file." % 543 (self.mirror_rp.path, ), 2) 544 return io.BytesIO(b'') 545 546 if not self.relevant_incs[-1].isreg(): 547 log.Log( 548 """Warning: Could not restore file %s! 549 550A regular file was indicated by the metadata, but could not be 551constructed from existing increments because last increment had type 552%s. Instead of the actual file's data, an empty length file will be 553created. This error is probably caused by data loss in the 554rdiff-backup destination directory, or a bug in rdiff-backup""" % 555 (self.mirror_rp.get_safeindexpath(), 556 self.relevant_incs[-1].lstat()), 2) 557 return io.BytesIO() 558 return robust.check_common_error(error_handler, get_fp) 559 560 def get_first_fp(self): 561 """Return first file object from relevant inc list""" 562 first_inc = self.relevant_incs[0] 563 assert first_inc.getinctype() == b'snapshot' 564 if not first_inc.isinccompressed(): 565 return first_inc.open("rb") 566 567 # current_fp must be a real (uncompressed) file 568 current_fp = tempfile.TemporaryFile() 569 fp = first_inc.open("rb", compress=1) 570 rpath.copyfileobj(fp, current_fp) 571 assert not fp.close() 572 current_fp.seek(0) 573 return current_fp 574 575 def yield_sub_rfs(self): 576 """Return RestoreFiles under current RestoreFile (which is dir)""" 577 if not self.mirror_rp.isdir() and not self.inc_rp.isdir(): 578 return 579 if self.mirror_rp.isdir(): 580 mirror_iter = self.yield_mirrorrps(self.mirror_rp) 581 else: 582 mirror_iter = iter([]) 583 if self.inc_rp.isdir(): 584 inc_pair_iter = self.yield_inc_complexes(self.inc_rp) 585 else: 586 inc_pair_iter = iter([]) 587 collated = rorpiter.Collate2Iters(mirror_iter, inc_pair_iter) 588 589 for mirror_rp, inc_pair in collated: 590 if not inc_pair: 591 inc_rp = self.inc_rp.new_index(mirror_rp.index) 592 inc_list = [] 593 else: 594 inc_rp, inc_list = inc_pair 595 if not mirror_rp: 596 mirror_rp = self.mirror_rp.new_index_empty(inc_rp.index) 597 yield self.__class__(mirror_rp, inc_rp, inc_list) 598 599 def yield_mirrorrps(self, mirrorrp): 600 """Yield mirrorrps underneath given mirrorrp""" 601 assert mirrorrp.isdir() 602 for filename in robust.listrp(mirrorrp): 603 rp = mirrorrp.append(filename) 604 if rp.index != (b'rdiff-backup-data', ): 605 yield rp 606 607 def yield_inc_complexes(self, inc_rpath): 608 """Yield (sub_inc_rpath, inc_list) IndexedTuples from given inc_rpath 609 610 Finds pairs under directory inc_rpath. sub_inc_rpath will just be 611 the prefix rp, while the rps in inc_list should actually exist. 612 613 """ 614 if not inc_rpath.isdir(): 615 return 616 617 def get_inc_pairs(): 618 """Return unsorted list of (basename, inc_filenames) pairs""" 619 inc_dict = {} # dictionary of basenames:inc_filenames 620 dirlist = robust.listrp(inc_rpath) 621 622 def add_to_dict(filename): 623 """Add filename to the inc tuple dictionary""" 624 rp = inc_rpath.append(filename) 625 if rp.isincfile() and rp.getinctype() != b'data': 626 basename = rp.getincbase_bname() 627 inc_filename_list = inc_dict.setdefault(basename, []) 628 inc_filename_list.append(filename) 629 elif rp.isdir(): 630 inc_dict.setdefault(filename, []) 631 632 for filename in dirlist: 633 add_to_dict(filename) 634 return list(inc_dict.items()) 635 636 def inc_filenames2incrps(filenames): 637 """Map list of filenames into increment rps""" 638 inc_list = [] 639 for filename in filenames: 640 rp = inc_rpath.append(filename) 641 assert rp.isincfile(), rp.path 642 inc_list.append(rp) 643 return inc_list 644 645 items = get_inc_pairs() 646 items.sort() # Sorting on basis of basename now 647 for (basename, inc_filenames) in items: 648 sub_inc_rpath = inc_rpath.append(basename) 649 yield rorpiter.IndexedTuple( 650 sub_inc_rpath.index, 651 (sub_inc_rpath, inc_filenames2incrps(inc_filenames))) 652 653 654class PatchITRB(rorpiter.ITRBranch): 655 """Patch an rpath with the given diff iters (use with IterTreeReducer) 656 657 The main complication here involves directories. We have to 658 finish processing the directory after what's in the directory, as 659 the directory may have inappropriate permissions to alter the 660 contents or the dir's mtime could change as we change the 661 contents. 662 663 This code was originally taken from backup.py. However, because 664 of different error correction requirements, it is repeated here. 665 666 """ 667 668 def __init__(self, basis_root_rp): 669 """Set basis_root_rp, the base of the tree to be incremented""" 670 self.basis_root_rp = basis_root_rp 671 assert basis_root_rp.conn is Globals.local_connection 672 self.dir_replacement, self.dir_update = None, None 673 self.cached_rp = None 674 675 def get_rp_from_root(self, index): 676 """Return RPath by adding index to self.basis_root_rp""" 677 if not self.cached_rp or self.cached_rp.index != index: 678 self.cached_rp = self.basis_root_rp.new_index(index) 679 return self.cached_rp 680 681 def can_fast_process(self, index, diff_rorp): 682 """True if diff_rorp and mirror are not directories""" 683 rp = self.get_rp_from_root(index) 684 return not diff_rorp.isdir() and not rp.isdir() 685 686 def fast_process(self, index, diff_rorp): 687 """Patch base_rp with diff_rorp (case where neither is directory)""" 688 rp = self.get_rp_from_root(index) 689 tf = TempFile.new(rp) 690 self.patch_to_temp(rp, diff_rorp, tf) 691 rpath.rename(tf, rp) 692 693 def check_hash(self, copy_report, diff_rorp): 694 """Check the hash in the copy_report with hash in diff_rorp""" 695 if not diff_rorp.isreg(): 696 return 697 if not diff_rorp.has_sha1(): 698 log.Log( 699 "Hash for %s missing, cannot check" % 700 (diff_rorp.get_safeindexpath()), 2) 701 elif copy_report.sha1_digest == diff_rorp.get_sha1(): 702 log.Log( 703 "Hash %s of %s verified" % (diff_rorp.get_sha1(), 704 diff_rorp.get_safeindexpath()), 6) 705 else: 706 log.Log( 707 "Warning: Hash %s of %s\ndoesn't match recorded hash %s!" % 708 (copy_report.sha1_digest, diff_rorp.get_safeindexpath(), 709 diff_rorp.get_sha1()), 2) 710 711 def patch_to_temp(self, basis_rp, diff_rorp, new): 712 """Patch basis_rp, writing output in new, which doesn't exist yet""" 713 if diff_rorp.isflaglinked(): 714 Hardlink.link_rp(diff_rorp, new, self.basis_root_rp) 715 return 716 if diff_rorp.get_attached_filetype() == 'snapshot': 717 copy_report = rpath.copy(diff_rorp, new) 718 else: 719 assert diff_rorp.get_attached_filetype() == 'diff' 720 copy_report = Rdiff.patch_local(basis_rp, diff_rorp, new) 721 self.check_hash(copy_report, diff_rorp) 722 if new.lstat(): 723 rpath.copy_attribs(diff_rorp, new) 724 725 def start_process(self, index, diff_rorp): 726 """Start processing directory - record information for later""" 727 base_rp = self.base_rp = self.get_rp_from_root(index) 728 assert diff_rorp.isdir() or base_rp.isdir() or not base_rp.index 729 if diff_rorp.isdir(): 730 self.prepare_dir(diff_rorp, base_rp) 731 else: 732 self.set_dir_replacement(diff_rorp, base_rp) 733 734 def set_dir_replacement(self, diff_rorp, base_rp): 735 """Set self.dir_replacement, which holds data until done with dir 736 737 This is used when base_rp is a dir, and diff_rorp is not. 738 739 """ 740 assert diff_rorp.get_attached_filetype() == 'snapshot' 741 self.dir_replacement = TempFile.new(base_rp) 742 rpath.copy_with_attribs(diff_rorp, self.dir_replacement) 743 if base_rp.isdir(): 744 base_rp.chmod(0o700) 745 746 def prepare_dir(self, diff_rorp, base_rp): 747 """Prepare base_rp to turn into a directory""" 748 self.dir_update = diff_rorp.getRORPath() # make copy in case changes 749 if not base_rp.isdir(): 750 if base_rp.lstat(): 751 base_rp.delete() 752 base_rp.mkdir() 753 base_rp.chmod(0o700) 754 755 def end_process(self): 756 """Finish processing directory""" 757 if self.dir_update: 758 assert self.base_rp.isdir() 759 rpath.copy_attribs(self.dir_update, self.base_rp) 760 else: 761 assert self.dir_replacement 762 self.base_rp.rmdir() 763 if self.dir_replacement.lstat(): 764 rpath.rename(self.dir_replacement, self.base_rp) 765 766 767class PermissionChanger: 768 """Change the permission of mirror files and directories 769 770 The problem is that mirror files and directories may need their 771 permissions changed in order to be read and listed, and then 772 changed back when we are done. This class hooks into the CachedRF 773 object to know when an rp is needed. 774 775 """ 776 777 def __init__(self, root_rp): 778 self.root_rp = root_rp 779 self.current_index = () 780 # Below is a list of (index, rp, old_perm) triples in reverse 781 # order that need clearing 782 self.open_index_list = [] 783 784 def __call__(self, index, mir_rorp=None): 785 """Given rpath, change permissions up to and including index""" 786 if mir_rorp and mir_rorp.has_alt_mirror_name(): 787 return 788 old_index = self.current_index 789 self.current_index = index 790 if not index or index <= old_index: 791 return 792 self.restore_old(index) 793 self.add_new(old_index, index) 794 795 def restore_old(self, index): 796 """Restore permissions for indices we are done with""" 797 while self.open_index_list: 798 old_index, old_rp, old_perms = self.open_index_list[0] 799 if index[:len(old_index)] > old_index: 800 old_rp.chmod(old_perms) 801 else: 802 break 803 del self.open_index_list[0] 804 805 def add_new(self, old_index, index): 806 """Change permissions of directories between old_index and index""" 807 for rp in self.get_new_rp_list(old_index, index): 808 if ((rp.isreg() and not rp.readable()) 809 or (rp.isdir() and not (rp.executable() and rp.readable()))): 810 old_perms = rp.getperms() 811 self.open_index_list.insert(0, (rp.index, rp, old_perms)) 812 if rp.isreg(): 813 rp.chmod(0o400 | old_perms) 814 else: 815 rp.chmod(0o700 | old_perms) 816 817 def get_new_rp_list(self, old_index, index): 818 """Return list of new rp's between old_index and index 819 820 Do this lazily so that the permissions on the outer 821 directories are fixed before we need the inner dirs. 822 823 """ 824 for i in range(len(index) - 1, -1, -1): 825 if old_index[:i] == index[:i]: 826 common_prefix_len = i 827 break 828 else: 829 assert 0 830 831 for total_len in range(common_prefix_len + 1, len(index) + 1): 832 yield self.root_rp.new_index(index[:total_len]) 833 834 def finish(self): 835 """Restore any remaining rps""" 836 for index, rp, perms in self.open_index_list: 837 rp.chmod(perms) 838 839 840from . import ( # noqa: E402 841 Globals, Rdiff, Hardlink, selection, rpath, 842 log, robust, metadata, TempFile, hash, longname 843) 844