1# Copyright 2002, 2003, 2004, 2005 Ben Escoto 2# 3# This file is part of rdiff-backup. 4# 5# rdiff-backup is free software; you can redistribute it and/or modify 6# under the terms of the GNU General Public License as published by the 7# Free Software Foundation; either version 2 of the License, or (at your 8# option) any later version. 9# 10# rdiff-backup is distributed in the hope that it will be useful, but 11# WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13# General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with rdiff-backup; if not, write to the Free Software 17# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 18# 02110-1301, USA 19"""Start (and end) here - read arguments, set global settings, etc.""" 20 21import getopt 22import sys 23import os 24import io 25import tempfile 26import time 27import errno 28import platform 29from .log import Log, LoggerError, ErrorLog 30from . import ( 31 Globals, Time, SetConnections, robust, rpath, 32 manage, backup, connection, restore, FilenameMapping, 33 Security, C, statistics, compare 34) 35 36action = None 37create_full_path = None 38remote_cmd, remote_schema = None, None 39force = None 40select_opts = [] 41select_files = [] 42user_mapping_filename, group_mapping_filename, preserve_numerical_ids = None, None, None 43 44# These are global because they are set while we are trying to figure 45# whether to restore or to backup 46restore_root, restore_index, restore_root_set = None, None, 0 47return_val = None # Set to cause exit code to be specified value 48 49 50def parse_cmdlineoptions(arglist): # noqa: C901 51 """Parse argument list and set global preferences""" 52 global args, action, create_full_path, force, restore_timestr, remote_cmd 53 global remote_schema, remove_older_than_string 54 global user_mapping_filename, group_mapping_filename, \ 55 preserve_numerical_ids 56 57 def sel_fl(filename): 58 """Helper function for including/excluding filelists below""" 59 try: 60 return open(filename, "rb") # files match paths hence bytes/bin 61 except IOError: 62 Log.FatalError("Error opening file %s" % filename) 63 64 def normalize_path(path): 65 """Used below to normalize the security paths before setting""" 66 return rpath.RPath(Globals.local_connection, path).normalize().path 67 68 try: 69 optlist, args = getopt.getopt(arglist, "blr:sv:V", [ 70 "allow-duplicate-timestamps", 71 "backup-mode", "calculate-average", "carbonfile", 72 "check-destination-dir", "compare", "compare-at-time=", 73 "compare-hash", "compare-hash-at-time=", "compare-full", 74 "compare-full-at-time=", "create-full-path", "current-time=", 75 "exclude=", "exclude-device-files", "exclude-fifos", 76 "exclude-filelist=", "exclude-symbolic-links", "exclude-sockets", 77 "exclude-filelist-stdin", "exclude-globbing-filelist=", 78 "exclude-globbing-filelist-stdin", "exclude-mirror=", 79 "exclude-other-filesystems", "exclude-regexp=", 80 "exclude-if-present=", "exclude-special-files", "force", 81 "group-mapping-file=", "include=", "include-filelist=", 82 "include-filelist-stdin", "include-globbing-filelist=", 83 "include-globbing-filelist-stdin", "include-regexp=", 84 "include-special-files", "include-symbolic-links", "list-at-time=", 85 "list-changed-since=", "list-increments", "list-increment-sizes", 86 "never-drop-acls", "max-file-size=", "min-file-size=", "no-acls", 87 "no-carbonfile", "no-compare-inode", "no-compression", 88 "no-compression-regexp=", "no-eas", "no-file-statistics", 89 "no-hard-links", "null-separator", "override-chars-to-quote=", 90 "parsable-output", "preserve-numerical-ids", "print-statistics", 91 "remote-cmd=", "remote-schema=", "remote-tempdir=", 92 "remove-older-than=", "restore-as-of=", "restrict=", 93 "restrict-read-only=", "restrict-update-only=", "server", 94 "ssh-no-compression", "tempdir=", "terminal-verbosity=", 95 "test-server", "use-compatible-timestamps", "user-mapping-file=", 96 "verbosity=", "verify", "verify-at-time=", "version", "no-fsync" 97 ]) 98 except getopt.error as e: 99 commandline_error("Bad commandline options: " + str(e)) 100 101 for opt, arg in optlist: 102 if opt == "-b" or opt == "--backup-mode": 103 action = "backup" 104 elif opt == "--calculate-average": 105 action = "calculate-average" 106 elif opt == "--carbonfile": 107 Globals.set("carbonfile_active", 1) 108 elif opt == "--check-destination-dir": 109 action = "check-destination-dir" 110 elif opt in ("--compare", "--compare-at-time", "--compare-hash", 111 "--compare-hash-at-time", "--compare-full", 112 "--compare-full-at-time"): 113 if opt[-8:] == "-at-time": 114 restore_timestr, opt = arg, opt[:-8] 115 else: 116 restore_timestr = "now" 117 action = opt[2:] 118 elif opt == "--create-full-path": 119 create_full_path = 1 120 elif opt == "--current-time": 121 Globals.set_integer('current_time', arg) 122 elif (opt == "--exclude" or opt == "--exclude-device-files" 123 or opt == "--exclude-fifos" 124 or opt == "--exclude-other-filesystems" 125 or opt == "--exclude-regexp" or opt == "--exclude-if-present" 126 or opt == "--exclude-special-files" or opt == "--exclude-sockets" 127 or opt == "--exclude-symbolic-links"): 128 select_opts.append((opt, arg)) 129 elif opt == "--exclude-filelist": 130 select_opts.append((opt, arg)) 131 select_files.append(sel_fl(arg)) 132 elif opt == "--exclude-filelist-stdin": 133 select_opts.append(("--exclude-filelist", "standard input")) 134 select_files.append(sys.stdin.buffer) 135 elif opt == "--exclude-globbing-filelist": 136 select_opts.append((opt, arg)) 137 select_files.append(sel_fl(arg)) 138 elif opt == "--exclude-globbing-filelist-stdin": 139 select_opts.append(("--exclude-globbing-filelist", 140 "standard input")) 141 select_files.append(sys.stdin.buffer) 142 elif opt == "--force": 143 force = 1 144 elif opt == "--group-mapping-file": 145 group_mapping_filename = os.fsencode(arg) 146 elif (opt == "--include" or opt == "--include-special-files" 147 or opt == "--include-symbolic-links"): 148 select_opts.append((opt, arg)) 149 elif opt == "--include-filelist": 150 select_opts.append((opt, arg)) 151 select_files.append(sel_fl(arg)) 152 elif opt == "--include-filelist-stdin": 153 select_opts.append(("--include-filelist", "standard input")) 154 select_files.append(sys.stdin.buffer) 155 elif opt == "--include-globbing-filelist": 156 select_opts.append((opt, arg)) 157 select_files.append(sel_fl(arg)) 158 elif opt == "--include-globbing-filelist-stdin": 159 select_opts.append(("--include-globbing-filelist", 160 "standard input")) 161 select_files.append(sys.stdin.buffer) 162 elif opt == "--include-regexp": 163 select_opts.append((opt, arg)) 164 elif opt == "--list-at-time": 165 restore_timestr, action = arg, "list-at-time" 166 elif opt == "--list-changed-since": 167 restore_timestr, action = arg, "list-changed-since" 168 elif opt == "-l" or opt == "--list-increments": 169 action = "list-increments" 170 elif opt == '--list-increment-sizes': 171 action = 'list-increment-sizes' 172 elif opt == "--max-file-size": 173 select_opts.append((opt, arg)) 174 elif opt == "--min-file-size": 175 select_opts.append((opt, arg)) 176 elif opt == "--never-drop-acls": 177 Globals.set("never_drop_acls", 1) 178 elif opt == "--no-acls": 179 Globals.set("acls_active", 0) 180 Globals.set("win_acls_active", 0) 181 elif opt == "--no-carbonfile": 182 Globals.set("carbonfile_active", 0) 183 elif opt == "--no-compare-inode": 184 Globals.set("compare_inode", 0) 185 elif opt == "--no-compression": 186 Globals.set("compression", None) 187 elif opt == "--no-compression-regexp": 188 Globals.set("no_compression_regexp_string", os.fsencode(arg)) 189 elif opt == "--no-eas": 190 Globals.set("eas_active", 0) 191 elif opt == "--no-file-statistics": 192 Globals.set('file_statistics', 0) 193 elif opt == "--no-hard-links": 194 Globals.set('preserve_hardlinks', 0) 195 elif opt == "--null-separator": 196 Globals.set("null_separator", 1) 197 elif opt == "--override-chars-to-quote": 198 Globals.set('chars_to_quote', os.fsencode(arg)) 199 elif opt == "--parsable-output": 200 Globals.set('parsable_output', 1) 201 elif opt == "--preserve-numerical-ids": 202 preserve_numerical_ids = 1 203 elif opt == "--print-statistics": 204 Globals.set('print_statistics', 1) 205 elif opt == "-r" or opt == "--restore-as-of": 206 restore_timestr, action = arg, "restore-as-of" 207 elif opt == "--remote-cmd": 208 remote_cmd = os.fsencode(arg) 209 elif opt == "--remote-schema": 210 remote_schema = os.fsencode(arg) 211 elif opt == "--remote-tempdir": 212 Globals.remote_tempdir = os.fsencode(arg) 213 elif opt == "--remove-older-than": 214 remove_older_than_string = arg 215 action = "remove-older-than" 216 elif opt == "--no-resource-forks": 217 Globals.set('resource_forks_active', 0) 218 elif opt == "--restrict": 219 Globals.restrict_path = normalize_path(arg) 220 elif opt == "--restrict-read-only": 221 Globals.security_level = "read-only" 222 Globals.restrict_path = normalize_path(arg) 223 elif opt == "--restrict-update-only": 224 Globals.security_level = "update-only" 225 Globals.restrict_path = normalize_path(arg) 226 elif opt == "-s" or opt == "--server": 227 action = "server" 228 Globals.server = 1 229 elif opt == "--ssh-no-compression": 230 Globals.set('ssh_compression', None) 231 elif opt == "--tempdir": 232 if (not os.path.isdir(arg)): 233 Log.FatalError("Temporary directory '%s' doesn't exist." % arg) 234 tempfile.tempdir = os.fsencode(arg) 235 elif opt == "--terminal-verbosity": 236 Log.setterm_verbosity(arg) 237 elif opt == "--test-server": 238 action = "test-server" 239 elif opt == "--use-compatible-timestamps": 240 Globals.set("use_compatible_timestamps", 1) 241 elif opt == "--allow-duplicate-timestamps": 242 Globals.set("allow_duplicate_timestamps", True) 243 elif opt == "--user-mapping-file": 244 user_mapping_filename = os.fsencode(arg) 245 elif opt == "-v" or opt == "--verbosity": 246 Log.setverbosity(arg) 247 elif opt == "--verify": 248 action, restore_timestr = "verify", "now" 249 elif opt == "--verify-at-time": 250 action, restore_timestr = "verify", arg 251 elif opt == "-V" or opt == "--version": 252 print("rdiff-backup " + Globals.version) 253 sys.exit(0) 254 elif opt == "--no-fsync": 255 Globals.do_fsync = False 256 else: 257 Log.FatalError("Unknown option %s" % opt) 258 Log("Using rdiff-backup version %s" % (Globals.version), 4) 259 Log("\twith %s %s version %s" % ( 260 sys.implementation.name, 261 sys.executable, 262 platform.python_version()), 4) 263 Log("\ton %s, fs encoding %s" % (platform.platform(), sys.getfilesystemencoding()), 4) 264 265 266def check_action(): 267 """Check to make sure action is compatible with args""" 268 global action 269 arg_action_dict = { 270 0: ['server'], 271 1: [ 272 'list-increments', 'list-increment-sizes', 'remove-older-than', 273 'list-at-time', 'list-changed-since', 'check-destination-dir', 274 'verify' 275 ], 276 2: [ 277 'backup', 'restore', 'restore-as-of', 'compare', 'compare-hash', 278 'compare-full' 279 ] 280 } 281 args_len = len(args) 282 if args_len == 0 and action not in arg_action_dict[args_len]: 283 commandline_error("No arguments given") 284 elif not action: 285 if args_len == 2: 286 pass # Will determine restore or backup later 287 else: 288 commandline_error("Switches missing or wrong number of arguments") 289 elif action == 'test-server' or action == 'calculate-average': 290 pass # these two take any number of args 291 elif args_len > 2 or action not in arg_action_dict[args_len]: 292 commandline_error("Wrong number of arguments given.") 293 294 295def final_set_action(rps): 296 """If no action set, decide between backup and restore at this point""" 297 global action 298 if action: 299 return 300 assert len(rps) == 2, rps 301 if restore_set_root(rps[0]): 302 action = "restore" 303 else: 304 action = "backup" 305 306 307def commandline_error(message): 308 Log.FatalError( 309 "%s\nSee the rdiff-backup manual page for more information." % message) 310 311 312def misc_setup(rps): 313 """Set default change ownership flag, umask, relay regexps""" 314 os.umask(0o77) 315 Time.setcurtime(Globals.current_time) 316 SetConnections.UpdateGlobal("client_conn", Globals.local_connection) 317 Globals.postset_regexp('no_compression_regexp', 318 Globals.no_compression_regexp_string) 319 for conn in Globals.connections: 320 conn.robust.install_signal_handlers() 321 conn.Hardlink.initialize_dictionaries() 322 323 324def init_user_group_mapping(destination_conn): 325 """Initialize user and group mapping on destination connection""" 326 global user_mapping_filename, group_mapping_filename, \ 327 preserve_numerical_ids 328 329 def get_string_from_file(filename): 330 if not filename: 331 return None 332 rp = rpath.RPath(Globals.local_connection, filename) 333 try: 334 return rp.get_string() 335 except OSError as e: 336 Log.FatalError( 337 "Error '%s' reading mapping file '%s'" % (str(e), filename)) 338 339 user_mapping_string = get_string_from_file(user_mapping_filename) 340 destination_conn.user_group.init_user_mapping(user_mapping_string, 341 preserve_numerical_ids) 342 group_mapping_string = get_string_from_file(group_mapping_filename) 343 destination_conn.user_group.init_group_mapping(group_mapping_string, 344 preserve_numerical_ids) 345 346 347def take_action(rps): 348 """Do whatever action says""" 349 if action == "server": 350 connection.PipeConnection(sys.stdin.buffer, sys.stdout.buffer).Server() 351 sys.exit(0) 352 elif action == "backup": 353 Backup(rps[0], rps[1]) 354 elif action == "calculate-average": 355 CalculateAverage(rps) 356 elif action == "check-destination-dir": 357 CheckDest(rps[0]) 358 elif action.startswith("compare"): 359 Compare(action, rps[0], rps[1]) 360 elif action == "list-at-time": 361 ListAtTime(rps[0]) 362 elif action == "list-changed-since": 363 ListChangedSince(rps[0]) 364 elif action == "list-increments": 365 ListIncrements(rps[0]) 366 elif action == 'list-increment-sizes': 367 ListIncrementSizes(rps[0]) 368 elif action == "remove-older-than": 369 RemoveOlderThan(rps[0]) 370 elif action == "restore": 371 Restore(*rps) 372 elif action == "restore-as-of": 373 Restore(rps[0], rps[1], 1) 374 elif action == "test-server": 375 SetConnections.TestConnections(rps) 376 elif action == "verify": 377 Verify(rps[0]) 378 else: 379 raise AssertionError("Unknown action " + action) 380 381 382def cleanup(): 383 """Do any last minute cleaning before exiting""" 384 Log("Cleaning up", 6) 385 if ErrorLog.isopen(): 386 ErrorLog.close() 387 Log.close_logfile() 388 if not Globals.server: 389 SetConnections.CloseConnections() 390 391 392def error_check_Main(arglist): 393 """Run Main on arglist, suppressing stack trace for routine errors""" 394 try: 395 Main(arglist) 396 except SystemExit: 397 raise 398 except (Exception, KeyboardInterrupt) as exc: 399 errmsg = robust.is_routine_fatal(exc) 400 if errmsg: 401 Log.exception(2, 6) 402 Log.FatalError(errmsg) 403 else: 404 Log.exception(2, 2) 405 raise 406 407 408def Main(arglist): 409 """Start everything up!""" 410 parse_cmdlineoptions(arglist) 411 check_action() 412 cmdpairs = SetConnections.get_cmd_pairs(args, remote_schema, remote_cmd) 413 Security.initialize(action or "mirror", cmdpairs) 414 rps = list(map(SetConnections.cmdpair2rp, cmdpairs)) 415 final_set_action(rps) 416 misc_setup(rps) 417 take_action(rps) 418 cleanup() 419 if return_val is not None: 420 sys.exit(return_val) 421 422 423def Backup(rpin, rpout): 424 """Backup, possibly incrementally, src_path to dest_path.""" 425 global incdir 426 SetConnections.BackupInitConnections(rpin.conn, rpout.conn) 427 backup_check_dirs(rpin, rpout) 428 backup_set_rbdir(rpin, rpout) 429 rpout.conn.fs_abilities.backup_set_globals(rpin, force) 430 if Globals.chars_to_quote: 431 rpout = backup_quoted_rpaths(rpout) 432 init_user_group_mapping(rpout.conn) 433 backup_final_init(rpout) 434 backup_set_select(rpin) 435 backup_warn_if_infinite_regress(rpin, rpout) 436 if prevtime: 437 Time.setprevtime(prevtime) 438 rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout) 439 backup.Mirror_and_increment(rpin, rpout, incdir) 440 rpout.conn.Main.backup_remove_curmirror_local() 441 else: 442 backup.Mirror(rpin, rpout) 443 rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout) 444 rpout.conn.Main.backup_close_statistics(time.time()) 445 446 447def backup_quoted_rpaths(rpout): 448 """Get QuotedRPath versions of important RPaths. Return rpout""" 449 global incdir 450 SetConnections.UpdateGlobal('rbdir', 451 FilenameMapping.get_quotedrpath(Globals.rbdir)) 452 incdir = FilenameMapping.get_quotedrpath(incdir) 453 return FilenameMapping.get_quotedrpath(rpout) 454 455 456def backup_set_select(rpin): 457 """Create Select objects on source connection""" 458 if rpin.conn.os.name == 'nt': 459 Log("Symbolic links excluded by default on Windows", 4) 460 select_opts.append(("--exclude-symbolic-links", None)) 461 rpin.conn.backup.SourceStruct.set_source_select(rpin, select_opts, 462 *select_files) 463 464 465def backup_check_dirs(rpin, rpout): 466 """Make sure in and out dirs exist and are directories""" 467 if rpout.lstat() and not rpout.isdir(): 468 if not force: 469 Log.FatalError("Destination %s exists and is not a " 470 "directory" % rpout.get_safepath()) 471 else: 472 Log("Deleting %s" % rpout.get_safepath(), 3) 473 rpout.delete() 474 if not rpout.lstat(): 475 try: 476 if create_full_path: 477 rpout.makedirs() 478 else: 479 rpout.mkdir() 480 except os.error: 481 Log.FatalError( 482 "Unable to create directory %s" % rpout.get_safepath()) 483 484 if not rpin.lstat(): 485 Log.FatalError( 486 "Source directory %s does not exist" % rpin.get_safepath()) 487 elif not rpin.isdir(): 488 Log.FatalError("Source %s is not a directory" % rpin.get_safepath()) 489 Globals.rbdir = rpout.append_path(b"rdiff-backup-data") 490 491 492def check_failed_initial_backup(): 493 """Returns true if it looks like initial backup failed.""" 494 if Globals.rbdir.lstat(): 495 rbdir_files = Globals.rbdir.listdir() 496 mirror_markers = [ 497 x for x in rbdir_files if x.startswith(b"current_mirror") 498 ] 499 error_logs = [x for x in rbdir_files if x.startswith(b"error_log")] 500 metadata_mirrors = [ 501 x for x in rbdir_files if x.startswith(b"mirror_metadata") 502 ] 503 # If we have no current_mirror marker, and the increments directory 504 # is empty, we most likely have a failed backup. 505 return not mirror_markers and len(error_logs) <= 1 and \ 506 len(metadata_mirrors) <= 1 507 return False 508 509 510def fix_failed_initial_backup(): 511 """Clear Globals.rbdir after a failed initial backup""" 512 Log("Found interrupted initial backup. Removing...", 2) 513 rbdir_files = Globals.rbdir.listdir() 514 # Try to delete the increments dir first 515 if b'increments' in rbdir_files: 516 rbdir_files.remove(b'increments') 517 rp = Globals.rbdir.append(b'increments') 518 try: 519 rp.conn.rpath.delete_dir_no_files(rp) 520 except rpath.RPathException: 521 Log("Increments dir contains files.", 4) 522 return 523 except Security.Violation: 524 Log("Server doesn't support resuming.", 2) 525 return 526 527 for file_name in rbdir_files: 528 rp = Globals.rbdir.append_path(file_name) 529 if not rp.isdir(): # Only remove files, not folders 530 rp.delete() 531 532 533def backup_set_rbdir(rpin, rpout): 534 """Initialize data dir and logging""" 535 global incdir 536 try: 537 incdir = Globals.rbdir.append_path(b"increments") 538 except IOError as exc: 539 if exc.errno == errno.EACCES: 540 print("\n") 541 Log.FatalError("Could not begin backup due to\n%s" % exc) 542 else: 543 raise 544 545 assert rpout.lstat(), (rpout.get_safepath(), rpout.lstat()) 546 if rpout.isdir() and not rpout.listdir(): # rpout is empty dir 547 try: 548 rpout.chmod(0o700) # just make sure permissions aren't too lax 549 except OSError: 550 Log("Cannot change permissions on target directory.", 2) 551 elif not Globals.rbdir.lstat() and not force: 552 Log.FatalError("""Destination directory 553 554%s 555 556exists, but does not look like a rdiff-backup directory. Running 557rdiff-backup like this could mess up what is currently in it. If you 558want to update or overwrite it, run rdiff-backup with the --force 559option.""" % rpout.get_safepath()) 560 elif check_failed_initial_backup(): 561 fix_failed_initial_backup() 562 563 if not Globals.rbdir.lstat(): 564 try: 565 Globals.rbdir.mkdir() 566 except (OSError, IOError) as exc: 567 Log.FatalError("""Could not create rdiff-backup directory 568 569%s 570 571due to 572 573%s 574 575Please check that the rdiff-backup user can create files and directories in the 576destination directory: %s""" % (Globals.rbdir.get_safepath(), exc, 577 rpout.get_safepath())) 578 SetConnections.UpdateGlobal('rbdir', Globals.rbdir) 579 580 581def backup_warn_if_infinite_regress(rpin, rpout): 582 """Warn user if destination area contained in source area""" 583 # Just a few heuristics, we don't have to get every case 584 if rpout.conn is not rpin.conn: 585 return 586 if len(rpout.path) <= len(rpin.path) + 1: 587 return 588 if rpout.path[:len(rpin.path) + 1] != rpin.path + b'/': 589 return 590 591 relative_rpout_comps = tuple(rpout.path[len(rpin.path) + 1:].split(b'/')) 592 relative_rpout = rpin.new_index(relative_rpout_comps) # noqa: F841 593 # FIXME: this fails currently because the selection object isn't stored 594 # but an iterable, the object not being pickable. 595 # Related to issue #296 596 #if not Globals.select_mirror.Select(relative_rpout): # noqa: E265 597 # return 598 599 Log( 600 """Warning: The destination directory '%s' may be contained in the 601source directory '%s'. This could cause an infinite regress. You 602may need to use the --exclude option (which you might already have done).""" 603 % (rpout.get_safepath(), rpin.get_safepath()), 2) 604 605 606def backup_get_mirrortime(): 607 """Return time in seconds of previous mirror, or None if cannot""" 608 incbase = Globals.rbdir.append_path(b"current_mirror") 609 mirror_rps = restore.get_inclist(incbase) 610 assert len(mirror_rps) <= 1, \ 611 "Found %s current_mirror rps, expected <=1" % (len(mirror_rps),) 612 if mirror_rps: 613 return mirror_rps[0].getinctime() 614 else: 615 return 0 # is always in the past 616 617 618def backup_final_init(rpout): 619 """Open the backup log and the error log, create increments dir""" 620 global prevtime, incdir 621 if Log.verbosity > 0: 622 Log.open_logfile(Globals.rbdir.append("backup.log")) 623 checkdest_if_necessary(rpout) 624 prevtime = backup_get_mirrortime() 625 if prevtime >= Time.curtime: 626 Log.FatalError( 627 """Time of Last backup is not in the past. This is probably caused 628by running two backups in less than a second. Wait a second and try again.""") 629 ErrorLog.open(Time.curtimestr, compress=Globals.compression) 630 if not incdir.lstat(): 631 incdir.mkdir() 632 633 634def backup_touch_curmirror_local(rpin, rpout): 635 """Make a file like current_mirror.time.data to record time 636 637 When doing an incremental backup, this should happen before any 638 other writes, and the file should be removed after all writes. 639 That way we can tell whether the previous session aborted if there 640 are two current_mirror files. 641 642 When doing the initial full backup, the file can be created after 643 everything else is in place. 644 645 """ 646 mirrorrp = Globals.rbdir.append(b'.'.join( 647 map(os.fsencode, (b"current_mirror", Time.curtimestr, "data")))) 648 Log("Writing mirror marker %s" % mirrorrp.get_safepath(), 6) 649 try: 650 pid = os.getpid() 651 except BaseException: 652 pid = "NA" 653 mirrorrp.write_string("PID %s\n" % (pid, )) 654 mirrorrp.fsync_with_dir() 655 656 657def backup_remove_curmirror_local(): 658 """Remove the older of the current_mirror files. Use at end of session""" 659 assert Globals.rbdir.conn is Globals.local_connection 660 curmir_incs = restore.get_inclist(Globals.rbdir.append(b"current_mirror")) 661 assert len(curmir_incs) == 2 662 if curmir_incs[0].getinctime() < curmir_incs[1].getinctime(): 663 older_inc = curmir_incs[0] 664 else: 665 older_inc = curmir_incs[1] 666 if Globals.do_fsync: 667 C.sync() # Make sure everything is written before curmirror is removed 668 older_inc.delete() 669 670 671def backup_close_statistics(end_time): 672 """Close out the tracking of the backup statistics. 673 674 Moved to run at this point so that only the clock of the system on which 675 rdiff-backup is run is used (set by passing in time.time() from that 676 system). Use at end of session. 677 678 """ 679 assert Globals.rbdir.conn is Globals.local_connection 680 if Globals.print_statistics: 681 statistics.print_active_stats(end_time) 682 if Globals.file_statistics: 683 statistics.FileStats.close() 684 statistics.write_active_statfileobj(end_time) 685 686 687def Restore(src_rp, dest_rp, restore_as_of=None): 688 """Main restoring function 689 690 Here src_rp should be the source file (either an increment or 691 mirror file), dest_rp should be the target rp to be written. 692 693 """ 694 if not restore_root_set and not restore_set_root(src_rp): 695 Log.FatalError("Could not find rdiff-backup repository at %s" % 696 src_rp.get_safepath()) 697 restore_check_paths(src_rp, dest_rp, restore_as_of) 698 try: 699 dest_rp.conn.fs_abilities.restore_set_globals(dest_rp) 700 except IOError as exc: 701 if exc.errno == errno.EACCES: 702 print("\n") 703 Log.FatalError("Could not begin restore due to\n%s" % exc) 704 else: 705 raise 706 init_user_group_mapping(dest_rp.conn) 707 src_rp = restore_init_quoting(src_rp) 708 restore_check_backup_dir(restore_root, src_rp, restore_as_of) 709 inc_rpath = Globals.rbdir.append_path(b'increments', restore_index) 710 if restore_as_of: 711 try: 712 time = Time.genstrtotime(restore_timestr, rp=inc_rpath) 713 except Time.TimeException as exc: 714 Log.FatalError(str(exc)) 715 else: 716 time = src_rp.getinctime() 717 restore_set_select(restore_root, dest_rp) 718 restore_start_log(src_rp, dest_rp, time) 719 try: 720 restore.Restore( 721 restore_root.new_index(restore_index), inc_rpath, dest_rp, time) 722 except IOError as exc: 723 if exc.errno == errno.EACCES: 724 print("\n") 725 Log.FatalError("Could not complete restore due to\n%s" % exc) 726 else: 727 raise 728 else: 729 Log("Restore finished", 4) 730 731 732def restore_init_quoting(src_rp): 733 """Change rpaths into quoted versions of themselves if necessary""" 734 global restore_root 735 if not Globals.chars_to_quote: 736 return src_rp 737 for conn in Globals.connections: 738 conn.FilenameMapping.set_init_quote_vals() 739 restore_root = FilenameMapping.get_quotedrpath(restore_root) 740 SetConnections.UpdateGlobal('rbdir', 741 FilenameMapping.get_quotedrpath(Globals.rbdir)) 742 return FilenameMapping.get_quotedrpath(src_rp) 743 744 745def restore_set_select(mirror_rp, target): 746 """Set the selection iterator on both side from command line args 747 748 We must set both sides because restore filtering is different from 749 select filtering. For instance, if a file is excluded it should 750 not be deleted from the target directory. 751 752 The BytesIO stuff is because filelists need to be read and then 753 duplicated, because we need two copies of them now. 754 755 """ 756 757 def fp2string(fp): 758 buf = fp.read() 759 assert not fp.close() 760 return buf 761 762 select_data = list(map(fp2string, select_files)) 763 if select_opts: 764 mirror_rp.conn.restore.MirrorStruct.set_mirror_select( 765 target, select_opts, *list(map(io.BytesIO, select_data))) 766 target.conn.restore.TargetStruct.set_target_select( 767 target, select_opts, *list(map(io.BytesIO, select_data))) 768 769 770def restore_start_log(rpin, target, time): 771 """Open restore log file, log initial message""" 772 try: 773 Log.open_logfile(Globals.rbdir.append("restore.log")) 774 except (LoggerError, Security.Violation) as e: 775 Log("Warning - Unable to open logfile: %s" % str(e), 2) 776 777 # Log following message at file verbosity 3, but term verbosity 4 778 log_message = ("Starting restore of %s to %s as it was as of %s." % ( 779 rpin.get_safepath(), target.get_safepath(), Time.timetopretty(time))) 780 if Log.term_verbosity >= 4: 781 Log.log_to_term(log_message, 4) 782 if Log.verbosity >= 3: 783 Log.log_to_file(log_message) 784 785 786def restore_check_paths(rpin, rpout, restoreasof=None): 787 """Make sure source and destination exist, and have appropriate type""" 788 if not restoreasof: 789 if not rpin.lstat(): 790 Log.FatalError( 791 "Source file %s does not exist" % rpin.get_safepath()) 792 if not force and rpout.lstat() and (not rpout.isdir() or rpout.listdir()): 793 Log.FatalError("Restore target %s already exists, " 794 "specify --force to overwrite." % rpout.get_safepath()) 795 if force and rpout.lstat() and not rpout.isdir(): 796 rpout.delete() 797 798 799def restore_check_backup_dir(mirror_root, src_rp=None, restore_as_of=1): 800 """Make sure backup dir root rpin is in consistent state""" 801 if not restore_as_of and not src_rp.isincfile(): 802 Log.FatalError("""File %s does not look like an increment file. 803 804Try restoring from an increment file (the filenames look like 805"foobar.2001-09-01T04:49:04-07:00.diff").""" % src_rp.get_safepath()) 806 807 result = checkdest_need_check(mirror_root) 808 if result is None: 809 Log.FatalError("%s does not appear to be an rdiff-backup directory." % 810 Globals.rbdir.get_safepath()) 811 elif result == 1: 812 Log.FatalError( 813 "Previous backup to %s seems to have failed.\nRerun rdiff-backup " 814 "with --check-destination-dir option to revert directory " 815 "to state before unsuccessful session." % 816 mirror_root.get_safepath()) 817 818 819def restore_set_root(rpin): 820 """Set data dir, restore_root and index, or return None if fail 821 822 The idea here is to keep backing up on the path until we find 823 a directory that contains "rdiff-backup-data". That is the 824 mirror root. If the path from there starts 825 "rdiff-backup-data/increments*", then the index is the 826 remainder minus that. Otherwise the index is just the path 827 minus the root. 828 829 All this could fail if the increment file is pointed to in a 830 funny way, using symlinks or somesuch. 831 832 """ 833 global restore_root, restore_index, restore_root_set 834 if rpin.isincfile(): 835 relpath = rpin.getincbase().path 836 else: 837 relpath = rpin.path 838 if rpin.conn is not Globals.local_connection: 839 # For security checking consistency, don't get absolute path 840 pathcomps = relpath.split(b'/') 841 else: 842 pathcomps = rpath.RORPath.path_join(rpath.RORPath.getcwdb(), 843 relpath).split(b'/') 844 if not pathcomps[0]: 845 min_len_pathcomps = 2 # treat abs paths differently 846 else: 847 min_len_pathcomps = 1 848 849 i = len(pathcomps) 850 while i >= min_len_pathcomps: 851 parent_dir = rpath.RPath(rpin.conn, b'/'.join(pathcomps[:i])) 852 if (parent_dir.isdir() and parent_dir.readable() 853 and b"rdiff-backup-data" in parent_dir.listdir()): 854 break 855 if parent_dir.path == rpin.conn.Globals.get('restrict_path'): 856 return None 857 i = i - 1 858 else: 859 return None 860 861 restore_root = parent_dir 862 Log("Using mirror root directory %s" % restore_root.get_safepath(), 6) 863 if restore_root.conn is Globals.local_connection: 864 Security.reset_restrict_path(restore_root) 865 SetConnections.UpdateGlobal('rbdir', 866 restore_root.append_path(b"rdiff-backup-data")) 867 if not Globals.rbdir.isdir(): 868 Log.FatalError("Unable to read rdiff-backup-data directory %s" % 869 Globals.rbdir.get_safepath()) 870 871 from_datadir = tuple(pathcomps[i:]) 872 if not from_datadir or from_datadir[0] != b"rdiff-backup-data": 873 restore_index = from_datadir # in mirror, not increments 874 else: 875 assert (from_datadir[1] == b"increments" 876 or (len(from_datadir) == 2 877 and from_datadir[1].startswith(b'increments'))), from_datadir 878 restore_index = from_datadir[2:] 879 restore_root_set = 1 880 return 1 881 882 883def ListIncrements(rp): 884 """Print out a summary of the increments and their times""" 885 rp = require_root_set(rp, 1) 886 restore_check_backup_dir(restore_root) 887 mirror_rp = restore_root.new_index(restore_index) 888 inc_rpath = Globals.rbdir.append_path(b'increments', restore_index) 889 incs = restore.get_inclist(inc_rpath) 890 mirror_time = restore.MirrorStruct.get_mirror_time() 891 if Globals.parsable_output: 892 print(manage.describe_incs_parsable(incs, mirror_time, mirror_rp)) 893 else: 894 print(manage.describe_incs_human(incs, mirror_time, mirror_rp)) 895 896 897def require_root_set(rp, read_only): 898 """Make sure rp is or is in a valid rdiff-backup dest directory. 899 900 Also initializes fs_abilities (read or read/write) and quoting and 901 return quoted rp if necessary. 902 903 """ 904 if not restore_set_root(rp): 905 Log.FatalError( 906 "Bad directory %s.\n" 907 "It doesn't appear to be an rdiff-backup destination dir." % 908 rp.get_safepath()) 909 try: 910 Globals.rbdir.conn.fs_abilities.single_set_globals( 911 Globals.rbdir, read_only) 912 except (OSError, IOError) as exc: 913 print("\n") 914 Log.FatalError("Could not open rdiff-backup directory\n\n%s\n\n" 915 "due to\n\n%s" % (Globals.rbdir.get_safepath(), exc)) 916 if Globals.chars_to_quote: 917 return restore_init_quoting(rp) 918 else: 919 return rp 920 921 922def ListIncrementSizes(rp): 923 """Print out a summary of the increments """ 924 rp = require_root_set(rp, 1) 925 print(manage.ListIncrementSizes(restore_root, restore_index)) 926 927 928def CalculateAverage(rps): 929 """Print out the average of the given statistics files""" 930 statobjs = [statistics.StatsObj().read_stats_from_rp(rp) for rp in rps] 931 average_stats = statistics.StatsObj().set_to_average(statobjs) 932 print(average_stats.get_stats_logstring( 933 "Average of %d stat files" % len(rps))) 934 935 936def RemoveOlderThan(rootrp): 937 """Remove all increment files older than a certain time""" 938 rootrp = require_root_set(rootrp, 0) 939 rot_require_rbdir_base(rootrp) 940 941 time = rot_check_time(remove_older_than_string) 942 if time is None: 943 return 944 Log("Actual remove older than time: %s" % (time, ), 6) 945 manage.delete_earlier_than(Globals.rbdir, time) 946 947 948def rot_check_time(time_string): 949 """Check remove older than time_string, return time in seconds""" 950 try: 951 time = Time.genstrtotime(time_string) 952 except Time.TimeException as exc: 953 Log.FatalError(str(exc)) 954 955 times_in_secs = [ 956 inc.getinctime() for inc in restore.get_inclist( 957 Globals.rbdir.append_path(b"increments")) 958 ] 959 times_in_secs = [t for t in times_in_secs if t < time] 960 if not times_in_secs: 961 Log( 962 "No increments older than %s found, exiting." % 963 (Time.timetopretty(time), ), 3) 964 return None 965 966 times_in_secs.sort() 967 inc_pretty_time = "\n".join(map(Time.timetopretty, times_in_secs)) 968 if len(times_in_secs) > 1 and not force: 969 Log.FatalError( 970 "Found %d relevant increments, dated:\n%s" 971 "\nIf you want to delete multiple increments in this way, " 972 "use the --force." % (len(times_in_secs), inc_pretty_time)) 973 if len(times_in_secs) == 1: 974 Log("Deleting increment at time:\n%s" % inc_pretty_time, 3) 975 else: 976 Log("Deleting increments at times:\n%s" % inc_pretty_time, 3) 977 return times_in_secs[-1] + 1 # make sure we don't delete current increment 978 979 980def rot_require_rbdir_base(rootrp): 981 """Make sure pointing to base of rdiff-backup dir""" 982 if restore_index != (): 983 Log.FatalError("Increments for directory %s cannot be removed " 984 "separately.\nInstead run on entire directory %s." % 985 (rootrp.get_safepath(), restore_root.get_safepath())) 986 987 988def ListChangedSince(rp): 989 """List all the files under rp that have changed since restoretime""" 990 rp = require_root_set(rp, 1) 991 try: 992 rest_time = Time.genstrtotime(restore_timestr) 993 except Time.TimeException as exc: 994 Log.FatalError(str(exc)) 995 mirror_rp = restore_root.new_index(restore_index) 996 inc_rp = mirror_rp.append_path(b"increments", restore_index) 997 for rorp in rp.conn.restore.ListChangedSince(mirror_rp, inc_rp, rest_time): 998 # This is a hack, see restore.ListChangedSince for rationale 999 print(rorp.get_safeindexpath()) 1000 1001 1002def ListAtTime(rp): 1003 """List files in archive under rp that are present at restoretime""" 1004 rp = require_root_set(rp, 1) 1005 try: 1006 rest_time = Time.genstrtotime(restore_timestr) 1007 except Time.TimeException as exc: 1008 Log.FatalError(str(exc)) 1009 mirror_rp = restore_root.new_index(restore_index) 1010 inc_rp = mirror_rp.append_path(b"increments", restore_index) 1011 for rorp in rp.conn.restore.ListAtTime(mirror_rp, inc_rp, rest_time): 1012 print(rorp.get_safeindexpath()) 1013 1014 1015def Compare(compare_type, src_rp, dest_rp, compare_time=None): 1016 """Compare metadata in src_rp with metadata of backup session 1017 1018 Prints to stdout whenever a file in the src_rp directory has 1019 different metadata than what is recorded in the metadata for the 1020 appropriate session. 1021 1022 Session time is read from restore_timestr if compare_time is None. 1023 1024 """ 1025 global return_val 1026 dest_rp = require_root_set(dest_rp, 1) 1027 if not compare_time: 1028 try: 1029 compare_time = Time.genstrtotime(restore_timestr) 1030 except Time.TimeException as exc: 1031 Log.FatalError(str(exc)) 1032 1033 mirror_rp = restore_root.new_index(restore_index) 1034 inc_rp = Globals.rbdir.append_path(b"increments", restore_index) 1035 backup_set_select(src_rp) # Sets source rorp iterator 1036 if compare_type == "compare": 1037 compare_func = compare.Compare 1038 elif compare_type == "compare-hash": 1039 compare_func = compare.Compare_hash 1040 else: 1041 assert compare_type == "compare-full", compare_type 1042 compare_func = compare.Compare_full 1043 return_val = compare_func(src_rp, mirror_rp, inc_rp, compare_time) 1044 1045 1046def Verify(dest_rp, verify_time=None): 1047 """Check the hashes of the regular files against mirror_metadata""" 1048 global return_val 1049 dest_rp = require_root_set(dest_rp, 1) 1050 if not verify_time: 1051 try: 1052 verify_time = Time.genstrtotime(restore_timestr) 1053 except Time.TimeException as exc: 1054 Log.FatalError(str(exc)) 1055 1056 mirror_rp = restore_root.new_index(restore_index) 1057 inc_rp = Globals.rbdir.append_path(b"increments", restore_index) 1058 return_val = dest_rp.conn.compare.Verify(mirror_rp, inc_rp, verify_time) 1059 1060 1061def CheckDest(dest_rp): 1062 """Check the destination directory, """ 1063 dest_rp = require_root_set(dest_rp, 0) 1064 need_check = checkdest_need_check(dest_rp) 1065 if need_check is None: 1066 Log.FatalError( 1067 "No destination dir found at %s" % dest_rp.get_safepath()) 1068 elif need_check == 0: 1069 Log.FatalError( 1070 "Destination dir %s does not need checking" % 1071 dest_rp.get_safepath(), 1072 no_fatal_message=1, 1073 errlevel=0) 1074 init_user_group_mapping(dest_rp.conn) 1075 dest_rp.conn.regress.Regress(dest_rp) 1076 1077 1078def checkdest_need_check(dest_rp): 1079 """Return None if no dest dir found, 1 if dest dir needs check, 0 o/w""" 1080 if not dest_rp.isdir() or not Globals.rbdir.isdir(): 1081 return None 1082 for filename in Globals.rbdir.listdir(): 1083 if filename not in [ 1084 b'chars_to_quote', b'special_escapes', b'backup.log' 1085 ]: 1086 break 1087 else: # This may happen the first backup just after we test for quoting 1088 return None 1089 curmirroot = Globals.rbdir.append(b"current_mirror") 1090 curmir_incs = restore.get_inclist(curmirroot) 1091 if not curmir_incs: 1092 Log.FatalError("""Bad rdiff-backup-data dir on destination side 1093 1094The rdiff-backup data directory 1095%s 1096exists, but we cannot find a valid current_mirror marker. You can 1097avoid this message by removing the rdiff-backup-data directory; 1098however any data in it will be lost. 1099 1100Probably this error was caused because the first rdiff-backup session 1101into a new directory failed. If this is the case it is safe to delete 1102the rdiff-backup-data directory because there is no important 1103information in it. 1104 1105""" % (Globals.rbdir.get_safepath(), )) 1106 elif len(curmir_incs) == 1: 1107 return 0 1108 else: 1109 if not force: 1110 try: 1111 curmir_incs[0].conn.regress.check_pids(curmir_incs) 1112 except (OSError, IOError) as exc: 1113 Log.FatalError("Could not check if rdiff-backup is currently" 1114 "running due to\n%s" % exc) 1115 assert len(curmir_incs) == 2, \ 1116 "Found too many current_mirror incs in %s!" % Globals.rbdir.get_safepath() 1117 return 1 1118 1119 1120def checkdest_if_necessary(dest_rp): 1121 """Check the destination dir if necessary. 1122 1123 This can/should be run before an incremental backup. 1124 1125 """ 1126 need_check = checkdest_need_check(dest_rp) 1127 if need_check == 1: 1128 Log( 1129 "Previous backup seems to have failed, regressing " 1130 "destination now.", 2) 1131 try: 1132 dest_rp.conn.regress.Regress(dest_rp) 1133 except Security.Violation: 1134 Log.FatalError("Security violation while attempting to regress " 1135 "destination, perhaps due to --restrict-read-only " 1136 "or --restrict-update-only.") 1137