1#!/usr/bin/perl 2# vim: sts=2 sw=2 ts=8 et 3 4# Copyright (c) 2007 OmniTI Computer Consulting, Inc. All rights reserved. 5# For information on licensing see: 6# https://labs.omniti.com/zetaback/trunk/LICENSE 7 8use strict; 9use Getopt::Long; 10use MIME::Base64; 11use POSIX qw/strftime/; 12use Fcntl qw/:flock/; 13use File::Path qw/mkpath/; 14use File::Copy; 15use IO::File; 16use Pod::Usage; 17 18use vars qw/%conf %locks $version_string $process_lock 19 $PREFIX $CONF $BLOCKSIZE $DEBUG $HOST $BACKUP 20 $RESTORE $RESTORE_HOST $RESTORE_ZFS $TIMESTAMP 21 $LIST $SUMMARY $SUMMARY_EXT $SUMMARY_VIOLATORS 22 $SUMMARY_VIOLATORS_VERBOSE $FORCE_FULL $FORCE_INC 23 $EXPUNGE $NEUTERED $ZFS $SHOW_FILENAMES $ARCHIVE 24 $VERSION $HELP/; 25$version_string = '1.0.6'; 26$PREFIX = q^__PREFIX__^; 27$CONF = qq^$PREFIX/etc/zetaback.conf^; 28$BLOCKSIZE = 1024*64; 29 30$conf{'default'}->{'time_format'} = "%Y-%m-%d %H:%M:%S"; 31$conf{'default'}->{'retention'} = 14 * 86400; 32$conf{'default'}->{'compressionlevel'} = 1; 33$conf{'default'}->{'dataset_backup'} = 0; 34$conf{'default'}->{'violator_grace_period'} = 21600; 35 36=pod 37 38=head1 NAME 39 40zetaback - perform backup, restore and retention policies for ZFS backups. 41 42=head1 SYNOPSIS 43 44 zetaback -v 45 46 zetaback [-l|-s|-sx|-sv|-svv] [--files] [-c conf] [-d] [-h host] [-z zfs] 47 48 zetaback -a [-c conf] [-d] [-h host] [-z zfs] 49 50 zetaback -b [-ff] [-fi] [-x] [-c conf] [-d] [-n] [-h host] [-z zfs] 51 52 zetaback -x [-b] [-c conf] [-d] [-n] [-h host] [-z zfs] 53 54 zetaback -r [-c conf] [-d] [-n] [-h host] [-z zfs] [-t timestamp] 55 [-rhost host] [-rzfs fs] 56 57=cut 58 59GetOptions( 60 "h=s" => \$HOST, 61 "z=s" => \$ZFS, 62 "c=s" => \$CONF, 63 "a" => \$ARCHIVE, 64 "b" => \$BACKUP, 65 "l" => \$LIST, 66 "s" => \$SUMMARY, 67 "sx" => \$SUMMARY_EXT, 68 "sv" => \$SUMMARY_VIOLATORS, 69 "svv" => \$SUMMARY_VIOLATORS_VERBOSE, 70 "r" => \$RESTORE, 71 "t=i" => \$TIMESTAMP, 72 "rhost=s" => \$RESTORE_HOST, 73 "rzfs=s" => \$RESTORE_ZFS, 74 "d" => \$DEBUG, 75 "n" => \$NEUTERED, 76 "x" => \$EXPUNGE, 77 "v" => \$VERSION, 78 "ff" => \$FORCE_FULL, 79 "fi" => \$FORCE_INC, 80 "files" => \$SHOW_FILENAMES, 81); 82 83# actions allowed together 'x' and 'b' all others are exclusive: 84my $actions = 0; 85$actions++ if($ARCHIVE); 86$actions++ if($BACKUP || $EXPUNGE); 87$actions++ if($RESTORE); 88$actions++ if($LIST); 89$actions++ if($SUMMARY); 90$actions++ if($SUMMARY_EXT); 91$actions++ if($SUMMARY_VIOLATORS); 92$actions++ if($SUMMARY_VIOLATORS_VERBOSE); 93$actions++ if($VERSION); 94$actions++ if($BACKUP && $FORCE_FULL && $FORCE_INC); 95if($actions != 1) { 96 pod2usage({ -verbose => 0 }); 97 exit -1; 98} 99 100=pod 101 102=head1 DESCRIPTION 103 104The B<zetaback> program orchestrates the backup (either full or 105incremental) of remote ZFS filesystems to a local store. It handles 106frequency requirements for both full and incemental backups as well 107as retention policies. In addition to backups, the B<zetaback> tool 108allows for the restore of any backup to a specified host and zfs 109filesystem. 110 111=head1 OPTIONS 112 113The non-optional action command line arguments define the invocation purpose 114of B<zetaback>. All other arguments are optional and refine the target 115of the action specified. 116 117=head2 Generic Options 118 119The following arguments have the same meaning over several actions: 120 121=over 122 123=item -c <conf> 124 125Use the specified file as the configuration file. The default file, if 126none is specified is /usr/local/etc/zetaback.conf. The prefix of this 127file may also be specified as an argument to the configure script. 128 129=item -d 130 131Enable debugging output. 132 133=item -n 134 135Don't actually perform any remote commands or expunging. This is useful with 136the -d argument to ascertain what would be done if the command was actually 137executed. 138 139=item -t <timestamp> 140 141Used during the restore process to specify a backup image from the desired 142point in time. If omitted, the command becomes interactive. This timestamp 143is a UNIX timestamp and is shown in the output of the -s and -sx actions. 144 145=item -rhost <host> 146 147Specify the remote host that is the target for a restore operation. If 148omitted the command becomes interactive. 149 150=item -rzfs <zfs> 151 152Specify the remote ZFS filesystem that is the target for a restore 153operation. If omitted the command becomes interactive. 154 155=item -h <host> 156 157Filters the operation to the host specified. If <host> is of the form 158/pattern/, it matches 'pattern' as a perl regular expression against available 159hosts. If omitted, no limit is enforced and all hosts are used for the action. 160 161=item -z <zfs> 162 163Filters the operation to the zfs filesystem specified. If <zfs> is of the 164form /pattern/, it matches 'pattern' as a perl regular expression against 165available zfs filesystems. If omitted, no filter is enforced and all zfs 166filesystems are used for the action. 167 168=back 169 170=head2 Actions 171 172=over 173 174=item -v 175 176Show the version. 177 178=item -l 179 180Show a brief listing of available backups. 181 182=item -s 183 184Like -l, -s will show a list of backups but provides additional information 185about the backups including timestamp, type (full or incremental) and the 186size on disk. 187 188=item -sx 189 190Shows an extended summary. In addition to the output provided by the -s 191action, the -sx action will show detail for each availble backup. For 192full backups, the detail will include any more recent full backups, if 193they exist. For incremental backups, the detail will include any 194incremental backups that are more recent than the last full backup. 195 196=item -sv 197 198Display all backups in the current store that violate the configured 199backup policy. This is where the most recent full backup is older than 200full_interval seconds ago, or the most recent incremental backup is older 201than backup_interval seconds ago. 202 203If, at the time of the most recent backup, a filesystem no longer exists on 204the server (because it was deleted), then backups of this filesystem are not 205included in the list of violators. To include these filesystems, use the -svv 206option instead. 207 208=item -svv 209 210The violators summary will exclude backups of filesystems that are no longer 211on the server in the list of violators. Use this option to include those 212filesystems. 213 214=item --files 215 216Display the on-disk file corresponding to each backup named in the output. 217This is useful with the -sv flag to name violating files. Often times, 218violators are filesystems that have been removed on the host machines and 219zetaback can no longer back them up. Be very careful if you choose to 220automate the removal of such backups as filesystems that would be backed up 221by the next regular zetaback run will often show up as violators. 222 223=item -a 224 225Performs an archive. This option will look at all eligible backup points 226(as restricted by -z and -h) and move those to the configured archive 227directory. The recommended use is to first issue -sx --files then 228carefully review available backup points and prune those that are 229unneeded. Then invoke with -a to move only the remaining "desired" 230backup points into the archives. Archived backups do not appear in any 231listings or in the list of policy violators generated by the -sv option. 232In effect, they are no longer "visible" to zetaback. 233 234=item -b 235 236Performs a backup. This option will investigate all eligible hosts, query 237the available filesystems from the remote agent and determine if any such 238filesystems require a new full or incremental backup to be taken. This 239option may be combined with the -x option (to clean up afterwards.) 240 241=item -ff 242 243Forces a full backup to be taken on each filesystem encountered. This is 244used in combination with -b. It is recommended to use this option only when 245targeting specific filesystems (via the -h and -z options.) Forcing a full 246backup across all machines will cause staggered backups to coalesce and 247could cause performance issues. 248 249=item -fi 250 251Forces an incremental backup to be taken on each filesystem encountered. 252This is used in combination with -b. It is recommended to use this option 253only when targeting specific filesystems (via the -h and -z options.) Forcing 254an incremental backup across all machines will cause staggered backups 255to coalesce and could cause performance issues. 256 257=item -x 258 259Perform an expunge. This option will determine which, if any, of the local 260backups may be deleted given the retention policy specified in the 261configuration. 262 263=item -r 264 265Perform a restore. This option will operate on the specified backup and 266restore it to the ZFS filesystem specified with -rzfs on the host specified 267with the -rhost option. The -h, -z and -t options may be used to filter 268the source backup list. If the filtered list contains more than one 269source backup image, the command will act interactively. If the -rhost 270and -rzfs command are not specified, the command will act interactively. 271 272When running interactively, you can choose multiple filesystems from the list 273using ranges. For example 1-4,5,10-11. If you do this, zetaback will enter 274multi-restore mode. In this mode it will automatically select the most recent 275backup, and restore filesystems in bulk. 276 277In multi-restore mode, you have the option to specify a base filesystem to 278restore to. This filesystem will be added as a prefix to the original 279filesystem name, so if you picked a prefix of data/restore, and one of the 280filesystems you are restoring is called data/set/myfilesystem, then the 281filesystem will be restored to data/restore/data/set/myfilesystem. 282 283Note that, just like in regular restore mode, zetaback won't create 284intermediate filesystems for you when restoring, and these should either exist 285beforehand, or you should make sure you pick a set of filesystems that will 286restore the entire tree for you, for example, you should restore data as well 287as data/set before restoring data/set/foo. 288 289=back 290 291=cut 292 293if($VERSION) { 294 print "zetaback: $version_string\n"; 295 exit 0; 296} 297 298=pod 299 300=head1 CONFIGURATION 301 302The zetaback configuration file consists of a default stanza, containing 303settings that can be overridden on a per-host basis. A stanza begins 304either with the string 'default', or a fully-qualified hostname, with 305settings enclosed in braces ({}). Single-line comments begin with a hash 306('#'), and whitespace is ignored, so feel free to indent for better 307readability. Every host to be backed up must have a host stanza in the 308configuration file. 309 310=head2 Storage Classes 311 312In addition to the default and host stanzas, the configuration file can also 313contain 'class' stanzas. Classes allow you to override settings on a 314per-filesystem basis rather than a per-host basis. A class stanza begins with 315the name of the class, and has a setting 'type = class'. For example: 316 317 myclass { 318 type = class 319 store = /path/to/alternate/store 320 } 321 322To add a filesystem to a class, set a zfs user property on the relevant 323filesystem. This must be done on the server that runs the zetaback agent, and 324not the zetaback server itself. 325 326 zfs set com.omniti.labs.zetaback:class=myclass pool/fs 327 328Note that user properties (and therefore classes) are are only available on 329Solaris 10 8/07 and newer, and on Solaris Express build 48 and newer. Only the 330server running the agent needs to have user property support, not the zetaback 331server itself. 332 333The following settings can be included in a class stanza. All other settings 334will be ignored, and their default (or per host) settings used instead: 335 336=over 337 338=item * 339 340store 341 342=item * 343 344full_interval 345 346=item * 347 348backup_interval 349 350=item * 351 352retention 353 354=item * 355 356dataset_backup 357 358=item * 359 360violator_grace_period 361 362=back 363 364=head2 Settings 365 366The following settings are valid in both the default and host scopes: 367 368=over 369 370=item store 371 372The base directory under which to keep backups. An interpolated variable 373'%h' can be used, which expands to the hostname. There is no default for 374this setting. 375 376=item archive 377 378The base directory under which archives are stored. The format is the same 379as the store setting. This is the destination to which files are relocated 380when issuing an archive action (-a). 381 382=item agent 383 384The location of the zetaback_agent binary on the host. There is no default 385for this setting. 386 387=item time_format 388 389All timestamps within zetaback are in UNIX timestamp format. This setting 390provides a string for formatting all timestamps on output. The sequences 391available are identical to those in strftime(3). If not specified, the 392default is '%Y-%m-%d %H:%M:%S'. 393 394=item backup_interval 395 396The frequency (in seconds) at which to perform incremental backups. An 397incremental backup will be performed if the current time is more than 398backup_interval since the last incremental backup. If there is no full backup 399for a particular filesystem, then a full backup is performed. There is no 400default for this setting. 401 402=item full_interval 403 404The frequency (in seconds) at which to perform full backups. A full backup will 405be performed if the current time is more than full_interval since the last full 406backup. 407 408=item retention 409 410The retention time (in seconds) for backups. This can be a simple number, in 411which case all backups older than this will be expunged. 412 413The retention specification can also be more complex, and consist of pairs of 414values separated by a comma. The first value is a time period in seconds, and 415the second value is how many backups should be retained within that period. 416For example: 417 418retention = 3600,4;86400,11 419 420This will keep up to 4 backups for the first hour, and an additional 11 421backups over 24 hours. The times do not stack. In other words, the 11 backups 422would be kept during the period from 1 hour old to 24 hours old, or one every 4232 hours. 424 425Any backups older than the largest time given are deleted. In the above 426example, all backups older than 24 hours are deleted. 427 428If a second number is not specified, then all backups are kept within that 429period. 430 431Note: Full backups are never deleted if they are depended upon by an 432incremental. In addition, the most recent backup is never deleted, regardless 433of how old it is. 434 435This value defaults to (14 * 86400), or two weeks. 436 437=item compressionlevel 438 439Compress files using gzip at the specified compression level. 0 means no 440compression. Accepted values are 1-9. Defaults to 1 (fastest/minimal 441compression.) 442 443=item ssh_config 444 445Full path to an alternate ssh client config. This is useful for specifying a 446less secure but faster cipher for some hosts, or using a different private 447key. There is no default for this setting. 448 449=item dataset_backup 450 451By default zetaback backs zfs filesystems up to files. This option lets you 452specify that the backup go be stored as a zfs dataset on the backup host. 453 454=item offline 455 456Setting this option to 1 for a host will mark it as being 'offline'. Hosts 457that are marked offline will not be backed up, will not have any old backups 458expunged and will not be included in the list of policy violators. However, 459the host will still be shown when listing backups and archiving. 460 461=item violator_grace_period 462 463This setting controls the grace period used when deciding if a backup has 464violated its backup window. It is used to prevent false positives in the case 465where a filesystem is still being backed up. For example, if it is 25 hours 466since the last daily backup, but the daily backup is in progress, the grace 467period will mean that it is not shown in the violators list. 468 469Like all intervals, this period is in seconds. The default is 21600 seconds (6 470hours). 471 472=back 473 474=head2 Global Settings 475 476The following settings are only valid in the default scope: 477 478=over 479 480=item process_limit 481 482This setting limits the number of concurrent zetaback processes that can run 483at one time. Zetaback already has locks on hosts and datasets to prevent 484conflicting backups, and this allows you to have multiple zetaback instances 485running in the event a backup takes some time to complete, while still keeping 486a limit on the resources used. If this configuration entry is missing, then no 487limiting will occur. 488 489=back 490 491=head1 CONFIGURATION EXAMPLES 492 493=head2 Uniform hosts 494 495This config results in backups stored in /var/spool/zfs_backups, with a 496subdirectory for each host. Incremental backups will be performed 497approximately once per day, assuming zetaback is run hourly. Full backups 498will be done once per week. Time format and retention are default. 499 500 default { 501 store = /var/spool/zfs_backups/%h 502 agent = /usr/local/bin/zetaback_agent 503 backup_interval = 83000 504 full_interval = 604800 505 } 506 507 host1 {} 508 509 host2 {} 510 511=head2 Non-uniform hosts 512 513Here, host1's and host2's agents are found in different places, and host2's 514backups should be stored in a different path. 515 516 default { 517 store = /var/spool/zfs_backups/%h 518 agent = /usr/local/bin/zetaback_agent 519 backup_interval = 83000 520 full_interval = 604800 521 } 522 523 host1 { 524 agent = /opt/local/bin/zetaback_agent 525 } 526 527 host2 { 528 store = /var/spool/alt_backups/%h 529 agent = /www/bin/zetaback_agent 530 } 531 532=cut 533 534# Make the parser more formal: 535# config => stanza* 536# stanza => string { kvp* } 537# kvp => string = string 538my $str_re = qr/(?:"(?:\\\\|\\"|[^"])*"|\S+)/; 539my $kvp_re = qr/($str_re)\s*=\s*($str_re)/; 540my $stanza_re = qr/($str_re)\s*\{((?:\s*$kvp_re)*)\s*\}/; 541 542sub parse_config() { 543 local($/); 544 $/ = undef; 545 open(CONF, "<$CONF") || die "Unable to open config file: $CONF"; 546 my $file = <CONF>; 547 # Rip comments 548 $file =~ s/^\s*#.*$//mg; 549 while($file =~ m/$stanza_re/gm) { 550 my $scope = $1; 551 my $filepart = $2; 552 $scope =~ s/^"(.*)"$/$1/; 553 $conf{$scope} ||= {}; 554 while($filepart =~ m/$kvp_re/gm) { 555 my $key = $1; 556 my $value = $2; 557 $key =~ s/^"(.*)"$/$1/; 558 $value =~ s/^"(.*)"$/$1/; 559 $conf{$scope}->{lc($key)} = $value; 560 } 561 } 562 close(CONF); 563} 564sub config_get($$;$) { 565 # Params: host, key, class 566 # Order of precedence: class, host, default 567 if ($_[2]) { 568 return $conf{$_[2]}->{$_[1]} || $conf{$_[0]}->{$_[1]} || 569 $conf{'default'}->{$_[1]}; 570 } else { 571 return $conf{$_[0]}->{$_[1]} || $conf{'default'}->{$_[1]}; 572 } 573} 574 575sub get_store($;$) { 576 my ($host, $class) = @_; 577 my $store = config_get($host, 'store', $class); 578 $store =~ s/%h/$host/g;; 579 return $store; 580} 581 582sub get_classes() { 583 my @classes = (""); # The default/blank class is always present 584 foreach my $key (keys %conf) { 585 if ($conf{$key}->{'type'} eq 'class') { 586 push @classes, $key; 587 } 588 } 589 return @classes; 590} 591 592sub fs_encode($) { 593 my $d = shift; 594 my @parts = split('@', $d); 595 my $e = encode_base64($parts[0], ''); 596 $e =~ s/\//_/g; 597 $e =~ s/=/-/g; 598 $e =~ s/\+/\./g; 599 if (exists $parts[1]) { 600 $e .= "\@$parts[1]"; 601 } 602 return $e; 603} 604sub fs_decode($) { 605 my $e = shift; 606 $e =~ s/_/\//g; 607 $e =~ s/-/=/g; 608 $e =~ s/\./\+/g; 609 return decode_base64($e); 610} 611sub dir_encode($) { 612 my $d = shift; 613 my $e = encode_base64($d, ''); 614 $e =~ s/\//_/; 615 return $e; 616} 617sub dir_decode($) { 618 my $e = shift; 619 $e =~ s/_/\//; 620 return decode_base64($e); 621} 622sub pretty_size($) { 623 my $bytes = shift; 624 if($bytes > 1024*1024*1024) { 625 return sprintf("%0.2f Gb", $bytes / (1024*1024*1024)); 626 } 627 if($bytes > 1024*1024) { 628 return sprintf("%0.2f Mb", $bytes / (1024*1024)); 629 } 630 if($bytes > 1024) { 631 return sprintf("%0.2f Kb", $bytes / (1024)); 632 } 633 return "$bytes b"; 634} 635sub lock($;$$) { 636 my ($host, $file, $nowait) = @_; 637 print "Acquiring lock for $host:$file\n" if($DEBUG); 638 $file ||= 'master.lock'; 639 my $store = get_store($host); # Don't take classes into account - not needed 640 mkpath($store) if(! -d $store); 641 return 1 if(exists($locks{"$host:$file"})); 642 unless (open(LOCK, "+>>$store/$file")) { 643 print STDERR "Cannot open: $store/$file\n" if $DEBUG; 644 return 0; 645 } 646 unless(flock(LOCK, LOCK_EX | ($nowait ? LOCK_NB : 0))) { 647 close(LOCK); 648 print STDERR "Lock failed: $host:$file\n" if $DEBUG; 649 return 0; 650 } 651 $locks{"$host:$file"} = \*LOCK; 652 return 1; 653} 654sub unlock($;$$) { 655 my ($host, $file, $remove) = @_; 656 print "Releasing lock for $host:$file\n" if($DEBUG); 657 $file ||= 'master.lock'; 658 my $store = get_store($host); # Don't take classes into account - not needed 659 mkpath($store) if(! -d $store); 660 return 0 unless(exists($locks{"$host:$file"})); 661 *UNLOCK = $locks{$file}; 662 unlink("$store/$file") if($remove); 663 flock(UNLOCK, LOCK_UN); 664 close(UNLOCK); 665 return 1; 666} 667sub limit_running_processes() { 668 my $max = $conf{'default'}->{'process_limit'}; 669 return unless defined($max); 670 print "Aquiring process lock\n" if $DEBUG; 671 for (my $i=0; $i < $max; $i++) { 672 my $file = "/tmp/.zetaback_$i.lock"; 673 print "$file\n" if $DEBUG; 674 open ($process_lock, "+>>$file") || next; 675 if (flock($process_lock, LOCK_EX | LOCK_NB)) { 676 print "Process lock succeeded: $file\n" if $DEBUG; 677 return 1; 678 } else { 679 close($process_lock); 680 } 681 } 682 print "Too many zetaback processes running. Exiting...\n" if $DEBUG; 683 exit 0; 684} 685sub scan_for_backups($) { 686 my %info = (); 687 my $dir = shift; 688 $info{last_full} = $info{last_incremental} = $info{last_backup} = 0; 689 # Look for standard file based backups first 690 opendir(D, $dir) || return \%info; 691 foreach my $file (readdir(D)) { 692 if($file =~ /^(\d+)\.([^\.]+)\.full$/) { 693 my $whence = $1; 694 my $fs = dir_decode($2); 695 $info{$fs}->{full}->{$whence}->{'file'} = "$dir/$file"; 696 $info{$fs}->{last_full} = $whence if($whence > $info{$fs}->{last_full}); 697 $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ? 698 $info{$fs}->{last_incremental} : $info{$fs}->{last_full}; 699 } 700 elsif($file =~ /^(\d+).([^\.]+)\.incremental.(\d+)$/) { 701 my $whence = $1; 702 my $fs = dir_decode($2); 703 $info{$fs}->{incremental}->{$whence}->{'depends'} = $3; 704 $info{$fs}->{incremental}->{$whence}->{'file'} = "$dir/$file"; 705 $info{$fs}->{last_incremental} = $whence if($whence > $info{$fs}->{last_incremental}); 706 $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ? 707 $info{$fs}->{last_incremental} : $info{$fs}->{last_full}; 708 } 709 } 710 closedir(D); 711 # Now look for zfs based backups 712 my $storefs; 713 eval { 714 $storefs = get_fs_from_mountpoint($dir); 715 }; 716 return \%info if ($@); 717 my $rv = open(ZFSLIST, "__ZFS__ list -H -r -t snapshot $storefs |"); 718 return \%info unless $rv; 719 while (<ZFSLIST>) { 720 my @F = split(' '); 721 my ($rawfs, $snap) = split('@', $F[0]); 722 my ($whence) = ($snap =~ /(\d+)/); 723 next unless $whence; 724 my @fsparts = split('/', $rawfs); 725 my $fs = fs_decode($fsparts[-1]); 726 # Treat a dataset backup as a full backup from the point of view of the 727 # backup lists 728 $info{$fs}->{full}->{$whence}->{'snapshot'} = $snap; 729 $info{$fs}->{full}->{$whence}->{'dataset'} = "$rawfs\@$snap"; 730 # Note - this field isn't set for file backups - we probably should do 731 # this 732 $info{$fs}->{full}->{$whence}->{'pretty_size'} = "$F[1]"; 733 $info{$fs}->{last_full} = $whence if ($whence > 734 $info{$fs}->{last_full}); 735 $info{$fs}->{last_backup} = $whence if ($whence > 736 $info{$fs}->{last_backup}); 737 } 738 close(ZFSLIST); 739 740 return \%info; 741} 742 743parse_config(); 744 745sub zetaback_log($$;@) { 746 my ($host, $mess, @args) = @_; 747 my $tf = config_get($host, 'time_format'); 748 my $file = config_get($host, 'logfile'); 749 my $fileh; 750 if(defined($file)) { 751 $fileh = IO::File->new(">>$file"); 752 } 753 $fileh ||= IO::File->new(">&STDERR"); 754 printf $fileh "%s: $mess", strftime($tf, localtime(time)), @args; 755 $fileh->close(); 756} 757 758sub zfs_remove_snap($$$) { 759 my ($host, $fs, $snap) = @_; 760 my $agent = config_get($host, 'agent'); 761 my $ssh_config = config_get($host, 'ssh_config'); 762 if($ssh_config) { 763 $ssh_config = "-F $ssh_config"; 764 print STDERR "Using custom ssh config file: $ssh_config\n" if($DEBUG); 765 } 766 return unless($snap); 767 print "Dropping $snap on $fs\n" if($DEBUG); 768 `ssh $ssh_config $host $agent -z $fs -d $snap`; 769} 770 771# Lots of args.. internally called. 772sub zfs_do_backup($$$$$$;$) { 773 my ($host, $fs, $type, $point, $store, $dumpname, $base) = @_; 774 my ($storefs, $encodedname); 775 my $agent = config_get($host, 'agent'); 776 my $ssh_config = config_get($host, 'ssh_config'); 777 if($ssh_config) { 778 $ssh_config = "-F $ssh_config"; 779 print STDERR "Using custom ssh config file: $ssh_config\n" if($DEBUG); 780 } 781 782 # compression is meaningless for dataset backups 783 if ($type ne "s") { 784 my $cl = config_get($host, 'compressionlevel'); 785 if ($cl >= 1 && $cl <= 9) { 786 open(LBACKUP, "|gzip -$cl >$store/.$dumpname") || 787 die "zfs_do_backup $host:$fs $type: cannot create dump\n"; 788 } else { 789 open(LBACKUP, ">$store/.$dumpname") || 790 die "zfs_do_backup $host:$fs $type: cannot create dump\n"; 791 } 792 } else { 793 # Dataset backup - pipe received filesystem to zfs recv 794 eval { 795 $storefs = get_fs_from_mountpoint($store); 796 }; 797 if ($@) { 798 # The zfs filesystem doesn't exist, so we have to work out what it 799 # would be 800 my $basestore = $store; 801 $basestore =~ s/\/?%h//g; 802 $storefs = get_fs_from_mountpoint($basestore); 803 $storefs="$storefs/$host"; 804 } 805 $encodedname = fs_encode($dumpname); 806 print STDERR "Receiving to zfs filesystem $storefs/$encodedname\n" 807 if($DEBUG); 808 zfs_create_intermediate_filesystems("$storefs/$encodedname"); 809 open(LBACKUP, "|__ZFS__ recv $storefs/$encodedname"); 810 } 811 # Do it. yeah. 812 eval { 813 if(my $pid = fork()) { 814 close(LBACKUP); 815 die "Errno: $!" if (waitpid($pid, 0) == -1); 816 my $ev = ($? >> 8); 817 my $sn = $? & 127; 818 die "Child signal number: $sn" if ($sn); 819 die "Child exit value: $ev" if ($ev); 820 } 821 else { 822 my @cmd = ('ssh', split(/ /, $ssh_config), $host, $agent, '-z', $fs); 823 if ($type eq "i" || ($type eq "s" && $base)) { 824 push @cmd, ("-i", $base); 825 } 826 if ($type eq "f" || $type eq "s") { 827 push @cmd, ("-$type", $point); 828 } 829 open STDIN, "/dev/null" || exit(-1); 830 open STDOUT, ">&LBACKUP" || exit(-1); 831 print STDERR " => @cmd\n" if($DEBUG); 832 unless (exec { $cmd[0] } @cmd) { 833 print STDERR "$cmd[0] failed: $!\n"; 834 exit(1); 835 } 836 } 837 if ($type ne "s") { 838 die "dump failed (zero bytes)\n" if(-z "$store/.$dumpname"); 839 rename("$store/.$dumpname", "$store/$dumpname") || die "cannot rename dump\n"; 840 } else { 841 # Check everything is ok 842 `__ZFS__ list $storefs/$encodedname`; 843 die "dump failed (received snapshot $storefs/$encodedname does not exist)\n" 844 if $?; 845 } 846 }; 847 if($@) { 848 if ($type ne "s") { 849 unlink("$store/.$dumpname"); 850 } 851 chomp(my $error = $@); 852 $error =~ s/[\r\n]+/ /gsm; 853 zetaback_log($host, "FAILED[$error] $host:$fs $type\n"); 854 die "zfs_do_backup $host:$fs $type: $error"; 855 } 856 my $size; 857 if ($type ne "s") { 858 my @st = stat("$store/$dumpname"); 859 $size = pretty_size($st[7]); 860 } else { 861 $size = `__ZFS__ get -Ho value used $storefs/$encodedname`; 862 chomp $size; 863 } 864 zetaback_log($host, "SUCCESS[$size] $host:$fs $type\n"); 865} 866 867sub zfs_create_intermediate_filesystems($) { 868 my ($fs) = @_; 869 my $idx=0; 870 while (($idx = index($fs, '/', $idx+1)) != -1) { 871 my $fspart = substr($fs, 0, $idx); 872 `__ZFS__ list $fspart 2>&1`; 873 if ($?) { 874 print STDERR "Creating intermediate zfs filesystem: $fspart\n" 875 if $DEBUG; 876 `__ZFS__ create $fspart`; 877 } 878 } 879} 880 881sub zfs_full_backup($$$) { 882 my ($host, $fs, $store) = @_; 883 884 # Translate into a proper dumpname 885 my $point = time(); 886 my $efs = dir_encode($fs); 887 my $dumpname = "$point.$efs.full"; 888 889 zfs_do_backup($host, $fs, 'f', $point, $store, $dumpname); 890} 891 892sub zfs_incremental_backup($$$$) { 893 my ($host, $fs, $base, $store) = @_; 894 my $agent = config_get($host, 'agent'); 895 896 # Translate into a proper dumpname 897 my $point = time(); 898 my $efs = dir_encode($fs); 899 my $dumpname = "$point.$efs.incremental.$base"; 900 901 zfs_do_backup($host, $fs, 'i', $point, $store, $dumpname, $base); 902} 903 904sub zfs_dataset_backup($$$$) { 905 my ($host, $fs, $base, $store) = @_; 906 my $agent = config_get($host, 'agent'); 907 908 my $point = time(); 909 my $dumpname = "$fs\@$point"; 910 911 zfs_do_backup($host, $fs, 's', $point, $store, $dumpname, $base); 912} 913 914sub perform_retention($) { 915 my ($host) = @_; 916 my $now = time(); 917 918 if ($DEBUG) { 919 print "Performing retention for $host\n"; 920 } 921 922 foreach my $class (get_classes()) { 923 if ($DEBUG) { 924 if ($class) { 925 print "=> Class: $class\n" if $class; 926 } else { 927 print "=> Class: (none)\n"; 928 } 929 } 930 my $retention = config_get($host, 'retention', $class); 931 my $store = get_store($host, $class); 932 my $backup_info = scan_for_backups($store); 933 foreach my $disk (sort keys %{$backup_info}) { 934 my $info = $backup_info->{$disk}; 935 next unless(ref($info) eq 'HASH'); 936 my %must_save; 937 938 if ($DEBUG) { 939 print " $disk\n"; 940 } 941 942 # Get a list of all the full and incrementals, sorts newest to oldest 943 my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}}); 944 @backup_points = sort { $b <=> $a } @backup_points; 945 946 # We _cannot_ throw away _all_ our backups, 947 # so save the most recent incremental and full no matter what 948 push(@{$must_save{$backup_points[0]}}, "most recent backup"); 949 my @fulls = grep { exists($info->{full}->{$_}) } @backup_points; 950 push(@{$must_save{$fulls[0]}}, "most recent full"); 951 952 # Process retention policy 953 my @parts = split(/;/, $retention); 954 my %retention_map; 955 foreach (@parts) { 956 my ($period, $amount) = split(/,/); 957 if (!defined($amount)) { 958 $amount = -1; 959 } 960 $retention_map{$period} = $amount; 961 } 962 my @periods = sort { $a <=> $b } keys(%retention_map); 963 my %backup_bins; 964 foreach(@periods) { 965 $backup_bins{$_} = (); 966 } 967 my $cutoff = $now - $periods[0]; 968 # Sort backups into time period sections 969 foreach (@backup_points) { 970 # @backup_points is in descending order (newest first) 971 while ($_ <= $cutoff) { 972 # Move to the next largest bin if the current backup is not in the 973 # current bin. However, if there is no larger bin, then don't 974 shift(@periods); 975 if (@periods) { 976 $cutoff = $now - $periods[0]; 977 } else { 978 last; 979 } 980 } 981 # Throw away all backups older than the largest time period specified 982 if (!@periods) { 983 last; 984 } 985 push(@{$backup_bins{$periods[0]}}, $_); 986 } 987 foreach (keys(%backup_bins)) { 988 my $keep = $retention_map{$_}; # How many backups to keep 989 if ($backup_bins{$_}) { 990 my @backups = @{$backup_bins{$_}}; 991 my $total = @backups; # How many backups we have 992 # If we didn't specify how many to keep, keep them all 993 if ($keep == -1) { $keep = $total }; 994 # If we have less backups than we should keep, keep them all 995 if ($total < $keep) { $keep = $total }; 996 for (my $i = 1; $i <= $keep; $i++) { 997 my $idx = int(($i * $total) / $keep) - 1; 998 push(@{$must_save{$backups[$idx]}}, "retention policy - $_"); 999 } 1000 } 1001 } 1002 if ($DEBUG) { 1003 print " => Backup bins:\n"; 1004 foreach my $a (keys(%backup_bins)) { 1005 print " => $a\n"; 1006 foreach my $i (@{$backup_bins{$a}}) { 1007 my $trans = $now - $i; 1008 print " => $i ($trans seconds old)"; 1009 if (exists($must_save{$i})) { print " => keep" }; 1010 print "\n"; 1011 } 1012 } 1013 } 1014 1015 # Look for dependencies 1016 foreach (@backup_points) { 1017 if(exists($info->{incremental}->{$_})) { 1018 print " => $_ depends on $info->{incremental}->{$_}->{depends}\n" if($DEBUG); 1019 if (exists($must_save{$_})) { 1020 push(@{$must_save{$info->{incremental}->{$_}->{depends}}}, 1021 "dependency"); 1022 } 1023 } 1024 } 1025 1026 my @removals = grep { !exists($must_save{$_}) } @backup_points; 1027 if($DEBUG) { 1028 my $tf = config_get($host, 'time_format'); 1029 print " => Candidates for removal:\n"; 1030 foreach (@backup_points) { 1031 print " => ". strftime($tf, localtime($_)); 1032 print " ($_)"; 1033 print " [". (exists($info->{full}->{$_}) ? "full":"incremental") ."]"; 1034 if (exists($must_save{$_})) { 1035 my $reason = join(", ", @{$must_save{$_}}); 1036 print " => keep ($reason)"; 1037 } else { 1038 print " => remove"; 1039 } 1040 print "\n"; 1041 } 1042 } 1043 foreach (@removals) { 1044 my $efs = dir_encode($disk); 1045 my $filename; 1046 my $dataset; 1047 if(exists($info->{full}->{$_}->{file})) { 1048 $filename = $info->{full}->{$_}->{file}; 1049 } elsif(exists($info->{incremental}->{$_}->{file})) { 1050 $filename = $info->{incremental}->{$_}->{file}; 1051 } elsif(exists($info->{full}->{$_}->{dataset})) { 1052 $dataset = $info->{full}->{$_}->{dataset}; 1053 } elsif(exists($info->{incremental}->{$_}->{dataset})) { 1054 $dataset = $info->{incremental}->{$_}->{dataset}; 1055 } else { 1056 print "ERROR: We tried to expunge $host $disk [$_], but couldn't find it.\n"; 1057 } 1058 print " => expunging ${filename}${dataset}\n" if($DEBUG); 1059 unless($NEUTERED) { 1060 if ($filename) { 1061 unlink($filename) || print "ERROR: unlink $filename: $?\n"; 1062 } elsif ($dataset) { 1063 `__ZFS__ destroy $dataset`; 1064 if ($?) { 1065 print "ERROR: zfs destroy $dataset: $?\n"; 1066 } 1067 } 1068 } 1069 } 1070 } 1071 } 1072} 1073 1074sub __default_sort($$) { return $_[0] cmp $_[1]; } 1075 1076sub choose($$;$$) { 1077 my($name, $obj, $many, $sort) = @_; 1078 $sort ||= \&__default_sort;; 1079 my @list; 1080 my $hash; 1081 if(ref $obj eq 'ARRAY') { 1082 @list = sort { $sort->($a,$b); } (@$obj); 1083 map { $hash->{$_} = $_; } @list; 1084 } 1085 elsif(ref $obj eq 'HASH') { 1086 @list = sort { $sort->($a,$b); } (keys %$obj); 1087 $hash = $obj; 1088 } 1089 else { 1090 die "choose passed bad object: " . ref($obj) . "\n"; 1091 } 1092 return \@list if(scalar(@list) == 1) && $many; 1093 return $list[0] if(scalar(@list) == 1) && !$many; 1094 print "\n"; 1095 my $i = 1; 1096 for (@list) { 1097 printf " %3d) $hash->{$_}\n", $i++; 1098 } 1099 if ($many) { 1100 my @selection; 1101 my $range; 1102 while(1) { 1103 print "$name: "; 1104 chomp($range = <>); 1105 next if ($range !~ /^[\d,-]+$/); 1106 my @parts = split(',', $range); 1107 foreach my $part (@parts) { 1108 my ($from, $to) = ($part =~ /(\d+)(?:-(\d+))?/); 1109 if ($from < 1 || $to > scalar(@list)) { 1110 print "Invalid range: $from-$to\n"; 1111 @selection = (); 1112 last; 1113 } 1114 if ($to) { 1115 push @selection, @list[$from - 1 .. $to - 1]; 1116 } else { 1117 push @selection, @list[$from - 1]; 1118 } 1119 } 1120 if (@selection) { 1121 last; 1122 } 1123 } 1124 return \@selection; 1125 } else { 1126 my $selection = 0; 1127 while($selection !~ /^\d+$/ or 1128 $selection < 1 or 1129 $selection >= $i) { 1130 print "$name: "; 1131 chomp($selection = <>); 1132 } 1133 return $list[$selection - 1]; 1134 } 1135} 1136 1137sub backup_chain($$) { 1138 my ($info, $ts) = @_; 1139 my @list; 1140 push @list, $info->{full}->{$ts} if(exists($info->{full}->{$ts})); 1141 if(exists($info->{incremental}->{$ts})) { 1142 push @list, $info->{incremental}->{$ts}; 1143 push @list, backup_chain($info, $info->{incremental}->{$ts}->{depends}); 1144 } 1145 return @list; 1146} 1147 1148sub get_fs_from_mountpoint($) { 1149 my ($mountpoint) = @_; 1150 my $fs; 1151 my $rv = open(ZFSLIST, "__ZFS__ list -t filesystem -H |"); 1152 die "Unable to determine zfs filesystem for $mountpoint" unless $rv; 1153 while (<ZFSLIST>) { 1154 my @F = split(' '); 1155 if ($F[-1] eq $mountpoint) { 1156 $fs = $F[0]; 1157 last; 1158 } 1159 } 1160 close(ZFSLIST); 1161 die "Unable to determine zfs filesystem for $mountpoint" unless $fs; 1162 return $fs; 1163} 1164 1165sub perform_restore() { 1166 my (%source, %classmap); 1167 1168 foreach my $host (grep { $_ ne "default" && $conf{$_}->{"type"} ne "class"} 1169 keys %conf) { 1170 # If -h was specific, we will skip this host if the arg isn't 1171 # an exact match or a pattern match 1172 if($HOST && 1173 !(($HOST eq $host) || 1174 ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) { 1175 next; 1176 } 1177 1178 foreach my $class (get_classes()) { 1179 if ($DEBUG) { 1180 if ($class) { 1181 print "=> Class: $class\n" if $class; 1182 } else { 1183 print "=> Class: (none)\n"; 1184 } 1185 } 1186 my $store = get_store($host, $class); 1187 my $backup_info = scan_for_backups($store); 1188 foreach my $disk (sort keys %{$backup_info}) { 1189 my $info = $backup_info->{$disk}; 1190 next unless(ref($info) eq 'HASH'); 1191 next 1192 if($ZFS && # if the pattern was specified it could 1193 !($disk eq $ZFS || # be a specific match or a 1194 ($ZFS =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex 1195 # We want to see this one 1196 my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}}); 1197 my @source_points; 1198 foreach (@backup_points) { 1199 push @source_points, $_ if(!$TIMESTAMP || $TIMESTAMP == $_) 1200 } 1201 if(@source_points) { 1202 $source{$host}->{$disk} = \@source_points; 1203 $classmap{$host}->{$disk} = $class; 1204 } 1205 } 1206 } 1207 } 1208 1209 if(! keys %source) { 1210 print "No matching backups found\n"; 1211 return; 1212 } 1213 1214 # Here goes the possibly interactive dialog 1215 my $host = choose("Restore from host", [keys %source]); 1216 my $disks = choose("Restore from ZFS", [keys %{$source{$host}}], 1); 1217 1218 if (scalar(@$disks) > 1) { 1219 # We selected multiple backups, only the latest backup of each should be 1220 # used 1221 print "Multiple filesystems selected, choosing latest backup for each\n"; 1222 my $backup_list = {}; 1223 foreach my $disk (@$disks) { 1224 my $store = get_store($host, $classmap{$host}->{$disk}); 1225 my $backup_info = scan_for_backups($store); 1226 $backup_list->{$disk} = [ reverse backup_chain($backup_info->{$disk}, 1227 $backup_info->{$disk}->{last_backup}) ]; 1228 } 1229 1230 if(!$RESTORE_HOST) { 1231 print "Restore to host [$host]:"; 1232 chomp(my $input = <>); 1233 $RESTORE_HOST = length($input) ? $input : $host; 1234 } 1235 if(!$RESTORE_ZFS) { 1236 print "Restore at base zfs (filesystem must exist) []:"; 1237 chomp(my $input = <>); 1238 $RESTORE_ZFS = $input; 1239 } 1240 1241 # show intentions 1242 print "Going to restore:\n"; 1243 print "\tfrom: $host\n"; 1244 foreach my $disk (@$disks) { 1245 print "\tfrom: $disk\n"; 1246 } 1247 print "\t to: $RESTORE_HOST\n"; 1248 print "\t at base zfs: $RESTORE_ZFS\n"; 1249 print "\n"; 1250 1251 foreach my $disk (@$disks) { 1252 print "Restoring: $disk\n"; 1253 foreach(@{$backup_list->{$disk}}) { 1254 my $restore_dataset = $disk; 1255 if ($RESTORE_ZFS) { 1256 $restore_dataset = "$RESTORE_ZFS/$restore_dataset"; 1257 } 1258 $_->{success} = zfs_restore_part($RESTORE_HOST, $restore_dataset, $_->{file}, $_->{dataset}, $_->{depends}); 1259 } 1260 } 1261 } else { 1262 my $disk = $disks->[0]; 1263 # Times are special. We build a human readable form and use a numerical 1264 # sort function instead of the default lexical one. 1265 my %times; 1266 my $tf = config_get($host, 'time_format'); 1267 map { $times{$_} = strftime($tf, localtime($_)); } @{$source{$host}->{$disk}}; 1268 my $timestamp = choose("Restore as of timestamp", \%times, 0, 1269 sub { $_[0] <=> $_[1]; }); 1270 1271 my $store = get_store($host, $classmap{$host}->{$disk}); 1272 my $backup_info = scan_for_backups($store); 1273 my @backup_list = reverse backup_chain($backup_info->{$disk}, $timestamp); 1274 1275 if(!$RESTORE_HOST) { 1276 print "Restore to host [$host]:"; 1277 chomp(my $input = <>); 1278 $RESTORE_HOST = length($input) ? $input : $host; 1279 } 1280 if(!$RESTORE_ZFS) { 1281 print "Restore to zfs [$disk]:"; 1282 chomp(my $input = <>); 1283 $RESTORE_ZFS = length($input) ? $input : $disk; 1284 } 1285 1286 # show intentions 1287 print "Going to restore:\n"; 1288 print "\tfrom: $host\n"; 1289 print "\tfrom: $disk\n"; 1290 print "\t at: $timestamp [" . strftime($tf, localtime($timestamp)) . "]\n"; 1291 print "\t to: $RESTORE_HOST\n"; 1292 print "\t to: $RESTORE_ZFS\n"; 1293 print "\n"; 1294 1295 foreach(@backup_list) { 1296 $_->{success} = zfs_restore_part($RESTORE_HOST, $RESTORE_ZFS, $_->{file}, $_->{dataset}, $_->{depends}); 1297 } 1298 } 1299 1300} 1301 1302sub zfs_restore_part($$$$;$) { 1303 my ($host, $fs, $file, $dataset, $dep) = @_; 1304 unless ($file || $dataset) { 1305 print STDERR "=> No dataset or filename given to restore. Bailing out."; 1306 return 1; 1307 } 1308 my $ssh_config = config_get($host, 'ssh_config'); 1309 if($ssh_config) { 1310 $ssh_config = "-F $ssh_config"; 1311 print STDERR "Using custom ssh config file: $ssh_config\n" if($DEBUG); 1312 } 1313 my $command; 1314 if(exists($conf{$host})) { 1315 my $agent = config_get($host, 'agent'); 1316 $command = "$agent -r -z $fs"; 1317 $command .= " -b $dep" if($dep); 1318 } 1319 else { 1320 $command = "__ZFS__ recv $fs"; 1321 } 1322 if ($file) { 1323 print " => piping $file to $command\n" if($DEBUG); 1324 print "gzip -dfc $file | ssh $ssh_config $host $command\n" if ($DEBUG && $NEUTERED); 1325 } elsif ($dataset) { 1326 print " => piping $dataset to $command using zfs send\n" if ($DEBUG); 1327 print "zfs send $dataset | ssh $ssh_config $host $command\n" if ($DEBUG && $NEUTERED); 1328 } 1329 unless($NEUTERED) { 1330 if ($file) { 1331 open(DUMP, "gzip -dfc $file |"); 1332 } elsif ($dataset) { 1333 open(DUMP, "__ZFS__ send $dataset |"); 1334 } 1335 eval { 1336 open(RECEIVER, "| ssh $ssh_config $host $command"); 1337 my $buffer; 1338 while(my $len = sysread(DUMP, $buffer, $BLOCKSIZE)) { 1339 if(syswrite(RECEIVER, $buffer, $len) != $len) { 1340 die "$!"; 1341 } 1342 } 1343 }; 1344 close(DUMP); 1345 close(RECEIVER); 1346 } 1347 return $?; 1348} 1349 1350sub pretty_print_backup($$$) { 1351 my ($info, $host, $point) = @_; 1352 my $tf = config_get($host, 'time_format'); 1353 print "\t" . strftime($tf, localtime($point)) . " [$point] "; 1354 if(exists($info->{full}->{$point})) { 1355 if ($info->{full}->{$point}->{file}) { 1356 my @st = stat($info->{full}->{$point}->{file}); 1357 print "FULL " . pretty_size($st[7]); 1358 print "\n\tfile: $info->{full}->{$point}->{file}" if($SHOW_FILENAMES); 1359 } elsif ($info->{full}->{$point}->{dataset}) { 1360 print "FULL $info->{full}->{$point}->{pretty_size}"; 1361 print "\n\tdataset: $info->{full}->{$point}->{dataset}" 1362 if($SHOW_FILENAMES); 1363 } 1364 } else { 1365 my @st = stat($info->{incremental}->{$point}->{file}); 1366 print "INCR from [$info->{incremental}->{$point}->{depends}] " . pretty_size($st[7]); 1367 print "\n\tfile: $info->{incremental}->{$point}->{file}" if($SHOW_FILENAMES); 1368 } 1369 print "\n"; 1370} 1371 1372sub show_backups($$) { 1373 my ($host, $diskpat) = @_; 1374 my (@files, @datasets, %classmap); 1375 my $tf = config_get($host, 'time_format'); 1376 foreach my $class (get_classes()) { 1377 if ($DEBUG) { 1378 if ($class) { 1379 print "=> Class: $class\n" if $class; 1380 } else { 1381 print "=> Class: (none)\n"; 1382 } 1383 } 1384 my $store = get_store($host, $class); 1385 my $backup_info = scan_for_backups($store); 1386 foreach my $disk (sort keys %{$backup_info}) { 1387 my $info = $backup_info->{$disk}; 1388 next unless(ref($info) eq 'HASH'); 1389 next 1390 if($diskpat && # if the pattern was specified it could 1391 !($disk eq $diskpat || # be a specific match or a 1392 ($diskpat =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex 1393 1394 my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}}); 1395 @backup_points = sort { $a <=> $b } @backup_points; 1396 @backup_points = (pop @backup_points) unless ($ARCHIVE || $SUMMARY_EXT); 1397 1398 # We want to see this one 1399 print "$host:$disk\n"; 1400 next unless($SUMMARY || $SUMMARY_EXT || $ARCHIVE); 1401 if($SUMMARY_EXT) { 1402 print "\tLast Full: ". ($info->{last_full} ? strftime($tf, localtime($info->{last_full})) : "Never") . "\n"; 1403 if($info->{last_full} < $info->{last_incremental}) { 1404 print "\tLast Incr: ". strftime($tf, localtime($info->{last_incremental})). "\n"; 1405 } 1406 } 1407 foreach (@backup_points) { 1408 pretty_print_backup($info, $host, $_); 1409 if(exists($info->{full}->{$_}->{file})) { 1410 push @files, $info->{full}->{$_}->{file}; 1411 $classmap{$info->{full}->{$_}->{file}} = $class; 1412 } elsif(exists($info->{incremental}->{$_}->{file})) { 1413 push @files, $info->{incremental}->{$_}->{file}; 1414 $classmap{$info->{incremental}->{$_}->{file}} = $class; 1415 } elsif(exists($info->{full}->{$_}->{dataset})) { 1416 push @datasets, $info->{full}->{$_}->{dataset}; 1417 $classmap{$info->{full}->{$_}->{dataset}} = $class; 1418 } 1419 } 1420 print "\n"; 1421 } 1422 } 1423 if($ARCHIVE && (scalar(@files) || scalar(@datasets))) { 1424 print "\nAre you sure you would like to archive ".scalar(@files). 1425 " file(s) and ".scalar(@datasets)." dataset(s)? "; 1426 while(($_ = <>) !~ /(?:y|n|yes|no)$/i) { 1427 print "\nAre you sure you would like to archive ".scalar(@files). 1428 " file(s) and ".scalar(@datasets)." dataset(s)? "; 1429 } 1430 if(/^y/i) { 1431 if (@files) { 1432 my $archive = config_get($host, 'archive'); 1433 $archive =~ s/%h/$host/g; 1434 if(! -d $archive) { 1435 mkdir $archive || die "Cannot mkdir($archive)\n"; 1436 } 1437 foreach my $file (@files) { 1438 my $store = get_store($host, $classmap{$file}); 1439 (my $afile = $file) =~ s/^$store/$archive/; 1440 move($file, $afile) || print "Error archiving $file: $!\n"; 1441 } 1442 } 1443 if (@datasets) { 1444 my $archive = config_get($host, 'archive'); 1445 (my $basearchive = $archive) =~ s/\/?%h//g; 1446 my $basearchivefs; 1447 eval { 1448 $basearchivefs = get_fs_from_mountpoint($basearchive); 1449 }; 1450 die "Unable to find archive filesystem. The archive directory must be the root of a zfs filesystem to archive datasets." if $@; 1451 my $archivefs = "$basearchivefs/$host"; 1452 `__ZFS__ create $archivefs`; # We don't care if this fails 1453 my %seen = (); 1454 foreach my $dataset (@datasets) { 1455 my $store = get_store($host, $classmap{$dataset}); 1456 my $storefs = get_fs_from_mountpoint($store); 1457 $dataset =~ s/@.*$//; # Only rename filesystems, not snapshots 1458 next if $seen{$dataset}++; # Only rename a filesystem once 1459 (my $adataset = $dataset) =~ s/^$storefs/$archivefs/; 1460 `__ZFS__ rename $dataset $adataset`; 1461 if ($?) { 1462 print "Error archiving $dataset\n"; 1463 } 1464 } 1465 } 1466 } 1467 } 1468} 1469 1470sub show_violators($$) { 1471 my ($host, $diskpat) = @_; 1472 my $host_store = get_store($host); 1473 my $filesystems = {}; 1474 if (open (my $fh, "$host_store/.fslist")) { 1475 while (<$fh>) { 1476 chomp; 1477 $filesystems->{$_} = 1; 1478 } 1479 close($fh); 1480 } elsif ($DEBUG) { 1481 print "=> $host_store/.fslist not present, skipping missing FS detection\n"; 1482 } 1483 foreach my $class (get_classes()) { 1484 if ($DEBUG) { 1485 if ($class) { 1486 print "=> Class: $class\n" if $class; 1487 } else { 1488 print "=> Class: (none)\n"; 1489 } 1490 } 1491 my $store = get_store($host, $class); 1492 my $backup_info = scan_for_backups($store); 1493 foreach my $disk (sort keys %{$backup_info}) { 1494 my $info = $backup_info->{$disk}; 1495 next unless(ref($info) eq 'HASH'); 1496 next if ( 1497 $diskpat && # if the pattern was specified it could 1498 !( 1499 $disk eq $diskpat || # be a specific match 1500 ($diskpat =~ /^\/(.+)\/$/ && $disk =~ /$1/) # or a regex 1501 ) 1502 ); # regex 1503 # Backups for filesystems that no longer exist aren't violators 1504 if (!$SUMMARY_VIOLATORS_VERBOSE && %{$filesystems} && 1505 !defined $filesystems->{$disk}) { 1506 print "=> $disk doesn't exist on server, not marking as violator\n" 1507 if ($DEBUG); 1508 next; 1509 } 1510 1511 1512 my @violators = (); 1513 1514 # No recent full 1515 if (time() > $info->{last_full} + 1516 config_get($host, 'full_interval', $class) + 1517 config_get($host, 'violator_grace_period', $class)) { 1518 push @violators, { 1519 "host" => $host, "disk" => $disk, 1520 "reason" => "No recent full backup", 1521 "backup" => "full" 1522 } 1523 } 1524 1525 # No recent incremental 1526 if (time() > $info->{last_backup} + 1527 config_get($host, 'backup_interval', $class) + 1528 config_get($host, 'violator_grace_period', $class)) { 1529 push @violators, { 1530 "host" => $host, "disk" => $disk, 1531 "reason" => "No recent incremental backup", 1532 "backup" => "backup" 1533 } 1534 } 1535 1536 for my $v (@violators) { 1537 print "$v->{host}:$v->{disk} - $v->{reason}\n"; 1538 pretty_print_backup($info, $host, $info->{"last_$v->{'backup'}"}); 1539 } 1540 } 1541 } 1542} 1543 1544sub plan_and_run($$) { 1545 my ($host, $diskpat) = @_; 1546 my $store; 1547 my $ssh_config = config_get($host, 'ssh_config'); 1548 $ssh_config = "-F $ssh_config" if($ssh_config); 1549 my %suppress; 1550 print "Planning '$host'\n" if($DEBUG); 1551 my $agent = config_get($host, 'agent'); 1552 my $took_action = 1; 1553 while($took_action) { 1554 $took_action = 0; 1555 my @disklist; 1556 1557 # We need a lock for the listing. 1558 return unless(lock($host, ".list")); 1559 1560 # Get list of zfs filesystems from the agent 1561 open(SILENT, ">&", \*STDERR); 1562 close(STDERR); 1563 my $rv = open(ZFSLIST, "ssh $ssh_config $host $agent -l |"); 1564 open(STDERR, ">&", \*SILENT); 1565 close(SILENT); 1566 next unless $rv; 1567 @disklist = grep { chomp } (<ZFSLIST>); 1568 close(ZFSLIST); 1569 # Write the filesystem list out to a file for use by the violators list 1570 my $store = get_store($host); # Don't take classes into account - not needed 1571 mkpath($store) if(! -d $store); 1572 open(my $fh, ">$store/.fslist"); 1573 foreach my $diskline (@disklist) { 1574 # Get only the filesystem and not the snapshots/classes 1575 (my $filesystem = $diskline) =~ s/ \[.*//; 1576 print $fh "$filesystem\n"; 1577 } 1578 close($fh); 1579 if ($DEBUG) { 1580 print " => Filesystems for $host (zetaback_agent -l output)\n"; 1581 foreach my $diskline (@disklist) { 1582 print " $diskline\n"; 1583 } 1584 } 1585 1586 foreach my $diskline (@disklist) { 1587 chomp($diskline); 1588 next unless($diskline =~ /^(\S+) \[([^\]]*)\](?: \{([^}]*)\})?/); 1589 my $diskname = $1; 1590 my %snaps; 1591 map { $snaps{$_} = 1 } (split(/,/, $2)); 1592 my $class = $3; 1593 1594 # We've just done this. 1595 next if($suppress{"$host:$diskname"}); 1596 # If we are being selective (via -z) now is the time. 1597 next 1598 if($diskpat && # if the pattern was specified it could 1599 !($diskname eq $diskpat || # be a specific match or a 1600 ($diskpat =~ /^\/(.+)\/$/ && $diskname =~ /$1/))); # regex 1601 1602 $store = get_store($host, $class); 1603 if ($DEBUG) { 1604 if ($class) { 1605 print STDERR "=> Class is $class\n"; 1606 } else { 1607 print STDERR "=> No/default class\n"; 1608 } 1609 } 1610 print " => Scanning '$store' for old backups of '$diskname'.\n" if($DEBUG); 1611 1612 # Make directory on demand 1613 mkpath($store) if(! -d $store); 1614 my $backup_info = scan_for_backups($store); 1615 # That gave us info on all backups, we just want this disk 1616 $backup_info = $backup_info->{$diskname} || {}; 1617 1618 # Should we do a backup? 1619 my $backup_type = 'no'; 1620 if(time() > $backup_info->{last_backup} + config_get($host, 1621 'backup_interval', $class)) { 1622 $backup_type = 'incremental'; 1623 } 1624 if(time() > $backup_info->{last_full} + config_get($host, 1625 'full_interval', $class)) { 1626 $backup_type = 'full'; 1627 } 1628 # If we want an incremental, but have no full, then we need to upgrade to full 1629 if($backup_type eq 'incremental') { 1630 my $have_full_locally = 0; 1631 # For each local full backup, see if the full backup still exists on the other end. 1632 foreach (keys %{$backup_info->{'full'}}) { 1633 $have_full_locally = 1 if(exists($snaps{'__zb_full_' . $_})); 1634 } 1635 $backup_type = 'full' unless($have_full_locally); 1636 } 1637 $backup_type = 'full' if($FORCE_FULL); 1638 $backup_type = 'incremental' if($FORCE_INC); 1639 $backup_type = 'dataset' if(config_get($host, 'dataset_backup', $class) 1640 eq 1 && $backup_type ne 'no'); 1641 1642 print " => doing $backup_type backup\n" if($DEBUG); 1643 # We need to drop a __zb_base snap or a __zb_incr snap before we proceed 1644 unless($NEUTERED || $backup_type eq 'no') { 1645 # attempt to lock this action, if it fails, skip -- someone else is working it. 1646 next unless(lock($host, dir_encode($diskname), 1)); 1647 unlock($host, '.list'); 1648 1649 if($backup_type eq 'full') { 1650 eval { zfs_full_backup($host, $diskname, $store); }; 1651 if ($@) { 1652 chomp(my $err = $@); 1653 print " => failure $err\n"; 1654 } 1655 else { 1656 # Unless there was an error backing up, remove all the other full snaps 1657 foreach (keys %snaps) { 1658 zfs_remove_snap($host, $diskname, $_) if(/^__zb_full_(\d+)/) 1659 } 1660 } 1661 $took_action = 1; 1662 } 1663 if($backup_type eq 'incremental') { 1664 eval { 1665 zfs_remove_snap($host, $diskname, '__zb_incr') if($snaps{'__zb_incr'}); 1666 # Find the newest full from which to do an incremental (NOTE: reverse numeric sort) 1667 my @fulls = sort { $b <=> $a } (keys %{$backup_info->{'full'}}); 1668 zfs_incremental_backup($host, $diskname, $fulls[0], $store); 1669 }; 1670 if ($@) { 1671 chomp(my $err = $@); 1672 print " => failure $err\n"; 1673 } 1674 else { 1675 $took_action = 1; 1676 } 1677 } 1678 if($backup_type eq 'dataset') { 1679 my @backups = sort { $b <=> $a } (keys %{$backup_info->{'full'}}); 1680 eval { zfs_dataset_backup($host, $diskname, $backups[0], $store); }; 1681 if ($@) { 1682 chomp(my $err = $@); 1683 print " => failure $err\n"; 1684 } 1685 else { 1686 # Unless there was an error backing up, remove all the other dset snaps 1687 foreach (keys %snaps) { 1688 zfs_remove_snap($host, $diskname, $_) if(/^__zb_dset_(\d+)/) 1689 } 1690 } 1691 $took_action = 1; 1692 } 1693 unlock($host, dir_encode($diskname), 1); 1694 } 1695 $suppress{"$host:$diskname"} = 1; 1696 last if($took_action); 1697 } 1698 unlock($host, '.list'); 1699 } 1700} 1701 1702## Start of main program 1703limit_running_processes; 1704 1705if($RESTORE) { 1706 perform_restore(); 1707} 1708else { 1709 foreach my $host (grep { $_ ne "default" && $conf{$_}->{"type"} ne "class"} 1710 keys %conf) { 1711 # If -h was specific, we will skip this host if the arg isn't 1712 # an exact match or a pattern match 1713 if($HOST && 1714 !(($HOST eq $host) || 1715 ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) { 1716 next; 1717 } 1718 1719 # Skip if the host is marked as 'offline' and we are not listing backups 1720 if (config_get($host, 'offline') == 1 && 1721 !$LIST && !$SUMMARY && !$SUMMARY_EXT && !$ARCHIVE) { 1722 next; 1723 } 1724 1725 if($LIST || $SUMMARY || $SUMMARY_EXT || $ARCHIVE) { 1726 show_backups($host, $ZFS); 1727 } 1728 if ($SUMMARY_VIOLATORS || $SUMMARY_VIOLATORS_VERBOSE) { 1729 show_violators($host, $ZFS); 1730 } 1731 if($BACKUP) { 1732 plan_and_run($host, $ZFS); 1733 } 1734 if($EXPUNGE) { 1735 perform_retention($host); 1736 } 1737 } 1738} 1739 1740exit 0; 1741 1742=pod 1743 1744=head1 FILES 1745 1746=over 1747 1748=item zetaback.conf 1749 1750The main zetaback configuration file. The location of the file can be 1751specified on the command line with the -c flag. The prefix of this 1752file may also be specified as an argument to the configure script. 1753 1754=back 1755 1756=head1 SEE ALSO 1757 1758zetaback_agent(1) 1759 1760=cut 1761