1# Copyright (c) 2010-2013 Zmanda Inc. All Rights Reserved. 2# 3# This program is free software; you can redistribute it and/or 4# modify it under the terms of the GNU General Public License 5# as published by the Free Software Foundation; either version 2 6# of the License, or (at your option) any later version. 7# 8# This program is distributed in the hope that it will be useful, but 9# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 10# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11# for more details. 12# 13# You should have received a copy of the GNU General Public License along 14# with this program; if not, write to the Free Software Foundation, Inc., 15# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 16# 17# Contact information: Zmanda Inc, 465 S. Mathilda Ave., Suite 300 18# Sunnyvale, CA 94086, USA, or: http://www.zmanda.com 19 20use Test::More tests => 21; 21use File::Path; 22use Data::Dumper; 23use strict; 24use warnings; 25 26use lib "@amperldir@"; 27use Installcheck::Config; 28use Installcheck::Dumpcache; 29use Amanda::Config qw( :init ); 30use Amanda::Changer; 31use Amanda::Device qw( :constants ); 32use Amanda::Debug; 33use Amanda::Header; 34use Amanda::DB::Catalog; 35use Amanda::Xfer qw( :constants ); 36use Amanda::Recovery::Clerk; 37use Amanda::Recovery::Scan; 38use Amanda::MainLoop; 39use Amanda::Util; 40use Amanda::Tapelist; 41 42# and disable Debug's die() and warn() overrides 43Amanda::Debug::disable_die_override(); 44 45# put the debug messages somewhere 46Amanda::Debug::dbopen("installcheck"); 47Installcheck::log_test_output(); 48 49my $testconf; 50$testconf = Installcheck::Config->new(); 51$testconf->add_param('debug_recovery', '9'); 52$testconf->write(); 53 54my $cfg_result = config_init($CONFIG_INIT_EXPLICIT_NAME, 'TESTCONF'); 55if ($cfg_result != $CFGERR_OK) { 56 my ($level, @errors) = Amanda::Config::config_errors(); 57 die(join "\n", @errors); 58} 59 60my $taperoot = "$Installcheck::TMP/Amanda_Recovery_Clerk"; 61my $datestamp = "20100101010203"; 62 63# set up a 2-tape disk changer with some spanned dumps in it, and add those 64# dumps to the catalog, too. To avoid re-implementing Amanda::Taper::Scribe, this 65# uses individual transfers for each part. 66sub setup_changer { 67 my ($finished_cb, $chg_name, $to_write, $part_len) = @_; 68 my $res; 69 my $chg; 70 my $label; 71 my ($slot, $xfer_info, $partnum); 72 73 my $steps = define_steps 74 cb_ref => \$finished_cb, 75 finalize => sub { $chg->quit() }; 76 77 step setup => sub { 78 $chg = Amanda::Changer->new($chg_name); 79 die "$chg" if $chg->isa("Amanda::Changer::Error"); 80 81 $steps->{'next'}->(); 82 }; 83 84 step next => sub { 85 return $steps->{'done'}->() unless @$to_write; 86 87 ($slot, $xfer_info, $partnum) = @{shift @$to_write}; 88 die "xfer len <= 0" if $xfer_info->[0] <= 0; 89 90 if (!$res || $res->{'this_slot'} != $slot) { 91 $steps->{'new_dev'}->(); 92 } else { 93 $steps->{'run_xfer'}->(); 94 } 95 }; 96 97 step new_dev => sub { 98 if ($res) { 99 $res->release(finished_cb => $steps->{'released'}); 100 } else { 101 $steps->{'released'}->(); 102 } 103 }; 104 105 step released => sub { 106 my ($err) = @_; 107 die "$err" if $err; 108 109 $chg->load(slot => $slot, res_cb => $steps->{'loaded'}); 110 }; 111 112 step loaded => sub { 113 (my $err, $res) = @_; 114 die "$err" if $err; 115 116 my $dev = $res->{'device'}; 117 118 # label the device 119 $label = "TESTCONF0" . $slot; 120 $dev->start($Amanda::Device::ACCESS_WRITE, $label, $datestamp) 121 or die("starting dev: " . $dev->error_or_status()); 122 123 $res->set_label(label => $label, finished_cb => $steps->{'run_xfer'}); 124 }; 125 126 step run_xfer => sub { 127 my $dev = $res->{'device'}; 128 my $name = $xfer_info->[2]; 129 130 my $hdr = Amanda::Header->new(); 131 # if the partnum is 0, write a DUMPFILE like Amanda < 3.1 did 132 $hdr->{'type'} = $partnum? $Amanda::Header::F_SPLIT_DUMPFILE : $Amanda::Header::F_DUMPFILE; 133 $hdr->{'datestamp'} = $datestamp; 134 $hdr->{'dumplevel'} = 0; 135 $hdr->{'name'} = $name; 136 $hdr->{'disk'} = "/$name"; 137 $hdr->{'program'} = "INSTALLCHECK"; 138 $hdr->{'partnum'} = $partnum; 139 $hdr->{'compressed'} = 0; 140 $hdr->{'comp_suffix'} = "N"; 141 142 $dev->start_file($hdr) 143 or die("starting file: " . $dev->error_or_status()); 144 145 my $len = $xfer_info->[0]; 146 $len = $part_len if $len > $part_len; 147 my $key = $xfer_info->[1]; 148 149 my $xsrc = Amanda::Xfer::Source::Random->new($len, $key); 150 my $xdst = Amanda::Xfer::Dest::Device->new($dev, 0); 151 my $xfer = Amanda::Xfer->new([$xsrc, $xdst]); 152 153 $xfer->start(sub { 154 my ($src, $msg, $xfer) = @_; 155 156 if ($msg->{'type'} == $XMSG_ERROR) { 157 die $msg->{'elt'} . " failed: " . $msg->{'message'}; 158 } elsif ($msg->{'type'} == $XMSG_DONE) { 159 # fix up $xfer_info 160 $xfer_info->[0] -= $len; 161 $xfer_info->[1] = $xsrc->get_seed(); 162 163 # add the dump to the catalog 164 Amanda::DB::Catalog::add_part({ 165 label => $label, 166 filenum => $dev->file() - 1, 167 dump_timestamp => $datestamp, 168 write_timestamp => $datestamp, 169 hostname => $name, 170 diskname => "/$name", 171 level => 0, 172 status => "OK", 173 # get the partnum right, even if this wasn't split 174 partnum => $partnum? $partnum : ($partnum+1), 175 nparts => -1, 176 kb => $len / 1024, 177 sec => 1.2, 178 }); 179 180 # and do the next part 181 $steps->{'next'}->(); 182 } 183 }); 184 }; 185 186 step done => sub { 187 if ($res) { 188 $res->release(finished_cb => $steps->{'done_released'}); 189 } else { 190 $steps->{'done_released'}->(); 191 } 192 }; 193 194 step done_released => sub { 195 $finished_cb->(); 196 }; 197} 198 199{ 200 # clean out the vtape root 201 if (-d $taperoot) { 202 rmtree($taperoot); 203 } 204 mkpath($taperoot); 205 206 for my $slot (1 .. 2) { 207 mkdir("$taperoot/slot$slot") 208 or die("Could not mkdir: $!"); 209 } 210 211 ## specification of the on-tape data 212 my @xfer_info = ( 213 # length, random, name ] 214 [ 1024*288, 0xF000, "home" ], 215 [ 1024*1088, 0xF001, "usr" ], 216 [ 1024*768, 0xF002, "games" ], 217 ); 218 my @to_write = ( 219 # slot xfer partnum 220 [ 1, $xfer_info[0], 0 ], # partnum 0 => old non-split header 221 [ 1, $xfer_info[1], 1 ], 222 [ 1, $xfer_info[1], 2 ], 223 [ 2, $xfer_info[1], 3 ], 224 [ 2, $xfer_info[2], 1 ], 225 [ 2, $xfer_info[2], 2 ], 226 ); 227 228 setup_changer(\&Amanda::MainLoop::quit, "chg-disk:$taperoot", \@to_write, 512*1024); 229 Amanda::MainLoop::run(); 230 pass("successfully set up test vtapes"); 231} 232 233# make a holding file 234my $holding_file = "$Installcheck::TMP/holding_file"; 235my $holding_key = 0x797; 236my $holding_kb = 64; 237{ 238 open(my $fh, ">", "$holding_file") or die("opening '$holding_file': $!"); 239 240 my $hdr = Amanda::Header->new(); 241 $hdr->{'type'} = $Amanda::Header::F_DUMPFILE; 242 $hdr->{'datestamp'} = '21001010101010'; 243 $hdr->{'dumplevel'} = 1; 244 $hdr->{'name'} = 'heldhost'; 245 $hdr->{'disk'} = '/to/holding'; 246 $hdr->{'program'} = "INSTALLCHECK"; 247 $hdr->{'is_partial'} = 0; 248 249 Amanda::Util::full_write(fileno($fh), $hdr->to_string(32768,32768), 32768); 250 251 # transfer some data to that file 252 my $xfer = Amanda::Xfer->new([ 253 Amanda::Xfer::Source::Random->new(1024*$holding_kb, $holding_key), 254 Amanda::Xfer::Dest::Fd->new($fh), 255 ]); 256 257 $xfer->start(sub { 258 my ($src, $msg, $xfer) = @_; 259 if ($msg->{type} == $XMSG_ERROR) { 260 die $msg->{elt} . " failed: " . $msg->{message}; 261 } elsif ($msg->{'type'} == $XMSG_DONE) { 262 $src->remove(); 263 Amanda::MainLoop::quit(); 264 } 265 }); 266 Amanda::MainLoop::run(); 267 close($fh); 268} 269 270# fill out a dump object like that returned from Amanda::DB::Catalog, with all 271# of the keys that we don't really need based on a much simpler description 272sub fake_dump { 273 my ($hostname, $diskname, $dump_timestamp, $level, @parts) = @_; 274 275 my $pldump = { 276 dump_timestamp => $dump_timestamp, 277 write_timestamp => $dump_timestamp, 278 hostname => $hostname, 279 diskname => $diskname, 280 level => $level, 281 status => 'OK', 282 message => '', 283 nparts => 0, # filled in later 284 kb => 128, # ignored by clerk anyway 285 secs => 10.0, # ditto 286 parts => [ undef ], 287 }; 288 289 for my $part (@parts) { 290 push @{$pldump->{'parts'}}, { 291 %$part, 292 dump => $pldump, 293 status => "OK", 294 partnum => scalar @{$pldump->{'parts'}}, 295 kb => 64, # ignored 296 sec => 1.0, # ignored 297 }; 298 $pldump->{'nparts'}++; 299 } 300 301 return $pldump; 302} 303 304package main::Feedback; 305 306use base 'Amanda::Recovery::Clerk::Feedback'; 307 308sub new { 309 my $class = shift; 310 my %params = @_; 311 312 return bless \%params, $class; 313} 314 315sub clerk_notif_part { 316 my $self = shift; 317 318 if (exists $self->{'clerk_notif_part'}) { 319 $self->{'clerk_notif_part'}->(@_); 320 } else { 321 $self->SUPER::clerk_notif_part(@_); 322 } 323} 324 325package main; 326 327# run a recovery with the given plan on the given clerk, expecting a bytestream with 328# the given random seed. 329sub try_recovery { 330 my %params = @_; 331 my $clerk = $params{'clerk'}; 332 my $result; 333 my $running_xfers = 0; 334 335 my $finished_cb = \&Amanda::MainLoop::quit; 336 my $steps = define_steps 337 cb_ref => \$finished_cb; 338 339 step start => sub { 340 $clerk->get_xfer_src( 341 dump => $params{'dump'}, 342 xfer_src_cb => $steps->{'xfer_src_cb'}); 343 }; 344 345 step xfer_src_cb => sub { 346 my ($errors, $header, $xfer_src, $dtcp_supp) = @_; 347 348 # simulate errors for xfail, below 349 if ($errors) { 350 $result = { result => "FAILED", errors => $errors }; 351 return $steps->{'verify'}->(); 352 } 353 354 # double-check the header; the Clerk should have checked this, so these 355 # are die's, for simplicity 356 die unless 357 $header->{'name'} eq $params{'dump'}->{'hostname'} && 358 $header->{'disk'} eq $params{'dump'}->{'diskname'} && 359 $header->{'datestamp'} eq $params{'dump'}->{'dump_timestamp'} && 360 $header->{'dumplevel'} == $params{'dump'}->{'level'}; 361 362 die if $params{'expect_directtcp_supported'} and !$dtcp_supp; 363 die if !$params{'expect_directtcp_supported'} and $dtcp_supp; 364 365 my $xfer; 366 my $xfer_dest; 367 if ($params{'directtcp'}) { 368 $xfer_dest = Amanda::Xfer::Dest::DirectTCPListen->new(); 369 } else { 370 $xfer_dest = Amanda::Xfer::Dest::Null->new($params{'seed'}); 371 } 372 373 $xfer = Amanda::Xfer->new([ $xfer_src, $xfer_dest ]); 374 $running_xfers++; 375 $xfer->start(sub { $clerk->handle_xmsg(@_); }); 376 377 if ($params{'directtcp'}) { 378 # use another xfer to read from that directtcp connection and verify 379 # it with Dest::Null 380 my $dest_xfer = Amanda::Xfer->new([ 381 Amanda::Xfer::Source::DirectTCPConnect->new($xfer_dest->get_addrs()), 382 Amanda::Xfer::Dest::Null->new($params{'seed'}), 383 ]); 384 $running_xfers++; 385 $dest_xfer->start(sub { 386 my ($src, $msg, $xfer) = @_; 387 if ($msg->{type} == $XMSG_ERROR) { 388 die $msg->{elt} . " failed: " . $msg->{message}; 389 } 390 if ($msg->{'type'} == $XMSG_DONE) { 391 $steps->{'maybe_done'}->(); 392 } 393 }); 394 } 395 396 $clerk->start_recovery( 397 xfer => $xfer, 398 recovery_cb => $steps->{'recovery_cb'}); 399 }; 400 401 step recovery_cb => sub { 402 $result = { @_ }; 403 $steps->{'maybe_done'}->(); 404 }; 405 406 step maybe_done => sub { 407 $steps->{'verify'}->() unless --$running_xfers; 408 }; 409 410 step verify => sub { 411 # verify the results 412 my $msg = $params{'msg'}; 413 if (@{$result->{'errors'}}) { 414 if ($params{'xfail'}) { 415 if ($result->{'result'} ne 'FAILED') { 416 diag("expected failure, but got $result->{result}"); 417 fail($msg); 418 } 419 is_deeply($result->{'errors'}, $params{'xfail'}, $msg); 420 } else { 421 diag("errors:"); 422 for (@{$result->{'errors'}}) { 423 diag("$_"); 424 } 425 if ($result->{'result'} ne 'FAILED') { 426 diag("XXX and result is " . $result->{'result'}); 427 } 428 fail($msg); 429 } 430 } else { 431 if ($result->{'result'} ne 'DONE') { 432 diag("XXX no errors but result is " . $result->{'result'}); 433 fail($msg); 434 } else { 435 pass($msg); 436 } 437 } 438 439 $finished_cb->(); 440 }; 441 442 Amanda::MainLoop::run(); 443} 444 445sub quit_clerk { 446 my ($clerk) = @_; 447 448 $clerk->quit(finished_cb => make_cb(finished_cb => sub { 449 my ($err) = @_; 450 die "$err" if $err; 451 452 Amanda::MainLoop::quit(); 453 })); 454 Amanda::MainLoop::run(); 455 pass("clerk quit"); 456} 457 458## 459## Tests! 460### 461 462my $clerk; 463my $feedback; 464my @clerk_notif_parts; 465my $chg = Amanda::Changer->new("chg-disk:$taperoot"); 466my $scan = Amanda::Recovery::Scan->new(chg => $chg); 467 468$clerk = Amanda::Recovery::Clerk->new(scan => $scan, debug => 1); 469 470try_recovery( 471 clerk => $clerk, 472 seed => 0xF000, 473 dump => fake_dump("home", "/home", $datestamp, 0, 474 { label => 'TESTCONF01', filenum => 1 }, 475 ), 476 msg => "one-part recovery successful"); 477 478try_recovery( 479 clerk => $clerk, 480 seed => 0xF001, 481 dump => fake_dump("usr", "/usr", $datestamp, 0, 482 { label => 'TESTCONF01', filenum => 2 }, 483 { label => 'TESTCONF01', filenum => 3 }, 484 { label => 'TESTCONF02', filenum => 1 }, 485 ), 486 msg => "multi-part recovery successful"); 487 488quit_clerk($clerk); 489 490# recover from TESTCONF02, then 01, and then 02 again 491 492@clerk_notif_parts = (); 493$feedback = main::Feedback->new( 494 clerk_notif_part => sub { 495 push @clerk_notif_parts, [ $_[0], $_[1] ], 496 }, 497); 498 499$chg = Amanda::Changer->new("chg-disk:$taperoot"); 500$scan = Amanda::Recovery::Scan->new(chg => $chg); 501$clerk = Amanda::Recovery::Clerk->new(scan => $scan, debug => 1, 502 feedback => $feedback); 503 504try_recovery( 505 clerk => $clerk, 506 seed => 0xF002, 507 dump => fake_dump("games", "/games", $datestamp, 0, 508 { label => 'TESTCONF02', filenum => 2 }, 509 { label => 'TESTCONF02', filenum => 3 }, 510 ), 511 msg => "two-part recovery from second tape successful"); 512 513is_deeply([ @clerk_notif_parts ], [ 514 [ 'TESTCONF02', 2 ], 515 [ 'TESTCONF02', 3 ], 516 ], "..and clerk_notif_part calls are correct"); 517 518try_recovery( 519 clerk => $clerk, 520 seed => 0xF001, 521 dump => fake_dump("usr", "/usr", $datestamp, 0, 522 { label => 'TESTCONF01', filenum => 2 }, 523 { label => 'TESTCONF01', filenum => 3 }, 524 { label => 'TESTCONF02', filenum => 1 }, 525 ), 526 msg => "multi-part recovery spanning tapes 1 and 2 successful"); 527 528try_recovery( 529 clerk => $clerk, 530 seed => 0xF001, 531 dump => fake_dump("usr", "/usr", $datestamp, 0, 532 { label => 'TESTCONF01', filenum => 2 }, 533 { label => 'TESTCONF01', filenum => 3 }, 534 { label => 'TESTCONF02', filenum => 1 }, 535 ), 536 directtcp => 1, 537 msg => "multi-part recovery spanning tapes 1 and 2 successful, with directtcp"); 538 539try_recovery( 540 clerk => $clerk, 541 seed => $holding_key, 542 dump => fake_dump("heldhost", "/to/holding", '21001010101010', 1, 543 { holding_file => $holding_file }, 544 ), 545 msg => "holding-disk recovery"); 546 547try_recovery( 548 clerk => $clerk, 549 seed => $holding_key, 550 dump => fake_dump("heldhost", "/to/holding", '21001010101010', 1, 551 { holding_file => $holding_file }, 552 ), 553 directtcp => 1, 554 msg => "holding-disk recovery, with directtcp"); 555 556# try some expected failures 557 558try_recovery( 559 clerk => $clerk, 560 seed => $holding_key, 561 dump => fake_dump("weldtoast", "/to/holding", '21001010101010', 1, 562 { holding_file => $holding_file }, 563 ), 564 xfail => [ "header on '$holding_file' does not match expectations: " . 565 "got hostname 'heldhost'; expected 'weldtoast'" ], 566 msg => "holding-disk recovery expected failure on header disagreement"); 567 568try_recovery( 569 clerk => $clerk, 570 seed => 0xF002, 571 dump => fake_dump("XXXgames", "/games", $datestamp, 0, 572 { label => 'TESTCONF02', filenum => 2 }, 573 ), 574 xfail => [ "header on 'TESTCONF02' file 2 does not match expectations: " . 575 "got hostname 'games'; expected 'XXXgames'" ], 576 msg => "mismatched hostname detected"); 577 578try_recovery( 579 clerk => $clerk, 580 seed => 0xF002, 581 dump => fake_dump("games", "XXX/games", $datestamp, 0, 582 { label => 'TESTCONF02', filenum => 2 }, 583 ), 584 xfail => [ "header on 'TESTCONF02' file 2 does not match expectations: " . 585 "got disk '/games'; expected 'XXX/games'" ], 586 msg => "mismatched disk detected"); 587 588try_recovery( 589 clerk => $clerk, 590 seed => 0xF002, 591 dump => fake_dump("games", "/games", "XXX", 0, 592 { label => 'TESTCONF02', filenum => 2 }, 593 ), 594 xfail => [ "header on 'TESTCONF02' file 2 does not match expectations: " . 595 "got datestamp '$datestamp'; expected 'XXX'" ], 596 msg => "mismatched datestamp detected"); 597 598try_recovery( 599 clerk => $clerk, 600 seed => 0xF002, 601 dump => fake_dump("games", "/games", $datestamp, 13, 602 { label => 'TESTCONF02', filenum => 2 }, 603 ), 604 xfail => [ "header on 'TESTCONF02' file 2 does not match expectations: " . 605 "got dumplevel '0'; expected '13'" ], 606 msg => "mismatched level detected"); 607 608quit_clerk($clerk); 609rmtree($taperoot); 610 611# try a recovery from a DirectTCP-capable device. Note that this is the only real 612# test of Amanda::Xfer::Source::Recovery's directtcp mode 613 614SKIP: { 615 skip "not built with ndmp and full client/server", 5 unless 616 Amanda::Util::built_with_component("ndmp") 617 and Amanda::Util::built_with_component("client") 618 and Amanda::Util::built_with_component("server"); 619 620 Installcheck::Dumpcache::load("ndmp"); 621 622 my $ndmp = Installcheck::Mock::NdmpServer->new(no_reset => 1); 623 624 $ndmp->edit_config(); 625 my $cfg_result = config_init($CONFIG_INIT_EXPLICIT_NAME, 'TESTCONF'); 626 if ($cfg_result != $CFGERR_OK) { 627 my ($level, @errors) = Amanda::Config::config_errors(); 628 die(join "\n", @errors); 629 } 630 631 my $tapelist = Amanda::Config::config_dir_relative("tapelist"); 632 my $tl = Amanda::Tapelist->new($tapelist); 633 634 my $chg = Amanda::Changer->new(); 635 my $scan = Amanda::Recovery::Scan->new(chg => $chg); 636 my $clerk = Amanda::Recovery::Clerk->new(scan => $scan, debug => 1); 637 638 try_recovery( 639 clerk => $clerk, 640 seed => 0, # no verification 641 dump => fake_dump("localhost", $Installcheck::Run::diskname, 642 $Installcheck::Dumpcache::timestamps[0], 0, 643 { label => 'TESTCONF01', filenum => 1 }, 644 ), 645 directtcp => 1, 646 expect_directtcp_supported => 1, 647 msg => "recovery of a real dump via NDMP and directtcp"); 648 quit_clerk($clerk); 649 650 ## specification of the on-tape data 651 my @xfer_info = ( 652 # length, random, name ] 653 [ 1024*160, 0xB000, "home" ], 654 ); 655 my @to_write = ( 656 # (note that slots 1 and 2 are i/e slots, and are initially empty) 657 # slot xfer partnum 658 [ 3, $xfer_info[0], 1 ], 659 [ 4, $xfer_info[0], 2 ], 660 [ 4, $xfer_info[0], 3 ], 661 ); 662 663 setup_changer(\&Amanda::MainLoop::quit, "ndmp_server", \@to_write, 64*1024); 664 Amanda::MainLoop::run(); 665 pass("successfully set up ndmp test data"); 666 667 $chg = Amanda::Changer->new(); 668 $scan = Amanda::Recovery::Scan->new(chg => $chg); 669 $clerk = Amanda::Recovery::Clerk->new(scan => $scan, debug => 1); 670 671 try_recovery( 672 clerk => $clerk, 673 seed => 0xB000, 674 dump => fake_dump("home", "/home", $datestamp, 0, 675 { label => 'TESTCONF03', filenum => 1 }, 676 { label => 'TESTCONF04', filenum => 1 }, 677 { label => 'TESTCONF04', filenum => 2 }, 678 ), 679 msg => "multi-part ndmp recovery successful", 680 expect_directtcp_supported => 1); 681 quit_clerk($clerk); 682} 683 684# cleanup 685rmtree($taperoot); 686unlink($holding_file); 687