1#################################################################################################################################### 2# HostBackupTest.pm - Backup host 3#################################################################################################################################### 4package pgBackRestTest::Env::Host::HostBackupTest; 5use parent 'pgBackRestTest::Env::Host::HostBaseTest'; 6 7#################################################################################################################################### 8# Perl includes 9#################################################################################################################################### 10use strict; 11use warnings FATAL => qw(all); 12use Carp qw(confess); 13 14use Exporter qw(import); 15 our @EXPORT = qw(); 16use Fcntl ':mode'; 17use File::Basename qw(dirname); 18use File::stat qw{lstat}; 19use Storable qw(dclone); 20 21use pgBackRestDoc::Common::Exception; 22use pgBackRestDoc::Common::Ini; 23use pgBackRestDoc::Common::Log; 24use pgBackRestDoc::Common::String; 25use pgBackRestDoc::ProjectInfo; 26 27use pgBackRestTest::Common::DbVersion; 28use pgBackRestTest::Common::StorageBase; 29use pgBackRestTest::Common::StorageRepo; 30use pgBackRestTest::Env::ArchiveInfo; 31use pgBackRestTest::Env::BackupInfo; 32use pgBackRestTest::Env::Host::HostAzureTest; 33use pgBackRestTest::Env::Host::HostGcsTest; 34use pgBackRestTest::Env::Host::HostBaseTest; 35use pgBackRestTest::Env::Host::HostS3Test; 36use pgBackRestTest::Env::Manifest; 37use pgBackRestTest::Common::ContainerTest; 38use pgBackRestTest::Common::ExecuteTest; 39use pgBackRestTest::Common::HostGroupTest; 40use pgBackRestTest::Common::RunTest; 41 42#################################################################################################################################### 43# Error constants 44#################################################################################################################################### 45use constant ERROR_REPO_INVALID => 103; 46push @EXPORT, qw(ERROR_REPO_INVALID); 47 48#################################################################################################################################### 49# Latest backup link constant 50#################################################################################################################################### 51use constant LINK_LATEST => 'latest'; 52 push @EXPORT, qw(LINK_LATEST); 53 54#################################################################################################################################### 55# Host defaults 56#################################################################################################################################### 57use constant HOST_PATH_LOCK => 'lock'; 58 push @EXPORT, qw(HOST_PATH_LOCK); 59use constant HOST_PATH_LOG => 'log'; 60 push @EXPORT, qw(HOST_PATH_LOG); 61use constant HOST_PATH_REPO => 'repo'; 62 63use constant HOST_PROTOCOL_TIMEOUT => 10; 64 push @EXPORT, qw(HOST_PROTOCOL_TIMEOUT); 65 66#################################################################################################################################### 67# Configuration constants 68#################################################################################################################################### 69use constant CFGDEF_SECTION_GLOBAL => 'global'; 70 push @EXPORT, qw(CFGDEF_SECTION_GLOBAL); 71use constant CFGDEF_SECTION_STANZA => 'stanza'; 72 push @EXPORT, qw(CFGDEF_SECTION_STANZA); 73 74use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full'; 75 push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL); 76use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff'; 77 push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF); 78use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr'; 79 push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR); 80 81use constant CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc'; 82 push @EXPORT, qw(CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC); 83 84use constant AZURE => 'azure'; 85 push @EXPORT, qw(AZURE); 86use constant CIFS => 'cifs'; 87 push @EXPORT, qw(CIFS); 88use constant GCS => 'gcs'; 89 push @EXPORT, qw(GCS); 90use constant POSIX => STORAGE_POSIX; 91 push @EXPORT, qw(POSIX); 92use constant S3 => 's3'; 93 push @EXPORT, qw(S3); 94 95use constant CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default'; 96 push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_DEFAULT); 97use constant CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate'; 98 push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_IMMEDIATE); 99use constant CFGOPTVAL_RESTORE_TYPE_NAME => 'name'; 100 push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_NAME); 101use constant CFGOPTVAL_RESTORE_TYPE_PRESERVE => 'preserve'; 102 push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_PRESERVE); 103use constant CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby'; 104 push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_STANDBY); 105use constant CFGOPTVAL_RESTORE_TYPE_TIME => 'time'; 106 push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_TIME); 107use constant CFGOPTVAL_RESTORE_TYPE_XID => 'xid'; 108 push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_XID); 109 110use constant NONE => 'none'; 111 push @EXPORT, qw(NONE); 112use constant BZ2 => 'bz2'; 113 push @EXPORT, qw(BZ2); 114use constant GZ => 'gz'; 115 push @EXPORT, qw(GZ); 116use constant LZ4 => 'lz4'; 117 push @EXPORT, qw(LZ4); 118use constant ZST => 'zst'; 119 push @EXPORT, qw(ZST); 120 121#################################################################################################################################### 122# new 123#################################################################################################################################### 124sub new 125{ 126 my $class = shift; # Class name 127 128 # Assign function parameters, defaults, and log debug info 129 my 130 ( 131 $strOperation, 132 $oParam, 133 ) = 134 logDebugParam 135 ( 136 __PACKAGE__ . '->new', \@_, 137 {name => 'oParam', required => false, trace => true}, 138 ); 139 140 # If params are not passed 141 my $oHostGroup = hostGroupGet(); 142 143 my ($strName, $strImage, $strUser); 144 145 if (!defined($$oParam{strName}) || $$oParam{strName} eq HOST_BACKUP) 146 { 147 $strName = HOST_BACKUP; 148 $strImage = containerRepo() . ':' . testRunGet()->vm() . '-test'; 149 } 150 else 151 { 152 $strName = $$oParam{strName}; 153 $strImage = $$oParam{strImage}; 154 } 155 156 $strUser = testRunGet()->pgUser(); 157 158 # Create the host 159 my $self = $class->SUPER::new($strName, {strImage => $strImage, strUser => $strUser}); 160 bless $self, $class; 161 162 # If repo is on local filesystem then set the repo-path locally 163 if ($oParam->{bRepoLocal}) 164 { 165 $self->{strRepoPath} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO; 166 } 167 # Else on KV store and repo will be in root 168 else 169 { 170 $self->{strRepoPath} = '/'; 171 } 172 173 # If there is a repo2 it will always be posix on the repo host 174 $self->{strRepo2Path} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO . "2"; 175 176 # Set log/lock paths 177 $self->{strLogPath} = $self->testPath() . '/' . HOST_PATH_LOG; 178 storageTest()->pathCreate($self->{strLogPath}, {strMode => '0770'}); 179 $self->{strLockPath} = $self->testPath() . '/' . HOST_PATH_LOCK; 180 181 # Set conf file 182 $self->{strBackRestConfig} = $self->testPath() . '/' . PROJECT_CONF; 183 184 # Set LogTest object 185 $self->{oLogTest} = $$oParam{oLogTest}; 186 187 # Set synthetic 188 $self->{bSynthetic} = defined($$oParam{bSynthetic}) && $$oParam{bSynthetic} ? true : false; 189 190 # Set the backup destination 191 $self->{strBackupDestination} = $$oParam{strBackupDestination}; 192 193 # Default hardlink to false 194 $self->{bHardLink} = false; 195 196 # By default there is no bogus host 197 $self->{bBogusHost} = false; 198 199 # Create a placeholder hash for file munging 200 $self->{hInfoFile} = {}; 201 202 # Set whether repo should be encrypted or not 203 $self->{bRepoEncrypt} = defined($$oParam{bRepoEncrypt}) ? $$oParam{bRepoEncrypt} : false; 204 205 # Return from function and log return values if any 206 return logDebugReturn 207 ( 208 $strOperation, 209 {name => 'self', value => $self, trace => true} 210 ); 211} 212 213#################################################################################################################################### 214# timestampFileFormat 215#################################################################################################################################### 216sub timestampFileFormat 217{ 218 my $strFormat = shift; 219 my $lTime = shift; 220 221 return timestampFormat(defined($strFormat) ? $strFormat : '%4d%02d%02d-%02d%02d%02d', $lTime); 222} 223 224push @EXPORT, qw(timestampFileFormat); 225 226#################################################################################################################################### 227# backupLabelFormat 228# 229# Format the label for a backup. 230#################################################################################################################################### 231sub backupLabelFormat 232{ 233 # Assign function parameters, defaults, and log debug info 234 my 235 ( 236 $strOperation, 237 $strType, 238 $strBackupLabelLast, 239 $lTimestampStart 240 ) = 241 logDebugParam 242 ( 243 __PACKAGE__ . '::backupLabelFormat', \@_, 244 {name => 'strType', trace => true}, 245 {name => 'strBackupLabelLast', required => false, trace => true}, 246 {name => 'lTimestampTart', trace => true} 247 ); 248 249 # Full backup label 250 my $strBackupLabel; 251 252 if ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL) 253 { 254 # Last backup label must not be defined 255 if (defined($strBackupLabelLast)) 256 { 257 confess &log(ASSERT, "strBackupLabelLast must not be defined when strType = '${strType}'"); 258 } 259 260 # Format the timestamp and add the full indicator 261 $strBackupLabel = timestampFileFormat(undef, $lTimestampStart) . 'F'; 262 } 263 # Else diff or incr label 264 else 265 { 266 # Last backup label must be defined 267 if (!defined($strBackupLabelLast)) 268 { 269 confess &log(ASSERT, "strBackupLabelLast must be defined when strType = '${strType}'"); 270 } 271 272 # Get the full backup portion of the last backup label 273 $strBackupLabel = substr($strBackupLabelLast, 0, 16); 274 275 # Format the timestamp 276 $strBackupLabel .= '_' . timestampFileFormat(undef, $lTimestampStart); 277 278 # Add the diff indicator 279 if ($strType eq CFGOPTVAL_BACKUP_TYPE_DIFF) 280 { 281 $strBackupLabel .= 'D'; 282 } 283 # Else incr indicator 284 else 285 { 286 $strBackupLabel .= 'I'; 287 } 288 } 289 290 # Return from function and log return values if any 291 return logDebugReturn 292 ( 293 $strOperation, 294 {name => 'strBackupLabel', value => $strBackupLabel, trace => true} 295 ); 296} 297 298push @EXPORT, qw(backupLabelFormat); 299 300#################################################################################################################################### 301# backupBegin 302#################################################################################################################################### 303sub backupBegin 304{ 305 my $self = shift; 306 307 # Assign function parameters, defaults, and log debug info 308 my 309 ( 310 $strOperation, 311 $strType, 312 $strComment, 313 $oParam, 314 ) = 315 logDebugParam 316 ( 317 __PACKAGE__ . '->backupBegin', \@_, 318 {name => 'strType', trace => true}, 319 {name => 'strComment', trace => true}, 320 {name => 'oParam', required => false, trace => true}, 321 ); 322 323 # Set defaults 324 my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? $$oParam{oExpectedManifest} : undef; 325 326 $strComment = 327 "${strType} backup" . (defined($strComment) ? " - ${strComment}" : '') . 328 ' (' . $self->nameGet() . ' host)'; 329 330 &log(INFO, " $strComment"); 331 332 # Execute the backup command 333 my $oExecuteBackup = $self->execute( 334 $self->backrestExe() . 335 ' --config=' . $self->backrestConfig() . 336 (defined($oExpectedManifest) ? " --no-online" : '') . 337 (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . 338 (defined($$oParam{bStandby}) && $$oParam{bStandby} ? " --backup-standby" : '') . 339 (defined($oParam->{strRepoType}) ? " --repo1-type=$oParam->{strRepoType}" : '') . 340 (defined($oParam->{iRepo}) ? ' --repo=' . $oParam->{iRepo} : '') . 341 ($strType ne 'incr' ? " --type=${strType}" : '') . 342 ' --stanza=' . (defined($oParam->{strStanza}) ? $oParam->{strStanza} : $self->stanza()) . ' backup', 343 {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, 344 oLogTest => $self->{oLogTest}, bLogOutput => $self->synthetic()}); 345 346 $oExecuteBackup->begin(); 347 348 # Return from function and log return values if any 349 return logDebugReturn 350 ( 351 $strOperation, 352 {name => 'oExecuteBackup', value => $oExecuteBackup, trace => true}, 353 ); 354} 355 356#################################################################################################################################### 357# backupEnd 358#################################################################################################################################### 359sub backupEnd 360{ 361 my $self = shift; 362 363 # Assign function parameters, defaults, and log debug info 364 my 365 ( 366 $strOperation, 367 $strType, 368 $oExecuteBackup, 369 $oParam, 370 $bManifestCompare, 371 ) = 372 logDebugParam 373 ( 374 __PACKAGE__ . '->backupEnd', \@_, 375 {name => 'strType', trace => true}, 376 {name => 'oExecuteBackup', trace => true}, 377 {name => 'oParam', required => false, trace => true}, 378 {name => 'bManifestCompare', required => false, default => true, trace => true}, 379 ); 380 381 # Set defaults 382 my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? dclone($$oParam{oExpectedManifest}) : undef; 383 384 my $iExitStatus = $oExecuteBackup->end(); 385 386 return if ($oExecuteBackup->{iExpectedExitStatus} != 0); 387 388 # If an alternate stanza was specified 389 if (defined($oParam->{strStanza})) 390 { 391 confess &log(ASSERT, 392 'if an alternate stanza is specified it must generate an error - the remaining code will not be aware of the stanza'); 393 } 394 395 my $strBackup = $self->backupLast($oParam->{iRepo}); 396 397 # Only compare backups that are in repo1. There is not a lot of value in comparing backups in other repos and it would require a 398 # lot of changes to the test harness. 399 if (!defined($oParam->{iRepo}) || $oParam->{iRepo} == 1) 400 { 401 # If a real backup then load the expected manifest from the actual manifest. An expected manifest can't be generated 402 # perfectly because a running database is always in flux. Even so, it allows us to test many things. 403 if (!$self->synthetic()) 404 { 405 $oExpectedManifest = iniParse( 406 ${storageRepo()->get( 407 storageRepo()->openRead( 408 'backup/' . $self->stanza() . "/${strBackup}/" . FILE_MANIFEST, 409 {strCipherPass => $self->cipherPassManifest()}))}); 410 } 411 412 # Make sure tablespace links are correct 413 if ($self->hasLink()) 414 { 415 if ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink()) 416 { 417 my $hTablespaceManifest = storageTest()->manifest( 418 $self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)); 419 420 # Remove . and .. 421 delete($hTablespaceManifest->{'.'}); 422 delete($hTablespaceManifest->{'..'}); 423 424 # Iterate file links 425 for my $strFile (sort(keys(%{$hTablespaceManifest}))) 426 { 427 # Make sure the link is in the expected manifest 428 my $hManifestTarget = 429 $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"}; 430 431 if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK || 432 $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile) 433 { 434 confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id"); 435 } 436 437 # Make sure the link really is a link 438 if ($hTablespaceManifest->{$strFile}{type} ne 'l') 439 { 440 confess &log(ERROR, "'${strFile}' in tablespace directory is not a link"); 441 } 442 443 # Make sure the link destination is correct 444 my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}"; 445 446 if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination) 447 { 448 confess &log(ERROR, 449 "'${strFile}' link should reference '${strLinkDestination}' but actually references " . 450 "'$hTablespaceManifest->{$strFile}{link_destination}'"); 451 } 452 } 453 454 # Iterate manifest targets 455 for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}}))) 456 { 457 my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}; 458 my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID}; 459 460 # Make sure the target exists as a link on disk 461 if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) && 462 !defined($hTablespaceManifest->{$strTablespaceId})) 463 { 464 confess &log(ERROR, 465 "target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'"); 466 } 467 } 468 } 469 # Else there should not be a tablespace directory at all. This is only valid for storage that supports links. 470 elsif (storageRepo()->capability(STORAGE_CAPABILITY_LINK) && 471 storageTest()->pathExists( 472 $self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))) 473 { 474 confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory'); 475 } 476 } 477 478 # Check that latest link exists unless repo links are disabled 479 my $strLatestLink = $self->repoBackupPath(LINK_LATEST); 480 my $bLatestLinkExists = storageRepo()->exists($strLatestLink); 481 482 if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq POSIX) && $self->hasLink()) 483 { 484 my $strLatestLinkDestination = readlink($strLatestLink); 485 486 if ($strLatestLinkDestination ne $strBackup) 487 { 488 confess &log(ERROR, "'" . LINK_LATEST . "' link should be '${strBackup}' but is '${strLatestLinkDestination}"); 489 } 490 } 491 elsif ($bLatestLinkExists) 492 { 493 confess &log(ERROR, "'" . LINK_LATEST . "' link should not exist"); 494 } 495 496 # Only do compare for synthetic backups since for real backups the expected manifest *is* the actual manifest. 497 if ($self->synthetic()) 498 { 499 # Compare only if expected to do so 500 if ($bManifestCompare) 501 { 502 # Set backup type in the expected manifest 503 ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE} = $strType; 504 505 $self->backupCompare($strBackup, $oExpectedManifest); 506 } 507 } 508 } 509 510 # Add files to expect log 511 if (defined($self->{oLogTest}) && (!defined($$oParam{bSupplemental}) || $$oParam{bSupplemental})) 512 { 513 my $oHostGroup = hostGroupGet(); 514 515 if (defined($oHostGroup->hostGet(HOST_DB_PRIMARY, true))) 516 { 517 $self->{oLogTest}->supplementalAdd($oHostGroup->hostGet(HOST_DB_PRIMARY)->testPath() . '/' . PROJECT_CONF); 518 } 519 520 if (defined($oHostGroup->hostGet(HOST_DB_STANDBY, true))) 521 { 522 $self->{oLogTest}->supplementalAdd($oHostGroup->hostGet(HOST_DB_STANDBY)->testPath() . '/' . PROJECT_CONF); 523 } 524 525 if (defined($oHostGroup->hostGet(HOST_BACKUP, true))) 526 { 527 $self->{oLogTest}->supplementalAdd($oHostGroup->hostGet(HOST_BACKUP)->testPath() . '/' . PROJECT_CONF); 528 } 529 530 if ($self->synthetic() && $bManifestCompare) 531 { 532 $self->{oLogTest}->supplementalAdd( 533 $self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), undef, 534 ${storageRepo()->get( 535 storageRepo()->openRead( 536 $self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), 537 {strCipherPass => $self->cipherPassManifest()}))}); 538 $self->{oLogTest}->supplementalAdd( 539 $self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo->get($self->repoBackupPath(FILE_BACKUP_INFO))}); 540 } 541 } 542 543 # Return from function and log return values if any 544 return logDebugReturn 545 ( 546 $strOperation, 547 {name => 'strBackup', value => $strBackup, trace => true}, 548 ); 549} 550 551#################################################################################################################################### 552# backup 553#################################################################################################################################### 554sub backup 555{ 556 my $self = shift; 557 558 # Assign function parameters, defaults, and log debug info 559 my 560 ( 561 $strOperation, 562 $strType, 563 $strComment, 564 $oParam, 565 $bManifestCompare, 566 ) = 567 logDebugParam 568 ( 569 __PACKAGE__ . '->backup', \@_, 570 {name => 'strType'}, 571 {name => 'strComment'}, 572 {name => 'oParam', required => false}, 573 {name => 'bManifestCompare', required => false, default => true}, 574 ); 575 576 my $oExecuteBackup = $self->backupBegin($strType, $strComment, $oParam); 577 my $strBackup = $self->backupEnd($strType, $oExecuteBackup, $oParam, $bManifestCompare); 578 579 # Return from function and log return values if any 580 return logDebugReturn 581 ( 582 $strOperation, 583 {name => 'strBackup', value => $strBackup}, 584 ); 585} 586 587#################################################################################################################################### 588# backupCompare 589#################################################################################################################################### 590sub backupCompare 591{ 592 my $self = shift; 593 594 # Assign function parameters, defaults, and log debug info 595 my 596 ( 597 $strOperation, 598 $strBackup, 599 $oExpectedManifest, 600 ) = 601 logDebugParam 602 ( 603 __PACKAGE__ . '->backupCompare', \@_, 604 {name => 'strBackup', trace => true}, 605 {name => 'oExpectedManifest', trace => true}, 606 ); 607 608 ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup; 609 610 my $oActualManifest = new pgBackRestTest::Env::Manifest( 611 $self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()}); 612 613 ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} = 614 $oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START); 615 ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP} = 616 $oActualManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP); 617 ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START} = 618 $oActualManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START); 619 ${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = 620 $oActualManifest->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM); 621 ${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = REPOSITORY_FORMAT + 0; 622 623 if (defined($oExpectedManifest->{&INI_SECTION_CIPHER}) && 624 defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}) && 625 $oActualManifest->test(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS)) 626 { 627 $oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS} = 628 $oActualManifest->get(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS); 629 } 630 631 # Update the expected manifest with whether the --delta option was used or not to perform the backup. 632 $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA} = 633 $oActualManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA) ? INI_TRUE : INI_FALSE; 634 635 my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH); 636 637 foreach my $strFileKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE)) 638 { 639 # Determine repo size if compression or encryption is enabled 640 my $strCompressType = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}; 641 642 if ($strCompressType ne NONE || 643 (defined($oExpectedManifest->{&INI_SECTION_CIPHER}) && 644 defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}))) 645 { 646 647 my $lRepoSize = 648 $oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ? 649 $oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) : 650 (storageRepo()->info( 651 $self->repoBackupPath("${strBackup}/${strFileKey}") . 652 ($strCompressType eq NONE ? '' : ".${strCompressType}")))->{size}; 653 654 if (defined($lRepoSize) && 655 $lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE}) 656 { 657 $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_REPO_SIZE} = $lRepoSize; 658 } 659 } 660 661 # If the backup does not have page checksums then no need to compare 662 if (!$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}) 663 { 664 delete($oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}); 665 delete($oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); 666 } 667 # Else make sure things that should have checks do have checks 668 elsif ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_CHECKSUM_PAGE) != 669 isChecksumPage($strFileKey)) 670 { 671 confess 672 "check-page actual for ${strFileKey} is " . 673 ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, 674 MANIFEST_SUBKEY_CHECKSUM_PAGE) ? 'set' : '[undef]') . 675 ' but isChecksumPage() says it should be ' . 676 (isChecksumPage($strFileKey) ? 'set' : 'undef') . '.'; 677 } 678 } 679 680 $self->manifestDefault($oExpectedManifest); 681 682 my $strTestPath = $self->testPath(); 683 684 storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent})); 685 storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifest)); 686 687 executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest"); 688 689 storageTest()->remove("${strTestPath}/expected.manifest"); 690 storageTest()->remove("${strTestPath}/actual.manifest"); 691 692 # Return from function and log return values if any 693 return logDebugReturn($strOperation); 694} 695 696#################################################################################################################################### 697# manifestDefault 698#################################################################################################################################### 699sub manifestDefault 700{ 701 my $self = shift; 702 my $oExpectedManifest = shift; 703 704 # Set defaults for subkeys that tend to repeat 705 foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK) 706 { 707 foreach my $strSubKey (&MANIFEST_SUBKEY_USER, &MANIFEST_SUBKEY_GROUP, &MANIFEST_SUBKEY_MODE, &MANIFEST_SUBKEY_MASTER) 708 { 709 my %oDefault; 710 my $iSectionTotal = 0; 711 712 next if !defined($oExpectedManifest->{$strSection}); 713 714 foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}})) 715 { 716 my $strValue = $oExpectedManifest->{$strSection}{$strFile}{$strSubKey}; 717 718 if (defined($strValue)) 719 { 720 if (defined($oDefault{$strValue})) 721 { 722 $oDefault{$strValue}++; 723 } 724 else 725 { 726 $oDefault{$strValue} = 1; 727 } 728 } 729 730 $iSectionTotal++; 731 } 732 733 my $strMaxValue; 734 my $iMaxValueTotal = 0; 735 736 foreach my $strValue (sort(keys(%oDefault))) 737 { 738 if ($oDefault{$strValue} > $iMaxValueTotal) 739 { 740 $iMaxValueTotal = $oDefault{$strValue}; 741 $strMaxValue = $strValue; 742 } 743 } 744 745 if (defined($strMaxValue) > 0 && $iMaxValueTotal > $iSectionTotal * MANIFEST_DEFAULT_MATCH_FACTOR) 746 { 747 if ($strSubKey eq MANIFEST_SUBKEY_MASTER) 748 { 749 $oExpectedManifest->{"${strSection}:default"}{$strSubKey} = $strMaxValue ? JSON::PP::true : JSON::PP::false; 750 } 751 else 752 { 753 $oExpectedManifest->{"${strSection}:default"}{$strSubKey} = $strMaxValue; 754 } 755 756 foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}})) 757 { 758 if (defined($oExpectedManifest->{$strSection}{$strFile}{$strSubKey}) && 759 $oExpectedManifest->{$strSection}{$strFile}{$strSubKey} eq $strMaxValue) 760 { 761 delete($oExpectedManifest->{$strSection}{$strFile}{$strSubKey}); 762 } 763 } 764 } 765 } 766 } 767} 768 769#################################################################################################################################### 770# backupLast 771#################################################################################################################################### 772sub backupLast 773{ 774 my $self = shift; 775 my $iRepo = shift; 776 777 my @stryBackup = storageRepo({iRepo => $iRepo})->list( 778 $self->repoBackupPath(undef, $iRepo), 779 {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'}); 780 781 if (!defined($stryBackup[0])) 782 { 783 confess 'no backup was found: ' . join(@stryBackup, ', '); 784 } 785 786 return $stryBackup[0]; 787} 788 789#################################################################################################################################### 790# check 791#################################################################################################################################### 792sub check 793{ 794 my $self = shift; 795 796 # Assign function parameters, defaults, and log debug info 797 my 798 ( 799 $strOperation, 800 $strComment, 801 $oParam, 802 ) = 803 logDebugParam 804 ( 805 __PACKAGE__ . '->check', \@_, 806 {name => 'strComment'}, 807 {name => 'oParam', required => false}, 808 ); 809 810 $strComment = 811 'check ' . $self->stanza() . ' - ' . $strComment . 812 ' (' . $self->nameGet() . ' host)'; 813 &log(INFO, " $strComment"); 814 815 $self->executeSimple( 816 $self->backrestExe() . 817 ' --config=' . $self->backrestConfig() . 818 (defined($$oParam{iTimeout}) ? " --archive-timeout=$$oParam{iTimeout}" : '') . 819 (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . 820 ' --stanza=' . $self->stanza() . ' check', 821 {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, oLogTest => $self->{oLogTest}, 822 bLogOutput => $self->synthetic()}); 823 824 # Return from function and log return values if any 825 return logDebugReturn($strOperation); 826} 827 828#################################################################################################################################### 829# expire 830#################################################################################################################################### 831sub expire 832{ 833 my $self = shift; 834 835 # Assign function parameters, defaults, and log debug info 836 my 837 ( 838 $strOperation, 839 $oParam, 840 ) = 841 logDebugParam 842 ( 843 __PACKAGE__ . '->check', \@_, 844 {name => 'oParam', required => false}, 845 ); 846 847 my $strComment = 848 'expire' . 849 (defined($$oParam{iRetentionFull}) ? " full=$$oParam{iRetentionFull}" : '') . 850 (defined($$oParam{iRetentionDiff}) ? " diff=$$oParam{iRetentionDiff}" : '') . 851 (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . 852 ' (' . $self->nameGet() . ' host)'; 853 &log(INFO, " ${strComment}"); 854 855 # Determine whether or not to expect an error 856 my $oHostGroup = hostGroupGet(); 857 858 $self->executeSimple( 859 $self->backrestExe() . 860 ' --config=' . $self->backrestConfig() . 861 (defined($$oParam{iRetentionFull}) ? " --repo1-retention-full=$$oParam{iRetentionFull}" : '') . 862 (defined($$oParam{iRetentionDiff}) ? " --repo1-retention-diff=$$oParam{iRetentionDiff}" : '') . 863 (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . 864 ' --repo=' . (defined($oParam->{iRepo}) ? $oParam->{iRepo} : '1') . 865 ' --stanza=' . $self->stanza() . ' expire', 866 {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, oLogTest => $self->{oLogTest}, 867 bLogOutput => $self->synthetic()}); 868} 869 870#################################################################################################################################### 871# info 872#################################################################################################################################### 873sub info 874{ 875 my $self = shift; 876 877 # Assign function parameters, defaults, and log debug info 878 my 879 ( 880 $strOperation, 881 $strComment, 882 $oParam, 883 ) = 884 logDebugParam 885 ( 886 __PACKAGE__ . '->info', \@_, 887 {name => 'strComment'}, 888 {name => 'oParam', required => false}, 889 ); 890 891 $strComment = 892 'info' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . ' - ' . $strComment . 893 ' (' . $self->nameGet() . ' host)'; 894 &log(INFO, " $strComment"); 895 896 $self->executeSimple( 897 $self->backrestExe() . 898 ' --config=' . $self->backrestConfig() . 899 ' --log-level-console=warn' . 900 (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . 901 (defined($$oParam{strOutput}) ? " --output=$$oParam{strOutput}" : '') . ' info', 902 {strComment => $strComment, oLogTest => $self->{oLogTest}, bLogOutput => $self->synthetic()}); 903 904 # Return from function and log return values if any 905 return logDebugReturn($strOperation); 906} 907 908#################################################################################################################################### 909# stanzaCreate 910#################################################################################################################################### 911sub stanzaCreate 912{ 913 my $self = shift; 914 915 # Assign function parameters, defaults, and log debug info 916 my 917 ( 918 $strOperation, 919 $strComment, 920 $oParam, 921 ) = 922 logDebugParam 923 ( 924 __PACKAGE__ . '->stanzaCreate', \@_, 925 {name => 'strComment'}, 926 {name => 'oParam', required => false}, 927 ); 928 929 $strComment = 930 'stanza-create ' . $self->stanza() . ' - ' . $strComment . 931 ' (' . $self->nameGet() . ' host)'; 932 &log(INFO, " $strComment"); 933 934 $self->executeSimple( 935 $self->backrestExe() . 936 ' --config=' . $self->backrestConfig() . 937 ' --stanza=' . $self->stanza() . 938 (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . 939 ' stanza-create', 940 {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, oLogTest => $self->{oLogTest}, 941 bLogOutput => $self->synthetic()}); 942 943 if (storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO)) 944 { 945 # If the info file was created, then add it to the expect log 946 if (defined($self->{oLogTest}) && $self->synthetic()) 947 { 948 $self->{oLogTest}->supplementalAdd( 949 $self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo()->get($self->repoBackupPath(FILE_BACKUP_INFO))}); 950 } 951 952 # Get the passphrase for accessing the manifest file 953 $self->{strCipherPassManifest} = (new pgBackRestTest::Env::BackupInfo($self->repoBackupPath()))->cipherPassSub(); 954 } 955 956 if (storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE)) 957 { 958 # If the info file was created, then add it to the expect log 959 if (defined($self->{oLogTest}) && $self->synthetic()) 960 { 961 $self->{oLogTest}->supplementalAdd( 962 $self->repoArchivePath(ARCHIVE_INFO_FILE), undef, ${storageRepo()->get($self->repoArchivePath(ARCHIVE_INFO_FILE))}); 963 } 964 965 # Get the passphrase for accessing the archived files 966 $self->{strCipherPassArchive} = 967 (new pgBackRestTest::Env::ArchiveInfo($self->repoArchivePath()))->cipherPassSub(); 968 } 969 970 # Return from function and log return values if any 971 return logDebugReturn($strOperation); 972} 973 974#################################################################################################################################### 975# stanzaUpgrade 976#################################################################################################################################### 977sub stanzaUpgrade 978{ 979 my $self = shift; 980 981 # Assign function parameters, defaults, and log debug info 982 my 983 ( 984 $strOperation, 985 $strComment, 986 $oParam, 987 ) = 988 logDebugParam 989 ( 990 __PACKAGE__ . '->stanzaUpgrade', \@_, 991 {name => 'strComment'}, 992 {name => 'oParam', required => false}, 993 ); 994 995 $strComment = 996 'stanza-upgrade ' . $self->stanza() . ' - ' . $strComment . 997 ' (' . $self->nameGet() . ' host)'; 998 &log(INFO, " $strComment"); 999 1000 $self->executeSimple( 1001 $self->backrestExe() . 1002 ' --config=' . $self->backrestConfig() . 1003 ' --stanza=' . $self->stanza() . 1004 (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . 1005 ' stanza-upgrade', 1006 {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, oLogTest => $self->{oLogTest}, 1007 bLogOutput => $self->synthetic()}); 1008 1009 # If the info file was created, then add it to the expect log 1010 if (defined($self->{oLogTest}) && $self->synthetic() && 1011 storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO)) 1012 { 1013 $self->{oLogTest}->supplementalAdd( 1014 $self->repoBackupPath(FILE_BACKUP_INFO), undef, ${storageRepo()->get($self->repoBackupPath(FILE_BACKUP_INFO))}); 1015 } 1016 1017 if (defined($self->{oLogTest}) && $self->synthetic() && 1018 storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE)) 1019 { 1020 $self->{oLogTest}->supplementalAdd( 1021 $self->repoArchivePath(ARCHIVE_INFO_FILE), undef, ${storageRepo()->get($self->repoArchivePath(ARCHIVE_INFO_FILE))}); 1022 } 1023 1024 # Return from function and log return values if any 1025 return logDebugReturn($strOperation); 1026} 1027 1028#################################################################################################################################### 1029# stanzaDelete 1030#################################################################################################################################### 1031sub stanzaDelete 1032{ 1033 my $self = shift; 1034 1035 # Assign function parameters, defaults, and log debug info 1036 my 1037 ( 1038 $strOperation, 1039 $strComment, 1040 $oParam, 1041 ) = 1042 logDebugParam 1043 ( 1044 __PACKAGE__ . '->stanzaDelete', \@_, 1045 {name => 'strComment'}, 1046 {name => 'oParam', required => false}, 1047 ); 1048 1049 $strComment = 1050 'stanza-delete ' . $self->stanza() . ' - ' . $strComment . 1051 ' (' . $self->nameGet() . ' host)'; 1052 &log(INFO, " $strComment"); 1053 1054 $self->executeSimple( 1055 $self->backrestExe() . 1056 ' --config=' . $self->backrestConfig() . 1057 ' --repo=' . (defined($oParam->{iRepo}) ? $oParam->{iRepo} : '1') . 1058 ' --stanza=' . $self->stanza() . 1059 (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . 1060 ' stanza-delete', 1061 {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, oLogTest => $self->{oLogTest}, 1062 bLogOutput => $self->synthetic()}); 1063 1064 if (defined($self->{oLogTest}) && $self->synthetic()) 1065 { 1066 $self->{oLogTest}->logAdd( 1067 'list backup', $self->stanza() . ' must not exist for successful delete', 1068 join("\n", storageRepo()->list('backup'))); 1069 $self->{oLogTest}->logAdd( 1070 'list archive', $self->stanza() . ' must not exist for successful delete', 1071 join("\n", storageRepo()->list('archive'))); 1072 } 1073 1074 # Return from function and log return values if any 1075 return logDebugReturn($strOperation); 1076} 1077 1078#################################################################################################################################### 1079# start 1080#################################################################################################################################### 1081sub start 1082{ 1083 my $self = shift; 1084 1085 # Assign function parameters, defaults, and log debug info 1086 my 1087 ( 1088 $strOperation, 1089 $oParam, 1090 ) = 1091 logDebugParam 1092 ( 1093 __PACKAGE__ . '->start', \@_, 1094 {name => 'oParam', required => false}, 1095 ); 1096 1097 my $strComment = 1098 'start' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . 1099 ' (' . $self->nameGet() . ' host)'; 1100 &log(INFO, " $strComment"); 1101 1102 $self->executeSimple( 1103 $self->backrestExe() . 1104 ' --config=' . $self->backrestConfig() . 1105 (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . ' start', 1106 {strComment => $strComment, oLogTest => $self->{oLogTest}, bLogOutput => $self->synthetic()}); 1107} 1108 1109#################################################################################################################################### 1110# stop 1111#################################################################################################################################### 1112sub stop 1113{ 1114 my $self = shift; 1115 1116 # Assign function parameters, defaults, and log debug info 1117 my 1118 ( 1119 $strOperation, 1120 $oParam, 1121 ) = 1122 logDebugParam 1123 ( 1124 __PACKAGE__ . '->stop', \@_, 1125 {name => 'oParam', required => false}, 1126 ); 1127 1128 my $strComment = 1129 'stop' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . 1130 ' (' . $self->nameGet() . ' host)'; 1131 &log(INFO, " $strComment"); 1132 1133 $self->executeSimple( 1134 $self->backrestExe() . 1135 ' --config=' . $self->backrestConfig() . 1136 (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . 1137 (defined($$oParam{bForce}) && $$oParam{bForce} ? ' --force' : '') . ' stop', 1138 {strComment => $strComment, oLogTest => $self->{oLogTest}, bLogOutput => $self->synthetic()}); 1139 1140 # Return from function and log return values if any 1141 return logDebugReturn($strOperation); 1142} 1143 1144#################################################################################################################################### 1145# configCreate 1146#################################################################################################################################### 1147sub configCreate 1148{ 1149 my $self = shift; 1150 1151 # Assign function parameters, defaults, and log debug info 1152 my 1153 ( 1154 $strOperation, 1155 $oParam, 1156 ) = 1157 logDebugParam 1158 ( 1159 __PACKAGE__ . '->stop', \@_, 1160 {name => 'oParam', required => false}, 1161 ); 1162 1163 my %oParamHash; 1164 my $strStanza = $self->stanza(); 1165 my $oHostGroup = hostGroupGet(); 1166 my $oHostBackup = $oHostGroup->hostGet($self->backupDestination()); 1167 my $oHostDbPrimary = $oHostGroup->hostGet(HOST_DB_PRIMARY); 1168 my $oHostDbStandby = $oHostGroup->hostGet(HOST_DB_STANDBY, true); 1169 1170 my $bArchiveAsync = defined($$oParam{bArchiveAsync}) ? $$oParam{bArchiveAsync} : false; 1171 1172 my $iRepoTotal = defined($oParam->{iRepoTotal}) ? $oParam->{iRepoTotal} : 1; 1173 1174 if ($iRepoTotal < 1 || $iRepoTotal > 2) 1175 { 1176 confess "invalid repo total ${iRepoTotal}"; 1177 } 1178 1179 # General options 1180 # ------------------------------------------------------------------------------------------------------------------------------ 1181 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'job-retry'} = 0; 1182 1183 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-console'} = lc(DETAIL); 1184 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-file'} = testRunGet()->logLevelTestFile(); 1185 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-stderr'} = lc(OFF); 1186 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-subprocess'} = 1187 testRunGet()->logLevelTestFile() eq lc(OFF) ? 'n' : 'y'; 1188 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-timestamp'} = 'n'; 1189 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'buffer-size'} = '64k'; 1190 1191 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath(); 1192 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath(); 1193 1194 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'protocol-timeout'} = 60; 1195 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'db-timeout'} = 45; 1196 1197 # Set to make sure that changing the default works and to speed compression for testing 1198 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level'} = 3; 1199 1200 # Only set network compress level if there is more than one host 1201 if ($oHostBackup != $oHostDbPrimary) 1202 { 1203 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level-network'} = 1; 1204 } 1205 1206 if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne 'gz') 1207 { 1208 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-type'} = $oParam->{strCompressType}; 1209 } 1210 1211 if ($self->isHostBackup()) 1212 { 1213 if ($self->repoEncrypt()) 1214 { 1215 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-type'} = CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC; 1216 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-pass'} = 'x'; 1217 } 1218 1219 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-path'} = $self->repoPath(); 1220 1221 # S3 settings 1222 if ($oParam->{strStorage} eq S3) 1223 { 1224 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = S3; 1225 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key'} = HOST_S3_ACCESS_KEY; 1226 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key-secret'} = HOST_S3_ACCESS_SECRET_KEY; 1227 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-bucket'} = HOST_S3_BUCKET; 1228 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-endpoint'} = HOST_S3_ENDPOINT; 1229 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-region'} = HOST_S3_REGION; 1230 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'} = 'n'; 1231 } 1232 elsif ($oParam->{strStorage} eq AZURE) 1233 { 1234 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = AZURE; 1235 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-account'} = HOST_AZURE_ACCOUNT; 1236 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-key'} = HOST_AZURE_KEY; 1237 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-container'} = HOST_AZURE_CONTAINER; 1238 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-host'} = HOST_AZURE; 1239 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-verify-tls'} = 'n'; 1240 } 1241 elsif ($oParam->{strStorage} eq GCS) 1242 { 1243 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = GCS; 1244 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-bucket'} = HOST_GCS_BUCKET; 1245 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-key-type'} = HOST_GCS_KEY_TYPE; 1246 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-key'} = HOST_GCS_KEY; 1247 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-endpoint'} = HOST_GCS . ':' . HOST_GCS_PORT; 1248 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-storage-verify-tls'} = 'n'; 1249 } 1250 1251 if ($iRepoTotal == 2) 1252 { 1253 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-path'} = $self->repo2Path(); 1254 } 1255 1256 if (defined($$oParam{bHardlink}) && $$oParam{bHardlink}) 1257 { 1258 $self->{bHardLink} = true; 1259 $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'repo1-s3-hardlink'} = 'y'; 1260 } 1261 1262 $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'archive-copy'} = 'y'; 1263 $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'start-fast'} = 'y'; 1264 } 1265 1266 # Host specific options 1267 # ------------------------------------------------------------------------------------------------------------------------------ 1268 1269 # If this is the backup host 1270 if ($self->isHostBackup()) 1271 { 1272 my $oHostDb1 = $oHostDbPrimary; 1273 my $oHostDb2 = $oHostDbStandby; 1274 1275 if ($self->nameTest(HOST_DB_STANDBY)) 1276 { 1277 $oHostDb1 = $oHostDbStandby; 1278 $oHostDb2 = $oHostDbPrimary; 1279 } 1280 1281 if ($self->nameTest(HOST_BACKUP)) 1282 { 1283 $oParamHash{$strStanza}{'pg1-host'} = $oHostDb1->nameGet(); 1284 $oParamHash{$strStanza}{'pg1-host-user'} = $oHostDb1->userGet(); 1285 $oParamHash{$strStanza}{'pg1-host-cmd'} = $oHostDb1->backrestExe(); 1286 $oParamHash{$strStanza}{'pg1-host-config'} = $oHostDb1->backrestConfig(); 1287 1288 # Port can't be configured for a synthetic host 1289 if (!$self->synthetic()) 1290 { 1291 $oParamHash{$strStanza}{'pg1-port'} = $oHostDb1->pgPort(); 1292 } 1293 } 1294 1295 $oParamHash{$strStanza}{'pg1-path'} = $oHostDb1->dbBasePath(); 1296 1297 if (defined($oHostDb2)) 1298 { 1299 # Add an invalid replica to simulate more than one replica. A warning should be thrown when a stanza is created and a 1300 # valid replica should be chosen. 1301 $oParamHash{$strStanza}{"pg2-host"} = BOGUS; 1302 $oParamHash{$strStanza}{"pg2-host-user"} = $oHostDb2->userGet(); 1303 $oParamHash{$strStanza}{"pg2-host-cmd"} = $oHostDb2->backrestExe(); 1304 $oParamHash{$strStanza}{"pg2-host-config"} = $oHostDb2->backrestConfig(); 1305 $oParamHash{$strStanza}{"pg2-path"} = $oHostDb2->dbBasePath(); 1306 1307 # Set a flag so we know there's a bogus host 1308 $self->{bBogusHost} = true; 1309 1310 # Set a valid replica to a higher index to ensure skipping indexes does not make a difference 1311 $oParamHash{$strStanza}{"pg8-host"} = $oHostDb2->nameGet(); 1312 $oParamHash{$strStanza}{"pg8-host-user"} = $oHostDb2->userGet(); 1313 $oParamHash{$strStanza}{"pg8-host-cmd"} = $oHostDb2->backrestExe(); 1314 $oParamHash{$strStanza}{"pg8-host-config"} = $oHostDb2->backrestConfig(); 1315 $oParamHash{$strStanza}{"pg8-path"} = $oHostDb2->dbBasePath(); 1316 1317 # Only test explicit ports on the backup server. This is so locally configured ports are also tested. 1318 if (!$self->synthetic() && $self->nameTest(HOST_BACKUP)) 1319 { 1320 $oParamHash{$strStanza}{"pg8-port"} = $oHostDb2->pgPort(); 1321 } 1322 } 1323 } 1324 1325 # If this is a database host 1326 if ($self->isHostDb()) 1327 { 1328 $oParamHash{$strStanza}{'pg1-path'} = $self->dbBasePath(); 1329 1330 if (!$self->synthetic()) 1331 { 1332 $oParamHash{$strStanza}{'pg1-socket-path'} = $self->pgSocketPath(); 1333 $oParamHash{$strStanza}{'pg1-port'} = $self->pgPort(); 1334 } 1335 1336 if ($bArchiveAsync) 1337 { 1338 $oParamHash{&CFGDEF_SECTION_GLOBAL . ':archive-push'}{'archive-async'} = 'y'; 1339 } 1340 1341 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'spool-path'} = $self->spoolPath(); 1342 1343 # If the backup host is remote 1344 if (!$self->isHostBackup()) 1345 { 1346 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host'} = $oHostBackup->nameGet(); 1347 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-user'} = $oHostBackup->userGet(); 1348 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-cmd'} = $oHostBackup->backrestExe(); 1349 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-config'} = $oHostBackup->backrestConfig(); 1350 1351 if ($iRepoTotal == 2) 1352 { 1353 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host'} = $oHostBackup->nameGet(); 1354 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-user'} = $oHostBackup->userGet(); 1355 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-cmd'} = $oHostBackup->backrestExe(); 1356 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-config'} = $oHostBackup->backrestConfig(); 1357 } 1358 1359 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath(); 1360 $oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath(); 1361 } 1362 } 1363 1364 # Write out the configuration file 1365 storageTest()->put($self->backrestConfig(), iniRender(\%oParamHash, true)); 1366} 1367 1368#################################################################################################################################### 1369# configUpdate - update configuration with new options 1370#################################################################################################################################### 1371sub configUpdate 1372{ 1373 my $self = shift; 1374 1375 # Assign function parameters, defaults, and log debug info 1376 my 1377 ( 1378 $strOperation, 1379 $hParam, 1380 ) = 1381 logDebugParam 1382 ( 1383 __PACKAGE__ . '->configUpdate', \@_, 1384 {name => 'hParam'}, 1385 ); 1386 1387 # Load db config file 1388 my $oConfig = iniParse(${storageTest()->get($self->backrestConfig())}, {bRelaxed => true}); 1389 1390 # Load params 1391 foreach my $strSection (keys(%{$hParam})) 1392 { 1393 foreach my $strKey (keys(%{$hParam->{$strSection}})) 1394 { 1395 if (defined($hParam->{$strSection}{$strKey})) 1396 { 1397 $oConfig->{$strSection}{$strKey} = $hParam->{$strSection}{$strKey}; 1398 } 1399 else 1400 { 1401 delete($oConfig->{$strSection}{$strKey}); 1402 } 1403 } 1404 } 1405 1406 storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); 1407 1408 # Return from function and log return values if any 1409 return logDebugReturn($strOperation); 1410} 1411 1412#################################################################################################################################### 1413# manifestMunge 1414# 1415# Allows for munging of the manifest while making it appear to be valid. This is used to create various error conditions that 1416# should be caught by the unit tests. 1417#################################################################################################################################### 1418sub manifestMunge 1419{ 1420 my $self = shift; 1421 1422 # Assign function parameters, defaults, and log debug info 1423 my 1424 ( 1425 $strOperation, 1426 $strBackup, 1427 $hParam, 1428 $bCache, 1429 ) = 1430 logDebugParam 1431 ( 1432 __PACKAGE__ . '->manifestMunge', \@_, 1433 {name => 'strBackup'}, 1434 {name => '$hParam'}, 1435 {name => 'bCache', default => true}, 1436 ); 1437 1438 $self->infoMunge($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $hParam, $bCache, true); 1439 1440 # Return from function and log return values if any 1441 return logDebugReturn($strOperation); 1442} 1443 1444#################################################################################################################################### 1445# manifestRestore 1446#################################################################################################################################### 1447sub manifestRestore 1448{ 1449 my $self = shift; 1450 1451 # Assign function parameters, defaults, and log debug info 1452 my 1453 ( 1454 $strOperation, 1455 $strBackup, 1456 $bSave, 1457 ) = 1458 logDebugParam 1459 ( 1460 __PACKAGE__ . '->manifestRestore', \@_, 1461 {name => 'strBackup'}, 1462 {name => 'bSave', default => true}, 1463 ); 1464 1465 $self->infoRestore($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $bSave); 1466 1467 # Return from function and log return values if any 1468 return logDebugReturn($strOperation); 1469} 1470 1471#################################################################################################################################### 1472# infoMunge 1473# 1474# With the file name specified (e.g. /repo/archive/db/archive.info) copy the current values from the file into the global hash and 1475# update the file with the new values passed. Later, using infoRestore, the global variable will be used to restore the file to its 1476# original state. 1477#################################################################################################################################### 1478sub infoMunge 1479{ 1480 my $self = shift; 1481 1482 # Assign function parameters, defaults, and log debug info 1483 my 1484 ( 1485 $strOperation, 1486 $strFileName, 1487 $hParam, 1488 $bCache, 1489 $bManifest, 1490 ) = 1491 logDebugParam 1492 ( 1493 __PACKAGE__ . '->infoMunge', \@_, 1494 {name => 'strFileName'}, 1495 {name => 'hParam'}, 1496 {name => 'bCache', default => true}, 1497 {name => 'bManifest', default => false}, 1498 ); 1499 1500 # If the original file content does not exist then load it 1501 if (!defined($self->{hInfoFile}{$strFileName})) 1502 { 1503 $self->{hInfoFile}{$strFileName} = new pgBackRestDoc::Common::Ini( 1504 storageRepo(), $strFileName, 1505 {strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()}); 1506 } 1507 1508 # Make a copy of the original file contents 1509 my $oMungeIni = new pgBackRestDoc::Common::Ini( 1510 storageRepo(), $strFileName, 1511 {bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}), 1512 strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()}); 1513 1514 # Load params 1515 foreach my $strSection (keys(%{$hParam})) 1516 { 1517 foreach my $strKey (keys(%{$hParam->{$strSection}})) 1518 { 1519 if (ref($hParam->{$strSection}{$strKey}) eq 'HASH') 1520 { 1521 foreach my $strSubKey (keys(%{$hParam->{$strSection}{$strKey}})) 1522 { 1523 # Munge the copy with the new parameter values 1524 if (defined($hParam->{$strSection}{$strKey}{$strSubKey})) 1525 { 1526 $oMungeIni->set($strSection, $strKey, $strSubKey, $hParam->{$strSection}{$strKey}{$strSubKey}); 1527 } 1528 else 1529 { 1530 $oMungeIni->remove($strSection, $strKey, $strSubKey); 1531 } 1532 } 1533 } 1534 else 1535 { 1536 # Munge the copy with the new parameter values 1537 if (defined($hParam->{$strSection}{$strKey})) 1538 { 1539 $oMungeIni->set($strSection, $strKey, undef, $hParam->{$strSection}{$strKey}); 1540 } 1541 else 1542 { 1543 $oMungeIni->remove($strSection, $strKey); 1544 } 1545 } 1546 } 1547 } 1548 1549 # Save the munged data to the file 1550 $oMungeIni->save(); 1551 1552 # Clear the cache is requested 1553 if (!$bCache) 1554 { 1555 delete($self->{hInfoFile}{$strFileName}); 1556 } 1557 1558 # Return from function and log return values if any 1559 return logDebugReturn($strOperation); 1560} 1561 1562#################################################################################################################################### 1563# infoRestore 1564# 1565# With the file name specified (e.g. /repo/archive/db/archive.info) use the original file contents in the global hash to restore the 1566# file to its original state after modifying the values with infoMunge. 1567#################################################################################################################################### 1568sub infoRestore 1569{ 1570 my $self = shift; 1571 1572 # Assign function parameters, defaults, and log debug info 1573 my 1574 ( 1575 $strOperation, 1576 $strFileName, 1577 $bSave, 1578 ) = 1579 logDebugParam 1580 ( 1581 __PACKAGE__ . '->infoRestore', \@_, 1582 {name => 'strFileName'}, 1583 {name => 'bSave', default => true}, 1584 ); 1585 1586 # If the original file content exists in the global hash, then save it to the file 1587 if (defined($self->{hInfoFile}{$strFileName})) 1588 { 1589 if ($bSave) 1590 { 1591 # Save the munged data to the file 1592 $self->{hInfoFile}{$strFileName}->{bModified} = true; 1593 $self->{hInfoFile}{$strFileName}->save(); 1594 } 1595 } 1596 else 1597 { 1598 confess &log(ASSERT, "There is no original data cached for $strFileName. infoMunge must be called first."); 1599 } 1600 1601 # Remove the element from the hash 1602 delete($self->{hInfoFile}{$strFileName}); 1603 1604 # Return from function and log return values if any 1605 return logDebugReturn($strOperation); 1606} 1607 1608#################################################################################################################################### 1609# configRecovery 1610#################################################################################################################################### 1611sub configRecovery 1612{ 1613 my $self = shift; 1614 my $oHostBackup = shift; 1615 my $oRecoveryHashRef = shift; 1616 1617 # Get stanza 1618 my $strStanza = $self->stanza(); 1619 1620 # Load db config file 1621 my $oConfig = iniParse(${storageTest->get($self->backrestConfig())}, {bRelaxed => true}); 1622 1623 # Rewrite recovery options 1624 my @stryRecoveryOption; 1625 1626 foreach my $strOption (sort(keys(%$oRecoveryHashRef))) 1627 { 1628 push (@stryRecoveryOption, "${strOption}=${$oRecoveryHashRef}{$strOption}"); 1629 } 1630 1631 if (@stryRecoveryOption) 1632 { 1633 $oConfig->{$strStanza}{'recovery-option'} = \@stryRecoveryOption; 1634 } 1635 1636 # Save db config file 1637 storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); 1638} 1639 1640#################################################################################################################################### 1641# configRemap 1642#################################################################################################################################### 1643sub configRemap 1644{ 1645 my $self = shift; 1646 my $oRemapHashRef = shift; 1647 my $oManifestRef = shift; 1648 1649 # Get stanza name 1650 my $strStanza = $self->stanza(); 1651 1652 # Load db config file 1653 my $oConfig = iniParse(${storageTest()->get($self->backrestConfig())}, {bRelaxed => true}); 1654 1655 # Load backup config file 1656 my $oRemoteConfig; 1657 my $oHostBackup = 1658 !$self->standby() && !$self->nameTest($self->backupDestination()) ? 1659 hostGroupGet()->hostGet($self->backupDestination()) : undef; 1660 1661 if (defined($oHostBackup)) 1662 { 1663 $oRemoteConfig = iniParse(${storageTest()->get($oHostBackup->backrestConfig())}, {bRelaxed => true}); 1664 } 1665 1666 # Rewrite recovery section 1667 delete($oConfig->{"${strStanza}:restore"}{'tablespace-map'}); 1668 my @stryTablespaceMap; 1669 1670 foreach my $strRemap (sort(keys(%$oRemapHashRef))) 1671 { 1672 my $strRemapPath = ${$oRemapHashRef}{$strRemap}; 1673 1674 if ($strRemap eq MANIFEST_TARGET_PGDATA) 1675 { 1676 $oConfig->{$strStanza}{'pg1-path'} = $strRemapPath; 1677 1678 ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; 1679 1680 if (defined($oHostBackup)) 1681 { 1682 $oRemoteConfig->{$strStanza}{'pg1-path'} = $strRemapPath; 1683 } 1684 } 1685 else 1686 { 1687 my $strTablespaceOid = (split('\/', $strRemap))[1]; 1688 push (@stryTablespaceMap, "${strTablespaceOid}=${strRemapPath}"); 1689 1690 ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strRemap}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; 1691 ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{MANIFEST_TARGET_PGDATA . "/${strRemap}"}{destination} = $strRemapPath; 1692 } 1693 } 1694 1695 if (@stryTablespaceMap) 1696 { 1697 $oConfig->{"${strStanza}:restore"}{'tablespace-map'} = \@stryTablespaceMap; 1698 } 1699 1700 # Save db config file 1701 storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); 1702 1703 # Save backup config file (but not if this is the standby which is not the source of backups) 1704 if (defined($oHostBackup)) 1705 { 1706 storageTest()->put($oHostBackup->backrestConfig(), iniRender($oRemoteConfig, true)); 1707 } 1708} 1709 1710#################################################################################################################################### 1711# restore 1712#################################################################################################################################### 1713sub restore 1714{ 1715 my $self = shift; 1716 1717 # Assign function parameters, defaults, and log debug info 1718 my 1719 ( 1720 $strOperation, 1721 $strComment, 1722 $strBackup, 1723 $rhExpectedManifest, 1724 $rhRemapHash, 1725 $bDelta, 1726 $bForce, 1727 $strType, 1728 $strTarget, 1729 $bTargetExclusive, 1730 $strTargetAction, 1731 $strTargetTimeline, 1732 $rhRecoveryHash, 1733 $iExpectedExitStatus, 1734 $strOptionalParam, 1735 $bTablespace, 1736 $strUser, 1737 $strBackupExpected, 1738 $iRepo, 1739 ) = 1740 logDebugParam 1741 ( 1742 __PACKAGE__ . '->restore', \@_, 1743 {name => 'strComment', required => false}, 1744 {name => 'strBackup'}, 1745 {name => 'rhExpectedManifest', optional => true}, 1746 {name => 'rhRemapHash', optional => true}, 1747 {name => 'bDelta', optional => true, default => false}, 1748 {name => 'bForce', optional => true, default => false}, 1749 {name => 'strType', optional => true}, 1750 {name => 'strTarget', optional => true}, 1751 {name => 'bTargetExclusive', optional => true, default => false}, 1752 {name => 'strTargetAction', optional => true}, 1753 {name => 'strTargetTimeline', optional => true}, 1754 {name => 'rhRecoveryHash', optional => true}, 1755 {name => 'iExpectedExitStatus', optional => true}, 1756 {name => 'strOptionalParam', optional => true}, 1757 {name => 'bTablespace', optional => true}, 1758 {name => 'strUser', optional => true}, 1759 {name => 'strBackupExpected', optional => true}, 1760 {name => 'iRepo', optional => true}, 1761 ); 1762 1763 # Build link map options 1764 my $strLinkMap; 1765 1766 foreach my $strTarget (sort(keys(%{$self->{hLinkRemap}}))) 1767 { 1768 $strLinkMap .= " --link-map=\"${strTarget}=${$self->{hLinkRemap}}{$strTarget}\""; 1769 } 1770 1771 $strComment = 'restore' . 1772 ($bDelta ? ' delta' : '') . 1773 ($bForce ? ', force' : '') . 1774 ($strBackup ne 'latest' ? ", backup '${strBackup}'" : '') . 1775 # This does not output 'default' for synthetic tests to make expect logs match up (may change later) 1776 ($strType ? ", type '${strType}'" : (defined($rhExpectedManifest) ? '' : ", type 'default'")) . 1777 ($strTarget ? ", target '${strTarget}'" : '') . 1778 ($strTargetTimeline ? ", timeline '${strTargetTimeline}'" : '') . 1779 ($bTargetExclusive ? ', exclusive' : '') . 1780 (defined($strTargetAction) && $strTargetAction ne 'pause' ? ", target-action=${strTargetAction}" : '') . 1781 (defined($rhRemapHash) ? ', remap' : '') . 1782 (defined($iExpectedExitStatus) ? ", expect exit ${iExpectedExitStatus}" : '') . 1783 (defined($strComment) ? " - ${strComment}" : '') . 1784 ' (' . $self->nameGet() . ' host)'; 1785 &log(INFO, " ${strComment}"); 1786 1787 # Get the backup host 1788 my $oHostGroup = hostGroupGet(); 1789 my $oHostBackup = defined($oHostGroup->hostGet(HOST_BACKUP, true)) ? $oHostGroup->hostGet(HOST_BACKUP) : $self; 1790 1791 # If the repo was not passed, then use repo1 as the repo for getting the expected manifest/backup 1792 my $iRepoDefault = !defined($iRepo) ? 1 : $iRepo; 1793 1794 # Load the expected manifest if it was not defined 1795 my $oExpectedManifest = undef; 1796 1797 # If an expected backup is defined, then the strBackup should be the default to allow the restore process to select the backup 1798 # - which should be the backup passed as strBackupExpected. If it is not defined, then set it based on the strBackup passed. 1799 if (!defined($strBackupExpected)) 1800 { 1801 $strBackupExpected = $strBackup eq 'latest' ? $oHostBackup->backupLast($iRepoDefault) : $strBackup; 1802 } 1803 1804 if (!defined($rhExpectedManifest)) 1805 { 1806 # Load the manifest from the backup expected to be chosen/processed by restore 1807 my $oExpectedManifest = new pgBackRestTest::Env::Manifest( 1808 $self->repoBackupPath($strBackupExpected . qw{/} . FILE_MANIFEST, $iRepoDefault), 1809 {strCipherPass => $oHostBackup->cipherPassManifest(), oStorage => storageRepo({iRepo => $iRepoDefault})}); 1810 1811 $rhExpectedManifest = $oExpectedManifest->{oContent}; 1812 1813 # Remap links in the expected manifest 1814 foreach my $strTarget (sort(keys(%{$self->{hLinkRemap}}))) 1815 { 1816 my $strDestination = ${$self->{hLinkRemap}}{$strTarget}; 1817 my $strTarget = 'pg_data/' . $strTarget; 1818 my $strTargetPath = $strDestination; 1819 1820 # If this link is to a file then the specified path must be split into file and path parts 1821 if ($oExpectedManifest->isTargetFile($strTarget)) 1822 { 1823 $strTargetPath = dirname($strTargetPath); 1824 1825 # Error when the path is not deep enough to be valid 1826 if (!defined($strTargetPath)) 1827 { 1828 confess &log(ERROR, "${strDestination} is not long enough to be target for ${strTarget}"); 1829 } 1830 1831 # Set the file part 1832 $oExpectedManifest->set( 1833 MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_FILE, 1834 substr($strDestination, length($strTargetPath) + 1)); 1835 1836 # Set the link target 1837 $oExpectedManifest->set( 1838 MANIFEST_SECTION_TARGET_LINK, $strTarget, MANIFEST_SUBKEY_DESTINATION, $strDestination); 1839 } 1840 else 1841 { 1842 # Set the link target 1843 $oExpectedManifest->set(MANIFEST_SECTION_TARGET_LINK, $strTarget, MANIFEST_SUBKEY_DESTINATION, $strTargetPath); 1844 } 1845 1846 # Set the target path 1847 $oExpectedManifest->set(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_PATH, $strTargetPath); 1848 } 1849 } 1850 1851 # Get the backup host 1852 if (defined($rhRemapHash)) 1853 { 1854 $self->configRemap($rhRemapHash, $rhExpectedManifest); 1855 } 1856 1857 if (defined($rhRecoveryHash)) 1858 { 1859 $self->configRecovery($oHostBackup, $rhRecoveryHash); 1860 } 1861 1862 # Create the restore command 1863 $self->executeSimple( 1864 $self->backrestExe() . 1865 ' --config=' . $self->backrestConfig() . 1866 ($bDelta ? ' --delta' : '') . 1867 ($bForce ? ' --force' : '') . 1868 ($strBackup ne 'latest' ? " --set=${strBackup}" : '') . 1869 (defined($strOptionalParam) ? " ${strOptionalParam} " : '') . 1870 (defined($strType) && $strType ne CFGOPTVAL_RESTORE_TYPE_DEFAULT ? " --type=${strType}" : '') . 1871 (defined($strTarget) ? " --target=\"${strTarget}\"" : '') . 1872 (defined($strTargetTimeline) ? " --target-timeline=\"${strTargetTimeline}\"" : '') . 1873 ($bTargetExclusive ? ' --target-exclusive' : '') . 1874 (defined($strLinkMap) ? $strLinkMap : '') . 1875 ($self->synthetic() ? '' : ' --link-all') . 1876 (defined($strTargetAction) && $strTargetAction ne 'pause' ? " --target-action=${strTargetAction}" : '') . 1877 (defined($iRepo) ? " --repo=${iRepo}" : '') . 1878 " --stanza=" . $self->stanza() . ' restore', 1879 {strComment => $strComment, iExpectedExitStatus => $iExpectedExitStatus, oLogTest => $self->{oLogTest}, 1880 bLogOutput => $self->synthetic()}, 1881 $strUser); 1882 1883 if (!defined($iExpectedExitStatus)) 1884 { 1885 # Only compare restores in repo1. There is not a lot of value in comparing restores in other repos and it would require a 1886 # lot of changes to the Perl test harness. 1887 if ($iRepoDefault == 1) 1888 { 1889 $self->restoreCompare($strBackupExpected, dclone($rhExpectedManifest), $bTablespace); 1890 } 1891 1892 if (defined($self->{oLogTest})) 1893 { 1894 $self->{oLogTest}->supplementalAdd( 1895 $rhExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} . 1896 "/recovery.conf"); 1897 } 1898 } 1899} 1900 1901#################################################################################################################################### 1902# restoreCompare 1903#################################################################################################################################### 1904sub restoreCompare 1905{ 1906 my $self = shift; 1907 my $strBackup = shift; 1908 my $oExpectedManifestRef = shift; 1909 my $bTablespace = shift; 1910 1911 my $strTestPath = $self->testPath(); 1912 1913 # Get the backup host 1914 my $oHostGroup = hostGroupGet(); 1915 my $oHostBackup = defined($oHostGroup->hostGet(HOST_BACKUP, true)) ? $oHostGroup->hostGet(HOST_BACKUP) : $self; 1916 1917 # Load the last manifest if it exists 1918 my $oLastManifest = undef; 1919 1920 if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR})) 1921 { 1922 my $oExpectedManifest = 1923 new pgBackRestTest::Env::Manifest( 1924 $self->repoBackupPath( 1925 ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST), 1926 {strCipherPass => $oHostBackup->cipherPassManifest()}); 1927 1928 # Get the --delta option from the backup manifest so the actual manifest can be built the same way for comparison 1929 $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA} = 1930 $oExpectedManifest->get(MANIFEST_SECTION_BACKUP_OPTION, &MANIFEST_KEY_DELTA); 1931 1932 $oLastManifest = 1933 new pgBackRestTest::Env::Manifest( 1934 $self->repoBackupPath( 1935 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST), 1936 {strCipherPass => $oHostBackup->cipherPassManifest()}); 1937 } 1938 1939 # Generate the tablespace map for real backups 1940 my $oTablespaceMap = undef; 1941 1942 if (!$self->synthetic()) 1943 { 1944 # Tablespace_map file is not restored in versions >= 9.5 because it interferes with internal remapping features. 1945 if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} >= PG_VERSION_95) 1946 { 1947 delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/tablespace_map'}); 1948 } 1949 1950 foreach my $strTarget (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}})) 1951 { 1952 if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) 1953 { 1954 my $iTablespaceId = 1955 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID}; 1956 1957 $$oTablespaceMap{$iTablespaceId} = 1958 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME}; 1959 } 1960 } 1961 } 1962 1963 # Generate the actual manifest 1964 my $strDbClusterPath = 1965 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH}; 1966 1967 if (defined($bTablespace) && !$bTablespace) 1968 { 1969 foreach my $strTarget (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}})) 1970 { 1971 if ($$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} eq 1972 MANIFEST_VALUE_LINK && 1973 defined($$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) 1974 { 1975 my $strRemapPath; 1976 my $iTablespaceName = 1977 $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME}; 1978 1979 $strRemapPath = "../../tablespace/${iTablespaceName}"; 1980 1981 $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; 1982 $$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK}{MANIFEST_TARGET_PGDATA . "/${strTarget}"} 1983 {&MANIFEST_SUBKEY_DESTINATION} = $strRemapPath; 1984 } 1985 } 1986 } 1987 1988 my $oActualManifest = new pgBackRestTest::Env::Manifest( 1989 "${strTestPath}/" . FILE_MANIFEST, 1990 {bLoad => false, strDbVersion => $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}, 1991 iDbCatalogVersion => $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}, 1992 oStorage => storageTest()}); 1993 1994 # Build the actual manifest using the delta setting that was actually used by the latest backup if one exists 1995 $oActualManifest->build(storageTest(), $strDbClusterPath, $oLastManifest, false, 1996 $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}, $oTablespaceMap); 1997 $oActualManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA, undef, 1998 $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}); 1999 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_TYPE, undef, 2000 $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}); 2001 2002 my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH); 2003 2004 foreach my $strName ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE)) 2005 { 2006 # If synthetic match checksum errors since they can't be verified here 2007 if ($self->synthetic) 2008 { 2009 my $bChecksumPage = $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}; 2010 2011 if (defined($bChecksumPage)) 2012 { 2013 $oActualManifest->boolSet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, $bChecksumPage); 2014 2015 if (!$bChecksumPage && 2016 defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR})) 2017 { 2018 $oActualManifest->set( 2019 MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR, 2020 $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); 2021 } 2022 } 2023 } 2024 # Else if page checksums are enabled make sure the correct files are being checksummed 2025 else 2026 { 2027 if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}) 2028 { 2029 if (defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}) != 2030 isChecksumPage($strName)) 2031 { 2032 confess 2033 "check-page actual for ${strName} is " . 2034 ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, 2035 MANIFEST_SUBKEY_CHECKSUM_PAGE) ? 'set' : '[undef]') . 2036 ' but isChecksumPage() says it should be ' . 2037 (isChecksumPage($strName) ? 'set' : '[undef]') . '.'; 2038 } 2039 2040 # Because the page checksum flag is copied to incr and diff from the previous backup but further processing is not 2041 # done, they can't be expected to match so delete them. 2042 delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}); 2043 $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE); 2044 } 2045 } 2046 2047 if (!$self->synthetic()) 2048 { 2049 $oActualManifest->set( 2050 MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE, 2051 ${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{size}); 2052 } 2053 2054 # Remove repo-size from the manifest. ??? This could be improved to get actual sizes from the backup. 2055 $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE); 2056 delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE}); 2057 2058 if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0) 2059 { 2060 my $oStat = lstat($oActualManifest->dbPathGet($strSectionPath, $strName)); 2061 2062 # When performing a selective restore, the files for the database(s) that are not restored are still copied but as empty 2063 # sparse files (blocks == 0). If the file is not a sparse file or is a link, then get the actual checksum for comparison 2064 if ($oStat->blocks > 0 || S_ISLNK($oStat->mode)) 2065 { 2066 my ($strHash) = storageTest()->hashSize($oActualManifest->dbPathGet($strSectionPath, $strName)); 2067 2068 $oActualManifest->set( 2069 MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM, $strHash); 2070 2071 # If the delta option was set, it is possible that the checksum on the file changed from the last manifest. If so, 2072 # then the file was expected to be copied by the backup and therefore the reference would have been removed. 2073 if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}) 2074 { 2075 # If the actual checksum and last manifest checksum don't match, remove the reference 2076 if (defined($oLastManifest) && 2077 $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM) && 2078 $strHash ne $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM)) 2079 { 2080 $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); 2081 } 2082 } 2083 } 2084 else 2085 { 2086 # If there is a sparse file, remove the checksum and reference since they may or may not match. In this case, it is 2087 # not important to check them since it is known that the file was intentionally not restored. 2088 $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM); 2089 delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM}); 2090 2091 $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); 2092 delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REFERENCE}); 2093 } 2094 } 2095 } 2096 2097 # If PostgreSQL >= 12 don't compare postgresql.auto.conf since it will have recovery settings written into it 2098 if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} >= PG_VERSION_12) 2099 { 2100 delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/postgresql.auto.conf'}); 2101 $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, 'pg_data/postgresql.auto.conf'); 2102 } 2103 2104 # If the link section is empty then delete it and the default section 2105 if (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_LINK}}) == 0) 2106 { 2107 delete($$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK}); 2108 delete($$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK . ':default'}); 2109 } 2110 2111 # Set actual to expected for settings that always change from backup to backup 2112 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK, undef, 2113 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_CHECK}); 2114 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY, undef, 2115 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_COPY}); 2116 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, 2117 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY}); 2118 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BUFFER_SIZE, undef, 2119 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE}); 2120 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS, undef, 2121 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS}); 2122 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_LEVEL, undef, 2123 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL}); 2124 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_LEVEL_NETWORK, undef, 2125 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK}); 2126 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK, undef, 2127 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK}); 2128 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, 2129 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE}); 2130 $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_PROCESS_MAX, undef, 2131 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX}); 2132 $oActualManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA, undef, 2133 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}); 2134 2135 $oActualManifest->set(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION, undef, 2136 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}); 2137 $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CONTROL, undef, 2138 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CONTROL}); 2139 $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG, undef, 2140 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}); 2141 $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_SYSTEM_ID, undef, 2142 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_SYSTEM_ID}); 2143 $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_ID, undef, 2144 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_ID}); 2145 2146 $oActualManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef, 2147 ${$oExpectedManifestRef}{&INI_SECTION_BACKREST}{&INI_KEY_VERSION}); 2148 2149 # Copy passphrase if one exists 2150 if (defined($oExpectedManifestRef->{&INI_SECTION_CIPHER}) && 2151 defined($oExpectedManifestRef->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS})) 2152 { 2153 $oActualManifest->set(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS, undef, 2154 $oExpectedManifestRef->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}); 2155 } 2156 2157 # This option won't be set in the actual manifest 2158 delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}); 2159 2160 if ($self->synthetic()) 2161 { 2162 $oActualManifest->remove(MANIFEST_SECTION_BACKUP); 2163 delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}); 2164 } 2165 else 2166 { 2167 $oActualManifest->set( 2168 INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef, $oExpectedManifestRef->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM}); 2169 $oActualManifest->set( 2170 MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, 2171 $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL}); 2172 $oActualManifest->set( 2173 MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, 2174 $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START}); 2175 $oActualManifest->set( 2176 MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef, 2177 $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START}); 2178 $oActualManifest->set( 2179 MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef, 2180 $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP}); 2181 $oActualManifest->set( 2182 MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, 2183 $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE}); 2184 2185 $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_START, undef, 2186 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_START}); 2187 $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_STOP, undef, 2188 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_STOP}); 2189 2190 if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_START})) 2191 { 2192 $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, 2193 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_START}); 2194 } 2195 2196 if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_STOP}) 2197 { 2198 $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, 2199 ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_STOP}); 2200 } 2201 } 2202 2203 # Check that archive status exists in the manifest for an online backup 2204 my $strArchiveStatusPath = MANIFEST_TARGET_PGDATA . qw{/} . $oActualManifest->walPath() . qw{/} . DB_PATH_ARCHIVESTATUS; 2205 2206 if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE} && 2207 !defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_PATH}{$strArchiveStatusPath})) 2208 { 2209 confess &log(ERROR, "${strArchiveStatusPath} expected for online backup", ERROR_ASSERT); 2210 } 2211 2212 # Delete the list of DBs 2213 delete($$oExpectedManifestRef{&MANIFEST_SECTION_DB}); 2214 2215 $self->manifestDefault($oExpectedManifestRef); 2216 2217 # Newer Perls will change this variable to a number whenever a numeric comparison is performed. It is expected to be a string so 2218 # make sure it is one before saving. 2219 $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} .= ''; 2220 2221 storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent})); 2222 storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifestRef)); 2223 2224 executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest"); 2225 2226 storageTest()->remove("${strTestPath}/expected.manifest"); 2227 storageTest()->remove("${strTestPath}/actual.manifest"); 2228} 2229 2230#################################################################################################################################### 2231# Get repo backup/archive path 2232#################################################################################################################################### 2233sub repoSubPath 2234{ 2235 my $self = shift; 2236 my $strSubPath = shift; 2237 my $strPath = shift; 2238 my $iRepo = shift; 2239 2240 my $strRepoPath = $self->repoPath(); 2241 2242 if (defined($iRepo) && $iRepo == 2) 2243 { 2244 $strRepoPath = $self->repo2Path(); 2245 } 2246 2247 return 2248 ($strRepoPath eq '/' ? '' : $strRepoPath) . "/${strSubPath}/" . $self->stanza() . 2249 (defined($strPath) ? "/${strPath}" : ''); 2250} 2251 2252#################################################################################################################################### 2253# Getters 2254#################################################################################################################################### 2255sub backrestConfig {return shift->{strBackRestConfig}} 2256sub backupDestination {return shift->{strBackupDestination}} 2257sub backrestExe {return testRunGet()->backrestExe()} 2258sub bogusHost {return shift->{bBogusHost}} 2259sub hardLink {return shift->{bHardLink}} 2260sub hasLink {storageRepo()->capability(STORAGE_CAPABILITY_LINK)} 2261sub isFS {storageRepo()->type() ne STORAGE_OBJECT} 2262sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()} 2263sub isHostDbPrimary {return shift->nameGet() eq HOST_DB_PRIMARY} 2264sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY} 2265sub isHostDb {my $self = shift; return $self->isHostDbPrimary() || $self->isHostDbStandby()} 2266sub lockPath {return shift->{strLockPath}} 2267sub logPath {return shift->{strLogPath}} 2268sub repoArchivePath {return shift->repoSubPath('archive', shift)} 2269sub repoBackupPath {return shift->repoSubPath('backup', shift, shift)} 2270sub repoPath {return shift->{strRepoPath}} 2271sub repo2Path {return shift->{strRepo2Path}} 2272sub repoEncrypt {return shift->{bRepoEncrypt}} 2273sub stanza {return testRunGet()->stanza()} 2274sub synthetic {return shift->{bSynthetic}} 2275sub cipherPassManifest {return shift->{strCipherPassManifest}} 2276sub cipherPassArchive {return shift->{strCipherPassArchive}} 2277 22781; 2279