1#!/usr/bin/perl 2#============================================================= -*-perl-*- 3# 4# BackupPC_refCountUpdate: Pool reference count updater 5# 6# DESCRIPTION 7# 8# BackupPC_refCountUpdate checks the pool reference counts 9# 10# Usage: BackupPC_refCountUpdate 11# 12# AUTHOR 13# Craig Barratt <cbarratt@users.sourceforge.net> 14# 15# COPYRIGHT 16# Copyright (C) 2001-2020 Craig Barratt 17# 18# This program is free software: you can redistribute it and/or modify 19# it under the terms of the GNU General Public License as published by 20# the Free Software Foundation, either version 3 of the License, or 21# (at your option) any later version. 22# 23# This program is distributed in the hope that it will be useful, 24# but WITHOUT ANY WARRANTY; without even the implied warranty of 25# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 26# GNU General Public License for more details. 27# 28# You should have received a copy of the GNU General Public License 29# along with this program. If not, see <http://www.gnu.org/licenses/>. 30# 31#======================================================================== 32# 33# Version 4.3.3, released 5 Apr 2020. 34# 35# See http://backuppc.sourceforge.net. 36# 37#======================================================================== 38 39use strict; 40no utf8; 41use lib "/usr/local/BackupPC/lib"; 42 43use Getopt::Std; 44use Fcntl qw(:mode); 45use File::Path; 46use Data::Dumper; 47 48use BackupPC::Lib; 49use BackupPC::XS; 50use BackupPC::DirOps qw( :BPC_DT_ALL ); 51 52select(STDOUT); $| = 1; 53 54my %opts; 55if ( !getopts("cFfnmpsvh:r:P:o:", \%opts) || @ARGV != 0 || !(defined($opts{h}) || $opts{m}) ) { 56 print <<EOF; 57Usage: 58 BackupPC_refCountUpdate -h HOST [-c] [-f] [-F] [-o N] [-p] [-v] 59 With no other args, updates count db on backups with poolCntDelta files 60 and computers the host's total reference counts. Also builds refCnt for 61 any >=4.0 backups without refCnts. 62 -f - do an fsck on this HOST, which involves a rebuild of the 63 last two backup refCnts. poolCntDelta files are ignored. 64 Also forces fsck if requested by needFsck flag files 65 in TopDir/pc/HOST/refCnt. Equivalent to -o 2. 66 -F - rebuild all the >=4.0 per-backup refCnt files for this 67 host. Equivalent to -o 3. 68 -c - compare current count db to new db before replacing 69 -o N - override \$Conf{RefCntFsck}. 70 -p - don't show progress 71 -v - verbose 72 Notes: in case there are legacy (ie: <=4.0.0alpha3) unapplied poolCntDelta 73 files in TopDir/pc/HOST/refCnt then the -f flag is turned on. 74 75 BackupPC_refCountUpdate -m [-f] [-p] [-c] [-r N-M] [-s] [-v] [-P phase] 76 -m Updates main count db, based on each HOST 77 -f - do an fsck on all the hosts, ignoring poolCntDelta files, 78 and replacing each host's count db. Will wait for backups 79 to finish if any are running. 80 -F - rebuild all the >=4.0 per-backup refCnt files. 81 -p - don't show progress 82 -c - clean pool files 83 -r N-M - process a subset of the main count db, 0 <= N <= M <= 255 84 -s - prints stats 85 -v - verbose 86 -P phase Phase from 0..15 each time we run BackupPC_nightly. Used 87 to compute exact pool size for portions of the pool based 88 on the phase and \$Conf{PoolSizeNightlyUpdatePeriod}. 89EOF 90 exit(1); 91} 92 93my $ErrorCnt = 0; 94my $refCntStart = 0; 95my $refCntEnd = 127; 96 97my $EmptyMD5 = pack("H*", "d41d8cd98f00b204e9800998ecf8427e"); 98 99die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); 100my $TopDir = $bpc->TopDir(); 101my $BinDir = $bpc->BinDir(); 102my $Hosts = $bpc->HostInfoRead(); 103my @Hosts = sort(keys(%$Hosts)); 104 105if ( $opts{h} ne "" ) { 106 if ( !defined($Hosts->{$opts{h}}) ) { 107 print(STDERR "BackupPC_refCountUpdate: host $opts{h} doesn't exist\n"); 108 exit(1); 109 } 110 $bpc->ConfigRead($opts{h}); 111} 112my %Conf = $bpc->Conf(); 113my $PoolStats = {}; 114 115if ( $opts{v} ) { 116 $Conf{XferLogLevel} = 8; 117 BackupPC::XS::Lib::logLevelSet(8); 118} 119 120# 121# Write new-style attrib files (<= 4.0.0beta3 uses old-style), which are 0-length 122# files with the digest encoded in the file name (eg: attrib_md5HexDigest). We 123# can still read the old-style files, and we upgrade them as we go. 124# 125BackupPC::XS::Attrib::backwardCompat(0, 0); 126 127print("__bpc_pidStart__ $$\n") if ( !$opts{p} ); 128 129if ( defined($opts{h}) ) { 130 updateHostPoolCnt($opts{h}, $opts{f}, $opts{F}, $opts{c}); 131} elsif ( $opts{m} ) { 132 if ( $opts{r} =~ /^(\d+)-(\d+)$/ && 0 <= $1 && $1 <= $2 && $2 <= 255 ) { 133 $refCntStart = int($1 / 2); 134 $refCntEnd = int($2 / 2); 135 } elsif ( defined($opts{r}) ) { 136 print(STDERR "BackupPC_refCountUpdate: -r arguments should be N-M where 0 <= N <= M <= 255\n"); 137 print("__bpc_pidEnd__ $$\n") if ( !$opts{p} ); 138 exit(1); 139 } 140 141 # 142 # update all the pool count changes for each host 143 # 144 if ( $opts{f} || $opts{F} ) { 145 updateAllHostPoolCnt($opts{f}, $opts{F}); 146 } 147 148 # 149 # rebuild the overall pool count database 150 # 151 updateTotalRefCnt(); 152 153 # 154 # clean files from the pool that are no longer referenced, 155 # and mark candidate files for future cleaning. This prints 156 # stats too, so if not cleaning, we need to print stats, if 157 # requested. 158 # 159 if ( $opts{c} ) { 160 cleanPoolFiles(); 161 } elsif ( $opts{s} ) { 162 statsPrint(); 163 } 164} 165 166print("BackupPC_refCountUpdate total errors: $ErrorCnt\n") if ( $ErrorCnt > 0 || $Conf{XferLogLevel} >= 2 ); 167print("__bpc_pidEnd__ $$\n") if ( !$opts{p} ); 168exit($ErrorCnt ? 1 : 0); 169 170# 171# For a given host, either update the poolCnt files using the poolCntDelta files, 172# or if $forceFsck, completely rebuild the poolCnt files from scratch. 173# 174sub updateHostPoolCnt 175{ 176 my($host, $forceLast2Fsck, $forceFullFsck, $compareCurr) = @_; 177 my $pcDir = "$TopDir/pc/$host"; 178 my $refCntDir = "$pcDir/refCnt"; 179 my $errorCntSave = $ErrorCnt; 180 my @fsckFiles; 181 my @Backups = $bpc->BackupInfoRead($host); 182 my $startClock = time; 183 184 mkdir($refCntDir, 0770) if ( !-d $refCntDir ); 185 186 # 187 # Grab the lock to make sure no dumps start. Try a non-blocking request 188 # first, so we can warn the user if we have to wait. 189 # 190 my $lockFd = BackupPC::XS::DirOps::lockRangeFile("$refCntDir/LOCK", 0, 1, 0); 191 if ( $lockFd < 0 ) { 192 print("Backup running on host $host.... waiting\n"); 193 $lockFd = BackupPC::XS::DirOps::lockRangeFile("$refCntDir/LOCK", 0, 1, 1); 194 } 195 if ( $lockFd < 0 ) { 196 print("Unable to get lock on $refCntDir/LOCK... skipping $host\n"); 197 $ErrorCnt++; 198 return; 199 } 200 201 # 202 # Clean up the HOST/refCnt directory to see if an fsck is needed 203 # 204 foreach my $e ( @{BackupPC::DirOps::dirRead($bpc, $refCntDir)} ) { 205 unlink("$refCntDir/$e->{name}") if ( $e->{name} =~ /^poolCntNew/ ); 206 if ( $e->{name} =~ /^t?poolCntDelta/ ) { 207 # 208 # legacy <= 4.0.0beta3 code left poolCntDelta files in the host's refCnt 209 # directory, so remove them and do a last2Fsck 210 # 211 unlink("$refCntDir/$e->{name}"); 212 $forceLast2Fsck = 1; 213 } 214 next if ( $e->{name} !~ /^needFsck/ ); 215 unlink("$refCntDir/$e->{name}"); 216 $forceLast2Fsck = 1; 217 } 218 219 # 220 # Apply policy in $Conf{RefCntFsck} (potentially overridden by -o N) 221 # 0: no additional fsck 222 # 1: do an fsck on the last backup if it is from a full backup (handled below) 223 # 2: do an fsck on the last two backups always 224 # 3: do a full fsck on all the backups 225 # 226 my $ConfRefCntFsck = $opts{o} =~ /^\d+$/ ? $opts{o} : $Conf{RefCntFsck}; 227 if ( $ConfRefCntFsck == 2 ) { 228 print("BackupPC_refCountUpdate: doing fsck on last two backups on $host since \$ConfRefCntFsck == 2\n"); 229 $forceLast2Fsck = 1; 230 } elsif ( $ConfRefCntFsck == 3 ) { 231 print("BackupPC_refCountUpdate: doing fsck on all backups on $host since \$ConfRefCntFsck == 3\n"); 232 $forceFullFsck = 1; 233 } 234 235 # 236 # Figure out which backups to rebuild or apply to deltas 237 # 238 my $bkupList = []; 239 for ( my $i = 0 ; $i < @Backups ; $i++ ) { 240 my $bkupNum = $Backups[$i]{num}; 241 my $compress = $Backups[$i]{compress}; 242 my $bkupRefCntDir = "$pcDir/$bkupNum/refCnt"; 243 my($gotFsck, $gotDelta, $gotPoolCnt, $gotNoPoolCntOk); 244 245 next if ( $Backups[$i]{version} eq "" || ($Backups[$i]{version} =~ /^[23]\./ && !-d $bkupRefCntDir) ); 246 247 if ( !-d $bkupRefCntDir ) { 248 print("BackupPC_refCountUpdate: doing fsck on $host #$bkupNum since $bkupRefCntDir doesn't exist\n"); 249 mkdir($bkupRefCntDir, 0770); 250 push(@$bkupList, {bkupNum => $bkupNum, fsck => 1, compress => $compress}); 251 next; 252 } 253 my $entries = BackupPC::DirOps::dirRead($bpc, $bkupRefCntDir); 254 foreach my $e ( @$entries ) { 255 unlink("$bkupRefCntDir/$e->{name}") if ( $e->{name} =~ /^poolCntNew/ ); 256 if ( $e->{name} =~ /^needFsck/ || $e->{name} =~ /^tpoolCntDelta/ ) { 257 unlink("$bkupRefCntDir/$e->{name}"); 258 $gotFsck = 1; 259 } 260 $gotDelta = 1 if ( $e->{name} =~ /^poolCntDelta/ ); 261 $gotPoolCnt = 1 if ( $e->{name} =~ /^poolCnt\./ ); 262 $gotNoPoolCntOk = 1 if ( $e->{name} =~ /^noPoolCntOk/ ); 263 } 264 print("BackupPC_refCountUpdate: host $host #$bkupNum: gotFsck = $gotFsck, gotDelta = $gotDelta," 265 . " gotPoolCnt = $gotPoolCnt, gotNoPoolCntOk = $gotNoPoolCntOk\n") 266 if ( $Conf{XferLogLevel} >= 4 ); 267 # 268 # Apply policy in $ConfRefCntFsck: 269 # 0: no additional fsck 270 # 1: do an fsck on the last backup if it is from a full backup 271 # 2: do an fsck on the last two backups always (handled above) 272 # 3: do a full fsck on all the backups (handled above) 273 # 274 if ( $ConfRefCntFsck == 1 && $i == @Backups - 1 && $Backups[$i]{type} eq "full" ) { 275 print("BackupPC_refCountUpdate: doing fsck on $host #$bkupNum (full) since \$ConfRefCntFsck == 1\n"); 276 $gotFsck = 1; 277 } 278 279 if ( !$gotPoolCnt && !$gotFsck && !$gotNoPoolCntOk ) { 280 # 281 # This shouldn't happen - we should have some pool files, so force an fsck 282 # 283 print("BackupPC_refCountUpdate: doing fsck on $host #$bkupNum since there are no poolCnt files\n"); 284 $gotFsck = 1; 285 } 286 if ( $forceFullFsck || ($forceLast2Fsck && $i >= @Backups - 2) || $gotFsck ) { 287 push(@$bkupList, {bkupNum => $bkupNum, fsck => 1, compress => $compress}); 288 } elsif ( $gotDelta ) { 289 push(@$bkupList, {bkupNum => $bkupNum, fsck => 0, compress => $compress}); 290 } 291 } 292 293 while ( (my $b = shift(@$bkupList)) ) { 294 my $errorCntSave0 = $ErrorCnt; 295 if ( processOneHostBackup($host, $b->{bkupNum}, $b->{fsck}, $b->{compress}) && !$b->{fsck} ) { 296 print("BackupPC_refCountUpdate: given errors, redoing host $host #$b->{bkupNum} with fsck (reset errorCnt to $errorCntSave0)\n"); 297 $ErrorCnt = $errorCntSave0; 298 unshift(@$bkupList, {bkupNum => $b->{bkupNum}, fsck => 1, compress => $b->{compress}}); 299 } 300 } 301 302 # 303 # Now add up all the per-backup counts to get the per-host totals 304 # 305 my $errorCntSave0 = $ErrorCnt; 306 print("BackupPC_refCountUpdate: computing totals for host $host\n") if ( $Conf{XferLogLevel} >= 2 ); 307 my $needFsck = updateHostRefCounts($host); 308 my $bkupRedoDone = {}; 309 310 while ( $needFsck ) { 311 my $listText = join(", ", @$needFsck); 312 print("BackupPC_refCountUpdate: due to errors, doing fsck on host $host backups $listText; (reset errorCnt to $errorCntSave0)\n"); 313 $ErrorCnt = $errorCntSave0; 314 foreach my $bkupNum ( @$needFsck ) { 315 next if ( $bkupRedoDone->{$bkupNum} ); 316 my $compress = 1; 317 for ( my $i = 0 ; $i < @Backups ; $i++ ) { 318 next if ( $Backups[$i]{num} != $bkupNum ); 319 $compress = $Backups[$i]{compress}; 320 last; 321 } 322 processOneHostBackup($host, $bkupNum, 1, $compress); 323 $bkupRedoDone->{$bkupNum} = 1; 324 } 325 $needFsck = updateHostRefCounts($host); 326 } 327 328 # 329 # and rename the new count files to replace the current ones 330 # 331 renameNewCntFiles($host, $refCntDir, $compareCurr, "total"); 332 333 # 334 # Now give the lock back 335 # 336 BackupPC::XS::DirOps::unlockRangeFile($lockFd); 337 printf("BackupPC_refCountUpdate: host %s got %d errors (took %d secs)\n", 338 $host, $ErrorCnt - $errorCntSave, time - $startClock); 339 $bpc->flushXSLibMesgs(); 340} 341 342sub processOneHostBackup 343{ 344 my($host, $bkupNum, $fsck, $compress) = @_; 345 my $pcDir = "$TopDir/pc/$host"; 346 my $bkupRefCntDir = "$pcDir/$bkupNum/refCnt"; 347 my $errorCntSave0 = $ErrorCnt; 348 my $fd; 349 350 # 351 # add a placeholder fsck request, in case we die unexpectedly 352 # 353 open($fd, ">", "$bkupRefCntDir/needFsck.refCountUpdate") && close($fd); 354 355 print("BackupPC_refCountUpdate: processing host $host #$bkupNum (fsck = $fsck)\n") if ( $Conf{XferLogLevel} >= 2 ); 356 357 if ( $fsck ) { 358 my $entries = BackupPC::DirOps::dirRead($bpc, $bkupRefCntDir); 359 # 360 # Remove any delta files since we are rebuilding 361 # 362 foreach my $e ( @$entries ) { 363 unlink("$bkupRefCntDir/$e->{name}") if ( $e->{name} =~ /^t?poolCntDelta/ || $e->{name} =~ /^poolCntNew/ ); 364 } 365 my $deltaInfo = BackupPC::XS::DeltaRefCnt::new("$pcDir/$bkupNum"); 366 print("__bpc_progress_state__ refCnt #$bkupNum\n") if ( !$opts{p} ); 367 my($err, $inodeMax) = BackupPC::XS::DirOps::refCountAllInodeMax("$pcDir/$bkupNum", $compress, 1, $deltaInfo); 368 $ErrorCnt += $err; 369 $deltaInfo->flush(); 370 $bpc->flushXSLibMesgs(); 371 # 372 # Update inodeLast for this backup if it is larger than what's stored in backups 373 # 374 my @backups = $bpc->BackupInfoRead($host); 375 for ( my $i = 0 ; $i < @backups ; $i++ ) { 376 next if ( $backups[$i]{num} != $bkupNum || $backups[$i]{inodeLast} > $inodeMax ); 377 $inodeMax += 2; 378 print("BackupPC_refCountUpdate: $host #$bkupNum inodeLast set to $inodeMax (was $backups[$i]{inodeLast})\n"); 379 $backups[$i]{inodeLast} = $inodeMax; 380 BackupPC::Storage->backupInfoWrite($pcDir, $bkupNum, $backups[$i], 1); 381 $bpc->BackupInfoWrite($host, @backups); 382 last; 383 } 384 } 385 print("__bpc_progress_state__ cntUpdate #$bkupNum\n") if ( !$opts{p} ); 386 updateHostDelta2Cnt($host, $bkupRefCntDir, $fsck, 0, $bkupNum); 387 $bpc->flushXSLibMesgs(); 388 389 # 390 # delete the noPoolCntOk if there are pool files, or create one 391 # if there are none 392 # 393 my($gotNoPoolCntOk, $gotPoolCnt); 394 foreach my $e ( @{BackupPC::DirOps::dirRead($bpc, $bkupRefCntDir)} ) { 395 $gotPoolCnt = 1 if ( $e->{name} =~ /^poolCnt\./ ); 396 $gotNoPoolCntOk = 1 if ( $e->{name} =~ /^noPoolCntOk/ ); 397 } 398 if ( $gotPoolCnt && $gotNoPoolCntOk ) { 399 unlink("$bkupRefCntDir/noPoolCntOk"); 400 } elsif ( !$gotPoolCnt ) { 401 open(my $fd, ">", "$bkupRefCntDir/noPoolCntOk") && close($fd); 402 } 403 unlink("$bkupRefCntDir/needFsck.refCountUpdate"); 404 return 1 if ( $errorCntSave0 != $ErrorCnt ); 405} 406 407# 408# Process all the per-backup ref counts to create the total ref count for the host. 409# Returns undef on success, or a listref of backup numbers that need fscks. 410# 411sub updateHostRefCounts 412{ 413 my($host) = @_; 414 415 my @Backups = $bpc->BackupInfoRead($host); 416 my $refCntDestDir = "$TopDir/pc/$host/refCnt"; 417 my $needFsck = {}; 418 419 print("__bpc_progress_state__ sumUpdate\n") if ( !$opts{p} ); 420 for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) { 421 print("__bpc_progress_fileCnt__ $refCntFile/128\n") if ( !$opts{p} && ($refCntFile & 0x7) == 0 ); 422 for ( my $compress = 0 ; $compress < 2 ; $compress++ ) { 423 my $count = BackupPC::XS::PoolRefCnt::new(); 424 my $countDirty; 425 my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", 426 $refCntDestDir, $compress, $refCntFile * 2); 427 for ( my $i = 0 ; $i < @Backups ; $i++ ) { 428 my $bkupNum = $Backups[$i]{num}; 429 my $refCntBkupDir = "$TopDir/pc/$host/$bkupNum/refCnt"; 430 next if ( $Backups[$i]{version} eq "" || ($Backups[$i]{version} =~ /^[23]\./ && !-d $refCntBkupDir) ); 431 my $poolCntHostFile = sprintf("%s/poolCnt.%d.%02x", $refCntBkupDir, $compress, $refCntFile * 2); 432 next if ( !-f $poolCntHostFile ); 433 434 my $countHost = BackupPC::XS::PoolRefCnt::new(); 435 if ( $countHost->read($poolCntHostFile) ) { 436 print("Can't open pool count file $poolCntHostFile\n"); 437 $ErrorCnt++; 438 next; 439 } 440 441 my($d, $c); 442 my $idx = 0; 443 while ( 1 ) { 444 ($d, $c, $idx) = $countHost->iterate($idx); 445 last if ( !defined($d) ); 446 447 if ( $c < 0 ) { 448 printf("BackupPC_refCountUpdate: host %s (%s) digest %s has negative count (%d); will do fsck on #%d\n", 449 $host, $poolCntHostFile, unpack("H*", $d), $c, $bkupNum); 450 $needFsck->{$bkupNum}++; 451 $ErrorCnt++; 452 } 453 $count->incr($d, $c); 454 $countDirty = 1; 455 } 456 } 457 if ( $countDirty && $count->write($poolCntFileNew) ) { 458 print("Can't write new host pool count file $poolCntFileNew\n"); 459 $ErrorCnt++; 460 } 461 $count = undef; 462 $bpc->flushXSLibMesgs(); 463 } 464 } 465 if ( keys(%$needFsck) ) { 466 return [sort({ $a <=> $b } keys(%$needFsck))]; 467 } 468} 469 470# 471# Process all the delta files in the given directory and update the poolCnt files 472# 473sub updateHostDelta2Cnt 474{ 475 my($host, $refCntDir, $forceFsck, $compareCurr, $bkupNum) = @_; 476 my $gotDeltas; 477 478 # 479 # Read all the poolCntDelta files, and update the host's poolCnt files 480 # (or overwrite them if $forceFsck) 481 # 482 my $entries = BackupPC::DirOps::dirRead($bpc, $refCntDir); 483 foreach my $e ( @$entries ) { 484 unlink("$refCntDir/$e->{name}") if ( $e->{name} =~ /^poolCntNew/ ); 485 } 486 foreach my $e ( @$entries ) { 487 next if ( $e->{name} !~ /^poolCntDelta/ ); 488 my $deltaFileName = "$refCntDir/$e->{name}"; 489 my $compress = 1; 490 $compress = $1 if ( $deltaFileName =~ m{/poolCntDelta_(\d)_} ); 491 $gotDeltas++; 492 updateHostPoolOneDelta2Cnt($host, $refCntDir, $deltaFileName, !$forceFsck, $compress, $bkupNum); 493 } 494 renameNewCntFiles($host, $refCntDir, $compareCurr, "#$bkupNum") if ( $gotDeltas ); 495} 496 497# 498# Rename all the new pool files, replacing the old ones 499# 500sub renameNewCntFiles 501{ 502 my($host, $refCntDir, $compareCurr, $statusText) = @_; 503 504 print("__bpc_progress_state__ rename $statusText\n") if ( !$opts{p} ); 505 for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) { 506 for ( my $compress = 0 ; $compress < 2 ; $compress++ ) { 507 my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", $refCntDir, 508 $compress, $refCntFile * 2); 509 my $poolCntFileCur = sprintf("%s/poolCnt.%d.%02x", $refCntDir, 510 $compress, $refCntFile * 2); 511 if ( $compareCurr ) { 512 $ErrorCnt += poolCountHostNewCompare($host, $compress, $poolCntFileCur, $poolCntFileNew); 513 } 514 if ( -f $poolCntFileNew ) { 515 if ( !rename($poolCntFileNew, $poolCntFileCur) ) { 516 print("BackupPC_refCountUpdate: can't rename $poolCntFileNew to $poolCntFileCur ($!)\n"); 517 unlink($poolCntFileNew); 518 $ErrorCnt++; 519 next; 520 } 521 } elsif ( -f $poolCntFileCur && !unlink($poolCntFileCur) ) { 522 print("BackupPC_refCountUpdate: can't unlink $poolCntFileCur ($!)\n"); 523 $ErrorCnt++; 524 next; 525 } 526 } 527 } 528} 529 530# 531# Apply a single pool delta file to the host's new count database. 532# 533sub updateHostPoolOneDelta2Cnt 534{ 535 my($host, $refCntDir, $deltaFileName, $accumCurr, $compress, $bkupNum) = @_; 536 537 my $count = BackupPC::XS::PoolRefCnt::new(); 538 if ( $count->read($deltaFileName) ) { 539 print("BackupPC_refCountUpdate: can't read pool count delta file $deltaFileName\n"); 540 $ErrorCnt++; 541 return; 542 } 543 my($delta, $d, $c, $entryCnt); 544 my $idx = 0; 545 while ( 1 ) { 546 ($d, $c, $idx) = $count->iterate($idx); 547 last if ( !defined($d) ); 548 $delta->[vec($d, 0, 8) >> 1]{$d} = $c; 549 $entryCnt++; 550 } 551 $count = undef; 552 print("BackupPC_refCountUpdate: processing host $host #$bkupNum deltaFile $deltaFileName with $entryCnt entries\n") 553 if ( $Conf{XferLogLevel} >= 2 ); 554 555 for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) { 556 my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", $refCntDir, 557 $compress, $refCntFile * 2); 558 my $poolCntFileCur = sprintf("%s/poolCnt.%d.%02x", $refCntDir, 559 $compress, $refCntFile * 2); 560 if ( !defined($delta->[$refCntFile]) || !%{$delta->[$refCntFile]} ) { 561 next if ( -f $poolCntFileNew || !-f $poolCntFileCur ); 562 } 563 # 564 # Read the existing count 565 # 566 my $count = BackupPC::XS::PoolRefCnt::new(); 567 if ( -f $poolCntFileNew ) { 568 if ( $count->read($poolCntFileNew) ) { 569 print("BackupPC_refCountUpdate: can't open new pool count file $poolCntFileNew\n"); 570 $ErrorCnt++; 571 next; 572 } 573 } else { 574 # 575 # Read in the existing counts if we are accumulating. 576 # 577 if ( $accumCurr && -f $poolCntFileCur && $count->read($poolCntFileCur) ) { 578 print("BackupPC_refCountUpdate: can't open cur pool count file $poolCntFileCur\n"); 579 $ErrorCnt++; 580 next; 581 } 582 } 583 # 584 # Apply the deltas and write the file back; it's an error for any count to 585 # be negative. 586 # 587 foreach my $d ( keys(%{$delta->[$refCntFile]}) ) { 588 my $c = $count->incr($d, $delta->[$refCntFile]{$d}); 589 next if ( $c >= 0 ); 590 printf("BackupPC_refCountUpdate: got a negative count for %s processing %s; redoing #%d\n", 591 unpack("H*", $d), $deltaFileName, $bkupNum); 592 $ErrorCnt++; 593 last; 594 } 595 if ( $count->write($poolCntFileNew) ) { 596 print("BackupPC_refCountUpdate: can't write new pool count file $poolCntFileNew\n"); 597 $ErrorCnt++; 598 next; 599 } 600 } 601 $delta = {}; 602 if ( unlink($deltaFileName) != 1 ) { 603 print("BackupPC_refCountUpdate: can't unlink $deltaFileName ($!)\n"); 604 $ErrorCnt++; 605 } 606} 607 608# 609# Compare a new pool count file with the existing one. Returns an error count. 610# After the compare, renames the new pool count to replace the existing 611# one. 612# 613sub poolCountHostNewCompare 614{ 615 my($host, $compress, $poolCntFileCur, $poolCntFileNew) = @_; 616 my $errorCnt = 0; 617 618 my $countNew = BackupPC::XS::PoolRefCnt::new(); 619 my $count = BackupPC::XS::PoolRefCnt::new(); 620 621 if ( -f $poolCntFileNew && $countNew->read($poolCntFileNew) ) { 622 print("BackupPC_refCountUpdate: can't open new pool count file $poolCntFileNew\n"); 623 $errorCnt++; 624 } 625 if ( -f $poolCntFileCur && $count->read($poolCntFileCur) ) { 626 print("BackupPC_refCountUpdate: can't open current pool count file $poolCntFileCur\n"); 627 $errorCnt++; 628 } 629 630 # 631 # Compare the new and existing counts 632 # 633 my($digest, $cnt); 634 my $idx = 0; 635 636 while ( 1 ) { 637 ($digest, $cnt, $idx) = $count->iterate($idx); 638 last if ( !defined($digest) ); 639 640 if ( !defined($countNew->get($digest)) && $cnt > 0 ) { 641 printf("BackupPC_refCountUpdate: host %s digest.%d %s count is %d, but should be 0\n", 642 $host, $compress, unpack("H*", $digest), $cnt); 643 $errorCnt++; 644 next; 645 } 646 if ( $cnt != $countNew->get($digest) ) { 647 printf("BackupPC_refCountUpdate: host %s digest.%d %s count is %d, but should be %d\n", 648 $host, $compress, unpack("H*", $digest), 649 $cnt, $countNew->get($digest)); 650 $errorCnt++; 651 } 652 $countNew->delete($digest); 653 } 654 655 $idx = 0; 656 while ( 1 ) { 657 ($digest, $cnt, $idx) = $countNew->iterate($idx); 658 last if ( !defined($digest) ); 659 printf("BackupPC_refCountUpdate: host %s digest.%d %s count missing, but should be %d\n", 660 $host, $compress, unpack("H*", $digest), $countNew->get($digest)); 661 $errorCnt++; 662 } 663 664 return $errorCnt; 665} 666 667sub updateAllHostPoolCnt 668{ 669 my($forceLast2Fsck, $forceFullFsck) = @_; 670 671 foreach my $host ( @Hosts ) { 672 updateHostPoolCnt($host, $forceLast2Fsck, $forceFullFsck, 0); 673 } 674} 675 676# 677# Accumulate all the host poolRefCnt files to create the 678# overall pool reference counts 679# 680sub updateTotalRefCnt 681{ 682 return if ( $ErrorCnt ); 683 684 print("__bpc_progress_state__ refCnt pool\n") if ( !$opts{p} ); 685 for ( my $compress = 0 ; $compress < 2 ; $compress++ ) { 686 my $poolName = $compress ? "cpool4" : "pool4"; 687 for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCntFile++ ) { 688 # 689 # Count the number of pool directories 690 # 691 print("__bpc_progress_fileCnt__ cntUpdate $refCntFile/$refCntEnd\n") if ( !$opts{p} ); 692 my $poolDir = sprintf("%s/%02x", 693 $compress ? $bpc->{CPoolDir} : $bpc->{PoolDir}, 694 $refCntFile * 2); 695 next if ( !-d $poolDir ); 696 $PoolStats->{$poolName}[$refCntFile]{dirCnt}++; 697 my $entries = BackupPC::DirOps::dirRead($bpc, $poolDir); 698 foreach my $e ( @$entries ) { 699 next if ( $e->{name} !~ /^[\da-f][\da-f]$/ ); 700 $PoolStats->{$poolName}[$refCntFile]{dirCnt}++; 701 } 702 703 # 704 # For each host update the per-host count 705 # 706 my $poolCntFile = "$poolDir/poolCnt"; 707 my $count = BackupPC::XS::PoolRefCnt::new(); 708 my $countCopy = BackupPC::XS::PoolRefCnt::new(); 709 my $countCurr = BackupPC::XS::PoolRefCnt::new(); 710 711 $countCurr->read($poolCntFile) if ( -f $poolCntFile ); 712 foreach my $host ( @Hosts ) { 713 my $refCntDir = "$TopDir/pc/$host/refCnt"; 714 my $hostCntFile = sprintf("%s/poolCnt.%d.%02x", $refCntDir, 715 $compress, $refCntFile * 2); 716 my $countHost = BackupPC::XS::PoolRefCnt::new(); 717 $countHost->read($hostCntFile) if ( -f $hostCntFile ); 718 my($d, $c); 719 my $idx = 0; 720 721 while ( 1 ) { 722 ($d, $c, $idx) = $countHost->iterate($idx); 723 last if ( !defined($d) ); 724 725 if ( $c < 0 ) { 726 printf("BackupPC_refCountUpdate: host %s (%s) digest %s has negative count (%d)\n", 727 $host, $hostCntFile, unpack("H*", $d), $c); 728 $ErrorCnt++; 729 } 730 if ( !defined($countCurr->get($d)) ) { 731 # 732 # add stats for the new pool file 733 # 734 my $poolFile = $bpc->MD52Path($d, $compress); 735 my @s = stat($poolFile); 736 if ( @s ) { 737 my $nBlks = $s[12]; 738 $PoolStats->{$poolName}[$refCntFile]{blkCnt} += $nBlks; 739 if ( $c > 0 ) { 740 if ( ($s[2] & S_IXOTH) && chmod(0444, $poolFile) != 1 ) { 741 print("BackupPC_refCountUpdate: can't chmod 0444 $poolFile\n"); 742 $ErrorCnt++; 743 } 744 } 745 } 746 } elsif ( $countCurr->get($d) == 0 && $c > 0 ) { 747 # 748 # remove S_IXOTH flag since this digest is now referenced 749 # 750 my $poolFile = $bpc->MD52Path($d, $compress); 751 my @s = stat($poolFile); 752 if ( @s ) { 753 if ( ($s[2] & S_IXOTH) && chmod(0444, $poolFile) != 1 ) { 754 print("BackupPC_refCountUpdate: can't chmod 0444 $poolFile\n"); 755 $ErrorCnt++; 756 } 757 } 758 } 759 $count->incr($d, $c); 760 $countCopy->incr($d, $c); 761 $countCurr->incr($d, $c); # make sure we only count the new pool file once. 762 } 763 } 764 765 # 766 # Add entries for any files in the existing pool count that aren't already in count/countCopy. 767 # 768 if ( -f $poolCntFile ) { 769 my($d, $c); 770 my $idx = 0; 771 772 while ( 1 ) { 773 ($d, $c, $idx) = $countCurr->iterate($idx); 774 last if ( !defined($d) ); 775 next if ( defined($count->get($d)) ); 776 $count->incr($d, 0); 777 $countCopy->incr($d, 0); 778 } 779 } 780 781 # 782 # Scan the pool to add any missing files that have a zero count 783 # 784 for ( my $subDir = 0 ; $subDir < 128 ; $subDir++ ) { 785 my $poolSubDir = sprintf("%s/%02x", $poolDir, $subDir * 2); 786 my $entries = BackupPC::DirOps::dirRead($bpc, $poolSubDir); 787 foreach my $e ( @$entries ) { 788 next if ( $e->{name} eq "." || $e->{name} eq ".." ); 789 if ( $e->{name} !~ /^[\da-f]{32,48}$/ && $e->{name} !~ /lock/i ) { 790 print("BackupPC_refCountUpdate: unknown pool file $poolSubDir/$e->{name} removed\n"); 791 unlink("$poolSubDir/$e->{name}"); 792 next; 793 } 794 my $d = pack("H*", $e->{name}); 795 my $b2 = vec($d, 0, 16); 796 if ( $refCntFile != (($b2 >> 8) & 0xfe) / 2 || $subDir != (($b2 >> 0) & 0xfe) / 2 ) { 797 print("BackupPC_refCountUpdate: unexpected pool file $poolSubDir/$e->{name} removed\n"); 798 unlink("$poolSubDir/$e->{name}"); 799 next; 800 } 801 if ( !defined($count->get($d)) ) { 802 my @s = stat("$poolSubDir/$e->{name}"); 803 print("BackupPC_refCountUpdate: adding pool file $e->{name} with count 0 (size $s[7])\n") 804 if ( $Conf{XferLogLevel} >= 5 ); 805 # 806 # add stats for new pool file 807 # 808 my $nBlks = $s[12]; 809 $PoolStats->{$poolName}[$refCntFile]{blkCnt} += $nBlks; 810 $count->incr($d, 0); 811 } else { 812 $countCopy->delete($d); 813 } 814 } 815 } 816 817 # 818 # Compute the full pool size periodically, in case the relative +/- above 819 # doesn't exactly track. 820 # 821 # Normally we only update relative changes to the pool size (when a new pool 822 # file is added, or an old one is deleted), which is a lot more efficient. 823 # 824 # So decide when to do a full pool size scan. $refCntFile goes from 0..127, 825 # phase ($opts{P}) is 0..15 and $Conf{PoolSizeNightlyUpdatePeriod} is 826 # 0, 1, 2, 4, 8, 16. 827 # 828 my $fullPoolScan; 829 if ( $Conf{PoolSizeNightlyUpdatePeriod} > 0 && defined($opts{P}) ) { 830 $fullPoolScan = (int($refCntFile / 8) % $Conf{PoolSizeNightlyUpdatePeriod}) 831 == ($opts{P} % $Conf{PoolSizeNightlyUpdatePeriod}); 832 } 833# print("BackupPC_refCountUpdate: computing full $poolName size for $refCntFile (phase = $opts{P}," 834# . " \$Conf{PoolSizeNightlyUpdatePeriod} = $Conf{PoolSizeNightlyUpdatePeriod})\n") 835# if ( $fullPoolScan ); 836 837 my $blkCnt = 0; # size of pool files 838 my $fileCntRep = 0; # total number of pool files with repeated md5 checksums 839 # (ie: digest > 16 bytes; first instance isn't counted) 840 my $fileRepMax = 0; # worse case chain length of pool files that have repeated 841 # checksums (ie: max(NNN) for all digests xxxxxxxxxxxxxxxxNNN) 842 my $fileLinkMax = 0; # maximum number of links on a pool file 843 my $fileLinkTotal = 0; # total number of links on entire pool 844 my($digest, $cnt); 845 my $idx = 0; 846 my $poolFileCnt = 0; 847 848 while ( 1 ) { 849 ($digest, $cnt, $idx) = $count->iterate($idx); 850 last if ( !defined($digest) ); 851 $poolFileCnt++; 852 $fileLinkTotal += $cnt; 853 $fileLinkMax = $cnt if ( $fileLinkMax < $cnt && $digest ne $EmptyMD5 ); 854 if ( $fullPoolScan ) { 855 my $poolFile = $bpc->MD52Path($digest, $compress); 856 my @s = stat($poolFile); 857 $blkCnt += $s[12] if ( @s ); 858 } 859 next if ( length($digest) <= 16 ); 860 my $ext = $bpc->digestExtGet($digest); 861 $fileCntRep++; 862 $fileRepMax = $ext if ( $fileRepMax < $ext ); 863 } 864 $PoolStats->{$poolName}[$refCntFile]{blkCnt} = $blkCnt if ( $fullPoolScan ); 865 $PoolStats->{$poolName}[$refCntFile]{fileCnt} = $poolFileCnt; 866 $PoolStats->{$poolName}[$refCntFile]{fileLinkMax} = $fileLinkMax; 867 $PoolStats->{$poolName}[$refCntFile]{fileLinkTotal} = $fileLinkTotal; 868 $PoolStats->{$poolName}[$refCntFile]{fileCntRep} = $fileCntRep; 869 $PoolStats->{$poolName}[$refCntFile]{fileRepMax} = $fileRepMax; 870 871 # 872 # Remove zero counts on pool files that don't exist. Report pool files 873 # that have non-zero counts and are missing. 874 # 875 $idx = 0; 876 while ( 1 ) { 877 ($digest, $cnt, $idx) = $countCopy->iterate($idx); 878 last if ( !defined($digest) ); 879 if ( $cnt == 0 ) { 880 $count->delete($digest); 881 next; 882 } 883 next if ( $digest eq $EmptyMD5 ); 884 my $digestStr = unpack("H*", $digest); 885 print("BackupPC_refCountUpdate: missing pool file $digestStr count $cnt\n"); 886 $ErrorCnt++; 887 } 888 889 890 if ( $count->write("$poolCntFile.$$") ) { 891 print("BackupPC_refCountUpdate: can't write new pool count file $poolCntFile.$$\n"); 892 unlink("$poolCntFile.$$"); 893 $ErrorCnt++; 894 return; 895 } elsif ( !rename("$poolCntFile.$$", $poolCntFile) ) { 896 print("BackupPC_refCountUpdate: can't rename $poolCntFile.$$ to $poolCntFile ($!)\n"); 897 unlink("$poolCntFile.$$"); 898 $ErrorCnt++; 899 return; 900 } 901 $bpc->flushXSLibMesgs(); 902 } 903 } 904} 905 906sub cleanPoolFiles 907{ 908 for ( my $compress = 0 ; $compress < 2 ; $compress++ ) { 909 my $poolName = $compress ? "cpool4" : "pool4"; 910 for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCntFile++ ) { 911 # 912 # Read the existing count 913 # 914 my $dirty = 0; 915 my $poolDir = sprintf("%s/%02x", 916 $compress ? $bpc->{CPoolDir} : $bpc->{PoolDir}, 917 $refCntFile * 2); 918 my $poolCntFile = "$poolDir/poolCnt"; 919 my $count = BackupPC::XS::PoolRefCnt::new(); 920 # 921 # Grab a lock to make sure BackupPC_dump won't unmark and use a pending 922 # delete file. 923 # 924 my $lockFd = BackupPC::XS::DirOps::lockRangeFile("$poolDir/LOCK", 0, 1, 1); 925 if ( -f $poolCntFile && $count->read($poolCntFile) ) { 926 print("BackupPC_refCountUpdate: can't read pool count file $poolCntFile\n"); 927 $dirty = 1; 928 $ErrorCnt++; 929 } 930 931 my($digest, $cnt); 932 my $idx = 0; 933 my $minMtime = time() - 7 * 24 * 3600; 934 while ( 1 ) { 935 ($digest, $cnt, $idx) = $count->iterate($idx); 936 last if ( !defined($digest) ); 937 next if ( $cnt > 0 ); 938 my $poolFile = $bpc->MD52Path($digest, $compress); 939 my @s = stat($poolFile); 940 next if ( !@s || $s[7] == 0 || $s[9] >= $minMtime ); 941 my $mode = $s[2]; 942 my $nBlks = $s[12]; 943 if ( $mode & S_IXOTH ) { 944 # 945 # figure out the next file in the sequence 946 # 947 my $ext = $bpc->digestExtGet($digest); 948 my($nextDigest, $nextPoolFile) = $bpc->digestConcat($digest, 949 $ext + 1, $compress); 950 if ( !-f $nextPoolFile ) { 951 # 952 # last in the chain (or no chain) - just delete it 953 # 954 print("BackupPC_refCountUpdate: removing pool file $poolFile\n") if ( $Conf{XferLogLevel} >= 4 ); 955 if ( unlink($poolFile) != 1 ) { 956 print("BackupPC_refCountUpdate: can't remove $poolFile\n"); 957 $ErrorCnt++; 958 next; 959 } 960 } else { 961 # 962 # in a chain of pool files we can't delete so 963 # we replace the file with an empty file. 964 # first remove S_IXOTH mode 965 # 966 print("BackupPC_refCountUpdate: zeroing pool file $poolFile (next $nextPoolFile exists)\n") if ( $Conf{XferLogLevel} >= 4 ); 967 if ( chmod(0644, $poolFile) != 1 ) { 968 print("BackupPC_refCountUpdate: can't chmod 0644 $poolFile\n"); 969 $ErrorCnt++; 970 } 971 if ( open(my $fh, ">", $poolFile) ) { 972 close($fh); 973 } else { 974 print("BackupPC_refCountUpdate: can't truncate $poolFile\n"); 975 $ErrorCnt++; 976 next; 977 } 978 } 979 $count->delete($digest); 980 $dirty = 1; 981 # 982 # update stats 983 # 984 $PoolStats->{$poolName}[$refCntFile]{fileCnt}--; 985 $PoolStats->{$poolName}[$refCntFile]{blkCnt} -= $nBlks; 986 $PoolStats->{$poolName}[$refCntFile]{fileCntRm}++; 987 $PoolStats->{$poolName}[$refCntFile]{blkCntRm} += $nBlks; 988 } else { 989 # 990 # mark the pool file so no one links to it 991 # 992 print("BackupPC_refCountUpdate: marking pool file $poolFile\n") if ( $Conf{XferLogLevel} >= 4 ); 993 if ( chmod(0445, $poolFile) != 1 ) { 994 print("BackupPC_refCountUpdate: can't chmod 0445 $poolFile\n"); 995 $ErrorCnt++; 996 } 997 } 998 } 999 if ( $dirty ) { 1000 # 1001 # rewrite the poolCnt file 1002 # 1003 if ( $count->write("$poolCntFile.$$") ) { 1004 print("BackupPC_refCountUpdate: can't write new pool count file $poolCntFile.$$\n"); 1005 $ErrorCnt++; 1006 } elsif ( !rename("$poolCntFile.$$", $poolCntFile) ) { 1007 print("BackupPC_refCountUpdate: can't rename $poolCntFile.$$ to $poolCntFile ($!)\n"); 1008 $ErrorCnt++; 1009 } 1010 } 1011 BackupPC::XS::DirOps::unlockRangeFile($lockFd); 1012 if ( $opts{s} ) { 1013 statsPrintSingle($poolName, $refCntFile); 1014 } 1015 } 1016 } 1017} 1018 1019sub statsPrintSingle 1020{ 1021 my($poolName, $refCntFile) = @_; 1022 1023 my $s = $PoolStats->{$poolName}[$refCntFile]; 1024 my $kb = $s->{blkCnt} >= 0 ? int($s->{blkCnt} / 2 + 0.5) : int($s->{blkCnt} / 2 - 0.5); 1025 my $kbRm = $s->{blkCntRm} >= 0 ? int($s->{blkCntRm} / 2 + 0.5) : int($s->{blkCntRm} / 2 - 0.5); 1026 printf("BackupPC_stats4 %d = %s,%d,%d,%d,%d,%d,%d,%d,%d,%d\n", 1027 $refCntFile, $poolName, 1028 $s->{fileCnt}, $s->{dirCnt}, $kb, $kbRm, 1029 $s->{fileCntRm}, $s->{fileCntRep}, $s->{fileRepMax}, 1030 $s->{fileLinkMax}, $s->{fileLinkTotal}); 1031} 1032 1033sub statsPrint 1034{ 1035 foreach my $poolName ( qw(pool4 cpool4) ) { 1036 for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCntFile++ ) { 1037 statsPrintSingle($poolName, $refCntFile); 1038 } 1039 } 1040} 1041