1#!/usr/bin/perl
2#============================================================= -*-perl-*-
3#
4# BackupPC_fsck: Pool reference count file system check
5#
6# DESCRIPTION
7#
8#   BackupPC_fsck checks the pool reference counts
9#
10#   Usage: BackupPC_fsck
11#
12# AUTHOR
13#   Craig Barratt  <cbarratt@users.sourceforge.net>
14#
15# COPYRIGHT
16#   Copyright (C) 2001-2020  Craig Barratt
17#
18#   This program is free software: you can redistribute it and/or modify
19#   it under the terms of the GNU General Public License as published by
20#   the Free Software Foundation, either version 3 of the License, or
21#   (at your option) any later version.
22#
23#   This program is distributed in the hope that it will be useful,
24#   but WITHOUT ANY WARRANTY; without even the implied warranty of
25#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
26#   GNU General Public License for more details.
27#
28#   You should have received a copy of the GNU General Public License
29#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
30#
31#========================================================================
32#
33# Version 4.4.0, released 20 Jun 2020.
34#
35# See http://backuppc.sourceforge.net.
36#
37#========================================================================
38
39use strict;
40no utf8;
41
42use lib "__INSTALLDIR__/lib";
43use BackupPC::Lib;
44use BackupPC::XS;
45use BackupPC::Storage;
46use BackupPC::DirOps;
47use Getopt::Std;
48
49use File::Path;
50use Data::Dumper;
51
52my(@Ref, $RefTotal);
53my $ErrorCnt = 0;
54my %opts;
55
56my $EmptyMD5 = pack("H*", "d41d8cd98f00b204e9800998ecf8427e");
57
58die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) );
59
60my $TopDir = $bpc->TopDir();
61my $BinDir = $bpc->BinDir();
62my %Conf   = $bpc->Conf();
63my $Hosts  = $bpc->HostInfoRead();
64my $s      = $bpc->{storage};
65
66if ( !getopts("fns", \%opts) || @ARGV >= 1 ) {
67    print STDERR <<EOF;
68usage: $0 [options]
69  Options:
70     -f              force regeneration of per-host reference counts
71     -n              don't remove zero count pool files - print only
72     -s              recompute pool stats
73EOF
74    exit(1);
75}
76
77#
78# We can't run if BackupPC is running
79#
80CheckIfServerRunning();
81
82my($Status, $Info);
83if ( $opts{s} ) {
84    ($Status, $Info) = $s->StatusDataRead();
85    if ( !defined($Info) && ref($Status) ne "HASH" ) {
86        print STDERR "$0: status.pl read failed: $Status\n";
87        $Info   = {};
88        $Status = {};
89    }
90    #
91    # Zero out the statistics
92    #
93    for my $p ( qw(pool4 cpool4) ) {
94        for ( my $i = 0 ; $i < 16 ; $i++ ) {
95            $Info->{pool}{$p}[$i]{FileCnt}     = 0;
96            $Info->{pool}{$p}[$i]{DirCnt}      = 0;
97            $Info->{pool}{$p}[$i]{KbRm}        = 0;
98            $Info->{pool}{$p}[$i]{Kb}          = 0;
99            $Info->{pool}{$p}[$i]{FileCntRm}   = 0;
100            $Info->{pool}{$p}[$i]{FileCntRep}  = 0;
101            $Info->{pool}{$p}[$i]{FileRepMax}  = 0;
102            $Info->{pool}{$p}[$i]{FileLinkMax} = 0;
103            $Info->{pool}{$p}[$i]{Time}        = 0;
104        }
105    }
106}
107
108if ( $opts{f} ) {
109    #
110    # Rebuild host count database
111    #
112    foreach my $host ( sort(keys(%$Hosts)) ) {
113        print("BackupPC_fsck: Rebuilding count database for host $host\n");
114        $ErrorCnt++ if ( system("$BinDir/BackupPC_refCountUpdate -h $host -F -p") );
115    }
116} else {
117    #
118    # Make sure each host count database is up to date
119    # (ie: process any delta files)
120    #
121    foreach my $host ( sort(keys(%$Hosts)) ) {
122        $ErrorCnt++ if ( system("$BinDir/BackupPC_refCountUpdate -o 0 -h $host -p") );
123    }
124}
125
126CheckIfServerRunning();
127
128print("BackupPC_fsck: building main count database\n");
129$ErrorCnt++ if ( system("$BinDir/BackupPC_refCountUpdate -m -p") );
130
131CheckIfServerRunning();
132
133print("BackupPC_fsck: Calling poolCountUpdate\n");
134poolCountUpdate();
135
136if ( $opts{s} ) {
137    print("$0: Rewriting $s->{LogDir}/status.pl\n");
138    $s->StatusDataWrite($Status, $Info);
139}
140
141print("$0: got $ErrorCnt errors\n");
142exit($ErrorCnt ? 1 : 0);
143
144sub poolCountUpdate
145{
146    for ( my $compress = 0 ; $compress < 2 ; $compress++ ) {
147        my $poolName = $compress ? "cpool4" : "pool4";
148        for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) {
149
150            my $fileCnt       = 0;    # total number of pool files
151            my $dirCnt        = 0;    # total number of pool directories
152            my $blkCnt        = 0;    # total block size of pool files
153            my $fileCntRm     = 0;    # total number of removed files
154            my $blkCntRm      = 0;    # total block size of removed pool files
155            my $fileCntRep    = 0;    # total number of pool files with repeated md5 checksums
156                                      # (ie: digest > 16 bytes; first instance isn't counted)
157            my $fileRepMax    = 0;    # worse case chain length of pool files that have repeated
158                                      # checksums (ie: max(NNN) for all digests xxxxxxxxxxxxxxxxNNN)
159            my $fileLinkMax   = 0;    # maximum number of links on a pool file
160            my $fileLinkTotal = 0;    # total number of links on entire pool
161
162            my $poolDir = sprintf("%s/%02x", $compress ? $bpc->{CPoolDir} : $bpc->{PoolDir}, $refCntFile * 2);
163
164            next if ( !-d $poolDir );
165            $dirCnt++;
166
167            my $count       = BackupPC::XS::PoolRefCnt::new();
168            my $dirty       = 0;
169            my $poolCntFile = "$poolDir/poolCnt";
170
171            #
172            # Count the number of pool directories
173            #
174            my $entries = BackupPC::DirOps::dirRead($bpc, $poolDir);
175            foreach my $e ( @$entries ) {
176                next if ( $e->{name} !~ /^[\da-f][\da-f]$/ );
177                $dirCnt++;
178            }
179
180            #
181            # Grab a lock to make sure BackupPC_dump won't unmark and use a pending
182            # delete file.
183            #
184            my $lockFd = BackupPC::XS::DirOps::lockRangeFile("$poolDir/LOCK", 0, 1, 1);
185            if ( -f $poolCntFile && $count->read($poolCntFile) ) {
186                print("Can't read pool count file $poolCntFile\n");
187                $dirty = 1;
188                $ErrorCnt++;
189            }
190
191            #
192            # Check that every file in the pool has a corresponding count.
193            # There are 128 subdirectories below this level.
194            #
195            for ( my $subDir = 0 ; $subDir < 128 ; $subDir++ ) {
196                my $subPoolDir = sprintf("%s/%02x", $poolDir, $subDir * 2);
197                next if ( !-d $subPoolDir );
198
199                my $entries = BackupPC::DirOps::dirRead($bpc, $subPoolDir);
200                next if ( !defined($entries) );
201
202                #
203                # traverse the files in reverse order, in case we can delete multiple files in
204                # a single chain.
205                #
206                foreach my $e ( sort { $b cmp $a } @$entries ) {
207                    next if ( $e->{name} eq "." || $e->{name} eq ".." || $e->{name} eq "LOCK" );
208                    my $digest   = pack("H*", $e->{name});
209                    my $poolFile = "$subPoolDir/$e->{name}";
210                    #printf("Got %s, digest = %s\n", $e->{name}, unpack("H*", $digest));
211                    my($nBlks, @s);
212                    if ( $opts{s} ) {
213                        @s     = stat($poolFile);
214                        $nBlks = $s[12];
215                        $blkCnt += $nBlks;
216                    }
217                    next if ( $count->get($digest) != 0 );
218
219                    #
220                    # figure out the next file in the chain to see how to
221                    # handle this one.
222                    #
223                    @s = stat($poolFile) if ( !$opts{s} );
224                    my $ext = $bpc->digestExtGet($digest);
225                    my($nextDigest, $nextPoolFile) = $bpc->digestConcat($digest, $ext + 1, $compress);
226                    if ( !-f $nextPoolFile ) {
227                        #
228                        # last in the chain (or no chain) - just delete it
229                        #
230                        print("Removing pool file $poolFile\n") if ( $Conf{XferLogLevel} >= 2 );
231                        if ( !$opts{n} ) {
232                            if ( unlink($poolFile) != 1 ) {
233                                print("Can't remove $poolFile\n");
234                                $ErrorCnt++;
235                                next;
236                            }
237                            $count->delete($digest);
238                            $fileCntRm++;
239                            $blkCntRm += $nBlks;
240                        }
241                    } elsif ( $s[7] > 0 ) {
242                        #
243                        # in the middle of a chain of pool files, so
244                        # we replace the file with an empty file.
245                        #
246                        print("Zeroing pool file $poolFile (next $nextPoolFile exists)\n")
247                          if ( $Conf{XferLogLevel} >= 2 );
248                        if ( !$opts{n} ) {
249                            if ( chmod(0644, $poolFile) != 1 ) {
250                                print("Can't chmod 0644 $poolFile\n");
251                                $ErrorCnt++;
252                            }
253                            if ( open(my $fh, ">", $poolFile) ) {
254                                close($fh);
255                            } else {
256                                print("Can't truncate $poolFile\n");
257                                $ErrorCnt++;
258                                next;
259                            }
260                            $count->delete($digest);
261                            $fileCntRm++;
262                            $blkCntRm += $nBlks;
263                        }
264                    }
265                }
266            }
267
268            if ( $opts{s} ) {
269                my($digest, $cnt);
270                my $idx = 0;
271                while ( 1 ) {
272                    ($digest, $cnt, $idx) = $count->iterate($idx);
273                    last if ( !defined($digest) );
274
275                    $fileCnt++;
276                    $fileLinkTotal += $cnt;
277                    $fileLinkMax = $cnt if ( $cnt > $fileLinkMax && $digest ne $EmptyMD5 );
278                    next if ( length($digest) <= 16 );
279                    my $ext = $bpc->digestExtGet($digest);
280                    $fileCntRep += $ext;
281                    $fileRepMax = $ext if ( $ext > $fileRepMax );
282                }
283                my $kb   = int($blkCnt / 2 + 0.5);
284                my $kbRm = int($blkCntRm / 2 + 0.5);
285                #print("BackupPC_stats4 $refCntFile = $poolName,$fileCnt,$dirCnt,$kb,$kbRm,"
286                #      . "$fileCntRm,$fileCntRep,$fileRepMax,$fileLinkMax,$fileLinkTotal\n");
287                my $chunk = int($refCntFile / 8);
288                $Info->{pool}{$poolName}[$chunk]{FileCnt}    += $fileCnt;
289                $Info->{pool}{$poolName}[$chunk]{DirCnt}     += $dirCnt;
290                $Info->{pool}{$poolName}[$chunk]{Kb}         += $kb;
291                $Info->{pool}{$poolName}[$chunk]{KbRm}       += $kbRm;
292                $Info->{pool}{$poolName}[$chunk]{FileCntRm}  += $fileCntRm;
293                $Info->{pool}{$poolName}[$chunk]{FileCntRep} += $fileCntRep;
294                $Info->{pool}{$poolName}[$chunk]{FileRepMax} = $fileRepMax
295                  if ( $Info->{pool}{$poolName}[$chunk]{FileRepMax} < $fileRepMax );
296                $Info->{pool}{$poolName}[$chunk]{FileLinkMax} = $fileLinkMax
297                  if ( $Info->{pool}{$poolName}[$chunk]{FileLinkMax} < $fileLinkMax );
298                $Info->{pool}{$poolName}[$chunk]{FileLinkTotal} += $fileLinkTotal;
299                $Info->{pool}{$poolName}[$chunk]{Time} = time;
300            }
301        }
302    }
303    if ( $opts{s} ) {
304        #
305        # Update the cumulative statistics for pool4 and cpool4
306        #
307        for my $p ( qw(pool4 cpool4) ) {
308            $Info->{"${p}FileCnt"}     = 0;
309            $Info->{"${p}DirCnt"}      = 0;
310            $Info->{"${p}Kb"}          = 0;
311            $Info->{"${p}KbRm"}        = 0;
312            $Info->{"${p}FileCntRm"}   = 0;
313            $Info->{"${p}FileCntRep"}  = 0;
314            $Info->{"${p}FileRepMax"}  = 0;
315            $Info->{"${p}FileLinkMax"} = 0;
316            $Info->{"${p}Time"}        = 0;
317            delete $Info->{"${p}FileCntRename"};
318
319            for ( my $i = 0 ; $i < 16 ; $i++ ) {
320                $Info->{"${p}FileCnt"}    += $Info->{pool}{$p}[$i]{FileCnt};
321                $Info->{"${p}DirCnt"}     += $Info->{pool}{$p}[$i]{DirCnt};
322                $Info->{"${p}Kb"}         += $Info->{pool}{$p}[$i]{Kb};
323                $Info->{"${p}KbRm"}       += $Info->{pool}{$p}[$i]{KbRm};
324                $Info->{"${p}FileCntRm"}  += $Info->{pool}{$p}[$i]{FileCntRm};
325                $Info->{"${p}FileCntRep"} += $Info->{pool}{$p}[$i]{FileCntRep};
326                $Info->{"${p}FileRepMax"} = $Info->{pool}{$p}[$i]{FileRepMax}
327                  if ( $Info->{"${p}FileRepMax"} < $Info->{pool}{$p}[$i]{FileRepMax} );
328                $Info->{"${p}FileLinkMax"} = $Info->{pool}{$p}[$i]{FileLinkMax}
329                  if ( $Info->{"${p}FileLinkMax"} < $Info->{pool}{$p}[$i]{FileLinkMax} );
330                $Info->{"${p}Time"} = $Info->{pool}{$p}[$i]{Time}
331                  if ( $Info->{"${p}Time"} < $Info->{pool}{$p}[$i]{Time} );
332            }
333            printf(
334                "%s%s BackupPC_fsck removed %d files of size %.2fGB\n",
335                $bpc->timeStamp, ucfirst($p),
336                $Info->{"${p}FileCntRm"},
337                $Info->{"${p}KbRm"} / (1000 * 1024)
338            );
339            printf(
340                "%s%s is %.2fGB, %d files (%d repeated, %d max chain, %d max links), %d directories\n",
341                $bpc->timeStamp, ucfirst($p),
342                $Info->{"${p}Kb"} / (1000 * 1024),
343                $Info->{"${p}FileCnt"},
344                $Info->{"${p}FileCntRep"},
345                $Info->{"${p}FileRepMax"},
346                $Info->{"${p}FileLinkMax"},
347                $Info->{"${p}DirCnt"}
348            );
349        }
350    }
351}
352
353sub CheckIfServerRunning
354{
355    my $err = $bpc->ServerConnect($Conf{ServerHost}, $Conf{ServerPort});
356    if ( !defined $err ) {
357        print STDERR "$0: can't run since BackupPC is running\n";
358        exit(1);
359    }
360}
361