1 // distribution boxbackup-0.11_trunk_2979 (svn version: 2979)
2 // Box Backup, http://www.boxbackup.org/
3 //
4 // Copyright (c) 2003-2010, Ben Summers and contributors.
5 // All rights reserved.
6 //
7 // Note that this project uses mixed licensing. Any file with this license
8 // attached, or where the code LICENSE-GPL appears on the first line, falls
9 // under the "Box Backup GPL" license. See the file COPYING.txt for more
10 // information about this license.
11 //
12 // ---------------------------------------------------------------------
13 // This program is free software; you can redistribute it and/or
14 // modify it under the terms of the GNU General Public License
15 // as published by the Free Software Foundation; either version 2
16 // of the License, or (at your option) any later version.
17 //
18 // This program is distributed in the hope that it will be useful,
19 // but WITHOUT ANY WARRANTY; without even the implied warranty of
20 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21 // GNU General Public License for more details.
22 //
23 // You should have received a copy of the GNU General Public License
24 // along with this program; if not, write to the Free Software
25 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
26 //
27 // [http://www.gnu.org/licenses/old-licenses/gpl-2.0.html#SEC4]
28 //
29 // As a special exception to the GPLv2, the Box Backup Project gives
30 // permission to link any code falling under this license (the Box Backup
31 // GPL) with any software that can be downloaded from
32 // the OpenSSL website [http://www.openssl.org] under either the
33 // "OpenSSL License" or the "Original SSLeay License", and to distribute
34 // the linked executables under the terms of the "Box Backup GPL" license.
35 //
36 // As a special exception to the GPLv2, the Box Backup Project gives
37 // permission to link any code falling under this license (the Box Backup
38 // GPL) with any version of Microsoft's Volume Shadow Copy Service 7.2 SDK
39 // or Microsoft Windows Software Development Kit (SDK), including
40 // vssapi.lib, that can be downloaded from the Microsoft website
41 // [*.microsoft.com], and to distribute the linked executables under the
42 // terms of the "Box Backup GPL" license.
43 // --------------------------------------------------------------------------
44 //
45 // File
46 //		Name:    BackupStoreCheck.cpp
47 //		Purpose: Check a store for consistency
48 //		Created: 21/4/04
49 //
50 // --------------------------------------------------------------------------
51 
52 #include "Box.h"
53 
54 #include <stdio.h>
55 #include <string.h>
56 
57 #ifdef HAVE_UNISTD_H
58 #	include <unistd.h>
59 #endif
60 
61 #include "BackupStoreCheck.h"
62 #include "StoreStructure.h"
63 #include "RaidFileRead.h"
64 #include "RaidFileWrite.h"
65 #include "autogen_BackupStoreException.h"
66 #include "BackupStoreObjectMagic.h"
67 #include "BackupStoreFile.h"
68 #include "BackupStoreDirectory.h"
69 #include "BackupStoreConstants.h"
70 
71 #include "MemLeakFindOn.h"
72 
73 
74 // --------------------------------------------------------------------------
75 //
76 // Function
77 //		Name:    BackupStoreCheck::BackupStoreCheck(const std::string &, int, int32_t, bool, bool)
78 //		Purpose: Constructor
79 //		Created: 21/4/04
80 //
81 // --------------------------------------------------------------------------
BackupStoreCheck(const std::string & rStoreRoot,int DiscSetNumber,int32_t AccountID,bool FixErrors,bool Quiet)82 BackupStoreCheck::BackupStoreCheck(const std::string &rStoreRoot, int DiscSetNumber, int32_t AccountID, bool FixErrors, bool Quiet)
83 	: mStoreRoot(rStoreRoot),
84 	  mDiscSetNumber(DiscSetNumber),
85 	  mAccountID(AccountID),
86 	  mFixErrors(FixErrors),
87 	  mQuiet(Quiet),
88 	  mNumberErrorsFound(0),
89 	  mLastIDInInfo(0),
90 	  mpInfoLastBlock(0),
91 	  mInfoLastBlockEntries(0),
92 	  mLostDirNameSerial(0),
93 	  mLostAndFoundDirectoryID(0),
94 	  mBlocksUsed(0),
95 	  mBlocksInCurrentFiles(0),
96 	  mBlocksInOldFiles(0),
97 	  mBlocksInDeletedFiles(0),
98 	  mBlocksInDirectories(0),
99 	  mNumFiles(0),
100 	  mNumOldFiles(0),
101 	  mNumDeletedFiles(0),
102 	  mNumDirectories(0)
103 {
104 }
105 
106 
107 // --------------------------------------------------------------------------
108 //
109 // Function
110 //		Name:    BackupStoreCheck::~BackupStoreCheck()
111 //		Purpose: Destructor
112 //		Created: 21/4/04
113 //
114 // --------------------------------------------------------------------------
~BackupStoreCheck()115 BackupStoreCheck::~BackupStoreCheck()
116 {
117 	// Clean up
118 	FreeInfo();
119 }
120 
121 
122 // --------------------------------------------------------------------------
123 //
124 // Function
125 //		Name:    BackupStoreCheck::Check()
126 //		Purpose: Perform the check on the given account
127 //		Created: 21/4/04
128 //
129 // --------------------------------------------------------------------------
Check()130 void BackupStoreCheck::Check()
131 {
132 	// Lock the account
133 	{
134 		std::string writeLockFilename;
135 		StoreStructure::MakeWriteLockFilename(mStoreRoot, mDiscSetNumber, writeLockFilename);
136 
137 		bool gotLock = false;
138 		int triesLeft = 8;
139 		do
140 		{
141 			gotLock = mAccountLock.TryAndGetLock(writeLockFilename.c_str(), 0600 /* restrictive file permissions */);
142 
143 			if(!gotLock)
144 			{
145 				--triesLeft;
146 				::sleep(1);
147 			}
148 		} while(!gotLock && triesLeft > 0);
149 
150 		if(!gotLock)
151 		{
152 			// Couldn't lock the account -- just stop now
153 			if(!mQuiet)
154 			{
155 				BOX_ERROR("Failed to lock the account -- did not check.\nTry again later after the client has disconnected.\nAlternatively, forcibly kill the server.");
156 			}
157 			THROW_EXCEPTION(BackupStoreException, CouldNotLockStoreAccount)
158 		}
159 	}
160 
161 	if(!mQuiet && mFixErrors)
162 	{
163 		BOX_NOTICE("Will fix errors encountered during checking.");
164 	}
165 
166 	// Phase 1, check objects
167 	if(!mQuiet)
168 	{
169 		BOX_INFO("Checking store account ID " <<
170 			BOX_FORMAT_ACCOUNT(mAccountID) << "...");
171 		BOX_INFO("Phase 1, check objects...");
172 	}
173 	CheckObjects();
174 
175 	// Phase 2, check directories
176 	if(!mQuiet)
177 	{
178 		BOX_INFO("Phase 2, check directories...");
179 	}
180 	CheckDirectories();
181 
182 	// Phase 3, check root
183 	if(!mQuiet)
184 	{
185 		BOX_INFO("Phase 3, check root...");
186 	}
187 	CheckRoot();
188 
189 	// Phase 4, check unattached objects
190 	if(!mQuiet)
191 	{
192 		BOX_INFO("Phase 4, fix unattached objects...");
193 	}
194 	CheckUnattachedObjects();
195 
196 	// Phase 5, fix bad info
197 	if(!mQuiet)
198 	{
199 		BOX_INFO("Phase 5, fix unrecovered inconsistencies...");
200 	}
201 	FixDirsWithWrongContainerID();
202 	FixDirsWithLostDirs();
203 
204 	// Phase 6, regenerate store info
205 	if(!mQuiet)
206 	{
207 		BOX_INFO("Phase 6, regenerate store info...");
208 	}
209 	WriteNewStoreInfo();
210 
211 //	DUMP_OBJECT_INFO
212 
213 	if(mNumberErrorsFound > 0)
214 	{
215 		BOX_WARNING("Finished checking store account ID " <<
216 			BOX_FORMAT_ACCOUNT(mAccountID) << ": " <<
217 			mNumberErrorsFound << " errors found");
218 		if(!mFixErrors)
219 		{
220 			BOX_WARNING("No changes to the store account "
221 				"have been made.");
222 		}
223 		if(!mFixErrors && mNumberErrorsFound > 0)
224 		{
225 			BOX_WARNING("Run again with fix option to "
226 				"fix these errors");
227 		}
228 		if(mFixErrors && mNumberErrorsFound > 0)
229 		{
230 			BOX_WARNING("You should now use bbackupquery "
231 				"on the client machine to examine the store.");
232 			if(mLostAndFoundDirectoryID != 0)
233 			{
234 				BOX_WARNING("A lost+found directory was "
235 					"created in the account root.\n"
236 					"This contains files and directories "
237 					"which could not be matched to "
238 					"existing directories.\n"\
239 					"bbackupd will delete this directory "
240 					"in a few days time.");
241 			}
242 		}
243 	}
244 	else
245 	{
246 		BOX_NOTICE("Finished checking store account ID " <<
247 			BOX_FORMAT_ACCOUNT(mAccountID) << ": "
248 			"no errors found");
249 	}
250 }
251 
252 
253 // --------------------------------------------------------------------------
254 //
255 // Function
256 //		Name:    static TwoDigitHexToInt(const char *, int &)
257 //		Purpose: Convert a two digit hex string to an int, returning whether it's valid or not
258 //		Created: 21/4/04
259 //
260 // --------------------------------------------------------------------------
TwoDigitHexToInt(const char * String,int & rNumberOut)261 static inline bool TwoDigitHexToInt(const char *String, int &rNumberOut)
262 {
263 	int n = 0;
264 	// Char 0
265 	if(String[0] >= '0' && String[0] <= '9')
266 	{
267 		n = (String[0] - '0') << 4;
268 	}
269 	else if(String[0] >= 'a' && String[0] <= 'f')
270 	{
271 		n = ((String[0] - 'a') + 0xa) << 4;
272 	}
273 	else
274 	{
275 		return false;
276 	}
277 	// Char 1
278 	if(String[1] >= '0' && String[1] <= '9')
279 	{
280 		n |= String[1] - '0';
281 	}
282 	else if(String[1] >= 'a' && String[1] <= 'f')
283 	{
284 		n |= (String[1] - 'a') + 0xa;
285 	}
286 	else
287 	{
288 		return false;
289 	}
290 
291 	// Return a valid number
292 	rNumberOut = n;
293 	return true;
294 }
295 
296 
297 // --------------------------------------------------------------------------
298 //
299 // Function
300 //		Name:    BackupStoreCheck::CheckObjects()
301 //		Purpose: Read in the contents of the directory, recurse to other levels,
302 //				 checking objects for sanity and readability
303 //		Created: 21/4/04
304 //
305 // --------------------------------------------------------------------------
CheckObjects()306 void BackupStoreCheck::CheckObjects()
307 {
308 	// Maximum start ID of directories -- worked out by looking at disc contents, not trusting anything
309 	int64_t maxDir = 0;
310 
311 	// Find the maximum directory starting ID
312 	{
313 		// Make sure the starting root dir doesn't end with '/'.
314 		std::string start(mStoreRoot);
315 		if(start.size() > 0 && start[start.size() - 1] == '/')
316 		{
317 			start.resize(start.size() - 1);
318 		}
319 
320 		maxDir = CheckObjectsScanDir(0, 1, mStoreRoot);
321 		BOX_TRACE("Max dir starting ID is " <<
322 			BOX_FORMAT_OBJECTID(maxDir));
323 	}
324 
325 	// Then go through and scan all the objects within those directories
326 	for(int64_t d = 0; d <= maxDir; d += (1<<STORE_ID_SEGMENT_LENGTH))
327 	{
328 		CheckObjectsDir(d);
329 	}
330 }
331 
332 // --------------------------------------------------------------------------
333 //
334 // Function
335 //		Name:    BackupStoreCheck::CheckObjectsScanDir(int64_t, int, int, const std::string &)
336 //		Purpose: Read in the contents of the directory, recurse to other levels,
337 //				 return the maximum starting ID of any directory found.
338 //		Created: 21/4/04
339 //
340 // --------------------------------------------------------------------------
CheckObjectsScanDir(int64_t StartID,int Level,const std::string & rDirName)341 int64_t BackupStoreCheck::CheckObjectsScanDir(int64_t StartID, int Level, const std::string &rDirName)
342 {
343 	//TRACE2("Scan directory for max dir starting ID %s, StartID %lld\n", rDirName.c_str(), StartID);
344 
345 	int64_t maxID = StartID;
346 
347 	// Read in all the directories, and recurse downwards
348 	{
349 		std::vector<std::string> dirs;
350 		RaidFileRead::ReadDirectoryContents(mDiscSetNumber, rDirName,
351 			RaidFileRead::DirReadType_DirsOnly, dirs);
352 
353 		for(std::vector<std::string>::const_iterator i(dirs.begin()); i != dirs.end(); ++i)
354 		{
355 			// Check to see if it's the right name
356 			int n = 0;
357 			if((*i).size() == 2 && TwoDigitHexToInt((*i).c_str(), n)
358 				&& n < (1<<STORE_ID_SEGMENT_LENGTH))
359 			{
360 				// Next level down
361 				int64_t mi = CheckObjectsScanDir(StartID | (n << (Level * STORE_ID_SEGMENT_LENGTH)), Level + 1,
362 					rDirName + DIRECTORY_SEPARATOR + *i);
363 				// Found a greater starting ID?
364 				if(mi > maxID)
365 				{
366 					maxID = mi;
367 				}
368 			}
369 			else
370 			{
371 				BOX_WARNING("Spurious or invalid directory " <<
372 					rDirName << DIRECTORY_SEPARATOR <<
373 					(*i) << " found, " <<
374 					(mFixErrors?"deleting":"delete manually"));
375 				++mNumberErrorsFound;
376 			}
377 		}
378 	}
379 
380 	return maxID;
381 }
382 
383 
384 // --------------------------------------------------------------------------
385 //
386 // Function
387 //		Name:    BackupStoreCheck::CheckObjectsDir(int64_t)
388 //		Purpose: Check all the files within this directory which has
389 //			 the given starting ID.
390 //		Created: 22/4/04
391 //
392 // --------------------------------------------------------------------------
CheckObjectsDir(int64_t StartID)393 void BackupStoreCheck::CheckObjectsDir(int64_t StartID)
394 {
395 	// Make directory name -- first generate the filename of an entry in it
396 	std::string dirName;
397 	StoreStructure::MakeObjectFilename(StartID, mStoreRoot, mDiscSetNumber, dirName, false /* don't make sure the dir exists */);
398 	// Check expectations
399 	ASSERT(dirName.size() > 4 &&
400 		dirName[dirName.size() - 4] == DIRECTORY_SEPARATOR_ASCHAR);
401 	// Remove the filename from it
402 	dirName.resize(dirName.size() - 4); // four chars for "/o00"
403 
404 	// Check directory exists
405 	if(!RaidFileRead::DirectoryExists(mDiscSetNumber, dirName))
406 	{
407 		BOX_WARNING("RaidFile dir " << dirName << " does not exist");
408 		return;
409 	}
410 
411 	// Read directory contents
412 	std::vector<std::string> files;
413 	RaidFileRead::ReadDirectoryContents(mDiscSetNumber, dirName,
414 		RaidFileRead::DirReadType_FilesOnly, files);
415 
416 	// Array of things present
417 	bool idsPresent[(1<<STORE_ID_SEGMENT_LENGTH)];
418 	for(int l = 0; l < (1<<STORE_ID_SEGMENT_LENGTH); ++l)
419 	{
420 		idsPresent[l] = false;
421 	}
422 
423 	// Parse each entry, building up a list of object IDs which are present in the dir.
424 	// This is done so that whatever order is retured from the directory, objects are scanned
425 	// in order.
426 	// Filename must begin with a 'o' and be three characters long, otherwise it gets deleted.
427 	for(std::vector<std::string>::const_iterator i(files.begin()); i != files.end(); ++i)
428 	{
429 		bool fileOK = true;
430 		int n = 0;
431 		if((*i).size() == 3 && (*i)[0] == 'o' && TwoDigitHexToInt((*i).c_str() + 1, n)
432 			&& n < (1<<STORE_ID_SEGMENT_LENGTH))
433 		{
434 			// Filename is valid, mark as existing
435 			idsPresent[n] = true;
436 		}
437 		// No other files should be present in subdirectories
438 		else if(StartID != 0)
439 		{
440 			fileOK = false;
441 		}
442 		// info and refcount databases are OK in the root directory
443 		else if(*i == "info" || *i == "refcount.db")
444 		{
445 			fileOK = true;
446 		}
447 		else
448 		{
449 			fileOK = false;
450 		}
451 
452 		if(!fileOK)
453 		{
454 			// Unexpected or bad file, delete it
455 			BOX_WARNING("Spurious file " << dirName <<
456 				DIRECTORY_SEPARATOR << (*i) << " found" <<
457 				(mFixErrors?", deleting":""));
458 			++mNumberErrorsFound;
459 			if(mFixErrors)
460 			{
461 				RaidFileWrite del(mDiscSetNumber, dirName + DIRECTORY_SEPARATOR + *i);
462 				del.Delete();
463 			}
464 		}
465 	}
466 
467 	// Check all the objects found in this directory
468 	for(int i = 0; i < (1<<STORE_ID_SEGMENT_LENGTH); ++i)
469 	{
470 		if(idsPresent[i])
471 		{
472 			// Check the object is OK, and add entry
473 			char leaf[8];
474 			::sprintf(leaf, DIRECTORY_SEPARATOR "o%02x", i);
475 			if(!CheckAndAddObject(StartID | i, dirName + leaf))
476 			{
477 				// File was bad, delete it
478 				BOX_WARNING("Corrupted file " << dirName <<
479 					leaf << " found" <<
480 					(mFixErrors?", deleting":""));
481 				++mNumberErrorsFound;
482 				if(mFixErrors)
483 				{
484 					RaidFileWrite del(mDiscSetNumber, dirName + leaf);
485 					del.Delete();
486 				}
487 			}
488 		}
489 	}
490 }
491 
492 
493 // --------------------------------------------------------------------------
494 //
495 // Function
496 //		Name:    BackupStoreCheck::CheckAndAddObject(int64_t,
497 //			 const std::string &)
498 //		Purpose: Check a specific object and add it to the list
499 //			 if it's OK. If there are any errors with the
500 //			 reading, return false and it'll be deleted.
501 //		Created: 21/4/04
502 //
503 // --------------------------------------------------------------------------
CheckAndAddObject(int64_t ObjectID,const std::string & rFilename)504 bool BackupStoreCheck::CheckAndAddObject(int64_t ObjectID,
505 	const std::string &rFilename)
506 {
507 	// Info on object...
508 	bool isFile = true;
509 	int64_t containerID = -1;
510 	int64_t size = -1;
511 
512 	// Debugging for Sune Molgaard's issue with non-existent files being
513 	// detected as unattached and crashing later in CheckUnattachedObjects()
514 	if (ObjectID == 0x90c1a)
515 	{
516 		BOX_INFO("Trying to open " << BOX_FORMAT_OBJECTID(ObjectID) <<
517 			" from " << rFilename << " on disc " << mDiscSetNumber);
518 	}
519 
520 	try
521 	{
522 		// Open file
523 		std::auto_ptr<RaidFileRead> file(
524 			RaidFileRead::Open(mDiscSetNumber, rFilename));
525 		size = file->GetDiscUsageInBlocks();
526 
527 		// Read in first four bytes -- don't have to worry about
528 		// retrying if not all bytes read as is RaidFile
529 		uint32_t signature;
530 		if(file->Read(&signature, sizeof(signature)) != sizeof(signature))
531 		{
532 			// Too short, can't read signature from it
533 			return false;
534 		}
535 		// Seek back to beginning
536 		file->Seek(0, IOStream::SeekType_Absolute);
537 
538 		// Then... check depending on the type
539 		switch(ntohl(signature))
540 		{
541 		case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
542 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
543 		case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
544 #endif
545 			// File... check
546 			containerID = CheckFile(ObjectID, *file);
547 			break;
548 
549 		case OBJECTMAGIC_DIR_MAGIC_VALUE:
550 			isFile = false;
551 			containerID = CheckDirInitial(ObjectID, *file);
552 			break;
553 
554 		default:
555 			// Unknown signature. Bad file. Very bad file.
556 			return false;
557 			break;
558 		}
559 
560 		// Add to usage counts
561 		mBlocksUsed += size;
562 		if(!isFile)
563 		{
564 			mBlocksInDirectories += size;
565 		}
566 	}
567 	catch(...)
568 	{
569 		// Error caught, not a good file then, let it be deleted
570 		return false;
571 	}
572 
573 	// Got a container ID? (ie check was successful)
574 	if(containerID == -1)
575 	{
576 		return false;
577 	}
578 
579 	// Debugging for Sune Molgaard's issue with non-existent files being
580 	// detected as unattached and crashing later in CheckUnattachedObjects()
581 	if (ObjectID == 0x90c1a)
582 	{
583 		BOX_INFO("Adding ID " << BOX_FORMAT_OBJECTID(ObjectID) <<
584 			" contained by " << BOX_FORMAT_OBJECTID(containerID) <<
585 			" with size " << size << " and isFile " << isFile);
586 	}
587 
588 	// Add to list of IDs known about
589 	AddID(ObjectID, containerID, size, isFile);
590 
591 	// Report success
592 	return true;
593 }
594 
595 
596 // --------------------------------------------------------------------------
597 //
598 // Function
599 //		Name:    BackupStoreCheck::CheckFile(int64_t, IOStream &)
600 //		Purpose: Do check on file, return original container ID
601 //			 if OK, or -1 on error
602 //		Created: 22/4/04
603 //
604 // --------------------------------------------------------------------------
CheckFile(int64_t ObjectID,IOStream & rStream)605 int64_t BackupStoreCheck::CheckFile(int64_t ObjectID, IOStream &rStream)
606 {
607 	// Check that it's not the root directory ID. Having a file as
608 	// the root directory would be bad.
609 	if(ObjectID == BACKUPSTORE_ROOT_DIRECTORY_ID)
610 	{
611 		// Get that dodgy thing deleted!
612 		BOX_ERROR("Have file as root directory. This is bad.");
613 		return -1;
614 	}
615 
616 	// Check the format of the file, and obtain the container ID
617 	int64_t originalContainerID = -1;
618 	if(!BackupStoreFile::VerifyEncodedFileFormat(rStream,
619 		0 /* don't want diffing from ID */,
620 		&originalContainerID))
621 	{
622 		// Didn't verify
623 		return -1;
624 	}
625 
626 	return originalContainerID;
627 }
628 
629 
630 // --------------------------------------------------------------------------
631 //
632 // Function
633 //		Name:    BackupStoreCheck::CheckDirInitial(int64_t, IOStream &)
634 //		Purpose: Do initial check on directory, return container ID
635 //			 if OK, or -1 on error
636 //		Created: 22/4/04
637 //
638 // --------------------------------------------------------------------------
CheckDirInitial(int64_t ObjectID,IOStream & rStream)639 int64_t BackupStoreCheck::CheckDirInitial(int64_t ObjectID, IOStream &rStream)
640 {
641 	// Simply attempt to read in the directory
642 	BackupStoreDirectory dir;
643 	dir.ReadFromStream(rStream, IOStream::TimeOutInfinite);
644 
645 	// Check object ID
646 	if(dir.GetObjectID() != ObjectID)
647 	{
648 		// Wrong object ID
649 		return -1;
650 	}
651 
652 	// Return container ID
653 	return dir.GetContainerID();
654 }
655 
656 
657 // --------------------------------------------------------------------------
658 //
659 // Function
660 //		Name:    BackupStoreCheck::CheckDirectories()
661 //		Purpose: Check the directories
662 //		Created: 22/4/04
663 //
664 // --------------------------------------------------------------------------
CheckDirectories()665 void BackupStoreCheck::CheckDirectories()
666 {
667 	// Phase 1 did this:
668 	//    Checked that all the directories are readable
669 	//    Built a list of all directories and files which exist on the store
670 	//
671 	// This phase will check all the files in the directories, make
672 	// a note of all directories which are missing, and do initial fixing.
673 
674 	// The root directory is not contained inside another directory, so
675 	// it has no directory entry to scan, but we have to count it
676 	// somewhere, so we'll count it here.
677 	mNumDirectories++;
678 
679 	// Scan all objects.
680 	for(Info_t::const_iterator i(mInfo.begin()); i != mInfo.end(); ++i)
681 	{
682 		IDBlock *pblock = i->second;
683 		int32_t bentries = (pblock == mpInfoLastBlock)?mInfoLastBlockEntries:BACKUPSTORECHECK_BLOCK_SIZE;
684 
685 		for(int e = 0; e < bentries; ++e)
686 		{
687 			uint8_t flags = GetFlags(pblock, e);
688 			if(flags & Flags_IsDir)
689 			{
690 				// Found a directory. Read it in.
691 				std::string filename;
692 				StoreStructure::MakeObjectFilename(pblock->mID[e], mStoreRoot, mDiscSetNumber, filename, false /* no dir creation */);
693 				BackupStoreDirectory dir;
694 				{
695 					std::auto_ptr<RaidFileRead> file(RaidFileRead::Open(mDiscSetNumber, filename));
696 					dir.ReadFromStream(*file, IOStream::TimeOutInfinite);
697 				}
698 
699 				// Flag for modifications
700 				bool isModified = false;
701 
702 				// Check for validity
703 				if(dir.CheckAndFix())
704 				{
705 					// Wasn't quite right, and has been modified
706 					BOX_WARNING("Directory ID " <<
707 						BOX_FORMAT_OBJECTID(pblock->mID[e]) <<
708 						" has bad structure");
709 					++mNumberErrorsFound;
710 					isModified = true;
711 				}
712 
713 				// Go through, and check that everything in that directory exists and is valid
714 				std::vector<int64_t> toDelete;
715 
716 				BackupStoreDirectory::Iterator i(dir);
717 				BackupStoreDirectory::Entry *en = 0;
718 				while((en = i.Next()) != 0)
719 				{
720 					// Lookup the item
721 					int32_t iIndex;
722 					IDBlock *piBlock = LookupID(en->GetObjectID(), iIndex);
723 					bool badEntry = false;
724 					if(piBlock != 0)
725 					{
726 						badEntry = !CheckDirectoryEntry(
727 							*en, pblock->mID[e],
728 							iIndex, isModified);
729 					}
730 					else
731 					{
732 						// Item can't be found. Is it a directory?
733 						if(en->IsDir())
734 						{
735 							// Store the directory for later attention
736 							mDirsWhichContainLostDirs[en->GetObjectID()] = pblock->mID[e];
737 						}
738 						else
739 						{
740 							// Just remove the entry
741 							badEntry = true;
742 							BOX_WARNING("Directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]) << " references object " << BOX_FORMAT_OBJECTID(en->GetObjectID()) << " which does not exist.");
743 						}
744 					}
745 
746 					// Is this entry worth keeping?
747 					if(badEntry)
748 					{
749 						toDelete.push_back(en->GetObjectID());
750 					}
751 					else if (en->IsFile())
752 					{
753 						// Add to sizes?
754 						if(en->IsOld())
755 						{
756 							mBlocksInOldFiles += en->GetSizeInBlocks();
757 						}
758 						if(en->IsDeleted())
759 						{
760 							mBlocksInDeletedFiles += en->GetSizeInBlocks();
761 						}
762 						if(!en->IsOld() &&
763 							!en->IsDeleted())
764 						{
765 							mBlocksInCurrentFiles += en->GetSizeInBlocks();
766 						}
767 					}
768 				}
769 
770 				if(toDelete.size() > 0)
771 				{
772 					// Delete entries from directory
773 					for(std::vector<int64_t>::const_iterator d(toDelete.begin()); d != toDelete.end(); ++d)
774 					{
775 						dir.DeleteEntry(*d);
776 					}
777 
778 					// Mark as modified
779 					isModified = true;
780 
781 					// Check the directory again, now that entries have been removed
782 					dir.CheckAndFix();
783 
784 					// Errors found
785 					++mNumberErrorsFound;
786 				}
787 
788 				if(isModified && mFixErrors)
789 				{
790 					BOX_WARNING("Fixing directory ID " << BOX_FORMAT_OBJECTID(pblock->mID[e]));
791 
792 					// Save back to disc
793 					RaidFileWrite fixed(mDiscSetNumber, filename);
794 					fixed.Open(true /* allow overwriting */);
795 					dir.WriteToStream(fixed);
796 					// Commit it
797 					fixed.Commit(true /* convert to raid representation now */);
798 				}
799 			}
800 		}
801 	}
802 
803 }
804 
CheckDirectoryEntry(BackupStoreDirectory::Entry & rEntry,int64_t DirectoryID,int32_t IndexInDirBlock,bool & rIsModified)805 bool BackupStoreCheck::CheckDirectoryEntry(BackupStoreDirectory::Entry& rEntry,
806 	int64_t DirectoryID, int32_t IndexInDirBlock, bool& rIsModified)
807 {
808 	IDBlock *piBlock = LookupID(rEntry.GetObjectID(), IndexInDirBlock);
809 	ASSERT(piBlock != 0);
810 
811 	uint8_t iflags = GetFlags(piBlock, IndexInDirBlock);
812 	bool badEntry = false;
813 
814 	// Is the type the same?
815 	if(((iflags & Flags_IsDir) == Flags_IsDir) != rEntry.IsDir())
816 	{
817 		// Entry is of wrong type
818 		BOX_WARNING("Directory ID " <<
819 			BOX_FORMAT_OBJECTID(DirectoryID) <<
820 			" references object " <<
821 			BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
822 			" which has a different type than expected.");
823 		badEntry = true;
824 	}
825 	// Check that the entry is not already contained.
826 	else if(iflags & Flags_IsContained)
827 	{
828 		BOX_WARNING("Directory ID " <<
829 			BOX_FORMAT_OBJECTID(DirectoryID) <<
830 			" references object " <<
831 			BOX_FORMAT_OBJECTID(rEntry.GetObjectID()) <<
832 			" which is already contained.");
833 		badEntry = true;
834 	}
835 	else
836 	{
837 		// Not already contained -- mark as contained
838 		SetFlags(piBlock, IndexInDirBlock, iflags | Flags_IsContained);
839 
840 		// Check that the container ID of the object is correct
841 		if(piBlock->mContainer[IndexInDirBlock] != DirectoryID)
842 		{
843 			// Needs fixing...
844 			if(iflags & Flags_IsDir)
845 			{
846 				// Add to will fix later list
847 				BOX_WARNING("Directory ID " <<
848 					BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
849 					<< " has wrong container ID.");
850 				mDirsWithWrongContainerID.push_back(rEntry.GetObjectID());
851 			}
852 			else
853 			{
854 				// This is OK for files, they might move
855 				BOX_WARNING("File ID " <<
856 					BOX_FORMAT_OBJECTID(rEntry.GetObjectID())
857 					<< " has different container ID, "
858 					"probably moved");
859 			}
860 
861 			// Fix entry for now
862 			piBlock->mContainer[IndexInDirBlock] = DirectoryID;
863 		}
864 	}
865 
866 	// Check the object size, if it's OK and a file
867 	if(!badEntry && !rEntry.IsDir())
868 	{
869 		if(rEntry.GetSizeInBlocks() != piBlock->mObjectSizeInBlocks[IndexInDirBlock])
870 		{
871 			// Wrong size, correct it.
872 			rEntry.SetSizeInBlocks(piBlock->mObjectSizeInBlocks[IndexInDirBlock]);
873 
874 			// Mark as changed
875 			rIsModified = true;
876 
877 			// Tell user
878 			BOX_WARNING("Directory ID " <<
879 				BOX_FORMAT_OBJECTID(DirectoryID) <<
880 				" has wrong size for object " <<
881 				BOX_FORMAT_OBJECTID(rEntry.GetObjectID()));
882 		}
883 	}
884 
885 	if (!badEntry)
886 	{
887 		if(rEntry.IsDir())
888 		{
889 			mNumDirectories++;
890 		}
891 		else
892 		{
893 			mNumFiles++;
894 
895 			if(rEntry.IsDeleted())
896 			{
897 				mNumDeletedFiles++;
898 			}
899 
900 			if(rEntry.IsOld())
901 			{
902 				mNumOldFiles++;
903 			}
904 		}
905 	}
906 
907 	return !badEntry;
908 }
909 
910