1 // distribution boxbackup-0.11_trunk_2979 (svn version: 2979)
2 // Box Backup, http://www.boxbackup.org/
3 //
4 // Copyright (c) 2003-2010, Ben Summers and contributors.
5 // All rights reserved.
6 //
7 // Note that this project uses mixed licensing. Any file with this license
8 // attached, or where the code LICENSE-GPL appears on the first line, falls
9 // under the "Box Backup GPL" license. See the file COPYING.txt for more
10 // information about this license.
11 //
12 // ---------------------------------------------------------------------
13 // This program is free software; you can redistribute it and/or
14 // modify it under the terms of the GNU General Public License
15 // as published by the Free Software Foundation; either version 2
16 // of the License, or (at your option) any later version.
17 //
18 // This program is distributed in the hope that it will be useful,
19 // but WITHOUT ANY WARRANTY; without even the implied warranty of
20 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21 // GNU General Public License for more details.
22 //
23 // You should have received a copy of the GNU General Public License
24 // along with this program; if not, write to the Free Software
25 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
26 //
27 // [http://www.gnu.org/licenses/old-licenses/gpl-2.0.html#SEC4]
28 //
29 // As a special exception to the GPLv2, the Box Backup Project gives
30 // permission to link any code falling under this license (the Box Backup
31 // GPL) with any software that can be downloaded from
32 // the OpenSSL website [http://www.openssl.org] under either the
33 // "OpenSSL License" or the "Original SSLeay License", and to distribute
34 // the linked executables under the terms of the "Box Backup GPL" license.
35 //
36 // As a special exception to the GPLv2, the Box Backup Project gives
37 // permission to link any code falling under this license (the Box Backup
38 // GPL) with any version of Microsoft's Volume Shadow Copy Service 7.2 SDK
39 // or Microsoft Windows Software Development Kit (SDK), including
40 // vssapi.lib, that can be downloaded from the Microsoft website
41 // [*.microsoft.com], and to distribute the linked executables under the
42 // terms of the "Box Backup GPL" license.
43 // --------------------------------------------------------------------------
44 //
45 // File
46 //		Name:    BackupStoreFileEncodeStream.cpp
47 //		Purpose: Implement stream-based file encoding for the backup store
48 //		Created: 12/1/04
49 //
50 // --------------------------------------------------------------------------
51 
52 #include "Box.h"
53 
54 #include <string.h>
55 
56 #include "BackupClientFileAttributes.h"
57 #include "BackupStoreConstants.h"
58 #include "BackupStoreException.h"
59 #include "BackupStoreFile.h"
60 #include "BackupStoreFileCryptVar.h"
61 #include "BackupStoreFileEncodeStream.h"
62 #include "BackupStoreFileWire.h"
63 #include "BackupStoreObjectMagic.h"
64 #include "BoxTime.h"
65 #include "FileStream.h"
66 #include "Random.h"
67 #include "RollingChecksum.h"
68 
69 #include "MemLeakFindOn.h"
70 
71 #include <cstring>
72 
73 using namespace BackupStoreFileCryptVar;
74 
75 
76 // --------------------------------------------------------------------------
77 //
78 // Function
79 //		Name:    BackupStoreFileEncodeStream::BackupStoreFileEncodeStream
80 //		Purpose: Constructor (opens file)
81 //		Created: 8/12/03
82 //
83 // --------------------------------------------------------------------------
BackupStoreFileEncodeStream()84 BackupStoreFileEncodeStream::BackupStoreFileEncodeStream()
85 	: mpRecipe(0),
86 	  mpFile(0),
87 	  mpLogging(0),
88 	  mpRunStatusProvider(NULL),
89 	  mStatus(Status_Header),
90 	  mSendData(true),
91 	  mTotalBlocks(0),
92 	  mAbsoluteBlockNumber(-1),
93 	  mInstructionNumber(-1),
94 	  mNumBlocks(0),
95 	  mCurrentBlock(-1),
96 	  mCurrentBlockEncodedSize(0),
97 	  mPositionInCurrentBlock(0),
98 	  mBlockSize(BACKUP_FILE_MIN_BLOCK_SIZE),
99 	  mLastBlockSize(0),
100 	  mTotalBytesSent(0),
101 	  mpRawBuffer(0),
102 	  mAllocatedBufferSize(0),
103 	  mEntryIVBase(0)
104 {
105 }
106 
107 // --------------------------------------------------------------------------
108 //
109 // Function
110 //		Name:    BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
111 //		Purpose: Destructor
112 //		Created: 8/12/03
113 //
114 // --------------------------------------------------------------------------
~BackupStoreFileEncodeStream()115 BackupStoreFileEncodeStream::~BackupStoreFileEncodeStream()
116 {
117 	// Free buffers
118 	if(mpRawBuffer)
119 	{
120 		::free(mpRawBuffer);
121 		mpRawBuffer = 0;
122 	}
123 
124 	// Close the file, which we might have open
125 	if(mpFile)
126 	{
127 		delete mpFile;
128 		mpFile = 0;
129 	}
130 
131 	// Clear up logging stream
132 	if(mpLogging)
133 	{
134 		delete mpLogging;
135 		mpLogging = 0;
136 	}
137 
138 	// Free the recipe
139 	if(mpRecipe != 0)
140 	{
141 		delete mpRecipe;
142 		mpRecipe = 0;
143 	}
144 }
145 
146 
147 // --------------------------------------------------------------------------
148 //
149 // Function
150 //		Name:    BackupStoreFileEncodeStream::Setup(const char *, Recipe *, int64_t, const BackupStoreFilename &, int64_t *)
151 //		Purpose: Reads file information, and builds file header reading for sending.
152 //				 Takes ownership of the Recipe.
153 //		Created: 8/12/03
154 //
155 // --------------------------------------------------------------------------
Setup(const char * Filename,BackupStoreFileEncodeStream::Recipe * pRecipe,int64_t ContainerID,const BackupStoreFilename & rStoreFilename,int64_t * pModificationTime,ReadLoggingStream::Logger * pLogger,RunStatusProvider * pRunStatusProvider)156 void BackupStoreFileEncodeStream::Setup(const char *Filename,
157 	BackupStoreFileEncodeStream::Recipe *pRecipe,
158 	int64_t ContainerID, const BackupStoreFilename &rStoreFilename,
159 	int64_t *pModificationTime, ReadLoggingStream::Logger* pLogger,
160 	RunStatusProvider* pRunStatusProvider)
161 {
162 	// Pointer to a blank recipe which we might create
163 	BackupStoreFileEncodeStream::Recipe *pblankRecipe = 0;
164 
165 	try
166 	{
167 		// Get file attributes
168 		box_time_t modTime = 0;
169 		int64_t fileSize = 0;
170 		BackupClientFileAttributes attr;
171 		attr.ReadAttributes(Filename, false /* no zeroing of modification times */, &modTime,
172 			0 /* not interested in attr mod time */, &fileSize);
173 
174 		// Might need to create a blank recipe...
175 		if(pRecipe == 0)
176 		{
177 			pblankRecipe = new BackupStoreFileEncodeStream::Recipe(0, 0);
178 
179 			BackupStoreFileEncodeStream::RecipeInstruction instruction;
180 			instruction.mSpaceBefore = fileSize; // whole file
181 			instruction.mBlocks = 0; // no blocks
182 			instruction.mpStartBlock = 0; // no block
183 			pblankRecipe->push_back(instruction);
184 
185 			pRecipe = pblankRecipe;
186 		}
187 
188 		// Tell caller?
189 		if(pModificationTime != 0)
190 		{
191 			*pModificationTime = modTime;
192 		}
193 
194 		// Go through each instruction in the recipe and work out how many blocks
195 		// it will add, and the max clear size of these blocks
196 		int maxBlockClearSize = 0;
197 		for(uint64_t inst = 0; inst < pRecipe->size(); ++inst)
198 		{
199 			if((*pRecipe)[inst].mSpaceBefore > 0)
200 			{
201 				// Calculate the number of blocks the space before requires
202 				int64_t numBlocks;
203 				int32_t blockSize, lastBlockSize;
204 				CalculateBlockSizes((*pRecipe)[inst].mSpaceBefore, numBlocks, blockSize, lastBlockSize);
205 				// Add to accumlated total
206 				mTotalBlocks += numBlocks;
207 				// Update maximum clear size
208 				if(blockSize > maxBlockClearSize) maxBlockClearSize = blockSize;
209 				if(lastBlockSize > maxBlockClearSize) maxBlockClearSize = lastBlockSize;
210 			}
211 
212 			// Add number of blocks copied from the previous file
213 			mTotalBlocks += (*pRecipe)[inst].mBlocks;
214 
215 			// Check for bad things
216 			if((*pRecipe)[inst].mBlocks < 0 || ((*pRecipe)[inst].mBlocks != 0 && (*pRecipe)[inst].mpStartBlock == 0))
217 			{
218 				THROW_EXCEPTION(BackupStoreException, Internal)
219 			}
220 
221 			// Run through blocks to get the max clear size
222 			for(int32_t b = 0; b < (*pRecipe)[inst].mBlocks; ++b)
223 			{
224 				if((*pRecipe)[inst].mpStartBlock[b].mSize > maxBlockClearSize) maxBlockClearSize = (*pRecipe)[inst].mpStartBlock[b].mSize;
225 			}
226 		}
227 
228 		// Send data? (symlinks don't have any data in them)
229 		mSendData = !attr.IsSymLink();
230 
231 		// If not data is being sent, then the max clear block size is zero
232 		if(!mSendData)
233 		{
234 			maxBlockClearSize = 0;
235 		}
236 
237 		// Header
238 		file_StreamFormat hdr;
239 		hdr.mMagicValue = htonl(OBJECTMAGIC_FILE_MAGIC_VALUE_V1);
240 		hdr.mNumBlocks = (mSendData)?(box_hton64(mTotalBlocks)):(0);
241 		hdr.mContainerID = box_hton64(ContainerID);
242 		hdr.mModificationTime = box_hton64(modTime);
243 		// add a bit to make it harder to tell what's going on -- try not to give away too much info about file size
244 		hdr.mMaxBlockClearSize = htonl(maxBlockClearSize + 128);
245 		hdr.mOptions = 0;		// no options defined yet
246 
247 		// Write header to stream
248 		mData.Write(&hdr, sizeof(hdr));
249 
250 		// Write filename to stream
251 		rStoreFilename.WriteToStream(mData);
252 
253 		// Write attributes to stream
254 		attr.WriteToStream(mData);
255 
256 		// Allocate some buffers for writing data
257 		if(mSendData)
258 		{
259 			// Open the file
260 			mpFile = new FileStream(Filename);
261 
262 			if (pLogger)
263 			{
264 				// Create logging stream
265 				mpLogging = new ReadLoggingStream(*mpFile,
266 					*pLogger);
267 			}
268 			else
269 			{
270 				// re-use FileStream instead
271 				mpLogging = mpFile;
272 				mpFile = NULL;
273 			}
274 
275 			// Work out the largest possible block required for the encoded data
276 			mAllocatedBufferSize = BackupStoreFile::MaxBlockSizeForChunkSize(maxBlockClearSize);
277 
278 			// Then allocate two blocks of this size
279 			mpRawBuffer = (uint8_t*)::malloc(mAllocatedBufferSize);
280 			if(mpRawBuffer == 0)
281 			{
282 				throw std::bad_alloc();
283 			}
284 #ifndef BOX_RELEASE_BUILD
285 			// In debug builds, make sure that the reallocation code is exercised.
286 			mEncodedBuffer.Allocate(mAllocatedBufferSize / 4);
287 #else
288 			mEncodedBuffer.Allocate(mAllocatedBufferSize);
289 #endif
290 		}
291 		else
292 		{
293 			// Write an empty block index for the symlink
294 			file_BlockIndexHeader blkhdr;
295 			blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
296 			blkhdr.mOtherFileID = box_hton64(0);	// not other file ID
297 			blkhdr.mEntryIVBase = box_hton64(0);
298 			blkhdr.mNumBlocks = box_hton64(0);
299 			mData.Write(&blkhdr, sizeof(blkhdr));
300 		}
301 
302 		// Ready for reading
303 		mData.SetForReading();
304 
305 		// Update stats
306 		BackupStoreFile::msStats.mBytesInEncodedFiles += fileSize;
307 
308 		// Finally, store the pointer to the recipe, when we know exceptions won't occur
309 		mpRecipe = pRecipe;
310 	}
311 	catch(...)
312 	{
313 		// Clean up any blank recipe
314 		if(pblankRecipe != 0)
315 		{
316 			delete pblankRecipe;
317 			pblankRecipe = 0;
318 		}
319 		throw;
320 	}
321 
322 	mpRunStatusProvider = pRunStatusProvider;
323 }
324 
325 
326 // --------------------------------------------------------------------------
327 //
328 // Function
329 //		Name:    BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t &, int32_t &, int32_t &)
330 //		Purpose: Calculates the sizes of blocks in a section of the file
331 //		Created: 16/1/04
332 //
333 // --------------------------------------------------------------------------
CalculateBlockSizes(int64_t DataSize,int64_t & rNumBlocksOut,int32_t & rBlockSizeOut,int32_t & rLastBlockSizeOut)334 void BackupStoreFileEncodeStream::CalculateBlockSizes(int64_t DataSize, int64_t &rNumBlocksOut, int32_t &rBlockSizeOut, int32_t &rLastBlockSizeOut)
335 {
336 	// How many blocks, and how big?
337 	rBlockSizeOut = BACKUP_FILE_MIN_BLOCK_SIZE / 2;
338 	do
339 	{
340 		rBlockSizeOut *= 2;
341 
342 		rNumBlocksOut = (DataSize + rBlockSizeOut - 1) / rBlockSizeOut;
343 
344 	} while(rBlockSizeOut < BACKUP_FILE_MAX_BLOCK_SIZE && rNumBlocksOut > BACKUP_FILE_INCREASE_BLOCK_SIZE_AFTER);
345 
346 	// Last block size
347 	rLastBlockSizeOut = DataSize - ((rNumBlocksOut - 1) * rBlockSizeOut);
348 
349 	// Avoid small blocks?
350 	if(rLastBlockSizeOut < BACKUP_FILE_AVOID_BLOCKS_LESS_THAN
351 		&& rNumBlocksOut > 1)
352 	{
353 		// Add the small bit of data to the last block
354 		--rNumBlocksOut;
355 		rLastBlockSizeOut += rBlockSizeOut;
356 	}
357 
358 	// checks!
359 	ASSERT((((rNumBlocksOut-1) * rBlockSizeOut) + rLastBlockSizeOut) == DataSize);
360 	//TRACE4("CalcBlockSize, sz %lld, num %lld, blocksize %d, last %d\n", DataSize, rNumBlocksOut, (int32_t)rBlockSizeOut, (int32_t)rLastBlockSizeOut);
361 }
362 
363 
364 
365 // --------------------------------------------------------------------------
366 //
367 // Function
368 //		Name:    BackupStoreFileEncodeStream::Read(void *, int, int)
369 //		Purpose: As interface -- generates encoded file data on the fly from the raw file
370 //		Created: 8/12/03
371 //
372 // --------------------------------------------------------------------------
Read(void * pBuffer,int NBytes,int Timeout)373 int BackupStoreFileEncodeStream::Read(void *pBuffer, int NBytes, int Timeout)
374 {
375 	// Check there's something to do.
376 	if(mStatus == Status_Finished)
377 	{
378 		return 0;
379 	}
380 
381 	if(mpRunStatusProvider && mpRunStatusProvider->StopRun())
382 	{
383 		THROW_EXCEPTION(BackupStoreException, SignalReceived);
384 	}
385 
386 	int bytesToRead = NBytes;
387 	uint8_t *buffer = (uint8_t*)pBuffer;
388 
389 	while(bytesToRead > 0 && mStatus != Status_Finished)
390 	{
391 		if(mStatus == Status_Header || mStatus == Status_BlockListing)
392 		{
393 			// Header or block listing phase -- send from the buffered stream
394 
395 			// Send bytes from the data buffer
396 			int b = mData.Read(buffer, bytesToRead, Timeout);
397 			bytesToRead -= b;
398 			buffer += b;
399 
400 			// Check to see if all the data has been used from this stream
401 			if(!mData.StreamDataLeft())
402 			{
403 				// Yes, move on to next phase (or finish, if there's no file data)
404 				if(!mSendData)
405 				{
406 					mStatus = Status_Finished;
407 				}
408 				else
409 				{
410 					// Reset the buffer so it can be used for the next phase
411 					mData.Reset();
412 
413 					// Get buffer ready for index?
414 					if(mStatus == Status_Header)
415 					{
416 						// Just finished doing the stream header, create the block index header
417 						file_BlockIndexHeader blkhdr;
418 						blkhdr.mMagicValue = htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1);
419 						ASSERT(mpRecipe != 0);
420 						blkhdr.mOtherFileID = box_hton64(mpRecipe->GetOtherFileID());
421 						blkhdr.mNumBlocks = box_hton64(mTotalBlocks);
422 
423 						// Generate the IV base
424 						Random::Generate(&mEntryIVBase, sizeof(mEntryIVBase));
425 						blkhdr.mEntryIVBase = box_hton64(mEntryIVBase);
426 
427 						mData.Write(&blkhdr, sizeof(blkhdr));
428 					}
429 
430 					++mStatus;
431 				}
432 			}
433 		}
434 		else if(mStatus == Status_Blocks)
435 		{
436 			// Block sending phase
437 
438 			if(mPositionInCurrentBlock >= mCurrentBlockEncodedSize)
439 			{
440 				// Next block!
441 				++mCurrentBlock;
442 				++mAbsoluteBlockNumber;
443 				if(mCurrentBlock >= mNumBlocks)
444 				{
445 					// Output extra blocks for this instruction and move forward in file
446 					if(mInstructionNumber >= 0)
447 					{
448 						SkipPreviousBlocksInInstruction();
449 					}
450 
451 					// Is there another instruction to go?
452 					++mInstructionNumber;
453 
454 					// Skip instructions which don't contain any data
455 					while(mInstructionNumber < static_cast<int64_t>(mpRecipe->size())
456 						&& (*mpRecipe)[mInstructionNumber].mSpaceBefore == 0)
457 					{
458 						SkipPreviousBlocksInInstruction();
459 						++mInstructionNumber;
460 					}
461 
462 					if(mInstructionNumber >= static_cast<int64_t>(mpRecipe->size()))
463 					{
464 						// End of blocks, go to next phase
465 						++mStatus;
466 
467 						// Set the data to reading so the index can be written
468 						mData.SetForReading();
469 					}
470 					else
471 					{
472 						// Get ready for this instruction
473 						SetForInstruction();
474 					}
475 				}
476 
477 				// Can't use 'else' here as SetForInstruction() will change this
478 				if(mCurrentBlock < mNumBlocks)
479 				{
480 					EncodeCurrentBlock();
481 				}
482 			}
483 
484 			// Send data from the current block (if there's data to send)
485 			if(mPositionInCurrentBlock < mCurrentBlockEncodedSize)
486 			{
487 				// How much data to put in the buffer?
488 				int s = mCurrentBlockEncodedSize - mPositionInCurrentBlock;
489 				if(s > bytesToRead) s = bytesToRead;
490 
491 				// Copy it in
492 				::memcpy(buffer, mEncodedBuffer.mpBuffer + mPositionInCurrentBlock, s);
493 
494 				// Update variables
495 				bytesToRead -= s;
496 				buffer += s;
497 				mPositionInCurrentBlock += s;
498 			}
499 		}
500 		else
501 		{
502 			// Should never get here, as it'd be an invalid status
503 			ASSERT(false);
504 		}
505 	}
506 
507 	// Add encoded size to stats
508 	BackupStoreFile::msStats.mTotalFileStreamSize += (NBytes - bytesToRead);
509 	mTotalBytesSent += (NBytes - bytesToRead);
510 
511 	// Return size of data to caller
512 	return NBytes - bytesToRead;
513 }
514 
515 
516 // --------------------------------------------------------------------------
517 //
518 // Function
519 //		Name:    BackupStoreFileEncodeStream::StorePreviousBlocksInInstruction()
520 //		Purpose: Private. Stores the blocks of the old file referenced in the current
521 //				 instruction into the index and skips over the data in the file
522 //		Created: 16/1/04
523 //
524 // --------------------------------------------------------------------------
SkipPreviousBlocksInInstruction()525 void BackupStoreFileEncodeStream::SkipPreviousBlocksInInstruction()
526 {
527 	// Check something is necessary
528 	if((*mpRecipe)[mInstructionNumber].mpStartBlock == 0 || (*mpRecipe)[mInstructionNumber].mBlocks == 0)
529 	{
530 		return;
531 	}
532 
533 	// Index of the first block in old file (being diffed from)
534 	int firstIndex = mpRecipe->BlockPtrToIndex((*mpRecipe)[mInstructionNumber].mpStartBlock);
535 
536 	int64_t sizeToSkip = 0;
537 
538 	for(int32_t b = 0; b < (*mpRecipe)[mInstructionNumber].mBlocks; ++b)
539 	{
540 		// Update stats
541 		BackupStoreFile::msStats.mBytesAlreadyOnServer += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
542 
543 		// Store the entry
544 		StoreBlockIndexEntry(0 - (firstIndex + b),
545 			(*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize,
546 			(*mpRecipe)[mInstructionNumber].mpStartBlock[b].mWeakChecksum,
547 			(*mpRecipe)[mInstructionNumber].mpStartBlock[b].mStrongChecksum);
548 
549 		// Increment the absolute block number -- kept encryption IV in sync
550 		++mAbsoluteBlockNumber;
551 
552 		// Add the size of this block to the size to skip
553 		sizeToSkip += (*mpRecipe)[mInstructionNumber].mpStartBlock[b].mSize;
554 	}
555 
556 	// Move forward in the stream
557 	mpLogging->Seek(sizeToSkip, IOStream::SeekType_Relative);
558 }
559 
560 
561 // --------------------------------------------------------------------------
562 //
563 // Function
564 //		Name:    BackupStoreFileEncodeStream::SetForInstruction()
565 //		Purpose: Private. Sets the state of the internal variables for the current instruction in the recipe
566 //		Created: 16/1/04
567 //
568 // --------------------------------------------------------------------------
SetForInstruction()569 void BackupStoreFileEncodeStream::SetForInstruction()
570 {
571 	// Calculate block sizes
572 	CalculateBlockSizes((*mpRecipe)[mInstructionNumber].mSpaceBefore, mNumBlocks, mBlockSize, mLastBlockSize);
573 
574 	// Set variables
575 	mCurrentBlock = 0;
576 	mCurrentBlockEncodedSize = 0;
577 	mPositionInCurrentBlock = 0;
578 }
579 
580 
581 
582 // --------------------------------------------------------------------------
583 //
584 // Function
585 //		Name:    BackupStoreFileEncodeStream::EncodeCurrentBlock()
586 //		Purpose: Private. Encodes the current block, and writes the block data to the index
587 //		Created: 8/12/03
588 //
589 // --------------------------------------------------------------------------
EncodeCurrentBlock()590 void BackupStoreFileEncodeStream::EncodeCurrentBlock()
591 {
592 	// How big is the block, raw?
593 	int blockRawSize = mBlockSize;
594 	if(mCurrentBlock == (mNumBlocks - 1))
595 	{
596 		blockRawSize = mLastBlockSize;
597 	}
598 	ASSERT(blockRawSize < mAllocatedBufferSize);
599 
600 	// Check file open
601 	if(mpLogging == 0)
602 	{
603 		// File should be open, but isn't. So logical error.
604 		THROW_EXCEPTION(BackupStoreException, Internal)
605 	}
606 
607 	// Read the data in
608 	if(!mpLogging->ReadFullBuffer(mpRawBuffer, blockRawSize,
609 		0 /* not interested in size if failure */))
610 	{
611 		// TODO: Do something more intelligent, and abort
612 		// this upload because the file has changed.
613 		THROW_EXCEPTION(BackupStoreException,
614 			Temp_FileEncodeStreamDidntReadBuffer)
615 	}
616 
617 	// Encode it
618 	mCurrentBlockEncodedSize = BackupStoreFile::EncodeChunk(mpRawBuffer,
619 		blockRawSize, mEncodedBuffer);
620 
621 	//TRACE2("Encode: Encoded size of block %d is %d\n", (int32_t)mCurrentBlock, (int32_t)mCurrentBlockEncodedSize);
622 
623 	// Create block listing data -- generate checksums
624 	RollingChecksum weakChecksum(mpRawBuffer, blockRawSize);
625 	MD5Digest strongChecksum;
626 	strongChecksum.Add(mpRawBuffer, blockRawSize);
627 	strongChecksum.Finish();
628 
629 	// Add entry to the index
630 	StoreBlockIndexEntry(mCurrentBlockEncodedSize, blockRawSize,
631 		weakChecksum.GetChecksum(), strongChecksum.DigestAsData());
632 
633 	// Set vars to reading this block
634 	mPositionInCurrentBlock = 0;
635 }
636 
637 // --------------------------------------------------------------------------
638 //
639 // Function
640 //		Name:    BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t, int32_t, uint32_t, uint8_t *)
641 //		Purpose: Private. Adds an entry to the index currently being stored for sending at end of the stream.
642 //		Created: 16/1/04
643 //
644 // --------------------------------------------------------------------------
StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex,int32_t ClearSize,uint32_t WeakChecksum,uint8_t * pStrongChecksum)645 void BackupStoreFileEncodeStream::StoreBlockIndexEntry(int64_t EncSizeOrBlkIndex, int32_t ClearSize, uint32_t WeakChecksum, uint8_t *pStrongChecksum)
646 {
647 	// First, the encrypted section
648 	file_BlockIndexEntryEnc entryEnc;
649 	entryEnc.mSize = htonl(ClearSize);
650 	entryEnc.mWeakChecksum = htonl(WeakChecksum);
651 	::memcpy(entryEnc.mStrongChecksum, pStrongChecksum, sizeof(entryEnc.mStrongChecksum));
652 
653 	// Then the clear section
654 	file_BlockIndexEntry entry;
655 	entry.mEncodedSize = box_hton64(((uint64_t)EncSizeOrBlkIndex));
656 
657 	// Then encrypt the encryted section
658 	// Generate the IV from the block number
659 	if(sBlowfishEncryptBlockEntry.GetIVLength() != sizeof(mEntryIVBase))
660 	{
661 		THROW_EXCEPTION(BackupStoreException, IVLengthForEncodedBlockSizeDoesntMeetLengthRequirements)
662 	}
663 	uint64_t iv = mEntryIVBase;
664 	iv += mAbsoluteBlockNumber;
665 	// Convert to network byte order before encrypting with it, so that restores work on
666 	// platforms with different endiannesses.
667 	iv = box_hton64(iv);
668 	sBlowfishEncryptBlockEntry.SetIV(&iv);
669 
670 	// Encode the data
671 	int encodedSize = sBlowfishEncryptBlockEntry.TransformBlock(entry.mEnEnc, sizeof(entry.mEnEnc), &entryEnc, sizeof(entryEnc));
672 	if(encodedSize != sizeof(entry.mEnEnc))
673 	{
674 		THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
675 	}
676 
677 	// Save to data block for sending at the end of the stream
678 	mData.Write(&entry, sizeof(entry));
679 }
680 
681 
682 // --------------------------------------------------------------------------
683 //
684 // Function
685 //		Name:    BackupStoreFileEncodeStream::Write(const void *, int)
686 //		Purpose: As interface. Exceptions.
687 //		Created: 8/12/03
688 //
689 // --------------------------------------------------------------------------
Write(const void * pBuffer,int NBytes)690 void BackupStoreFileEncodeStream::Write(const void *pBuffer, int NBytes)
691 {
692 	THROW_EXCEPTION(BackupStoreException, CantWriteToEncodedFileStream)
693 }
694 
695 // --------------------------------------------------------------------------
696 //
697 // Function
698 //		Name:    BackupStoreFileEncodeStream::StreamDataLeft()
699 //		Purpose: As interface -- end of stream reached?
700 //		Created: 8/12/03
701 //
702 // --------------------------------------------------------------------------
StreamDataLeft()703 bool BackupStoreFileEncodeStream::StreamDataLeft()
704 {
705 	return (mStatus != Status_Finished);
706 }
707 
708 
709 // --------------------------------------------------------------------------
710 //
711 // Function
712 //		Name:    BackupStoreFileEncodeStream::StreamClosed()
713 //		Purpose: As interface
714 //		Created: 8/12/03
715 //
716 // --------------------------------------------------------------------------
StreamClosed()717 bool BackupStoreFileEncodeStream::StreamClosed()
718 {
719 	return true;
720 }
721 
722 
723 // --------------------------------------------------------------------------
724 //
725 // Function
726 //		Name:    BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *, int64_t)
727 //		Purpose: Constructor. Takes ownership of the block index, and will delete it when it's deleted
728 //		Created: 15/1/04
729 //
730 // --------------------------------------------------------------------------
Recipe(BackupStoreFileCreation::BlocksAvailableEntry * pBlockIndex,int64_t NumBlocksInIndex,int64_t OtherFileID)731 BackupStoreFileEncodeStream::Recipe::Recipe(BackupStoreFileCreation::BlocksAvailableEntry *pBlockIndex,
732 		int64_t NumBlocksInIndex, int64_t OtherFileID)
733 	: mpBlockIndex(pBlockIndex),
734 	  mNumBlocksInIndex(NumBlocksInIndex),
735 	  mOtherFileID(OtherFileID)
736 {
737 	ASSERT((mpBlockIndex == 0) || (NumBlocksInIndex != 0))
738 }
739 
740 // --------------------------------------------------------------------------
741 //
742 // Function
743 //		Name:    BackupStoreFileEncodeStream::Recipe::~Recipe()
744 //		Purpose: Destructor
745 //		Created: 15/1/04
746 //
747 // --------------------------------------------------------------------------
~Recipe()748 BackupStoreFileEncodeStream::Recipe::~Recipe()
749 {
750 	// Free the block index, if there is one
751 	if(mpBlockIndex != 0)
752 	{
753 		::free(mpBlockIndex);
754 	}
755 }
756 
757 
758 
759 
760