1 // distribution boxbackup-0.11_trunk_2979 (svn version: 2979)
2 // Box Backup, http://www.boxbackup.org/
3 //
4 // Copyright (c) 2003-2010, Ben Summers and contributors.
5 // All rights reserved.
6 //
7 // Note that this project uses mixed licensing. Any file with this license
8 // attached, or where the code LICENSE-GPL appears on the first line, falls
9 // under the "Box Backup GPL" license. See the file COPYING.txt for more
10 // information about this license.
11 //
12 // ---------------------------------------------------------------------
13 // This program is free software; you can redistribute it and/or
14 // modify it under the terms of the GNU General Public License
15 // as published by the Free Software Foundation; either version 2
16 // of the License, or (at your option) any later version.
17 //
18 // This program is distributed in the hope that it will be useful,
19 // but WITHOUT ANY WARRANTY; without even the implied warranty of
20 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 // GNU General Public License for more details.
22 //
23 // You should have received a copy of the GNU General Public License
24 // along with this program; if not, write to the Free Software
25 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
26 //
27 // [http://www.gnu.org/licenses/old-licenses/gpl-2.0.html#SEC4]
28 //
29 // As a special exception to the GPLv2, the Box Backup Project gives
30 // permission to link any code falling under this license (the Box Backup
31 // GPL) with any software that can be downloaded from
32 // the OpenSSL website [http://www.openssl.org] under either the
33 // "OpenSSL License" or the "Original SSLeay License", and to distribute
34 // the linked executables under the terms of the "Box Backup GPL" license.
35 //
36 // As a special exception to the GPLv2, the Box Backup Project gives
37 // permission to link any code falling under this license (the Box Backup
38 // GPL) with any version of Microsoft's Volume Shadow Copy Service 7.2 SDK
39 // or Microsoft Windows Software Development Kit (SDK), including
40 // vssapi.lib, that can be downloaded from the Microsoft website
41 // [*.microsoft.com], and to distribute the linked executables under the
42 // terms of the "Box Backup GPL" license.
43 // --------------------------------------------------------------------------
44 //
45 // File
46 // Name: BackupStoreFile.cpp
47 // Purpose: Utils for manipulating files
48 // Created: 2003/08/28
49 //
50 // --------------------------------------------------------------------------
51
52 #include "Box.h"
53
54 #ifdef HAVE_UNISTD_H
55 #include <unistd.h>
56 #endif
57
58 #include <sys/stat.h>
59 #include <string.h>
60 #include <new>
61 #include <string.h>
62
63 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
64 #include <stdio.h>
65 #endif
66
67 #include "BackupStoreFile.h"
68 #include "BackupStoreFileWire.h"
69 #include "BackupStoreFileCryptVar.h"
70 #include "BackupStoreFilename.h"
71 #include "BackupStoreException.h"
72 #include "IOStream.h"
73 #include "Guards.h"
74 #include "FileModificationTime.h"
75 #include "FileStream.h"
76 #include "BackupClientFileAttributes.h"
77 #include "BackupStoreObjectMagic.h"
78 #include "Compress.h"
79 #include "CipherContext.h"
80 #include "CipherBlowfish.h"
81 #include "CipherAES.h"
82 #include "BackupStoreConstants.h"
83 #include "CollectInBufferStream.h"
84 #include "RollingChecksum.h"
85 #include "MD5Digest.h"
86 #include "ReadGatherStream.h"
87 #include "Random.h"
88 #include "BackupStoreFileEncodeStream.h"
89 #include "Logging.h"
90
91 #include "MemLeakFindOn.h"
92
93 using namespace BackupStoreFileCryptVar;
94
95 // How big a buffer to use for copying files
96 #define COPY_BUFFER_SIZE (8*1024)
97
98 // Statistics
99 BackupStoreFileStats BackupStoreFile::msStats = {0,0,0};
100
101 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
102 bool sWarnedAboutBackwardsCompatiblity = false;
103 #endif
104
105 // --------------------------------------------------------------------------
106 //
107 // Function
108 // Name: BackupStoreFile::EncodeFile(IOStream &, IOStream &)
109 // Purpose: Encode a file into something for storing on file server.
110 // Requires a real filename so full info can be stored.
111 //
112 // Returns a stream. Most of the work is done by the stream
113 // when data is actually requested -- the file will be held
114 // open until the stream is deleted or the file finished.
115 // Created: 2003/08/28
116 //
117 // --------------------------------------------------------------------------
EncodeFile(const char * Filename,int64_t ContainerID,const BackupStoreFilename & rStoreFilename,int64_t * pModificationTime,ReadLoggingStream::Logger * pLogger,RunStatusProvider * pRunStatusProvider)118 std::auto_ptr<IOStream> BackupStoreFile::EncodeFile(
119 const char *Filename, int64_t ContainerID,
120 const BackupStoreFilename &rStoreFilename,
121 int64_t *pModificationTime,
122 ReadLoggingStream::Logger* pLogger,
123 RunStatusProvider* pRunStatusProvider)
124 {
125 // Create the stream
126 std::auto_ptr<IOStream> stream(new BackupStoreFileEncodeStream);
127
128 // Do the initial setup
129 ((BackupStoreFileEncodeStream*)stream.get())->Setup(Filename,
130 0 /* no recipe, just encode */,
131 ContainerID, rStoreFilename, pModificationTime, pLogger,
132 pRunStatusProvider);
133
134 // Return the stream for the caller
135 return stream;
136 }
137
138 // --------------------------------------------------------------------------
139 //
140 // Function
141 // Name: BackupStoreFile::VerifyEncodedFileFormat(IOStream &)
142 // Purpose: Verify that an encoded file meets the format
143 // requirements. Doesn't verify that the data is intact
144 // and can be decoded. Optionally returns the ID of the
145 // file which it is diffed from, and the (original)
146 // container ID.
147 // Created: 2003/08/28
148 //
149 // --------------------------------------------------------------------------
VerifyEncodedFileFormat(IOStream & rFile,int64_t * pDiffFromObjectIDOut,int64_t * pContainerIDOut)150 bool BackupStoreFile::VerifyEncodedFileFormat(IOStream &rFile, int64_t *pDiffFromObjectIDOut, int64_t *pContainerIDOut)
151 {
152 // Get the size of the file
153 int64_t fileSize = rFile.BytesLeftToRead();
154 if(fileSize == IOStream::SizeOfStreamUnknown)
155 {
156 THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
157 }
158
159 // Get the header...
160 file_StreamFormat hdr;
161 if(!rFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */))
162 {
163 // Couldn't read header
164 return false;
165 }
166
167 // Check magic number
168 if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
169 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
170 && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
171 #endif
172 )
173 {
174 return false;
175 }
176
177 // Get a filename, see if it loads OK
178 try
179 {
180 BackupStoreFilename fn;
181 fn.ReadFromStream(rFile, IOStream::TimeOutInfinite);
182 }
183 catch(...)
184 {
185 // an error occured while reading it, so that's not good
186 return false;
187 }
188
189 // Skip the attributes -- because they're encrypted, the server can't tell whether they're OK or not
190 try
191 {
192 int32_t size_s;
193 if(!rFile.ReadFullBuffer(&size_s, sizeof(size_s), 0 /* not interested in bytes read if this fails */))
194 {
195 THROW_EXCEPTION(CommonException, StreamableMemBlockIncompleteRead)
196 }
197 int size = ntohl(size_s);
198 // Skip forward the size
199 rFile.Seek(size, IOStream::SeekType_Relative);
200 }
201 catch(...)
202 {
203 // an error occured while reading it, so that's not good
204 return false;
205 }
206
207 // Get current position in file -- the end of the header
208 int64_t headerEnd = rFile.GetPosition();
209
210 // Get number of blocks
211 int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
212
213 // Calculate where the block index will be, check it's reasonable
214 int64_t blockIndexLoc = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
215 if(blockIndexLoc < headerEnd)
216 {
217 // Not enough space left for the block index, let alone the blocks themselves
218 return false;
219 }
220
221 // Load the block index header
222 rFile.Seek(blockIndexLoc, IOStream::SeekType_Absolute);
223 file_BlockIndexHeader blkhdr;
224 if(!rFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */))
225 {
226 // Couldn't read block index header -- assume bad file
227 return false;
228 }
229
230 // Check header
231 if((ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
232 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
233 && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
234 #endif
235 )
236 || (int64_t)box_ntoh64(blkhdr.mNumBlocks) != numBlocks)
237 {
238 // Bad header -- either magic value or number of blocks is wrong
239 return false;
240 }
241
242 // Flag for recording whether a block is referenced from another file
243 bool blockFromOtherFileReferenced = false;
244
245 // Read the index, checking that the length values all make sense
246 int64_t currentBlockStart = headerEnd;
247 for(int64_t b = 0; b < numBlocks; ++b)
248 {
249 // Read block entry
250 file_BlockIndexEntry blk;
251 if(!rFile.ReadFullBuffer(&blk, sizeof(blk), 0 /* not interested in bytes read if this fails */))
252 {
253 // Couldn't read block index entry -- assume bad file
254 return false;
255 }
256
257 // Check size and location
258 int64_t blkSize = box_ntoh64(blk.mEncodedSize);
259 if(blkSize <= 0)
260 {
261 // Mark that this file references another file
262 blockFromOtherFileReferenced = true;
263 }
264 else
265 {
266 // This block is actually in this file
267 if((currentBlockStart + blkSize) > blockIndexLoc)
268 {
269 // Encoded size makes the block run over the index
270 return false;
271 }
272
273 // Move the current block start ot the end of this block
274 currentBlockStart += blkSize;
275 }
276 }
277
278 // Check that there's no empty space
279 if(currentBlockStart != blockIndexLoc)
280 {
281 return false;
282 }
283
284 // Check that if another block is references, then the ID is there, and if one isn't there is no ID.
285 int64_t otherID = box_ntoh64(blkhdr.mOtherFileID);
286 if((otherID != 0 && blockFromOtherFileReferenced == false)
287 || (otherID == 0 && blockFromOtherFileReferenced == true))
288 {
289 // Doesn't look good!
290 return false;
291 }
292
293 // Does the caller want the other ID?
294 if(pDiffFromObjectIDOut)
295 {
296 *pDiffFromObjectIDOut = otherID;
297 }
298
299 // Does the caller want the container ID?
300 if(pContainerIDOut)
301 {
302 *pContainerIDOut = box_ntoh64(hdr.mContainerID);
303 }
304
305 // Passes all tests
306 return true;
307 }
308
309 // --------------------------------------------------------------------------
310 //
311 // Function
312 // Name: BackupStoreFile::DecodeFile(IOStream &, const char *)
313 // Purpose: Decode a file. Will set file attributes. File must not exist.
314 // Created: 2003/08/28
315 //
316 // --------------------------------------------------------------------------
DecodeFile(IOStream & rEncodedFile,const char * DecodedFilename,int Timeout,const BackupClientFileAttributes * pAlterativeAttr)317 void BackupStoreFile::DecodeFile(IOStream &rEncodedFile, const char *DecodedFilename, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
318 {
319 // Does file exist?
320 EMU_STRUCT_STAT st;
321 if(EMU_STAT(DecodedFilename, &st) == 0)
322 {
323 THROW_EXCEPTION(BackupStoreException, OutputFileAlreadyExists)
324 }
325
326 // Try, delete output file if error
327 try
328 {
329 // Make a stream for outputting this file
330 FileStream out(DecodedFilename, O_WRONLY | O_CREAT | O_EXCL);
331
332 // Get the decoding stream
333 std::auto_ptr<DecodedStream> stream(DecodeFileStream(rEncodedFile, Timeout, pAlterativeAttr));
334
335 // Is it a symlink?
336 if(!stream->IsSymLink())
337 {
338 // Copy it out to the file
339 stream->CopyStreamTo(out);
340 }
341
342 out.Close();
343
344 // The stream might have uncertain size, in which case
345 // we need to drain it to get the
346 // Protocol::ProtocolStreamHeader_EndOfStream byte
347 // out of our connection stream.
348 char buffer[1];
349 int drained = rEncodedFile.Read(buffer, 1);
350
351 // The Read will return 0 if we are actually at the end
352 // of the stream, but some tests decode files directly,
353 // in which case we are actually positioned at the start
354 // of the block index. I hope that reading an extra byte
355 // doesn't hurt!
356 // ASSERT(drained == 0);
357
358 // Write the attributes
359 try
360 {
361 stream->GetAttributes().WriteAttributes(DecodedFilename);
362 }
363 catch (std::exception& e)
364 {
365 BOX_WARNING("Failed to restore attributes on " <<
366 DecodedFilename << ": " << e.what());
367 }
368 }
369 catch(...)
370 {
371 ::unlink(DecodedFilename);
372 throw;
373 }
374 }
375
376
377 // --------------------------------------------------------------------------
378 //
379 // Function
380 // Name: BackupStoreFile::DecodeFileStream(IOStream &, int, const BackupClientFileAttributes *)
381 // Purpose: Return a stream which will decode the encrypted file data on the fly.
382 // Accepts streams in block index first, or main header first, order. In the latter case,
383 // the stream must be Seek()able.
384 //
385 // Before you use the returned stream, call IsSymLink() -- symlink streams won't allow
386 // you to read any data to enforce correct logic. See BackupStoreFile::DecodeFile() implementation.
387 // Created: 9/12/03
388 //
389 // --------------------------------------------------------------------------
DecodeFileStream(IOStream & rEncodedFile,int Timeout,const BackupClientFileAttributes * pAlterativeAttr)390 std::auto_ptr<BackupStoreFile::DecodedStream> BackupStoreFile::DecodeFileStream(IOStream &rEncodedFile, int Timeout, const BackupClientFileAttributes *pAlterativeAttr)
391 {
392 // Create stream
393 std::auto_ptr<DecodedStream> stream(new DecodedStream(rEncodedFile, Timeout));
394
395 // Get it ready
396 stream->Setup(pAlterativeAttr);
397
398 // Return to caller
399 return stream;
400 }
401
402
403 // --------------------------------------------------------------------------
404 //
405 // Function
406 // Name: BackupStoreFile::DecodedStream::DecodedStream(IOStream &, int)
407 // Purpose: Constructor
408 // Created: 9/12/03
409 //
410 // --------------------------------------------------------------------------
DecodedStream(IOStream & rEncodedFile,int Timeout)411 BackupStoreFile::DecodedStream::DecodedStream(IOStream &rEncodedFile, int Timeout)
412 : mrEncodedFile(rEncodedFile),
413 mTimeout(Timeout),
414 mNumBlocks(0),
415 mpBlockIndex(0),
416 mpEncodedData(0),
417 mpClearData(0),
418 mClearDataSize(0),
419 mCurrentBlock(-1),
420 mCurrentBlockClearSize(0),
421 mPositionInCurrentBlock(0),
422 mEntryIVBase(42) // different to default value in the encoded stream!
423 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
424 , mIsOldVersion(false)
425 #endif
426 {
427 }
428
429
430 // --------------------------------------------------------------------------
431 //
432 // Function
433 // Name: BackupStoreFile::DecodedStream::~DecodedStream()
434 // Purpose: Desctructor
435 // Created: 9/12/03
436 //
437 // --------------------------------------------------------------------------
~DecodedStream()438 BackupStoreFile::DecodedStream::~DecodedStream()
439 {
440 // Free any allocated memory
441 if(mpBlockIndex)
442 {
443 ::free(mpBlockIndex);
444 }
445 if(mpEncodedData)
446 {
447 BackupStoreFile::CodingChunkFree(mpEncodedData);
448 }
449 if(mpClearData)
450 {
451 ::free(mpClearData);
452 }
453 }
454
455
456 // --------------------------------------------------------------------------
457 //
458 // Function
459 // Name: BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *)
460 // Purpose: Get the stream ready to decode -- reads in headers
461 // Created: 9/12/03
462 //
463 // --------------------------------------------------------------------------
Setup(const BackupClientFileAttributes * pAlterativeAttr)464 void BackupStoreFile::DecodedStream::Setup(const BackupClientFileAttributes *pAlterativeAttr)
465 {
466 // Get the size of the file
467 int64_t fileSize = mrEncodedFile.BytesLeftToRead();
468
469 // Get the magic number to work out which order the stream is in
470 int32_t magic;
471 if(!mrEncodedFile.ReadFullBuffer(&magic, sizeof(magic), 0 /* not interested in bytes read if this fails */, mTimeout))
472 {
473 // Couldn't read magic value
474 THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
475 }
476
477 bool inFileOrder = true;
478 switch(ntohl(magic))
479 {
480 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
481 case OBJECTMAGIC_FILE_MAGIC_VALUE_V0:
482 mIsOldVersion = true;
483 // control flows on
484 #endif
485 case OBJECTMAGIC_FILE_MAGIC_VALUE_V1:
486 inFileOrder = true;
487 break;
488
489 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
490 case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0:
491 mIsOldVersion = true;
492 // control flows on
493 #endif
494 case OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1:
495 inFileOrder = false;
496 break;
497
498 default:
499 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
500 }
501
502 // If not in file order, then the index list must be read now
503 if(!inFileOrder)
504 {
505 ReadBlockIndex(true /* have already read and verified the magic number */);
506 }
507
508 // Get header
509 file_StreamFormat hdr;
510 if(inFileOrder)
511 {
512 // Read the header, without the magic number
513 if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&hdr) + sizeof(magic), sizeof(hdr) - sizeof(magic),
514 0 /* not interested in bytes read if this fails */, mTimeout))
515 {
516 // Couldn't read header
517 THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
518 }
519 // Put in magic number
520 hdr.mMagicValue = magic;
521 }
522 else
523 {
524 // Not in file order, so need to read the full header
525 if(!mrEncodedFile.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, mTimeout))
526 {
527 // Couldn't read header
528 THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
529 }
530 }
531
532 // Check magic number
533 if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
534 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
535 && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
536 #endif
537 )
538 {
539 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
540 }
541
542 // Get the filename
543 mFilename.ReadFromStream(mrEncodedFile, mTimeout);
544
545 // Get the attributes (either from stream, or supplied attributes)
546 if(pAlterativeAttr != 0)
547 {
548 // Read dummy attributes
549 BackupClientFileAttributes attr;
550 attr.ReadFromStream(mrEncodedFile, mTimeout);
551
552 // Set to supplied attributes
553 mAttributes = *pAlterativeAttr;
554 }
555 else
556 {
557 // Read the attributes from the stream
558 mAttributes.ReadFromStream(mrEncodedFile, mTimeout);
559 }
560
561 // If it is in file order, go and read the file attributes
562 // Requires that the stream can seek
563 if(inFileOrder)
564 {
565 // Make sure the file size is known
566 if(fileSize == IOStream::SizeOfStreamUnknown)
567 {
568 THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
569 }
570
571 // Store current location (beginning of encoded blocks)
572 int64_t endOfHeaderPos = mrEncodedFile.GetPosition();
573
574 // Work out where the index is
575 int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
576 int64_t blockHeaderPos = fileSize - ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
577
578 // Seek to that position
579 mrEncodedFile.Seek(blockHeaderPos, IOStream::SeekType_Absolute);
580
581 // Read the block index
582 ReadBlockIndex(false /* magic number still to be read */);
583
584 // Seek back to the end of header position, ready for reading the chunks
585 mrEncodedFile.Seek(endOfHeaderPos, IOStream::SeekType_Absolute);
586 }
587
588 // Check view of blocks from block header and file header match
589 if(mNumBlocks != (int64_t)box_ntoh64(hdr.mNumBlocks))
590 {
591 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
592 }
593
594 // Need to allocate some memory for the two blocks for reading encoded data, and clear data
595 if(mNumBlocks > 0)
596 {
597 // Find the maximum encoded data size
598 int32_t maxEncodedDataSize = 0;
599 const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
600 ASSERT(entry != 0);
601 for(int64_t e = 0; e < mNumBlocks; e++)
602 {
603 // Get the clear and encoded size
604 int32_t encodedSize = box_ntoh64(entry[e].mEncodedSize);
605 ASSERT(encodedSize > 0);
606
607 // Larger?
608 if(encodedSize > maxEncodedDataSize) maxEncodedDataSize = encodedSize;
609 }
610
611 // Allocate those blocks!
612 mpEncodedData = (uint8_t*)BackupStoreFile::CodingChunkAlloc(maxEncodedDataSize + 32);
613
614 // Allocate the block for the clear data, using the hint from the header.
615 // If this is wrong, things will exception neatly later on, so it can't be used
616 // to do anything more than cause an error on downloading.
617 mClearDataSize = OutputBufferSizeForKnownOutputSize(ntohl(hdr.mMaxBlockClearSize)) + 32;
618 mpClearData = (uint8_t*)::malloc(mClearDataSize);
619 }
620 }
621
622
623 // --------------------------------------------------------------------------
624 //
625 // Function
626 // Name: BackupStoreFile::DecodedStream::ReadBlockIndex(bool)
627 // Purpose: Read the block index from the stream, and store in internal buffer (minus header)
628 // Created: 9/12/03
629 //
630 // --------------------------------------------------------------------------
ReadBlockIndex(bool MagicAlreadyRead)631 void BackupStoreFile::DecodedStream::ReadBlockIndex(bool MagicAlreadyRead)
632 {
633 // Header
634 file_BlockIndexHeader blkhdr;
635
636 // Read it in -- way depends on how whether the magic number has already been read
637 if(MagicAlreadyRead)
638 {
639 // Read the header, without the magic number
640 if(!mrEncodedFile.ReadFullBuffer(((uint8_t*)&blkhdr) + sizeof(blkhdr.mMagicValue), sizeof(blkhdr) - sizeof(blkhdr.mMagicValue),
641 0 /* not interested in bytes read if this fails */, mTimeout))
642 {
643 // Couldn't read header
644 THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
645 }
646 }
647 else
648 {
649 // Magic not already read, so need to read the full header
650 if(!mrEncodedFile.ReadFullBuffer(&blkhdr, sizeof(blkhdr), 0 /* not interested in bytes read if this fails */, mTimeout))
651 {
652 // Couldn't read header
653 THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
654 }
655
656 // Check magic value
657 if(ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1
658 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
659 && ntohl(blkhdr.mMagicValue) != OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0
660 #endif
661 )
662 {
663 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
664 }
665 }
666
667 // Get the number of blocks out of the header
668 mNumBlocks = box_ntoh64(blkhdr.mNumBlocks);
669
670 // Read the IV base
671 mEntryIVBase = box_ntoh64(blkhdr.mEntryIVBase);
672
673 // Load the block entries in?
674 if(mNumBlocks > 0)
675 {
676 // How big is the index?
677 int64_t indexSize = sizeof(file_BlockIndexEntry) * mNumBlocks;
678
679 // Allocate some memory
680 mpBlockIndex = ::malloc(indexSize);
681 if(mpBlockIndex == 0)
682 {
683 throw std::bad_alloc();
684 }
685
686 // Read it in
687 if(!mrEncodedFile.ReadFullBuffer(mpBlockIndex, indexSize, 0 /* not interested in bytes read if this fails */, mTimeout))
688 {
689 // Couldn't read header
690 THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
691 }
692 }
693 }
694
695
696 // --------------------------------------------------------------------------
697 //
698 // Function
699 // Name: BackupStoreFile::DecodedStream::Read(void *, int, int)
700 // Purpose: As interface. Reads decrpyted data.
701 // Created: 9/12/03
702 //
703 // --------------------------------------------------------------------------
Read(void * pBuffer,int NBytes,int Timeout)704 int BackupStoreFile::DecodedStream::Read(void *pBuffer, int NBytes, int Timeout)
705 {
706 // Symlinks don't have data. So can't read it. Not even zero bytes.
707 if(IsSymLink())
708 {
709 // Don't allow reading in this case
710 THROW_EXCEPTION(BackupStoreException, ThereIsNoDataInASymLink);
711 }
712
713 // Already finished?
714 if(mCurrentBlock >= mNumBlocks)
715 {
716 // At end of stream, nothing to do
717 return 0;
718 }
719
720 int bytesToRead = NBytes;
721 uint8_t *output = (uint8_t*)pBuffer;
722
723 while(bytesToRead > 0 && mCurrentBlock < mNumBlocks)
724 {
725 // Anything left in the current block?
726 if(mPositionInCurrentBlock < mCurrentBlockClearSize)
727 {
728 // Copy data out of this buffer
729 int s = mCurrentBlockClearSize - mPositionInCurrentBlock;
730 if(s > bytesToRead) s = bytesToRead; // limit to requested data
731
732 // Copy
733 ::memcpy(output, mpClearData + mPositionInCurrentBlock, s);
734
735 // Update positions
736 output += s;
737 mPositionInCurrentBlock += s;
738 bytesToRead -= s;
739 }
740
741 // Need to get some more data?
742 if(bytesToRead > 0 && mPositionInCurrentBlock >= mCurrentBlockClearSize)
743 {
744 // Number of next block
745 ++mCurrentBlock;
746 if(mCurrentBlock >= mNumBlocks)
747 {
748 // Stop now!
749 break;
750 }
751
752 // Get the size from the block index
753 const file_BlockIndexEntry *entry = (file_BlockIndexEntry *)mpBlockIndex;
754 int32_t encodedSize = box_ntoh64(entry[mCurrentBlock].mEncodedSize);
755 if(encodedSize <= 0)
756 {
757 // The caller is attempting to decode a file which is the direct result of a diff
758 // operation, and so does not contain all the data.
759 // It needs to be combined with the previous version first.
760 THROW_EXCEPTION(BackupStoreException, CannotDecodeDiffedFilesWithoutCombining)
761 }
762
763 // Load in next block
764 if(!mrEncodedFile.ReadFullBuffer(mpEncodedData, encodedSize, 0 /* not interested in bytes read if this fails */, mTimeout))
765 {
766 // Couldn't read header
767 THROW_EXCEPTION(BackupStoreException, WhenDecodingExpectedToReadButCouldnt)
768 }
769
770 // Decode the data
771 mCurrentBlockClearSize = BackupStoreFile::DecodeChunk(mpEncodedData, encodedSize, mpClearData, mClearDataSize);
772
773 // Calculate IV for this entry
774 uint64_t iv = mEntryIVBase;
775 iv += mCurrentBlock;
776 // Convert to network byte order before encrypting with it, so that restores work on
777 // platforms with different endiannesses.
778 iv = box_hton64(iv);
779 sBlowfishDecryptBlockEntry.SetIV(&iv);
780
781 // Decrypt the encrypted section
782 file_BlockIndexEntryEnc entryEnc;
783 int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
784 entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
785 if(sectionSize != sizeof(entryEnc))
786 {
787 THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
788 }
789
790 // Make sure this is the right size
791 if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
792 {
793 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
794 if(!mIsOldVersion)
795 {
796 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
797 }
798 // Versions 0.05 and previous of Box Backup didn't properly handle endianess of the
799 // IV for the encrypted section. Try again, with the thing the other way round
800 iv = box_swap64(iv);
801 sBlowfishDecryptBlockEntry.SetIV(&iv);
802 int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
803 entry[mCurrentBlock].mEnEnc, sizeof(entry[mCurrentBlock].mEnEnc));
804 if(sectionSize != sizeof(entryEnc))
805 {
806 THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
807 }
808 if(mCurrentBlockClearSize != (int32_t)ntohl(entryEnc.mSize))
809 {
810 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
811 }
812 else
813 {
814 // Warn and log this issue
815 if(!sWarnedAboutBackwardsCompatiblity)
816 {
817 BOX_WARNING("WARNING: Decoded one or more files using backwards compatibility mode for block index.");
818 sWarnedAboutBackwardsCompatiblity = true;
819 }
820 }
821 #else
822 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
823 #endif
824 }
825
826 // Check the digest
827 MD5Digest md5;
828 md5.Add(mpClearData, mCurrentBlockClearSize);
829 md5.Finish();
830 if(!md5.DigestMatches((uint8_t*)entryEnc.mStrongChecksum))
831 {
832 THROW_EXCEPTION(BackupStoreException, BackupStoreFileFailedIntegrityCheck)
833 }
834
835 // Set vars to say what's happening
836 mPositionInCurrentBlock = 0;
837 }
838 }
839
840 ASSERT(bytesToRead >= 0);
841 ASSERT(bytesToRead <= NBytes);
842
843 return NBytes - bytesToRead;
844 }
845
846
847 // --------------------------------------------------------------------------
848 //
849 // Function
850 // Name: BackupStoreFile::DecodedStream::IsSymLink()
851 // Purpose: Is the unencoded file actually a symlink?
852 // Created: 10/12/03
853 //
854 // --------------------------------------------------------------------------
IsSymLink()855 bool BackupStoreFile::DecodedStream::IsSymLink()
856 {
857 // First, check in with the attributes
858 if(!mAttributes.IsSymLink())
859 {
860 return false;
861 }
862
863 // So the attributes think it is a symlink.
864 // Consistency check...
865 if(mNumBlocks != 0)
866 {
867 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
868 }
869
870 return true;
871 }
872
873
874 // --------------------------------------------------------------------------
875 //
876 // Function
877 // Name: BackupStoreFile::DecodedStream::Write(const void *, int)
878 // Purpose: As interface. Throws exception, as you can't write to this stream.
879 // Created: 9/12/03
880 //
881 // --------------------------------------------------------------------------
Write(const void * pBuffer,int NBytes)882 void BackupStoreFile::DecodedStream::Write(const void *pBuffer, int NBytes)
883 {
884 THROW_EXCEPTION(BackupStoreException, CantWriteToDecodedFileStream)
885 }
886
887
888 // --------------------------------------------------------------------------
889 //
890 // Function
891 // Name: BackupStoreFile::DecodedStream::StreamDataLeft()
892 // Purpose: As interface. Any data left?
893 // Created: 9/12/03
894 //
895 // --------------------------------------------------------------------------
StreamDataLeft()896 bool BackupStoreFile::DecodedStream::StreamDataLeft()
897 {
898 return mCurrentBlock < mNumBlocks;
899 }
900
901
902 // --------------------------------------------------------------------------
903 //
904 // Function
905 // Name: BackupStoreFile::DecodedStream::StreamClosed()
906 // Purpose: As interface. Always returns true, no writing allowed.
907 // Created: 9/12/03
908 //
909 // --------------------------------------------------------------------------
StreamClosed()910 bool BackupStoreFile::DecodedStream::StreamClosed()
911 {
912 // Can't write to this stream!
913 return true;
914 }
915
916
917
918
919
920 // --------------------------------------------------------------------------
921 //
922 // Function
923 // Name: BackupStoreFile::SetBlowfishKey(const void *, int)
924 // Purpose: Static. Sets the key to use for encryption and decryption.
925 // Created: 7/12/03
926 //
927 // --------------------------------------------------------------------------
SetBlowfishKeys(const void * pKey,int KeyLength,const void * pBlockEntryKey,int BlockEntryKeyLength)928 void BackupStoreFile::SetBlowfishKeys(const void *pKey, int KeyLength, const void *pBlockEntryKey, int BlockEntryKeyLength)
929 {
930 // IVs set later
931 sBlowfishEncrypt.Reset();
932 sBlowfishEncrypt.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
933 sBlowfishDecrypt.Reset();
934 sBlowfishDecrypt.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pKey, KeyLength));
935
936 sBlowfishEncryptBlockEntry.Reset();
937 sBlowfishEncryptBlockEntry.Init(CipherContext::Encrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
938 sBlowfishEncryptBlockEntry.UsePadding(false);
939 sBlowfishDecryptBlockEntry.Reset();
940 sBlowfishDecryptBlockEntry.Init(CipherContext::Decrypt, CipherBlowfish(CipherDescription::Mode_CBC, pBlockEntryKey, BlockEntryKeyLength));
941 sBlowfishDecryptBlockEntry.UsePadding(false);
942 }
943
944
945 #ifndef HAVE_OLD_SSL
946 // --------------------------------------------------------------------------
947 //
948 // Function
949 // Name: BackupStoreFile::SetAESKey(const void *, int)
950 // Purpose: Sets the AES key to use for file data encryption. Will select AES as
951 // the cipher to use when encrypting.
952 // Created: 27/4/04
953 //
954 // --------------------------------------------------------------------------
SetAESKey(const void * pKey,int KeyLength)955 void BackupStoreFile::SetAESKey(const void *pKey, int KeyLength)
956 {
957 // Setup context
958 sAESEncrypt.Reset();
959 sAESEncrypt.Init(CipherContext::Encrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
960 sAESDecrypt.Reset();
961 sAESDecrypt.Init(CipherContext::Decrypt, CipherAES(CipherDescription::Mode_CBC, pKey, KeyLength));
962
963 // Set encryption to use this key, instead of the "default" blowfish key
964 spEncrypt = &sAESEncrypt;
965 sEncryptCipherType = HEADER_AES_ENCODING;
966 }
967 #endif
968
969
970 // --------------------------------------------------------------------------
971 //
972 // Function
973 // Name: BackupStoreFile::MaxBlockSizeForChunkSize(int)
974 // Purpose: The maximum output size of a block, given the chunk size
975 // Created: 7/12/03
976 //
977 // --------------------------------------------------------------------------
MaxBlockSizeForChunkSize(int ChunkSize)978 int BackupStoreFile::MaxBlockSizeForChunkSize(int ChunkSize)
979 {
980 // Calculate... the maximum size of output by first the largest it could be after compression,
981 // which is encrypted, and has a 1 bytes header and the IV added, plus 1 byte for luck
982 // And then on top, add 128 bytes just to make sure. (Belts and braces approach to fixing
983 // an problem where a rather non-compressable file didn't fit in a block buffer.)
984 return sBlowfishEncrypt.MaxOutSizeForInBufferSize(Compress_MaxSizeForCompressedData(ChunkSize)) + 1 + 1
985 + sBlowfishEncrypt.GetIVLength() + 128;
986 }
987
988
989
990 // --------------------------------------------------------------------------
991 //
992 // Function
993 // Name: BackupStoreFile::EncodeChunk(const void *, int, BackupStoreFile::EncodingBuffer &)
994 // Purpose: Encodes a chunk (encryption, possible compressed beforehand)
995 // Created: 8/12/03
996 //
997 // --------------------------------------------------------------------------
EncodeChunk(const void * Chunk,int ChunkSize,BackupStoreFile::EncodingBuffer & rOutput)998 int BackupStoreFile::EncodeChunk(const void *Chunk, int ChunkSize, BackupStoreFile::EncodingBuffer &rOutput)
999 {
1000 ASSERT(spEncrypt != 0);
1001
1002 // Check there's some space in the output block
1003 if(rOutput.mBufferSize < 256)
1004 {
1005 rOutput.Reallocate(256);
1006 }
1007
1008 // Check alignment of the block
1009 ASSERT((((uint32_t)(long)rOutput.mpBuffer) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
1010
1011 // Want to compress it?
1012 bool compressChunk = (ChunkSize >= BACKUP_FILE_MIN_COMPRESSED_CHUNK_SIZE);
1013
1014 // Build header
1015 uint8_t header = sEncryptCipherType << HEADER_ENCODING_SHIFT;
1016 if(compressChunk) header |= HEADER_CHUNK_IS_COMPRESSED;
1017
1018 // Store header
1019 rOutput.mpBuffer[0] = header;
1020 int outOffset = 1;
1021
1022 // Setup cipher, and store the IV
1023 int ivLen = 0;
1024 const void *iv = spEncrypt->SetRandomIV(ivLen);
1025 ::memcpy(rOutput.mpBuffer + outOffset, iv, ivLen);
1026 outOffset += ivLen;
1027
1028 // Start encryption process
1029 spEncrypt->Begin();
1030
1031 #define ENCODECHUNK_CHECK_SPACE(ToEncryptSize) \
1032 { \
1033 if((rOutput.mBufferSize - outOffset) < ((ToEncryptSize) + 128)) \
1034 { \
1035 rOutput.Reallocate(rOutput.mBufferSize + (ToEncryptSize) + 128); \
1036 } \
1037 }
1038
1039 // Encode the chunk
1040 if(compressChunk)
1041 {
1042 // buffer to compress into
1043 uint8_t buffer[2048];
1044
1045 // Set compressor with all the chunk as an input
1046 Compress<true> compress;
1047 compress.Input(Chunk, ChunkSize);
1048 compress.FinishInput();
1049
1050 // Get and encrypt output
1051 while(!compress.OutputHasFinished())
1052 {
1053 int s = compress.Output(buffer, sizeof(buffer));
1054 if(s > 0)
1055 {
1056 ENCODECHUNK_CHECK_SPACE(s)
1057 outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, buffer, s);
1058 }
1059 else
1060 {
1061 // Should never happen, as we put all the input in in one go.
1062 // So if this happens, it means there's a logical problem somewhere
1063 THROW_EXCEPTION(BackupStoreException, Internal)
1064 }
1065 }
1066 ENCODECHUNK_CHECK_SPACE(16)
1067 outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
1068 }
1069 else
1070 {
1071 // Straight encryption
1072 ENCODECHUNK_CHECK_SPACE(ChunkSize)
1073 outOffset += spEncrypt->Transform(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset, Chunk, ChunkSize);
1074 ENCODECHUNK_CHECK_SPACE(16)
1075 outOffset += spEncrypt->Final(rOutput.mpBuffer + outOffset, rOutput.mBufferSize - outOffset);
1076 }
1077
1078 ASSERT(outOffset < rOutput.mBufferSize); // first check should have sorted this -- merely logic check
1079
1080 return outOffset;
1081 }
1082
1083 // --------------------------------------------------------------------------
1084 //
1085 // Function
1086 // Name: BackupStoreFile::DecodeChunk(const void *, int, void *, int)
1087 // Purpose: Decode an encoded chunk -- use OutputBufferSizeForKnownOutputSize() to find
1088 // the extra output buffer size needed before calling.
1089 // See notes in EncodeChunk() for notes re alignment of the
1090 // encoded data.
1091 // Created: 8/12/03
1092 //
1093 // --------------------------------------------------------------------------
DecodeChunk(const void * Encoded,int EncodedSize,void * Output,int OutputSize)1094 int BackupStoreFile::DecodeChunk(const void *Encoded, int EncodedSize, void *Output, int OutputSize)
1095 {
1096 // Check alignment of the encoded block
1097 ASSERT((((uint32_t)(long)Encoded) % BACKUPSTOREFILE_CODING_BLOCKSIZE) == BACKUPSTOREFILE_CODING_OFFSET);
1098
1099 // First check
1100 if(EncodedSize < 1)
1101 {
1102 THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
1103 }
1104
1105 const uint8_t *input = (uint8_t*)Encoded;
1106
1107 // Get header, make checks, etc
1108 uint8_t header = input[0];
1109 bool chunkCompressed = (header & HEADER_CHUNK_IS_COMPRESSED) == HEADER_CHUNK_IS_COMPRESSED;
1110 uint8_t encodingType = (header >> HEADER_ENCODING_SHIFT);
1111 if(encodingType != HEADER_BLOWFISH_ENCODING && encodingType != HEADER_AES_ENCODING)
1112 {
1113 THROW_EXCEPTION(BackupStoreException, ChunkHasUnknownEncoding)
1114 }
1115
1116 #ifndef HAVE_OLD_SSL
1117 // Choose cipher
1118 CipherContext &cipher((encodingType == HEADER_AES_ENCODING)?sAESDecrypt:sBlowfishDecrypt);
1119 #else
1120 // AES not supported with this version of OpenSSL
1121 if(encodingType == HEADER_AES_ENCODING)
1122 {
1123 THROW_EXCEPTION(BackupStoreException, AEScipherNotSupportedByInstalledOpenSSL)
1124 }
1125 CipherContext &cipher(sBlowfishDecrypt);
1126 #endif
1127
1128 // Check enough space for header, an IV and one byte of input
1129 int ivLen = cipher.GetIVLength();
1130 if(EncodedSize < (1 + ivLen + 1))
1131 {
1132 THROW_EXCEPTION(BackupStoreException, BadEncodedChunk)
1133 }
1134
1135 // Set IV in decrypt context, and start
1136 cipher.SetIV(input + 1);
1137 cipher.Begin();
1138
1139 // Setup vars for code
1140 int inOffset = 1 + ivLen;
1141 uint8_t *output = (uint8_t*)Output;
1142 int outOffset = 0;
1143
1144 // Do action
1145 if(chunkCompressed)
1146 {
1147 // Do things in chunks
1148 uint8_t buffer[2048];
1149 int inputBlockLen = cipher.InSizeForOutBufferSize(sizeof(buffer));
1150
1151 // Decompressor
1152 Compress<false> decompress;
1153
1154 while(inOffset < EncodedSize)
1155 {
1156 // Decrypt a block
1157 int bl = inputBlockLen;
1158 if(bl > (EncodedSize - inOffset)) bl = EncodedSize - inOffset; // not too long
1159 int s = cipher.Transform(buffer, sizeof(buffer), input + inOffset, bl);
1160 inOffset += bl;
1161
1162 // Decompress the decrypted data
1163 if(s > 0)
1164 {
1165 decompress.Input(buffer, s);
1166 int os = 0;
1167 do
1168 {
1169 os = decompress.Output(output + outOffset, OutputSize - outOffset);
1170 outOffset += os;
1171 } while(os > 0);
1172
1173 // Check that there's space left in the output buffer -- there always should be
1174 if(outOffset >= OutputSize)
1175 {
1176 THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
1177 }
1178 }
1179 }
1180
1181 // Get any compressed data remaining in the cipher context and compression
1182 int s = cipher.Final(buffer, sizeof(buffer));
1183 decompress.Input(buffer, s);
1184 decompress.FinishInput();
1185 while(!decompress.OutputHasFinished())
1186 {
1187 int os = decompress.Output(output + outOffset, OutputSize - outOffset);
1188 outOffset += os;
1189
1190 // Check that there's space left in the output buffer -- there always should be
1191 if(outOffset >= OutputSize)
1192 {
1193 THROW_EXCEPTION(BackupStoreException, NotEnoughSpaceToDecodeChunk)
1194 }
1195 }
1196 }
1197 else
1198 {
1199 // Easy decryption
1200 outOffset += cipher.Transform(output + outOffset, OutputSize - outOffset, input + inOffset, EncodedSize - inOffset);
1201 outOffset += cipher.Final(output + outOffset, OutputSize - outOffset);
1202 }
1203
1204 return outOffset;
1205 }
1206
1207
1208
1209 // --------------------------------------------------------------------------
1210 //
1211 // Function
1212 // Name: BackupStoreFile::ReorderFileToStreamOrder(IOStream *, bool)
1213 // Purpose: Returns a stream which gives a Stream order version of the encoded file.
1214 // If TakeOwnership == true, then the input stream will be deleted when the
1215 // returned stream is deleted.
1216 // The input stream must be seekable.
1217 // Created: 10/12/03
1218 //
1219 // --------------------------------------------------------------------------
ReorderFileToStreamOrder(IOStream * pStream,bool TakeOwnership)1220 std::auto_ptr<IOStream> BackupStoreFile::ReorderFileToStreamOrder(IOStream *pStream, bool TakeOwnership)
1221 {
1222 ASSERT(pStream != 0);
1223
1224 // Get the size of the file
1225 int64_t fileSize = pStream->BytesLeftToRead();
1226 if(fileSize == IOStream::SizeOfStreamUnknown)
1227 {
1228 THROW_EXCEPTION(BackupStoreException, StreamDoesntHaveRequiredFeatures)
1229 }
1230
1231 // Read the header
1232 int bytesRead = 0;
1233 file_StreamFormat hdr;
1234 bool readBlock = pStream->ReadFullBuffer(&hdr, sizeof(hdr), &bytesRead);
1235
1236 // Seek backwards to put the file pointer back where it was before we started this
1237 pStream->Seek(0 - bytesRead, IOStream::SeekType_Relative);
1238
1239 // Check we got a block
1240 if(!readBlock)
1241 {
1242 // Couldn't read header -- assume file bad
1243 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
1244 }
1245
1246 // Check magic number
1247 if(ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V1
1248 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
1249 && ntohl(hdr.mMagicValue) != OBJECTMAGIC_FILE_MAGIC_VALUE_V0
1250 #endif
1251 )
1252 {
1253 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
1254 }
1255
1256 // Get number of blocks
1257 int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
1258
1259 // Calculate where the block index will be, check it's reasonable
1260 int64_t blockIndexSize = ((numBlocks * sizeof(file_BlockIndexEntry)) + sizeof(file_BlockIndexHeader));
1261 int64_t blockIndexLoc = fileSize - blockIndexSize;
1262 if(blockIndexLoc < 0)
1263 {
1264 // Doesn't look good!
1265 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
1266 }
1267
1268 // Build a reordered stream
1269 std::auto_ptr<IOStream> reordered(new ReadGatherStream(TakeOwnership));
1270
1271 // Set it up...
1272 ReadGatherStream &rreordered(*((ReadGatherStream*)reordered.get()));
1273 int component = rreordered.AddComponent(pStream);
1274 // Send out the block index
1275 rreordered.AddBlock(component, blockIndexSize, true, blockIndexLoc);
1276 // And then the rest of the file
1277 rreordered.AddBlock(component, blockIndexLoc, true, 0);
1278
1279 return reordered;
1280 }
1281
1282
1283
1284 // --------------------------------------------------------------------------
1285 //
1286 // Function
1287 // Name: BackupStoreFile::ResetStats()
1288 // Purpose: Reset the gathered statistics
1289 // Created: 20/1/04
1290 //
1291 // --------------------------------------------------------------------------
ResetStats()1292 void BackupStoreFile::ResetStats()
1293 {
1294 msStats.mBytesInEncodedFiles = 0;
1295 msStats.mBytesAlreadyOnServer = 0;
1296 msStats.mTotalFileStreamSize = 0;
1297 }
1298
1299
1300
1301 // --------------------------------------------------------------------------
1302 //
1303 // Function
1304 // Name: BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *, IOStream &)
1305 // Purpose: Compares the contents of a file against the checksums contained in the
1306 // block index. Returns true if the checksums match, meaning the file is
1307 // extremely likely to match the original. Will always consume the entire index.
1308 // Created: 21/1/04
1309 //
1310 // --------------------------------------------------------------------------
CompareFileContentsAgainstBlockIndex(const char * Filename,IOStream & rBlockIndex,int Timeout)1311 bool BackupStoreFile::CompareFileContentsAgainstBlockIndex(const char *Filename, IOStream &rBlockIndex, int Timeout)
1312 {
1313 // is it a symlink?
1314 bool sourceIsSymlink = false;
1315 {
1316 EMU_STRUCT_STAT st;
1317 if(EMU_LSTAT(Filename, &st) == -1)
1318 {
1319 THROW_EXCEPTION(CommonException, OSFileError)
1320 }
1321 if((st.st_mode & S_IFMT) == S_IFLNK)
1322 {
1323 sourceIsSymlink = true;
1324 }
1325 }
1326
1327 // Open file, if it's not a symlink
1328 std::auto_ptr<FileStream> in;
1329 if(!sourceIsSymlink)
1330 {
1331 in.reset(new FileStream(Filename));
1332 }
1333
1334 // Read header
1335 file_BlockIndexHeader hdr;
1336 if(!rBlockIndex.ReadFullBuffer(&hdr, sizeof(hdr), 0 /* not interested in bytes read if this fails */, Timeout))
1337 {
1338 // Couldn't read header
1339 THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
1340 }
1341
1342 // Check magic
1343 if(hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V1)
1344 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
1345 && hdr.mMagicValue != (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0)
1346 #endif
1347 )
1348 {
1349 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
1350 }
1351
1352 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
1353 bool isOldVersion = hdr.mMagicValue == (int32_t)htonl(OBJECTMAGIC_FILE_BLOCKS_MAGIC_VALUE_V0);
1354 #endif
1355
1356 // Get basic information
1357 int64_t numBlocks = box_ntoh64(hdr.mNumBlocks);
1358 uint64_t entryIVBase = box_ntoh64(hdr.mEntryIVBase);
1359
1360 //TODO: Verify that these sizes look reasonable
1361
1362 // setup
1363 void *data = 0;
1364 int32_t dataSize = -1;
1365 bool matches = true;
1366 int64_t totalSizeInBlockIndex = 0;
1367
1368 try
1369 {
1370 for(int64_t b = 0; b < numBlocks; ++b)
1371 {
1372 // Read an entry from the stream
1373 file_BlockIndexEntry entry;
1374 if(!rBlockIndex.ReadFullBuffer(&entry, sizeof(entry), 0 /* not interested in bytes read if this fails */, Timeout))
1375 {
1376 // Couldn't read entry
1377 THROW_EXCEPTION(BackupStoreException, CouldntReadEntireStructureFromStream)
1378 }
1379
1380 // Calculate IV for this entry
1381 uint64_t iv = entryIVBase;
1382 iv += b;
1383 iv = box_hton64(iv);
1384 #ifndef BOX_DISABLE_BACKWARDS_COMPATIBILITY_BACKUPSTOREFILE
1385 if(isOldVersion)
1386 {
1387 // Reverse the IV for compatibility
1388 iv = box_swap64(iv);
1389 }
1390 #endif
1391 sBlowfishDecryptBlockEntry.SetIV(&iv);
1392
1393 // Decrypt the encrypted section
1394 file_BlockIndexEntryEnc entryEnc;
1395 int sectionSize = sBlowfishDecryptBlockEntry.TransformBlock(&entryEnc, sizeof(entryEnc),
1396 entry.mEnEnc, sizeof(entry.mEnEnc));
1397 if(sectionSize != sizeof(entryEnc))
1398 {
1399 THROW_EXCEPTION(BackupStoreException, BlockEntryEncodingDidntGiveExpectedLength)
1400 }
1401
1402 // Size of block
1403 int32_t blockClearSize = ntohl(entryEnc.mSize);
1404 if(blockClearSize < 0 || blockClearSize > (BACKUP_FILE_MAX_BLOCK_SIZE + 1024))
1405 {
1406 THROW_EXCEPTION(BackupStoreException, BadBackupStoreFile)
1407 }
1408 totalSizeInBlockIndex += blockClearSize;
1409
1410 // Make sure there's enough memory allocated to load the block in
1411 if(dataSize < blockClearSize)
1412 {
1413 // Too small, free the block if it's already allocated
1414 if(data != 0)
1415 {
1416 ::free(data);
1417 data = 0;
1418 }
1419 // Allocate a block
1420 data = ::malloc(blockClearSize + 128);
1421 if(data == 0)
1422 {
1423 throw std::bad_alloc();
1424 }
1425 dataSize = blockClearSize + 128;
1426 }
1427
1428 // Load in the block from the file, if it's not a symlink
1429 if(!sourceIsSymlink)
1430 {
1431 if(in->Read(data, blockClearSize) != blockClearSize)
1432 {
1433 // Not enough data left in the file, can't possibly match
1434 matches = false;
1435 }
1436 else
1437 {
1438 // Check the checksum
1439 MD5Digest md5;
1440 md5.Add(data, blockClearSize);
1441 md5.Finish();
1442 if(!md5.DigestMatches(entryEnc.mStrongChecksum))
1443 {
1444 // Checksum didn't match
1445 matches = false;
1446 }
1447 }
1448 }
1449
1450 // Keep on going regardless, to make sure the entire block index stream is read
1451 // -- must always be consistent about what happens with the stream.
1452 }
1453 }
1454 catch(...)
1455 {
1456 // clean up in case of errors
1457 if(data != 0)
1458 {
1459 ::free(data);
1460 data = 0;
1461 }
1462 throw;
1463 }
1464
1465 // free block
1466 if(data != 0)
1467 {
1468 ::free(data);
1469 data = 0;
1470 }
1471
1472 // Check for data left over if it's not a symlink
1473 if(!sourceIsSymlink)
1474 {
1475 // Anything left to read in the file?
1476 if(in->BytesLeftToRead() != 0)
1477 {
1478 // File has extra data at the end
1479 matches = false;
1480 }
1481 }
1482
1483 // Symlinks must have zero size on server
1484 if(sourceIsSymlink)
1485 {
1486 matches = (totalSizeInBlockIndex == 0);
1487 }
1488
1489 return matches;
1490 }
1491
1492
1493 // --------------------------------------------------------------------------
1494 //
1495 // Function
1496 // Name: BackupStoreFile::EncodingBuffer::EncodingBuffer()
1497 // Purpose: Constructor
1498 // Created: 25/11/04
1499 //
1500 // --------------------------------------------------------------------------
EncodingBuffer()1501 BackupStoreFile::EncodingBuffer::EncodingBuffer()
1502 : mpBuffer(0),
1503 mBufferSize(0)
1504 {
1505 }
1506
1507
1508 // --------------------------------------------------------------------------
1509 //
1510 // Function
1511 // Name: BackupStoreFile::EncodingBuffer::~EncodingBuffer()
1512 // Purpose: Destructor
1513 // Created: 25/11/04
1514 //
1515 // --------------------------------------------------------------------------
~EncodingBuffer()1516 BackupStoreFile::EncodingBuffer::~EncodingBuffer()
1517 {
1518 if(mpBuffer != 0)
1519 {
1520 BackupStoreFile::CodingChunkFree(mpBuffer);
1521 mpBuffer = 0;
1522 }
1523 }
1524
1525
1526 // --------------------------------------------------------------------------
1527 //
1528 // Function
1529 // Name: BackupStoreFile::EncodingBuffer::Allocate(int)
1530 // Purpose: Do initial allocation of block
1531 // Created: 25/11/04
1532 //
1533 // --------------------------------------------------------------------------
Allocate(int Size)1534 void BackupStoreFile::EncodingBuffer::Allocate(int Size)
1535 {
1536 ASSERT(mpBuffer == 0);
1537 uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(Size);
1538 if(buffer == 0)
1539 {
1540 throw std::bad_alloc();
1541 }
1542 mpBuffer = buffer;
1543 mBufferSize = Size;
1544 }
1545
1546
1547 // --------------------------------------------------------------------------
1548 //
1549 // Function
1550 // Name: BackupStoreFile::EncodingBuffer::Reallocate(int)
1551 // Purpose: Reallocate the block. Try not to call this, it has to copy
1552 // the entire contents as the block can't be reallocated straight.
1553 // Created: 25/11/04
1554 //
1555 // --------------------------------------------------------------------------
Reallocate(int NewSize)1556 void BackupStoreFile::EncodingBuffer::Reallocate(int NewSize)
1557 {
1558 BOX_TRACE("Reallocating EncodingBuffer from " << mBufferSize <<
1559 " to " << NewSize);
1560 ASSERT(mpBuffer != 0);
1561 uint8_t *buffer = (uint8_t*)BackupStoreFile::CodingChunkAlloc(NewSize);
1562 if(buffer == 0)
1563 {
1564 throw std::bad_alloc();
1565 }
1566 // Copy data
1567 ::memcpy(buffer, mpBuffer, (NewSize > mBufferSize)?mBufferSize:NewSize);
1568
1569 // Free old
1570 BackupStoreFile::CodingChunkFree(mpBuffer);
1571
1572 // Store new buffer
1573 mpBuffer = buffer;
1574 mBufferSize = NewSize;
1575 }
1576
1577
1578 // --------------------------------------------------------------------------
1579 //
1580 // Function
1581 // Name: DiffTimer::DiffTimer();
1582 // Purpose: Constructor
1583 // Created: 2005/02/01
1584 //
1585 // --------------------------------------------------------------------------
DiffTimer()1586 DiffTimer::DiffTimer()
1587 {
1588 }
1589
1590
1591 // --------------------------------------------------------------------------
1592 //
1593 // Function
1594 // Name: DiffTimer::DiffTimer();
1595 // Purpose: Destructor
1596 // Created: 2005/02/01
1597 //
1598 // --------------------------------------------------------------------------
~DiffTimer()1599 DiffTimer::~DiffTimer()
1600 {
1601 }
1602