1 // This file is part of par2cmdline (a PAR 2.0 compatible file verification and
2 // repair tool). See http://parchive.sourceforge.net for details of PAR 2.0.
3 //
4 // Copyright (c) 2003 Peter Brian Clements
5 //
6 // par2cmdline is free software; you can redistribute it and/or modify
7 // it under the terms of the GNU General Public License as published by
8 // the Free Software Foundation; either version 2 of the License, or
9 // (at your option) any later version.
10 //
11 // par2cmdline is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
15 //
16 // You should have received a copy of the GNU General Public License
17 // along with this program; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
19
20 #include "par2cmdline.h"
21
22 #ifdef _MSC_VER
23 #ifdef _DEBUG
24 #undef THIS_FILE
25 static char THIS_FILE[]=__FILE__;
26 #define new DEBUG_NEW
27 #endif
28 #endif
29
Par2Creator(void)30 Par2Creator::Par2Creator(void)
31 : noiselevel(CommandLine::nlUnknown)
32 , blocksize(0)
33 , chunksize(0)
34 , inputbuffer(0)
35 , outputbuffer(0)
36
37 , sourcefilecount(0)
38 , sourceblockcount(0)
39
40 , largestfilesize(0)
41 , recoveryfilescheme(CommandLine::scUnknown)
42 , recoveryfilecount(0)
43 , recoveryblockcount(0)
44 , firstrecoveryblock(0)
45
46 , mainpacket(0)
47 , creatorpacket(0)
48
49 , deferhashcomputation(false)
50 {
51 }
52
~Par2Creator(void)53 Par2Creator::~Par2Creator(void)
54 {
55 delete mainpacket;
56 delete creatorpacket;
57
58 delete [] (u8*)inputbuffer;
59 delete [] (u8*)outputbuffer;
60
61 vector<Par2CreatorSourceFile*>::iterator sourcefile = sourcefiles.begin();
62 while (sourcefile != sourcefiles.end())
63 {
64 delete *sourcefile;
65 ++sourcefile;
66 }
67 }
68
Process(const CommandLine & commandline)69 Result Par2Creator::Process(const CommandLine &commandline)
70 {
71 // Get information from commandline
72 noiselevel = commandline.GetNoiseLevel();
73 blocksize = commandline.GetBlockSize();
74 sourceblockcount = commandline.GetBlockCount();
75 const list<CommandLine::ExtraFile> extrafiles = commandline.GetExtraFiles();
76 sourcefilecount = (u32)extrafiles.size();
77 u32 redundancy = commandline.GetRedundancy();
78 recoveryblockcount = commandline.GetRecoveryBlockCount();
79 recoveryfilecount = commandline.GetRecoveryFileCount();
80 firstrecoveryblock = commandline.GetFirstRecoveryBlock();
81 recoveryfilescheme = commandline.GetRecoveryFileScheme();
82 string par2filename = commandline.GetParFilename();
83 size_t memorylimit = commandline.GetMemoryLimit();
84 largestfilesize = commandline.GetLargestSourceSize();
85
86 // Compute block size from block count or vice versa depending on which was
87 // specified on the command line
88 if (!ComputeBlockSizeAndBlockCount(extrafiles))
89 return eInvalidCommandLineArguments;
90
91 // Determine how many recovery blocks to create based on the source block
92 // count and the requested level of redundancy.
93 if (redundancy > 0 && !ComputeRecoveryBlockCount(redundancy))
94 return eInvalidCommandLineArguments;
95
96 // Determine how much recovery data can be computed on one pass
97 if (!CalculateProcessBlockSize(memorylimit))
98 return eLogicError;
99
100 // Determine how many recovery files to create.
101 if (!ComputeRecoveryFileCount())
102 return eInvalidCommandLineArguments;
103
104 if (noiselevel > CommandLine::nlQuiet)
105 {
106 // Display information.
107 cout << "Block size: " << blocksize << endl;
108 cout << "Source file count: " << sourcefilecount << endl;
109 cout << "Source block count: " << sourceblockcount << endl;
110 if (redundancy>0 || recoveryblockcount==0)
111 cout << "Redundancy: " << redundancy << '%' << endl;
112 cout << "Recovery block count: " << recoveryblockcount << endl;
113 cout << "Recovery file count: " << recoveryfilecount << endl;
114 cout << endl;
115 }
116
117 // Open all of the source files, compute the Hashes and CRC values, and store
118 // the results in the file verification and file description packets.
119 if (!OpenSourceFiles(extrafiles))
120 return eFileIOError;
121
122 // Create the main packet and determine the setid to use with all packets
123 if (!CreateMainPacket())
124 return eLogicError;
125
126 // Create the creator packet.
127 if (!CreateCreatorPacket())
128 return eLogicError;
129
130 // Initialise all of the source blocks ready to start reading data from the source files.
131 if (!CreateSourceBlocks())
132 return eLogicError;
133
134 // Create all of the output files and allocate all packets to appropriate file offets.
135 if (!InitialiseOutputFiles(par2filename))
136 return eFileIOError;
137
138 if (recoveryblockcount > 0)
139 {
140 // Allocate memory buffers for reading and writing data to disk.
141 if (!AllocateBuffers())
142 return eMemoryError;
143
144 // Compute the Reed Solomon matrix
145 if (!ComputeRSMatrix())
146 return eLogicError;
147
148 // Set the total amount of data to be processed.
149 progress = 0;
150 totaldata = blocksize * sourceblockcount * recoveryblockcount;
151
152 // Start at an offset of 0 within a block.
153 u64 blockoffset = 0;
154 while (blockoffset < blocksize) // Continue until the end of the block.
155 {
156 // Work out how much data to process this time.
157 size_t blocklength = (size_t)min((u64)chunksize, blocksize-blockoffset);
158
159 // Read source data, process it through the RS matrix and write it to disk.
160 if (!ProcessData(blockoffset, blocklength))
161 return eFileIOError;
162
163 blockoffset += blocklength;
164 }
165
166 if (noiselevel > CommandLine::nlQuiet)
167 cout << "Writing recovery packets" << endl;
168
169 // Finish computation of the recovery packets and write the headers to disk.
170 if (!WriteRecoveryPacketHeaders())
171 return eFileIOError;
172
173 // Finish computing the full file hash values of the source files
174 if (!FinishFileHashComputation())
175 return eLogicError;
176 }
177
178 // Fill in all remaining details in the critical packets.
179 if (!FinishCriticalPackets())
180 return eLogicError;
181
182 if (noiselevel > CommandLine::nlQuiet)
183 cout << "Writing verification packets" << endl;
184
185 // Write all other critical packets to disk.
186 if (!WriteCriticalPackets())
187 return eFileIOError;
188
189 // Close all files.
190 if (!CloseFiles())
191 return eFileIOError;
192
193 if (noiselevel > CommandLine::nlSilent)
194 cout << "Done" << endl;
195
196 return eSuccess;
197 }
198
199 // Compute block size from block count or vice versa depending on which was
200 // specified on the command line
ComputeBlockSizeAndBlockCount(const list<CommandLine::ExtraFile> & extrafiles)201 bool Par2Creator::ComputeBlockSizeAndBlockCount(const list<CommandLine::ExtraFile> &extrafiles)
202 {
203 // Determine blocksize from sourceblockcount or vice-versa
204 if (blocksize > 0)
205 {
206 u64 count = 0;
207
208 for (ExtraFileIterator i=extrafiles.begin(); i!=extrafiles.end(); i++)
209 {
210 count += (i->FileSize() + blocksize-1) / blocksize;
211 }
212
213 if (count > 32768)
214 {
215 cerr << "Block size is too small. It would require " << count << "blocks." << endl;
216 return false;
217 }
218
219 sourceblockcount = (u32)count;
220 }
221 else if (sourceblockcount > 0)
222 {
223 if (sourceblockcount < extrafiles.size())
224 {
225 // The block count cannot be less that the number of files.
226
227 cerr << "Block count is too small." << endl;
228 return false;
229 }
230 else if (sourceblockcount == extrafiles.size())
231 {
232 // If the block count is the same as the number of files, then the block
233 // size is the size of the largest file (rounded up to a multiple of 4).
234
235 u64 largestsourcesize = 0;
236 for (ExtraFileIterator i=extrafiles.begin(); i!=extrafiles.end(); i++)
237 {
238 if (largestsourcesize < i->FileSize())
239 {
240 largestsourcesize = i->FileSize();
241 }
242 }
243
244 blocksize = (largestsourcesize + 3) & ~3;
245 }
246 else
247 {
248 u64 totalsize = 0;
249 for (ExtraFileIterator i=extrafiles.begin(); i!=extrafiles.end(); i++)
250 {
251 totalsize += (i->FileSize() + 3) / 4;
252 }
253
254 if (sourceblockcount > totalsize)
255 {
256 sourceblockcount = (u32)totalsize;
257 blocksize = 4;
258 }
259 else
260 {
261 // Absolute lower bound and upper bound on the source block size that will
262 // result in the requested source block count.
263 u64 lowerBound = totalsize / sourceblockcount;
264 u64 upperBound = (totalsize + sourceblockcount - extrafiles.size() - 1) / (sourceblockcount - extrafiles.size());
265
266 u64 bestsize = lowerBound;
267 u64 bestdistance = 1000000;
268 u64 bestcount = 0;
269
270 u64 count;
271 u64 size;
272
273 // Work out how many blocks you get for the lower bound block size
274 {
275 size = lowerBound;
276
277 count = 0;
278 for (ExtraFileIterator i=extrafiles.begin(); i!=extrafiles.end(); i++)
279 {
280 count += ((i->FileSize()+3)/4 + size-1) / size;
281 }
282
283 if (bestdistance > (count>sourceblockcount ? count-sourceblockcount : sourceblockcount-count))
284 {
285 bestdistance = (count>sourceblockcount ? count-sourceblockcount : sourceblockcount-count);
286 bestcount = count;
287 bestsize = size;
288 }
289 }
290
291 // Work out how many blocks you get for the upper bound block size
292 {
293 size = upperBound;
294
295 count = 0;
296 for (ExtraFileIterator i=extrafiles.begin(); i!=extrafiles.end(); i++)
297 {
298 count += ((i->FileSize()+3)/4 + size-1) / size;
299 }
300
301 if (bestdistance > (count>sourceblockcount ? count-sourceblockcount : sourceblockcount-count))
302 {
303 bestdistance = (count>sourceblockcount ? count-sourceblockcount : sourceblockcount-count);
304 bestcount = count;
305 bestsize = size;
306 }
307 }
308
309 // Use binary search to find best block size
310 while (lowerBound+1 < upperBound)
311 {
312 size = (lowerBound + upperBound)/2;
313
314 count = 0;
315 for (ExtraFileIterator i=extrafiles.begin(); i!=extrafiles.end(); i++)
316 {
317 count += ((i->FileSize()+3)/4 + size-1) / size;
318 }
319
320 if (bestdistance > (count>sourceblockcount ? count-sourceblockcount : sourceblockcount-count))
321 {
322 bestdistance = (count>sourceblockcount ? count-sourceblockcount : sourceblockcount-count);
323 bestcount = count;
324 bestsize = size;
325 }
326
327 if (count < sourceblockcount)
328 {
329 upperBound = size;
330 }
331 else if (count > sourceblockcount)
332 {
333 lowerBound = size;
334 }
335 else
336 {
337 upperBound = size;
338 }
339 }
340
341 size = bestsize;
342 count = bestcount;
343
344 if (count > 32768)
345 {
346 cerr << "Error calculating block size." << endl;
347 return false;
348 }
349
350 sourceblockcount = (u32)count;
351 blocksize = size*4;
352 }
353 }
354 }
355
356 return true;
357 }
358
359
360 // Determine how many recovery blocks to create based on the source block
361 // count and the requested level of redundancy.
ComputeRecoveryBlockCount(u32 redundancy)362 bool Par2Creator::ComputeRecoveryBlockCount(u32 redundancy)
363 {
364 // Determine recoveryblockcount
365 recoveryblockcount = (sourceblockcount * redundancy + 50) / 100;
366
367 // Force valid values if necessary
368 if (recoveryblockcount == 0 && redundancy > 0)
369 recoveryblockcount = 1;
370
371 if (recoveryblockcount > 65536)
372 {
373 cerr << "Too many recovery blocks requested." << endl;
374 return false;
375 }
376
377 // Check that the last recovery block number would not be too large
378 if (firstrecoveryblock + recoveryblockcount >= 65536)
379 {
380 cerr << "First recovery block number is too high." << endl;
381 return false;
382 }
383
384 return true;
385 }
386
387 // Determine how much recovery data can be computed on one pass
CalculateProcessBlockSize(size_t memorylimit)388 bool Par2Creator::CalculateProcessBlockSize(size_t memorylimit)
389 {
390 // Are we computing any recovery blocks
391 if (recoveryblockcount == 0)
392 {
393 deferhashcomputation = false;
394 }
395 else
396 {
397 // Would single pass processing use too much memory
398 if (blocksize * recoveryblockcount > memorylimit)
399 {
400 // Pick a size that is small enough
401 chunksize = ~3 & (memorylimit / recoveryblockcount);
402
403 deferhashcomputation = false;
404 }
405 else
406 {
407 chunksize = (size_t)blocksize;
408
409 deferhashcomputation = true;
410 }
411 }
412
413 return true;
414 }
415
416 // Determine how many recovery files to create.
ComputeRecoveryFileCount(void)417 bool Par2Creator::ComputeRecoveryFileCount(void)
418 {
419 // Are we computing any recovery blocks
420 if (recoveryblockcount == 0)
421 {
422 recoveryfilecount = 0;
423 return true;
424 }
425
426 switch (recoveryfilescheme)
427 {
428 case CommandLine::scUnknown:
429 {
430 assert(false);
431 return false;
432 }
433 break;
434 case CommandLine::scVariable:
435 case CommandLine::scUniform:
436 {
437 if (recoveryfilecount == 0)
438 {
439 // If none specified then then filecount is roughly log2(blockcount)
440 // This prevents you getting excessively large numbers of files
441 // when the block count is high and also allows the files to have
442 // sizes which vary exponentially.
443
444 for (u32 blocks=recoveryblockcount; blocks>0; blocks>>=1)
445 {
446 recoveryfilecount++;
447 }
448 }
449
450 if (recoveryfilecount > recoveryblockcount)
451 {
452 // You cannot have move recovery files that there are recovery blocks
453 // to put in them.
454 cerr << "Too many recovery files specified." << endl;
455 return false;
456 }
457 }
458 break;
459
460 case CommandLine::scLimited:
461 {
462 // No recovery file will contain more recovery blocks than would
463 // be required to reconstruct the largest source file if it
464 // were missing. Other recovery files will have recovery blocks
465 // distributed in an exponential scheme.
466
467 u32 largest = (u32)((largestfilesize + blocksize-1) / blocksize);
468 u32 whole = recoveryblockcount / largest;
469 whole = (whole >= 1) ? whole-1 : 0;
470
471 u32 extra = recoveryblockcount - whole * largest;
472 recoveryfilecount = whole;
473 for (u32 blocks=extra; blocks>0; blocks>>=1)
474 {
475 recoveryfilecount++;
476 }
477 }
478 break;
479 }
480
481 return true;
482 }
483
484 // Open all of the source files, compute the Hashes and CRC values, and store
485 // the results in the file verification and file description packets.
OpenSourceFiles(const list<CommandLine::ExtraFile> & extrafiles)486 bool Par2Creator::OpenSourceFiles(const list<CommandLine::ExtraFile> &extrafiles)
487 {
488 ExtraFileIterator extrafile = extrafiles.begin();
489 while (extrafile != extrafiles.end())
490 {
491 Par2CreatorSourceFile *sourcefile = new Par2CreatorSourceFile;
492
493 string path;
494 string name;
495 DiskFile::SplitFilename(extrafile->FileName(), path, name);
496
497 if (noiselevel > CommandLine::nlSilent)
498 cout << "Opening: " << name << endl;
499
500 // Open the source file and compute its Hashes and CRCs.
501 if (!sourcefile->Open(noiselevel, *extrafile, blocksize, deferhashcomputation))
502 {
503 delete sourcefile;
504 return false;
505 }
506
507 // Record the file verification and file description packets
508 // in the critical packet list.
509 sourcefile->RecordCriticalPackets(criticalpackets);
510
511 // Add the source file to the sourcefiles array.
512 sourcefiles.push_back(sourcefile);
513
514 // Close the source file until its needed
515 sourcefile->Close();
516
517 ++extrafile;
518 }
519
520 return true;
521 }
522
523 // Create the main packet and determine the setid to use with all packets
CreateMainPacket(void)524 bool Par2Creator::CreateMainPacket(void)
525 {
526 // Construct the main packet from the list of source files and the block size.
527 mainpacket = new MainPacket;
528
529 // Add the main packet to the list of critical packets.
530 criticalpackets.push_back(mainpacket);
531
532 // Create the packet (sourcefiles will get sorted into FileId order).
533 return mainpacket->Create(sourcefiles, blocksize);
534 }
535
536 // Create the creator packet.
CreateCreatorPacket(void)537 bool Par2Creator::CreateCreatorPacket(void)
538 {
539 // Construct the creator packet
540 creatorpacket = new CreatorPacket;
541
542 // Create the packet
543 return creatorpacket->Create(mainpacket->SetId());
544 }
545
546 // Initialise all of the source blocks ready to start reading data from the source files.
CreateSourceBlocks(void)547 bool Par2Creator::CreateSourceBlocks(void)
548 {
549 // Allocate the array of source blocks
550 sourceblocks.resize(sourceblockcount);
551
552 vector<DataBlock>::iterator sourceblock = sourceblocks.begin();
553
554 for (vector<Par2CreatorSourceFile*>::iterator sourcefile = sourcefiles.begin();
555 sourcefile!= sourcefiles.end();
556 sourcefile++)
557 {
558 // Allocate the appopriate number of source blocks to each source file.
559 // sourceblock will be advanced.
560
561 (*sourcefile)->InitialiseSourceBlocks(sourceblock, blocksize);
562 }
563
564 return true;
565 }
566
567 class FileAllocation
568 {
569 public:
FileAllocation(void)570 FileAllocation(void)
571 {
572 filename = "";
573 exponent = 0;
574 count = 0;
575 }
576
577 string filename;
578 u32 exponent;
579 u32 count;
580 };
581
582 // Create all of the output files and allocate all packets to appropriate file offets.
InitialiseOutputFiles(string par2filename)583 bool Par2Creator::InitialiseOutputFiles(string par2filename)
584 {
585 // Allocate the recovery packets
586 recoverypackets.resize(recoveryblockcount);
587
588 // Choose filenames and decide which recovery blocks to place in each file
589 vector<FileAllocation> fileallocations;
590 fileallocations.resize(recoveryfilecount+1); // One extra file with no recovery blocks
591 {
592 // Decide how many recovery blocks to place in each file
593 u32 exponent = firstrecoveryblock;
594 if (recoveryfilecount > 0)
595 {
596 switch (recoveryfilescheme)
597 {
598 case CommandLine::scUnknown:
599 {
600 assert(false);
601 return false;
602 }
603 break;
604 case CommandLine::scUniform:
605 {
606 // Files will have roughly the same number of recovery blocks each.
607
608 u32 base = recoveryblockcount / recoveryfilecount;
609 u32 remainder = recoveryblockcount % recoveryfilecount;
610
611 for (u32 filenumber=0; filenumber<recoveryfilecount; filenumber++)
612 {
613 fileallocations[filenumber].exponent = exponent;
614 fileallocations[filenumber].count = (filenumber<remainder) ? base+1 : base;
615 exponent += fileallocations[filenumber].count;
616 }
617 }
618 break;
619
620 case CommandLine::scVariable:
621 {
622 // Files will have recovery blocks allocated in an exponential fashion.
623
624 // Work out how many blocks to place in the smallest file
625 u32 lowblockcount = 1;
626 u32 maxrecoveryblocks = (1 << recoveryfilecount) - 1;
627 while (maxrecoveryblocks < recoveryblockcount)
628 {
629 lowblockcount <<= 1;
630 maxrecoveryblocks <<= 1;
631 }
632
633 // Allocate the blocks.
634 u32 blocks = recoveryblockcount;
635 for (u32 filenumber=0; filenumber<recoveryfilecount; filenumber++)
636 {
637 u32 number = min(lowblockcount, blocks);
638 fileallocations[filenumber].exponent = exponent;
639 fileallocations[filenumber].count = number;
640 exponent += number;
641 blocks -= number;
642 lowblockcount <<= 1;
643 }
644 }
645 break;
646
647 case CommandLine::scLimited:
648 {
649 // Files will be allocated in an exponential fashion but the
650 // Maximum file size will be limited.
651
652 u32 largest = (u32)((largestfilesize + blocksize-1) / blocksize);
653 u32 filenumber = recoveryfilecount;
654 u32 blocks = recoveryblockcount;
655
656 exponent = firstrecoveryblock + recoveryblockcount;
657
658 // Allocate uniformly at the top
659 while (blocks >= 2*largest && filenumber > 0)
660 {
661 filenumber--;
662 exponent -= largest;
663 blocks -= largest;
664
665 fileallocations[filenumber].exponent = exponent;
666 fileallocations[filenumber].count = largest;
667 }
668 assert(blocks > 0 && filenumber > 0);
669
670 exponent = firstrecoveryblock;
671 u32 count = 1;
672 u32 files = filenumber;
673
674 // Allocate exponentially at the bottom
675 for (filenumber=0; filenumber<files; filenumber++)
676 {
677 u32 number = min(count, blocks);
678 fileallocations[filenumber].exponent = exponent;
679 fileallocations[filenumber].count = number;
680
681 exponent += number;
682 blocks -= number;
683 count <<= 1;
684 }
685 }
686 break;
687 }
688 }
689
690 // There will be an extra file with no recovery blocks.
691 fileallocations[recoveryfilecount].exponent = exponent;
692 fileallocations[recoveryfilecount].count = 0;
693
694 // Determine the format to use for filenames of recovery files
695 char filenameformat[300];
696 {
697 u32 limitLow = 0;
698 u32 limitCount = 0;
699 for (u32 filenumber=0; filenumber<=recoveryfilecount; filenumber++)
700 {
701 if (limitLow < fileallocations[filenumber].exponent)
702 {
703 limitLow = fileallocations[filenumber].exponent;
704 }
705 if (limitCount < fileallocations[filenumber].count)
706 {
707 limitCount = fileallocations[filenumber].count;
708 }
709 }
710
711 u32 digitsLow = 1;
712 for (u32 t=limitLow; t>=10; t/=10)
713 {
714 digitsLow++;
715 }
716
717 u32 digitsCount = 1;
718 for (u32 t=limitCount; t>=10; t/=10)
719 {
720 digitsCount++;
721 }
722
723 sprintf(filenameformat, "%%s.vol%%0%dd+%%0%dd.par2", digitsLow, digitsCount);
724 }
725
726 // Set the filenames
727 for (u32 filenumber=0; filenumber<recoveryfilecount; filenumber++)
728 {
729 char filename[300];
730 snprintf(filename, sizeof(filename), filenameformat, par2filename.c_str(), fileallocations[filenumber].exponent, fileallocations[filenumber].count);
731 fileallocations[filenumber].filename = filename;
732 }
733 fileallocations[recoveryfilecount].filename = par2filename + ".par2";
734 }
735
736 // Allocate the recovery files
737 {
738 recoveryfiles.resize(recoveryfilecount+1);
739
740 // Allocate packets to the output files
741 {
742 const MD5Hash &setid = mainpacket->SetId();
743 vector<RecoveryPacket>::iterator recoverypacket = recoverypackets.begin();
744
745 vector<DiskFile>::iterator recoveryfile = recoveryfiles.begin();
746 vector<FileAllocation>::iterator fileallocation = fileallocations.begin();
747
748 // For each recovery file:
749 while (recoveryfile != recoveryfiles.end())
750 {
751 // How many recovery blocks in this file
752 u32 count = fileallocation->count;
753
754 // start at the beginning of the recovery file
755 u64 offset = 0;
756
757 if (count == 0)
758 {
759 // Write one set of critical packets
760 list<CriticalPacket*>::const_iterator nextCriticalPacket = criticalpackets.begin();
761
762 while (nextCriticalPacket != criticalpackets.end())
763 {
764 criticalpacketentries.push_back(CriticalPacketEntry(&*recoveryfile,
765 offset,
766 *nextCriticalPacket));
767 offset += (*nextCriticalPacket)->PacketLength();
768
769 ++nextCriticalPacket;
770 }
771 }
772 else
773 {
774 // How many copies of each critical packet
775 u32 copies = 0;
776 for (u32 t=count; t>0; t>>=1)
777 {
778 copies++;
779 }
780
781 // Get ready to iterate through the critical packets
782 u32 packetCount = 0;
783 list<CriticalPacket*>::const_iterator nextCriticalPacket = criticalpackets.end();
784
785 // What is the first exponent
786 u32 exponent = fileallocation->exponent;
787
788 // Start allocating the recovery packets
789 u32 limit = exponent + count;
790 while (exponent < limit)
791 {
792 // Add the next recovery packet
793 recoverypacket->Create(&*recoveryfile, offset, blocksize, exponent, setid);
794
795 offset += recoverypacket->PacketLength();
796 ++recoverypacket;
797 ++exponent;
798
799 // Add some critical packets
800 packetCount += copies * criticalpackets.size();
801 while (packetCount >= count)
802 {
803 if (nextCriticalPacket == criticalpackets.end()) nextCriticalPacket = criticalpackets.begin();
804 criticalpacketentries.push_back(CriticalPacketEntry(&*recoveryfile,
805 offset,
806 *nextCriticalPacket));
807 offset += (*nextCriticalPacket)->PacketLength();
808 ++nextCriticalPacket;
809
810 packetCount -= count;
811 }
812 }
813 }
814
815 // Add one copy of the creator packet
816 criticalpacketentries.push_back(CriticalPacketEntry(&*recoveryfile,
817 offset,
818 creatorpacket));
819 offset += creatorpacket->PacketLength();
820
821 // Create the file on disk and make it the required size
822 if (!recoveryfile->Create(fileallocation->filename, offset))
823 return false;
824
825 ++recoveryfile;
826 ++fileallocation;
827 }
828 }
829 }
830
831 return true;
832 }
833
834 // Allocate memory buffers for reading and writing data to disk.
AllocateBuffers(void)835 bool Par2Creator::AllocateBuffers(void)
836 {
837 inputbuffer = new u8[chunksize];
838 outputbuffer = new u8[chunksize * recoveryblockcount];
839
840 if (inputbuffer == NULL || outputbuffer == NULL)
841 {
842 cerr << "Could not allocate buffer memory." << endl;
843 return false;
844 }
845
846 return true;
847 }
848
849 // Compute the Reed Solomon matrix
ComputeRSMatrix(void)850 bool Par2Creator::ComputeRSMatrix(void)
851 {
852 // Set the number of input blocks
853 if (!rs.SetInput(sourceblockcount))
854 return false;
855
856 // Set the number of output blocks to be created
857 if (!rs.SetOutput(false,
858 (u16)firstrecoveryblock,
859 (u16)firstrecoveryblock + (u16)(recoveryblockcount-1)))
860 return false;
861
862 // Compute the RS matrix
863 if (!rs.Compute(noiselevel))
864 return false;
865
866 return true;
867 }
868
869 // Read source data, process it through the RS matrix and write it to disk.
ProcessData(u64 blockoffset,size_t blocklength)870 bool Par2Creator::ProcessData(u64 blockoffset, size_t blocklength)
871 {
872 // Clear the output buffer
873 memset(outputbuffer, 0, chunksize * recoveryblockcount);
874
875 // If we have defered computation of the file hash and block crc and hashes
876 // sourcefile and sourceindex will be used to update them during
877 // the main recovery block computation
878 vector<Par2CreatorSourceFile*>::iterator sourcefile = sourcefiles.begin();
879 u32 sourceindex = 0;
880
881 vector<DataBlock>::iterator sourceblock;
882 u32 inputblock;
883
884 DiskFile *lastopenfile = NULL;
885
886 // For each input block
887 for ((sourceblock=sourceblocks.begin()),(inputblock=0);
888 sourceblock != sourceblocks.end();
889 ++sourceblock, ++inputblock)
890 {
891 // Are we reading from a new file?
892 if (lastopenfile != (*sourceblock).GetDiskFile())
893 {
894 // Close the last file
895 if (lastopenfile != NULL)
896 {
897 lastopenfile->Close();
898 }
899
900 // Open the new file
901 lastopenfile = (*sourceblock).GetDiskFile();
902 if (!lastopenfile->Open())
903 {
904 return false;
905 }
906 }
907
908 // Read data from the current input block
909 if (!sourceblock->ReadData(blockoffset, blocklength, inputbuffer))
910 return false;
911
912 if (deferhashcomputation)
913 {
914 assert(blockoffset == 0 && blocklength == blocksize);
915 assert(sourcefile != sourcefiles.end());
916
917 (*sourcefile)->UpdateHashes(sourceindex, inputbuffer, blocklength);
918 }
919
920 // For each output block
921 for (u32 outputblock=0; outputblock<recoveryblockcount; outputblock++)
922 {
923 // Select the appropriate part of the output buffer
924 void *outbuf = &((u8*)outputbuffer)[chunksize * outputblock];
925
926 // Process the data through the RS matrix
927 rs.Process(blocklength, inputblock, inputbuffer, outputblock, outbuf);
928
929 if (noiselevel > CommandLine::nlQuiet)
930 {
931 // Update a progress indicator
932 u32 oldfraction = (u32)(1000 * progress / totaldata);
933 progress += blocklength;
934 u32 newfraction = (u32)(1000 * progress / totaldata);
935
936 if (oldfraction != newfraction)
937 {
938 cout << "Processing: " << newfraction/10 << '.' << newfraction%10 << "%\r" << flush;
939 }
940 }
941 }
942
943 // Work out which source file the next block belongs to
944 if (++sourceindex >= (*sourcefile)->BlockCount())
945 {
946 sourceindex = 0;
947 ++sourcefile;
948 }
949 }
950
951 // Close the last file
952 if (lastopenfile != NULL)
953 {
954 lastopenfile->Close();
955 }
956
957 if (noiselevel > CommandLine::nlQuiet)
958 cout << "Writing recovery packets\r";
959
960 // For each output block
961 for (u32 outputblock=0; outputblock<recoveryblockcount;outputblock++)
962 {
963 // Select the appropriate part of the output buffer
964 char *outbuf = &((char*)outputbuffer)[chunksize * outputblock];
965
966 // Write the data to the recovery packet
967 if (!recoverypackets[outputblock].WriteData(blockoffset, blocklength, outbuf))
968 return false;
969 }
970
971 if (noiselevel > CommandLine::nlQuiet)
972 cout << "Wrote " << recoveryblockcount * blocklength << " bytes to disk" << endl;
973
974 return true;
975 }
976
977 // Finish computation of the recovery packets and write the headers to disk.
WriteRecoveryPacketHeaders(void)978 bool Par2Creator::WriteRecoveryPacketHeaders(void)
979 {
980 // For each recovery packet
981 for (vector<RecoveryPacket>::iterator recoverypacket = recoverypackets.begin();
982 recoverypacket != recoverypackets.end();
983 ++recoverypacket)
984 {
985 // Finish the packet header and write it to disk
986 if (!recoverypacket->WriteHeader())
987 return false;
988 }
989
990 return true;
991 }
992
FinishFileHashComputation(void)993 bool Par2Creator::FinishFileHashComputation(void)
994 {
995 // If we defered the computation of the full file hash, then we finish it now
996 if (deferhashcomputation)
997 {
998 // For each source file
999 vector<Par2CreatorSourceFile*>::iterator sourcefile = sourcefiles.begin();
1000
1001 while (sourcefile != sourcefiles.end())
1002 {
1003 (*sourcefile)->FinishHashes();
1004
1005 ++sourcefile;
1006 }
1007 }
1008
1009 return true;
1010 }
1011
1012 // Fill in all remaining details in the critical packets.
FinishCriticalPackets(void)1013 bool Par2Creator::FinishCriticalPackets(void)
1014 {
1015 // Get the setid from the main packet
1016 const MD5Hash &setid = mainpacket->SetId();
1017
1018 for (list<CriticalPacket*>::iterator criticalpacket=criticalpackets.begin();
1019 criticalpacket!=criticalpackets.end();
1020 criticalpacket++)
1021 {
1022 // Store the setid in each of the critical packets
1023 // and compute the packet_hash of each one.
1024
1025 (*criticalpacket)->FinishPacket(setid);
1026 }
1027
1028 return true;
1029 }
1030
1031 // Write all other critical packets to disk.
WriteCriticalPackets(void)1032 bool Par2Creator::WriteCriticalPackets(void)
1033 {
1034 list<CriticalPacketEntry>::const_iterator packetentry = criticalpacketentries.begin();
1035
1036 // For each critical packet
1037 while (packetentry != criticalpacketentries.end())
1038 {
1039 // Write it to disk
1040 if (!packetentry->WritePacket())
1041 return false;
1042
1043 ++packetentry;
1044 }
1045
1046 return true;
1047 }
1048
1049 // Close all files.
CloseFiles(void)1050 bool Par2Creator::CloseFiles(void)
1051 {
1052 // // Close each source file.
1053 // for (vector<Par2CreatorSourceFile*>::iterator sourcefile = sourcefiles.begin();
1054 // sourcefile != sourcefiles.end();
1055 // ++sourcefile)
1056 // {
1057 // (*sourcefile)->Close();
1058 // }
1059
1060 // Close each recovery file.
1061 for (vector<DiskFile>::iterator recoveryfile = recoveryfiles.begin();
1062 recoveryfile != recoveryfiles.end();
1063 ++recoveryfile)
1064 {
1065 recoveryfile->Close();
1066 }
1067
1068 return true;
1069 }
1070