1 // 7zUpdate.cpp
2
3 #include "StdAfx.h"
4
5 #include "../../../../C/CpuArch.h"
6
7 #include "../../../Common/Wildcard.h"
8
9 #include "../../Common/CreateCoder.h"
10 #include "../../Common/LimitedStreams.h"
11 #include "../../Common/ProgressUtils.h"
12
13 #include "../../Compress/CopyCoder.h"
14
15 #include "../Common/ItemNameUtils.h"
16
17 #include "7zDecode.h"
18 #include "7zEncode.h"
19 #include "7zFolderInStream.h"
20 #include "7zHandler.h"
21 #include "7zOut.h"
22 #include "7zUpdate.h"
23
24 namespace NArchive {
25 namespace N7z {
26
27
28 #define k_X86 k_BCJ
29
30 struct CFilterMode
31 {
32 UInt32 Id;
33 UInt32 Delta;
34
CFilterModeNArchive::N7z::CFilterMode35 CFilterMode(): Id(0), Delta(0) {}
36
SetDeltaNArchive::N7z::CFilterMode37 void SetDelta()
38 {
39 if (Id == k_IA64)
40 Delta = 16;
41 else if (Id == k_ARM || Id == k_PPC || Id == k_SPARC)
42 Delta = 4;
43 else if (Id == k_ARMT)
44 Delta = 2;
45 else
46 Delta = 0;
47 }
48 };
49
50
51 /* ---------- PE ---------- */
52
53 #define MZ_SIG 0x5A4D
54
55 #define PE_SIG 0x00004550
56 #define PE_OptHeader_Magic_32 0x10B
57 #define PE_OptHeader_Magic_64 0x20B
58 #define PE_SectHeaderSize 40
59 #define PE_SECT_EXECUTE 0x20000000
60
Parse_EXE(const Byte * buf,size_t size,CFilterMode * filterMode)61 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode)
62 {
63 if (size < 512 || GetUi16(buf) != MZ_SIG)
64 return 0;
65
66 const Byte *p;
67 UInt32 peOffset, optHeaderSize, filterId;
68
69 peOffset = GetUi32(buf + 0x3C);
70 if (peOffset >= 0x1000 || peOffset + 512 > size || (peOffset & 7) != 0)
71 return 0;
72 p = buf + peOffset;
73 if (GetUi32(p) != PE_SIG)
74 return 0;
75 p += 4;
76
77 switch (GetUi16(p))
78 {
79 case 0x014C:
80 case 0x8664: filterId = k_X86; break;
81
82 /*
83 IMAGE_FILE_MACHINE_ARM 0x01C0 // ARM LE
84 IMAGE_FILE_MACHINE_THUMB 0x01C2 // ARM Thumb / Thumb-2 LE
85 IMAGE_FILE_MACHINE_ARMNT 0x01C4 // ARM Thumb-2, LE
86 Note: We use ARM filter for 0x01C2. (WinCE 5 - 0x01C2) files mostly contain ARM code (not Thumb/Thumb-2).
87 */
88
89 case 0x01C0: // WinCE old
90 case 0x01C2: filterId = k_ARM; break; // WinCE new
91 case 0x01C4: filterId = k_ARMT; break; // WinRT
92
93 case 0x0200: filterId = k_IA64; break;
94 default: return 0;
95 }
96
97 optHeaderSize = GetUi16(p + 16);
98 if (optHeaderSize > (1 << 10))
99 return 0;
100
101 p += 20; /* headerSize */
102
103 switch (GetUi16(p))
104 {
105 case PE_OptHeader_Magic_32:
106 case PE_OptHeader_Magic_64:
107 break;
108 default:
109 return 0;
110 }
111
112 filterMode->Id = filterId;
113 return 1;
114 }
115
116
117 /* ---------- ELF ---------- */
118
119 #define ELF_SIG 0x464C457F
120
121 #define ELF_CLASS_32 1
122 #define ELF_CLASS_64 2
123
124 #define ELF_DATA_2LSB 1
125 #define ELF_DATA_2MSB 2
126
Get16(const Byte * p,Bool be)127 static UInt16 Get16(const Byte *p, Bool be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); }
Get32(const Byte * p,Bool be)128 static UInt32 Get32(const Byte *p, Bool be) { if (be) return GetBe32(p); return GetUi32(p); }
129 // static UInt64 Get64(const Byte *p, Bool be) { if (be) return GetBe64(p); return GetUi64(p); }
130
Parse_ELF(const Byte * buf,size_t size,CFilterMode * filterMode)131 static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode)
132 {
133 Bool /* is32, */ be;
134 UInt32 filterId;
135
136 if (size < 512 || buf[6] != 1) /* ver */
137 return 0;
138
139 if (GetUi32(buf) != ELF_SIG)
140 return 0;
141
142 switch (buf[4])
143 {
144 case ELF_CLASS_32: /* is32 = True; */ break;
145 case ELF_CLASS_64: /* is32 = False; */ break;
146 default: return 0;
147 }
148
149 switch (buf[5])
150 {
151 case ELF_DATA_2LSB: be = False; break;
152 case ELF_DATA_2MSB: be = True; break;
153 default: return 0;
154 }
155
156 switch (Get16(buf + 0x12, be))
157 {
158 case 3:
159 case 6:
160 case 62: filterId = k_X86; break;
161 case 2:
162 case 18:
163 case 43: filterId = k_SPARC; break;
164 case 20:
165 case 21: if (!be) return 0; filterId = k_PPC; break;
166 case 40: if ( be) return 0; filterId = k_ARM; break;
167
168 /* Some IA-64 ELF exacutable have size that is not aligned for 16 bytes.
169 So we don't use IA-64 filter for IA-64 ELF */
170 // case 50: if ( be) return 0; filterId = k_IA64; break;
171
172 default: return 0;
173 }
174
175 filterMode->Id = filterId;
176 return 1;
177 }
178
179
180
181 /* ---------- Mach-O ---------- */
182
183 #define MACH_SIG_BE_32 0xCEFAEDFE
184 #define MACH_SIG_BE_64 0xCFFAEDFE
185 #define MACH_SIG_LE_32 0xFEEDFACE
186 #define MACH_SIG_LE_64 0xFEEDFACF
187
188 #define MACH_ARCH_ABI64 (1 << 24)
189 #define MACH_MACHINE_386 7
190 #define MACH_MACHINE_ARM 12
191 #define MACH_MACHINE_SPARC 14
192 #define MACH_MACHINE_PPC 18
193 #define MACH_MACHINE_PPC64 (MACH_ARCH_ABI64 | MACH_MACHINE_PPC)
194 #define MACH_MACHINE_AMD64 (MACH_ARCH_ABI64 | MACH_MACHINE_386)
195
Parse_MACH(const Byte * buf,size_t size,CFilterMode * filterMode)196 static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode)
197 {
198 UInt32 filterId, numCommands, commandsSize;
199
200 if (size < 512)
201 return 0;
202
203 Bool /* mode64, */ be;
204 switch (GetUi32(buf))
205 {
206 case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break;
207 case MACH_SIG_BE_64: /* mode64 = True; */ be = True; break;
208 case MACH_SIG_LE_32: /* mode64 = False; */ be = False; break;
209 case MACH_SIG_LE_64: /* mode64 = True; */ be = False; break;
210 default: return 0;
211 }
212
213 switch (Get32(buf + 4, be))
214 {
215 case MACH_MACHINE_386:
216 case MACH_MACHINE_AMD64: filterId = k_X86; break;
217 case MACH_MACHINE_ARM: if ( be) return 0; filterId = k_ARM; break;
218 case MACH_MACHINE_SPARC: if (!be) return 0; filterId = k_SPARC; break;
219 case MACH_MACHINE_PPC:
220 case MACH_MACHINE_PPC64: if (!be) return 0; filterId = k_PPC; break;
221 default: return 0;
222 }
223
224 numCommands = Get32(buf + 0x10, be);
225 commandsSize = Get32(buf + 0x14, be);
226
227 if (commandsSize > (1 << 24) || numCommands > (1 << 18))
228 return 0;
229
230 filterMode->Id = filterId;
231 return 1;
232 }
233
234
235 /* ---------- WAV ---------- */
236
237 #define WAV_SUBCHUNK_fmt 0x20746D66
238 #define WAV_SUBCHUNK_data 0x61746164
239
240 #define RIFF_SIG 0x46464952
241
Parse_WAV(const Byte * buf,size_t size,CFilterMode * filterMode)242 static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode)
243 {
244 UInt32 subChunkSize, pos;
245 if (size < 0x2C)
246 return False;
247
248 if (GetUi32(buf + 0) != RIFF_SIG ||
249 GetUi32(buf + 8) != 0x45564157 || // WAVE
250 GetUi32(buf + 0xC) != WAV_SUBCHUNK_fmt)
251 return False;
252 subChunkSize = GetUi32(buf + 0x10);
253 /* [0x14 = format] = 1 (PCM) */
254 if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1)
255 return False;
256
257 unsigned numChannels = GetUi16(buf + 0x16);
258 unsigned bitsPerSample = GetUi16(buf + 0x22);
259
260 if ((bitsPerSample & 0x7) != 0 || bitsPerSample >= 256 || numChannels >= 256)
261 return False;
262
263 pos = 0x14 + subChunkSize;
264
265 const int kNumSubChunksTests = 10;
266 // Do we need to scan more than 3 sub-chunks?
267 for (int i = 0; i < kNumSubChunksTests; i++)
268 {
269 if (pos + 8 > size)
270 return False;
271 subChunkSize = GetUi32(buf + pos + 4);
272 if (GetUi32(buf + pos) == WAV_SUBCHUNK_data)
273 {
274 unsigned delta = numChannels * (bitsPerSample >> 3);
275 if (delta >= 256)
276 return False;
277 filterMode->Id = k_Delta;
278 filterMode->Delta = delta;
279 return True;
280 }
281 if (subChunkSize > (1 << 16))
282 return False;
283 pos += subChunkSize + 8;
284 }
285 return False;
286 }
287
ParseFile(const Byte * buf,size_t size,CFilterMode * filterMode)288 static Bool ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode)
289 {
290 filterMode->Id = 0;
291 filterMode->Delta = 0;
292
293 if (Parse_EXE(buf, size, filterMode)) return True;
294 if (Parse_ELF(buf, size, filterMode)) return True;
295 if (Parse_MACH(buf, size, filterMode)) return True;
296 return Parse_WAV(buf, size, filterMode);
297 }
298
299
300
301
302 struct CFilterMode2: public CFilterMode
303 {
304 bool Encrypted;
305 unsigned GroupIndex;
306
CFilterMode2NArchive::N7z::CFilterMode2307 CFilterMode2(): Encrypted(false) {}
308
CompareNArchive::N7z::CFilterMode2309 int Compare(const CFilterMode2 &m) const
310 {
311 if (!Encrypted)
312 {
313 if (m.Encrypted)
314 return -1;
315 }
316 else if (!m.Encrypted)
317 return 1;
318
319 if (Id < m.Id) return -1;
320 if (Id > m.Id) return 1;
321
322 if (Delta < m.Delta) return -1;
323 if (Delta > m.Delta) return 1;
324
325 return 0;
326 }
327
operator ==NArchive::N7z::CFilterMode2328 bool operator ==(const CFilterMode2 &m) const
329 {
330 return Id == m.Id && Delta == m.Delta && Encrypted == m.Encrypted;
331 }
332 };
333
GetGroup(CRecordVector<CFilterMode2> & filters,const CFilterMode2 & m)334 static unsigned GetGroup(CRecordVector<CFilterMode2> &filters, const CFilterMode2 &m)
335 {
336 unsigned i;
337 for (i = 0; i < filters.Size(); i++)
338 {
339 const CFilterMode2 &m2 = filters[i];
340 if (m == m2)
341 return i;
342 /*
343 if (m.Encrypted != m2.Encrypted)
344 {
345 if (!m.Encrypted)
346 break;
347 continue;
348 }
349
350 if (m.Id < m2.Id) break;
351 if (m.Id != m2.Id) continue;
352
353 if (m.Delta < m2.Delta) break;
354 if (m.Delta != m2.Delta) continue;
355 */
356 }
357 // filters.Insert(i, m);
358 // return i;
359 return filters.Add(m);
360 }
361
Is86Filter(CMethodId m)362 static inline bool Is86Filter(CMethodId m)
363 {
364 return (m == k_BCJ || m == k_BCJ2);
365 }
366
IsExeFilter(CMethodId m)367 static inline bool IsExeFilter(CMethodId m)
368 {
369 switch (m)
370 {
371 case k_BCJ:
372 case k_BCJ2:
373 case k_ARM:
374 case k_ARMT:
375 case k_PPC:
376 case k_SPARC:
377 case k_IA64:
378 return true;
379 }
380 return false;
381 }
382
Get_FilterGroup_for_Folder(CRecordVector<CFilterMode2> & filters,const CFolderEx & f,bool extractFilter)383 static unsigned Get_FilterGroup_for_Folder(
384 CRecordVector<CFilterMode2> &filters, const CFolderEx &f, bool extractFilter)
385 {
386 CFilterMode2 m;
387 m.Id = 0;
388 m.Delta = 0;
389 m.Encrypted = f.IsEncrypted();
390
391 if (extractFilter)
392 {
393 const CCoderInfo &coder = f.Coders[f.UnpackCoder];
394
395 if (coder.MethodID == k_Delta)
396 {
397 if (coder.Props.Size() == 1)
398 {
399 m.Delta = (unsigned)coder.Props[0] + 1;
400 m.Id = k_Delta;
401 }
402 }
403 else if (IsExeFilter(coder.MethodID))
404 {
405 m.Id = (UInt32)coder.MethodID;
406 if (m.Id == k_BCJ2)
407 m.Id = k_BCJ;
408 m.SetDelta();
409 }
410 }
411
412 return GetGroup(filters, m);
413 }
414
415
416
417
WriteRange(IInStream * inStream,ISequentialOutStream * outStream,UInt64 position,UInt64 size,ICompressProgressInfo * progress)418 static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
419 UInt64 position, UInt64 size, ICompressProgressInfo *progress)
420 {
421 RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0));
422 CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream;
423 CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec);
424 streamSpec->SetStream(inStream);
425 streamSpec->Init(size);
426
427 NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder;
428 CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec;
429 RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress));
430 return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL);
431 }
432
433 /*
434 unsigned CUpdateItem::GetExtensionPos() const
435 {
436 int slashPos = Name.ReverseFind_PathSepar();
437 int dotPos = Name.ReverseFind_Dot();
438 if (dotPos <= slashPos)
439 return Name.Len();
440 return dotPos + 1;
441 }
442
443 UString CUpdateItem::GetExtension() const
444 {
445 return Name.Ptr(GetExtensionPos());
446 }
447 */
448
449 #define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
450
451 #define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
452
453 /*
454 static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
455 {
456 size_t c1 = a1.GetCapacity();
457 size_t c2 = a2.GetCapacity();
458 RINOZ_COMP(c1, c2);
459 for (size_t i = 0; i < c1; i++)
460 RINOZ_COMP(a1[i], a2[i]);
461 return 0;
462 }
463
464 static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
465 {
466 RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
467 RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
468 RINOZ_COMP(c1.MethodID, c2.MethodID);
469 return CompareBuffers(c1.Props, c2.Props);
470 }
471
472 static int CompareBonds(const CBond &b1, const CBond &b2)
473 {
474 RINOZ_COMP(b1.InIndex, b2.InIndex);
475 return MyCompare(b1.OutIndex, b2.OutIndex);
476 }
477
478 static int CompareFolders(const CFolder &f1, const CFolder &f2)
479 {
480 int s1 = f1.Coders.Size();
481 int s2 = f2.Coders.Size();
482 RINOZ_COMP(s1, s2);
483 int i;
484 for (i = 0; i < s1; i++)
485 RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
486 s1 = f1.Bonds.Size();
487 s2 = f2.Bonds.Size();
488 RINOZ_COMP(s1, s2);
489 for (i = 0; i < s1; i++)
490 RINOZ(CompareBonds(f1.Bonds[i], f2.Bonds[i]));
491 return 0;
492 }
493 */
494
495 /*
496 static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
497 {
498 return CompareFileNames(f1.Name, f2.Name);
499 }
500 */
501
502 struct CFolderRepack
503 {
504 unsigned FolderIndex;
505 CNum NumCopyFiles;
506 };
507
508 /*
509 static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *)
510 {
511 int i1 = p1->FolderIndex;
512 int i2 = p2->FolderIndex;
513 // In that version we don't want to parse folders here, so we don't compare folders
514 // probably it must be improved in future
515 // const CDbEx &db = *(const CDbEx *)param;
516 // RINOZ(CompareFolders(
517 // db.Folders[i1],
518 // db.Folders[i2]));
519
520 return MyCompare(i1, i2);
521
522 // RINOZ_COMP(
523 // db.NumUnpackStreamsVector[i1],
524 // db.NumUnpackStreamsVector[i2]);
525 // if (db.NumUnpackStreamsVector[i1] == 0)
526 // return 0;
527 // return CompareFiles(
528 // db.Files[db.FolderStartFileIndex[i1]],
529 // db.Files[db.FolderStartFileIndex[i2]]);
530 }
531 */
532
533 /*
534 we sort empty files and dirs in such order:
535 - Dir.NonAnti (name sorted)
536 - File.NonAnti (name sorted)
537 - File.Anti (name sorted)
538 - Dir.Anti (reverse name sorted)
539 */
540
CompareEmptyItems(const unsigned * p1,const unsigned * p2,void * param)541 static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param)
542 {
543 const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
544 const CUpdateItem &u1 = updateItems[*p1];
545 const CUpdateItem &u2 = updateItems[*p2];
546 // NonAnti < Anti
547 if (u1.IsAnti != u2.IsAnti)
548 return (u1.IsAnti ? 1 : -1);
549 if (u1.IsDir != u2.IsDir)
550 {
551 // Dir.NonAnti < File < Dir.Anti
552 if (u1.IsDir)
553 return (u1.IsAnti ? 1 : -1);
554 return (u2.IsAnti ? -1 : 1);
555 }
556 int n = CompareFileNames(u1.Name, u2.Name);
557 return (u1.IsDir && u1.IsAnti) ? -n : n;
558 }
559
560 static const char *g_Exts =
561 " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
562 " zip jar ear war msi"
563 " 3gp avi mov mpeg mpg mpe wmv"
564 " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
565 " swf"
566 " chm hxi hxs"
567 " gif jpeg jpg jp2 png tiff bmp ico psd psp"
568 " awg ps eps cgm dxf svg vrml wmf emf ai md"
569 " cad dwg pps key sxi"
570 " max 3ds"
571 " iso bin nrg mdf img pdi tar cpio xpi"
572 " vfd vhd vud vmc vsv"
573 " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
574 " inl inc idl acf asa"
575 " h hpp hxx c cpp cxx m mm go swift"
576 " rc java cs rs pas bas vb cls ctl frm dlg def"
577 " f77 f f90 f95"
578 " asm s"
579 " sql manifest dep"
580 " mak clw csproj vcproj sln dsp dsw"
581 " class"
582 " bat cmd bash sh"
583 " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
584 " awk sed hta js json php php3 php4 php5 phptml pl pm py pyo rb tcl ts vbs"
585 " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
586 " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
587 " abw afp cwk lwp wpd wps wpt wrf wri"
588 " abf afm bdf fon mgf otf pcf pfa snf ttf"
589 " dbf mdb nsf ntf wdb db fdb gdb"
590 " exe dll ocx vbx sfx sys tlb awx com obj lib out o so"
591 " pdb pch idb ncb opt";
592
GetExtIndex(const char * ext)593 static unsigned GetExtIndex(const char *ext)
594 {
595 unsigned extIndex = 1;
596 const char *p = g_Exts;
597 for (;;)
598 {
599 char c = *p++;
600 if (c == 0)
601 return extIndex;
602 if (c == ' ')
603 continue;
604 unsigned pos = 0;
605 for (;;)
606 {
607 char c2 = ext[pos++];
608 if (c2 == 0 && (c == 0 || c == ' '))
609 return extIndex;
610 if (c != c2)
611 break;
612 c = *p++;
613 }
614 extIndex++;
615 for (;;)
616 {
617 if (c == 0)
618 return extIndex;
619 if (c == ' ')
620 break;
621 c = *p++;
622 }
623 }
624 }
625
626 struct CRefItem
627 {
628 const CUpdateItem *UpdateItem;
629 UInt32 Index;
630 unsigned ExtensionPos;
631 unsigned NamePos;
632 unsigned ExtensionIndex;
633
CRefItemNArchive::N7z::CRefItem634 CRefItem() {};
CRefItemNArchive::N7z::CRefItem635 CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
636 UpdateItem(&ui),
637 Index(index),
638 ExtensionPos(0),
639 NamePos(0),
640 ExtensionIndex(0)
641 {
642 if (sortByType)
643 {
644 int slashPos = ui.Name.ReverseFind_PathSepar();
645 NamePos = slashPos + 1;
646 int dotPos = ui.Name.ReverseFind_Dot();
647 if (dotPos <= slashPos)
648 ExtensionPos = ui.Name.Len();
649 else
650 {
651 ExtensionPos = dotPos + 1;
652 if (ExtensionPos != ui.Name.Len())
653 {
654 AString s;
655 for (unsigned pos = ExtensionPos;; pos++)
656 {
657 wchar_t c = ui.Name[pos];
658 if (c >= 0x80)
659 break;
660 if (c == 0)
661 {
662 ExtensionIndex = GetExtIndex(s);
663 break;
664 }
665 s += (char)MyCharLower_Ascii((char)c);
666 }
667 }
668 }
669 }
670 }
671 };
672
673 struct CSortParam
674 {
675 // const CObjectVector<CTreeFolder> *TreeFolders;
676 bool SortByType;
677 };
678
679 /*
680 we sort files in such order:
681 - Dir.NonAnti (name sorted)
682 - alt streams
683 - Dirs
684 - Dir.Anti (reverse name sorted)
685 */
686
687
CompareUpdateItems(const CRefItem * p1,const CRefItem * p2,void * param)688 static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
689 {
690 const CRefItem &a1 = *p1;
691 const CRefItem &a2 = *p2;
692 const CUpdateItem &u1 = *a1.UpdateItem;
693 const CUpdateItem &u2 = *a2.UpdateItem;
694
695 /*
696 if (u1.IsAltStream != u2.IsAltStream)
697 return u1.IsAltStream ? 1 : -1;
698 */
699
700 // Actually there are no dirs that time. They were stored in other steps
701 // So that code is unused?
702 if (u1.IsDir != u2.IsDir)
703 return u1.IsDir ? 1 : -1;
704 if (u1.IsDir)
705 {
706 if (u1.IsAnti != u2.IsAnti)
707 return (u1.IsAnti ? 1 : -1);
708 int n = CompareFileNames(u1.Name, u2.Name);
709 return -n;
710 }
711
712 // bool sortByType = *(bool *)param;
713 const CSortParam *sortParam = (const CSortParam *)param;
714 bool sortByType = sortParam->SortByType;
715 if (sortByType)
716 {
717 RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex);
718 RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos)));
719 RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos)));
720 if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
721 if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
722 if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime);
723 RINOZ_COMP(u1.Size, u2.Size);
724 }
725 /*
726 int par1 = a1.UpdateItem->ParentFolderIndex;
727 int par2 = a2.UpdateItem->ParentFolderIndex;
728 const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1];
729 const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2];
730
731 int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd;
732 int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd;
733 if (b1 < b2)
734 {
735 if (e1 <= b2)
736 return -1;
737 // p2 in p1
738 int par = par2;
739 for (;;)
740 {
741 const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
742 par = tf.Parent;
743 if (par == par1)
744 {
745 RINOZ(CompareFileNames(u1.Name, tf.Name));
746 break;
747 }
748 }
749 }
750 else if (b2 < b1)
751 {
752 if (e2 <= b1)
753 return 1;
754 // p1 in p2
755 int par = par1;
756 for (;;)
757 {
758 const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
759 par = tf.Parent;
760 if (par == par2)
761 {
762 RINOZ(CompareFileNames(tf.Name, u2.Name));
763 break;
764 }
765 }
766 }
767 */
768 // RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex);
769 RINOK(CompareFileNames(u1.Name, u2.Name));
770 RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient);
771 RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive);
772 return 0;
773 }
774
775 struct CSolidGroup
776 {
777 CRecordVector<UInt32> Indices;
778
779 CRecordVector<CFolderRepack> folderRefs;
780 };
781
782 static const char * const g_ExeExts[] =
783 {
784 "dll"
785 , "exe"
786 , "ocx"
787 , "sfx"
788 , "sys"
789 };
790
IsExeExt(const wchar_t * ext)791 static bool IsExeExt(const wchar_t *ext)
792 {
793 for (unsigned i = 0; i < ARRAY_SIZE(g_ExeExts); i++)
794 if (StringsAreEqualNoCase_Ascii(ext, g_ExeExts[i]))
795 return true;
796 return false;
797 }
798
799 struct CAnalysis
800 {
801 CMyComPtr<IArchiveUpdateCallbackFile> Callback;
802 CByteBuffer Buffer;
803
804 bool ParseWav;
805 bool ParseExe;
806 bool ParseAll;
807
CAnalysisNArchive::N7z::CAnalysis808 CAnalysis():
809 ParseWav(true),
810 ParseExe(false),
811 ParseAll(false)
812 {}
813
814 HRESULT GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode);
815 };
816
817 static const size_t kAnalysisBufSize = 1 << 14;
818
GetFilterGroup(UInt32 index,const CUpdateItem & ui,CFilterMode & filterMode)819 HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode)
820 {
821 filterMode.Id = 0;
822 filterMode.Delta = 0;
823
824 CFilterMode filterModeTemp = filterMode;
825
826 int slashPos = ui.Name.ReverseFind_PathSepar();
827 int dotPos = ui.Name.ReverseFind_Dot();
828
829 // if (dotPos > slashPos)
830 {
831 bool needReadFile = ParseAll;
832
833 bool probablyIsSameIsa = false;
834
835 if (!needReadFile || !Callback)
836 {
837 const wchar_t *ext;
838 if (dotPos > slashPos)
839 ext = ui.Name.Ptr(dotPos + 1);
840 else
841 ext = ui.Name.RightPtr(0);
842
843 // p7zip uses the trick to store posix attributes in high 16 bits
844 if (ui.Attrib & 0x8000)
845 {
846 unsigned st_mode = ui.Attrib >> 16;
847 // st_mode = 00111;
848 if ((st_mode & 00111) && (ui.Size >= 2048))
849 {
850 #ifndef _WIN32
851 probablyIsSameIsa = true;
852 #endif
853 needReadFile = true;
854 }
855 }
856
857 if (IsExeExt(ext))
858 {
859 needReadFile = true;
860 #ifdef _WIN32
861 probablyIsSameIsa = true;
862 needReadFile = ParseExe;
863 #endif
864 }
865 else if (StringsAreEqualNoCase_Ascii(ext, "wav"))
866 {
867 needReadFile = ParseWav;
868 }
869 /*
870 else if (!needReadFile && ParseUnixExt)
871 {
872 if (StringsAreEqualNoCase_Ascii(ext, "so")
873 || StringsAreEqualNoCase_Ascii(ext, ""))
874
875 needReadFile = true;
876 }
877 */
878 }
879
880 if (needReadFile && Callback)
881 {
882 if (Buffer.Size() != kAnalysisBufSize)
883 {
884 Buffer.Alloc(kAnalysisBufSize);
885 }
886 {
887 CMyComPtr<ISequentialInStream> stream;
888 HRESULT result = Callback->GetStream2(index, &stream, NUpdateNotifyOp::kAnalyze);
889 if (result == S_OK && stream)
890 {
891 size_t size = kAnalysisBufSize;
892 result = ReadStream(stream, Buffer, &size);
893 stream.Release();
894 // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK));
895 if (result == S_OK)
896 {
897 Bool parseRes = ParseFile(Buffer, size, &filterModeTemp);
898 if (parseRes && filterModeTemp.Delta == 0)
899 {
900 filterModeTemp.SetDelta();
901 if (filterModeTemp.Delta != 0 && filterModeTemp.Id != k_Delta)
902 {
903 if (ui.Size % filterModeTemp.Delta != 0)
904 {
905 parseRes = false;
906 }
907 }
908 }
909 if (!parseRes)
910 {
911 filterModeTemp.Id = 0;
912 filterModeTemp.Delta = 0;
913 }
914 }
915 }
916 }
917 }
918 else if ((needReadFile && !Callback) || probablyIsSameIsa)
919 {
920 #ifdef MY_CPU_X86_OR_AMD64
921 if (probablyIsSameIsa)
922 filterModeTemp.Id = k_X86;
923 #endif
924 }
925 }
926
927 filterMode = filterModeTemp;
928 return S_OK;
929 }
930
GetMethodFull(UInt64 methodID,UInt32 numStreams,CMethodFull & m)931 static inline void GetMethodFull(UInt64 methodID, UInt32 numStreams, CMethodFull &m)
932 {
933 m.Id = methodID;
934 m.NumStreams = numStreams;
935 }
936
AddBondForFilter(CCompressionMethodMode & mode)937 static HRESULT AddBondForFilter(CCompressionMethodMode &mode)
938 {
939 for (unsigned c = 1; c < mode.Methods.Size(); c++)
940 {
941 if (!mode.IsThereBond_to_Coder(c))
942 {
943 CBond2 bond;
944 bond.OutCoder = 0;
945 bond.OutStream = 0;
946 bond.InCoder = c;
947 mode.Bonds.Add(bond);
948 return S_OK;
949 }
950 }
951 return E_INVALIDARG;
952 }
953
AddFilterBond(CCompressionMethodMode & mode)954 static HRESULT AddFilterBond(CCompressionMethodMode &mode)
955 {
956 if (!mode.Bonds.IsEmpty())
957 return AddBondForFilter(mode);
958 return S_OK;
959 }
960
AddBcj2Methods(CCompressionMethodMode & mode)961 static HRESULT AddBcj2Methods(CCompressionMethodMode &mode)
962 {
963 // mode.Methods[0] must be k_BCJ2 method !
964
965 CMethodFull m;
966 GetMethodFull(k_LZMA, 1, m);
967
968 m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20);
969 m.AddProp32(NCoderPropID::kNumFastBytes, 128);
970 m.AddProp32(NCoderPropID::kNumThreads, 1);
971 m.AddProp32(NCoderPropID::kLitPosBits, 2);
972 m.AddProp32(NCoderPropID::kLitContextBits, 0);
973 // m.AddProp_Ascii(NCoderPropID::kMatchFinder, "BT2");
974
975 unsigned methodIndex = mode.Methods.Size();
976
977 if (mode.Bonds.IsEmpty())
978 {
979 for (unsigned i = 1; i + 1 < mode.Methods.Size(); i++)
980 {
981 CBond2 bond;
982 bond.OutCoder = i;
983 bond.OutStream = 0;
984 bond.InCoder = i + 1;
985 mode.Bonds.Add(bond);
986 }
987 }
988
989 mode.Methods.Add(m);
990 mode.Methods.Add(m);
991
992 RINOK(AddBondForFilter(mode));
993 CBond2 bond;
994 bond.OutCoder = 0;
995 bond.InCoder = methodIndex; bond.OutStream = 1; mode.Bonds.Add(bond);
996 bond.InCoder = methodIndex + 1; bond.OutStream = 2; mode.Bonds.Add(bond);
997 return S_OK;
998 }
999
MakeExeMethod(CCompressionMethodMode & mode,const CFilterMode & filterMode,bool bcj2Filter)1000 static HRESULT MakeExeMethod(CCompressionMethodMode &mode,
1001 const CFilterMode &filterMode, /* bool addFilter, */ bool bcj2Filter)
1002 {
1003 if (mode.Filter_was_Inserted)
1004 {
1005 const CMethodFull &m = mode.Methods[0];
1006 CMethodId id = m.Id;
1007 if (id == k_BCJ2)
1008 return AddBcj2Methods(mode);
1009 if (!m.IsSimpleCoder())
1010 return E_NOTIMPL;
1011 // if (Bonds.IsEmpty()) we can create bonds later
1012 return AddFilterBond(mode);
1013 }
1014
1015 if (filterMode.Id == 0)
1016 return S_OK;
1017
1018 CMethodFull &m = mode.Methods.InsertNew(0);
1019
1020 {
1021 FOR_VECTOR(k, mode.Bonds)
1022 {
1023 CBond2 &bond = mode.Bonds[k];
1024 bond.InCoder++;
1025 bond.OutCoder++;
1026 }
1027 }
1028
1029 HRESULT res;
1030
1031 if (bcj2Filter && Is86Filter(filterMode.Id))
1032 {
1033 GetMethodFull(k_BCJ2, 4, m);
1034 res = AddBcj2Methods(mode);
1035 }
1036 else
1037 {
1038 GetMethodFull(filterMode.Id, 1, m);
1039 if (filterMode.Id == k_Delta)
1040 m.AddProp32(NCoderPropID::kDefaultProp, filterMode.Delta);
1041 res = AddFilterBond(mode);
1042
1043 int alignBits = -1;
1044 if (filterMode.Id == k_Delta || filterMode.Delta != 0)
1045 {
1046 if (filterMode.Delta == 1) alignBits = 0;
1047 else if (filterMode.Delta == 2) alignBits = 1;
1048 else if (filterMode.Delta == 4) alignBits = 2;
1049 else if (filterMode.Delta == 8) alignBits = 3;
1050 else if (filterMode.Delta == 16) alignBits = 4;
1051 }
1052 else
1053 {
1054 // alignBits = GetAlignForFilterMethod(filterMode.Id);
1055 }
1056
1057 if (res == S_OK && alignBits >= 0)
1058 {
1059 unsigned nextCoder = 1;
1060 if (!mode.Bonds.IsEmpty())
1061 {
1062 nextCoder = mode.Bonds.Back().InCoder;
1063 }
1064 if (nextCoder < mode.Methods.Size())
1065 {
1066 CMethodFull &nextMethod = mode.Methods[nextCoder];
1067 if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2)
1068 {
1069 if (!nextMethod.Are_Lzma_Model_Props_Defined())
1070 {
1071 if (alignBits != 0)
1072 {
1073 if (alignBits > 2 || filterMode.Id == k_Delta)
1074 nextMethod.AddProp32(NCoderPropID::kPosStateBits, alignBits);
1075 unsigned lc = 0;
1076 if (alignBits < 3)
1077 lc = 3 - alignBits;
1078 nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc);
1079 nextMethod.AddProp32(NCoderPropID::kLitPosBits, alignBits);
1080 }
1081 }
1082 }
1083 }
1084 }
1085 }
1086
1087 return res;
1088 }
1089
1090
UpdateItem_To_FileItem2(const CUpdateItem & ui,CFileItem2 & file2)1091 static void UpdateItem_To_FileItem2(const CUpdateItem &ui, CFileItem2 &file2)
1092 {
1093 file2.Attrib = ui.Attrib; file2.AttribDefined = ui.AttribDefined;
1094 file2.CTime = ui.CTime; file2.CTimeDefined = ui.CTimeDefined;
1095 file2.ATime = ui.ATime; file2.ATimeDefined = ui.ATimeDefined;
1096 file2.MTime = ui.MTime; file2.MTimeDefined = ui.MTimeDefined;
1097 file2.IsAnti = ui.IsAnti;
1098 // file2.IsAux = false;
1099 file2.StartPosDefined = false;
1100 // file2.StartPos = 0;
1101 }
1102
1103
UpdateItem_To_FileItem(const CUpdateItem & ui,CFileItem & file,CFileItem2 & file2)1104 static void UpdateItem_To_FileItem(const CUpdateItem &ui,
1105 CFileItem &file, CFileItem2 &file2)
1106 {
1107 UpdateItem_To_FileItem2(ui, file2);
1108
1109 file.Size = ui.Size;
1110 file.IsDir = ui.IsDir;
1111 file.HasStream = ui.HasStream();
1112 // file.IsAltStream = ui.IsAltStream;
1113 }
1114
1115
1116
1117 class CRepackInStreamWithSizes:
1118 public ISequentialInStream,
1119 public ICompressGetSubStreamSize,
1120 public CMyUnknownImp
1121 {
1122 CMyComPtr<ISequentialInStream> _stream;
1123 // UInt64 _size;
1124 const CBoolVector *_extractStatuses;
1125 UInt32 _startIndex;
1126 public:
1127 const CDbEx *_db;
1128
Init(ISequentialInStream * stream,UInt32 startIndex,const CBoolVector * extractStatuses)1129 void Init(ISequentialInStream *stream, UInt32 startIndex, const CBoolVector *extractStatuses)
1130 {
1131 _startIndex = startIndex;
1132 _extractStatuses = extractStatuses;
1133 // _size = 0;
1134 _stream = stream;
1135 }
1136 // UInt64 GetSize() const { return _size; }
1137
1138 MY_UNKNOWN_IMP2(ISequentialInStream, ICompressGetSubStreamSize)
1139
1140 STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
1141
1142 STDMETHOD(GetSubStreamSize)(UInt64 subStream, UInt64 *value);
1143 };
1144
Read(void * data,UInt32 size,UInt32 * processedSize)1145 STDMETHODIMP CRepackInStreamWithSizes::Read(void *data, UInt32 size, UInt32 *processedSize)
1146 {
1147 return _stream->Read(data, size, processedSize);
1148 /*
1149 UInt32 realProcessedSize;
1150 HRESULT result = _stream->Read(data, size, &realProcessedSize);
1151 _size += realProcessedSize;
1152 if (processedSize)
1153 *processedSize = realProcessedSize;
1154 return result;
1155 */
1156 }
1157
GetSubStreamSize(UInt64 subStream,UInt64 * value)1158 STDMETHODIMP CRepackInStreamWithSizes::GetSubStreamSize(UInt64 subStream, UInt64 *value)
1159 {
1160 *value = 0;
1161 if (subStream >= _extractStatuses->Size())
1162 return S_FALSE; // E_FAIL;
1163 unsigned index = (unsigned)subStream;
1164 if ((*_extractStatuses)[index])
1165 {
1166 const CFileItem &fi = _db->Files[_startIndex + index];
1167 if (fi.HasStream)
1168 *value = fi.Size;
1169 }
1170 return S_OK;
1171 }
1172
1173
1174 class CRepackStreamBase
1175 {
1176 protected:
1177 bool _needWrite;
1178 bool _fileIsOpen;
1179 bool _calcCrc;
1180 UInt32 _crc;
1181 UInt64 _rem;
1182
1183 const CBoolVector *_extractStatuses;
1184 UInt32 _startIndex;
1185 unsigned _currentIndex;
1186
1187 HRESULT OpenFile();
1188 HRESULT CloseFile();
1189 HRESULT ProcessEmptyFiles();
1190
1191 public:
1192 const CDbEx *_db;
1193 CMyComPtr<IArchiveUpdateCallbackFile> _opCallback;
1194 CMyComPtr<IArchiveExtractCallbackMessage> _extractCallback;
1195
1196 HRESULT Init(UInt32 startIndex, const CBoolVector *extractStatuses);
CheckFinishedState() const1197 HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
1198 };
1199
Init(UInt32 startIndex,const CBoolVector * extractStatuses)1200 HRESULT CRepackStreamBase::Init(UInt32 startIndex, const CBoolVector *extractStatuses)
1201 {
1202 _startIndex = startIndex;
1203 _extractStatuses = extractStatuses;
1204
1205 _currentIndex = 0;
1206 _fileIsOpen = false;
1207
1208 return ProcessEmptyFiles();
1209 }
1210
OpenFile()1211 HRESULT CRepackStreamBase::OpenFile()
1212 {
1213 UInt32 arcIndex = _startIndex + _currentIndex;
1214 const CFileItem &fi = _db->Files[arcIndex];
1215
1216 _needWrite = (*_extractStatuses)[_currentIndex];
1217 if (_opCallback)
1218 {
1219 RINOK(_opCallback->ReportOperation(
1220 NEventIndexType::kInArcIndex, arcIndex,
1221 _needWrite ?
1222 NUpdateNotifyOp::kRepack :
1223 NUpdateNotifyOp::kSkip));
1224 }
1225
1226 _crc = CRC_INIT_VAL;
1227 _calcCrc = (fi.CrcDefined && !fi.IsDir);
1228
1229 _fileIsOpen = true;
1230 _rem = fi.Size;
1231 return S_OK;
1232 }
1233
1234 const HRESULT k_My_HRESULT_CRC_ERROR = 0x20000002;
1235
CloseFile()1236 HRESULT CRepackStreamBase::CloseFile()
1237 {
1238 UInt32 arcIndex = _startIndex + _currentIndex;
1239 const CFileItem &fi = _db->Files[arcIndex];
1240 _fileIsOpen = false;
1241 _currentIndex++;
1242 if (!_calcCrc || fi.Crc == CRC_GET_DIGEST(_crc))
1243 return S_OK;
1244
1245 if (_extractCallback)
1246 {
1247 RINOK(_extractCallback->ReportExtractResult(
1248 NEventIndexType::kInArcIndex, arcIndex,
1249 NExtract::NOperationResult::kCRCError));
1250 }
1251 // return S_FALSE;
1252 return k_My_HRESULT_CRC_ERROR;
1253 }
1254
ProcessEmptyFiles()1255 HRESULT CRepackStreamBase::ProcessEmptyFiles()
1256 {
1257 while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
1258 {
1259 RINOK(OpenFile());
1260 RINOK(CloseFile());
1261 }
1262 return S_OK;
1263 }
1264
1265
1266
1267 #ifndef _7ZIP_ST
1268
1269 class CFolderOutStream2:
1270 public CRepackStreamBase,
1271 public ISequentialOutStream,
1272 public CMyUnknownImp
1273 {
1274 public:
1275 CMyComPtr<ISequentialOutStream> _stream;
1276
1277 MY_UNKNOWN_IMP
1278
1279 STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
1280 };
1281
Write(const void * data,UInt32 size,UInt32 * processedSize)1282 STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize)
1283 {
1284 if (processedSize)
1285 *processedSize = 0;
1286
1287 while (size != 0)
1288 {
1289 if (_fileIsOpen)
1290 {
1291 UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1292 HRESULT result = S_OK;
1293 if (_needWrite)
1294 result = _stream->Write(data, cur, &cur);
1295 if (_calcCrc)
1296 _crc = CrcUpdate(_crc, data, cur);
1297 if (processedSize)
1298 *processedSize += cur;
1299 data = (const Byte *)data + cur;
1300 size -= cur;
1301 _rem -= cur;
1302 if (_rem == 0)
1303 {
1304 RINOK(CloseFile());
1305 RINOK(ProcessEmptyFiles());
1306 }
1307 RINOK(result);
1308 if (cur == 0)
1309 break;
1310 continue;
1311 }
1312
1313 RINOK(ProcessEmptyFiles());
1314 if (_currentIndex == _extractStatuses->Size())
1315 {
1316 // we don't support write cut here
1317 return E_FAIL;
1318 }
1319 RINOK(OpenFile());
1320 }
1321
1322 return S_OK;
1323 }
1324
1325 #endif
1326
1327
1328
1329 static const UInt32 kTempBufSize = 1 << 16;
1330
1331 class CFolderInStream2:
1332 public CRepackStreamBase,
1333 public ISequentialInStream,
1334 public CMyUnknownImp
1335 {
1336 Byte *_buf;
1337 public:
1338 CMyComPtr<ISequentialInStream> _inStream;
1339 HRESULT Result;
1340
1341 MY_UNKNOWN_IMP
1342
CFolderInStream2()1343 CFolderInStream2():
1344 Result(S_OK)
1345 {
1346 _buf = new Byte[kTempBufSize];
1347 }
1348
~CFolderInStream2()1349 ~CFolderInStream2()
1350 {
1351 delete []_buf;
1352 }
1353
Init()1354 void Init() { Result = S_OK; }
1355 STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
1356 };
1357
Read(void * data,UInt32 size,UInt32 * processedSize)1358 STDMETHODIMP CFolderInStream2::Read(void *data, UInt32 size, UInt32 *processedSize)
1359 {
1360 if (processedSize)
1361 *processedSize = 0;
1362
1363 while (size != 0)
1364 {
1365 if (_fileIsOpen)
1366 {
1367 UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1368
1369 void *buf;
1370 if (_needWrite)
1371 buf = data;
1372 else
1373 {
1374 buf = _buf;
1375 if (cur > kTempBufSize)
1376 cur = kTempBufSize;
1377 }
1378
1379 HRESULT result = _inStream->Read(buf, cur, &cur);
1380 _crc = CrcUpdate(_crc, buf, cur);
1381 _rem -= cur;
1382
1383 if (_needWrite)
1384 {
1385 data = (Byte *)data + cur;
1386 size -= cur;
1387 if (processedSize)
1388 *processedSize += cur;
1389 }
1390
1391 if (result != S_OK)
1392 Result = result;
1393
1394 if (_rem == 0)
1395 {
1396 RINOK(CloseFile());
1397 RINOK(ProcessEmptyFiles());
1398 }
1399
1400 RINOK(result);
1401
1402 if (cur == 0)
1403 return E_FAIL;
1404
1405 continue;
1406 }
1407
1408 RINOK(ProcessEmptyFiles());
1409 if (_currentIndex == _extractStatuses->Size())
1410 {
1411 return S_OK;
1412 }
1413 RINOK(OpenFile());
1414 }
1415
1416 return S_OK;
1417 }
1418
1419
1420 class CThreadDecoder
1421 #ifndef _7ZIP_ST
1422 : public CVirtThread
1423 #endif
1424 {
1425 public:
1426 CDecoder Decoder;
1427
CThreadDecoder(bool multiThreadMixer)1428 CThreadDecoder(bool multiThreadMixer):
1429 Decoder(multiThreadMixer)
1430 {
1431 #ifndef _7ZIP_ST
1432 if (multiThreadMixer)
1433 {
1434 MtMode = false;
1435 NumThreads = 1;
1436 FosSpec = new CFolderOutStream2;
1437 Fos = FosSpec;
1438 Result = E_FAIL;
1439 }
1440 #endif
1441 // UnpackSize = 0;
1442 // send_UnpackSize = false;
1443 }
1444
1445 #ifndef _7ZIP_ST
1446
1447 bool dataAfterEnd_Error;
1448 HRESULT Result;
1449 CMyComPtr<IInStream> InStream;
1450
1451 CFolderOutStream2 *FosSpec;
1452 CMyComPtr<ISequentialOutStream> Fos;
1453
1454 UInt64 StartPos;
1455 const CFolders *Folders;
1456 int FolderIndex;
1457
1458 // bool send_UnpackSize;
1459 // UInt64 UnpackSize;
1460
1461 #ifndef _NO_CRYPTO
1462 CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1463 #endif
1464
1465 DECL_EXTERNAL_CODECS_LOC_VARS2;
1466
1467 #ifndef _7ZIP_ST
1468 bool MtMode;
1469 UInt32 NumThreads;
1470 #endif
1471
1472
~CThreadDecoder()1473 ~CThreadDecoder() { CVirtThread::WaitThreadFinish(); }
1474 virtual void Execute();
1475
1476 #endif
1477 };
1478
1479 #ifndef _7ZIP_ST
1480
Execute()1481 void CThreadDecoder::Execute()
1482 {
1483 try
1484 {
1485 #ifndef _NO_CRYPTO
1486 bool isEncrypted = false;
1487 bool passwordIsDefined = false;
1488 UString password;
1489 #endif
1490
1491 dataAfterEnd_Error = false;
1492
1493 Result = Decoder.Decode(
1494 EXTERNAL_CODECS_LOC_VARS
1495 InStream,
1496 StartPos,
1497 *Folders, FolderIndex,
1498
1499 // send_UnpackSize ? &UnpackSize : NULL,
1500 NULL, // unpackSize : FULL unpack
1501
1502 Fos,
1503 NULL, // compressProgress
1504
1505 NULL // *inStreamMainRes
1506 , dataAfterEnd_Error
1507
1508 _7Z_DECODER_CRYPRO_VARS
1509 #ifndef _7ZIP_ST
1510 , MtMode, NumThreads,
1511 0 // MemUsage
1512 #endif
1513
1514 );
1515 }
1516 catch(...)
1517 {
1518 Result = E_FAIL;
1519 }
1520
1521 /*
1522 if (Result == S_OK)
1523 Result = FosSpec->CheckFinishedState();
1524 */
1525 FosSpec->_stream.Release();
1526 }
1527
1528 #endif
1529
1530 #ifndef _NO_CRYPTO
1531
1532 class CCryptoGetTextPassword:
1533 public ICryptoGetTextPassword,
1534 public CMyUnknownImp
1535 {
1536 public:
1537 UString Password;
1538
1539 MY_UNKNOWN_IMP
1540 STDMETHOD(CryptoGetTextPassword)(BSTR *password);
1541 };
1542
CryptoGetTextPassword(BSTR * password)1543 STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password)
1544 {
1545 return StringToBstr(Password, password);
1546 }
1547
1548 #endif
1549
1550
GetFile(const CDatabase & inDb,unsigned index,CFileItem & file,CFileItem2 & file2)1551 static void GetFile(const CDatabase &inDb, unsigned index, CFileItem &file, CFileItem2 &file2)
1552 {
1553 file = inDb.Files[index];
1554 file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime);
1555 file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime);
1556 file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime);
1557 file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos);
1558 file2.AttribDefined = inDb.Attrib.GetItem(index, file2.Attrib);
1559 file2.IsAnti = inDb.IsItemAnti(index);
1560 // file2.IsAux = inDb.IsItemAux(index);
1561 }
1562
Update(DECL_EXTERNAL_CODECS_LOC_VARS IInStream * inStream,const CDbEx * db,const CObjectVector<CUpdateItem> & updateItems,COutArchive & archive,CArchiveDatabaseOut & newDatabase,ISequentialOutStream * seqOutStream,IArchiveUpdateCallback * updateCallback,const CUpdateOptions & options,ICryptoGetTextPassword * getDecoderPassword)1563 HRESULT Update(
1564 DECL_EXTERNAL_CODECS_LOC_VARS
1565 IInStream *inStream,
1566 const CDbEx *db,
1567 const CObjectVector<CUpdateItem> &updateItems,
1568 // const CObjectVector<CTreeFolder> &treeFolders,
1569 // const CUniqBlocks &secureBlocks,
1570 COutArchive &archive,
1571 CArchiveDatabaseOut &newDatabase,
1572 ISequentialOutStream *seqOutStream,
1573 IArchiveUpdateCallback *updateCallback,
1574 const CUpdateOptions &options
1575 #ifndef _NO_CRYPTO
1576 , ICryptoGetTextPassword *getDecoderPassword
1577 #endif
1578 )
1579 {
1580 UInt64 numSolidFiles = options.NumSolidFiles;
1581 if (numSolidFiles == 0)
1582 numSolidFiles = 1;
1583
1584 CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
1585 updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
1586
1587 CMyComPtr<IArchiveExtractCallbackMessage> extractCallback;
1588 updateCallback->QueryInterface(IID_IArchiveExtractCallbackMessage, (void **)&extractCallback);
1589
1590 // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes();
1591
1592 /*
1593 CMyComPtr<IOutStream> outStream;
1594 RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream));
1595 if (!outStream)
1596 return E_NOTIMPL;
1597 */
1598
1599 UInt64 startBlockSize = db ? db->ArcInfo.StartPosition: 0;
1600 if (startBlockSize > 0 && !options.RemoveSfxBlock)
1601 {
1602 RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL));
1603 }
1604
1605 CIntArr fileIndexToUpdateIndexMap;
1606 UInt64 complexity = 0;
1607 UInt64 inSizeForReduce2 = 0;
1608 bool needEncryptedRepack = false;
1609
1610 CRecordVector<CFilterMode2> filters;
1611 CObjectVector<CSolidGroup> groups;
1612 bool thereAreRepacks = false;
1613
1614 bool useFilters = options.UseFilters;
1615 if (useFilters)
1616 {
1617 const CCompressionMethodMode &method = *options.Method;
1618
1619 FOR_VECTOR (i, method.Methods)
1620 if (IsFilterMethod(method.Methods[i].Id))
1621 {
1622 useFilters = false;
1623 break;
1624 }
1625 }
1626
1627 if (db)
1628 {
1629 fileIndexToUpdateIndexMap.Alloc(db->Files.Size());
1630 unsigned i;
1631
1632 for (i = 0; i < db->Files.Size(); i++)
1633 fileIndexToUpdateIndexMap[i] = -1;
1634
1635 for (i = 0; i < updateItems.Size(); i++)
1636 {
1637 int index = updateItems[i].IndexInArchive;
1638 if (index != -1)
1639 fileIndexToUpdateIndexMap[(unsigned)index] = i;
1640 }
1641
1642 for (i = 0; i < db->NumFolders; i++)
1643 {
1644 CNum indexInFolder = 0;
1645 CNum numCopyItems = 0;
1646 CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
1647 UInt64 repackSize = 0;
1648
1649 for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
1650 {
1651 const CFileItem &file = db->Files[fi];
1652 if (file.HasStream)
1653 {
1654 indexInFolder++;
1655 int updateIndex = fileIndexToUpdateIndexMap[fi];
1656 if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
1657 {
1658 numCopyItems++;
1659 repackSize += file.Size;
1660 }
1661 }
1662 }
1663
1664 if (numCopyItems == 0)
1665 continue;
1666
1667 CFolderRepack rep;
1668 rep.FolderIndex = i;
1669 rep.NumCopyFiles = numCopyItems;
1670 CFolderEx f;
1671 db->ParseFolderEx(i, f);
1672
1673 const bool isEncrypted = f.IsEncrypted();
1674 const bool needCopy = (numCopyItems == numUnpackStreams);
1675 const bool extractFilter = (useFilters || needCopy);
1676
1677 unsigned groupIndex = Get_FilterGroup_for_Folder(filters, f, extractFilter);
1678
1679 while (groupIndex >= groups.Size())
1680 groups.AddNew();
1681
1682 groups[groupIndex].folderRefs.Add(rep);
1683
1684 if (needCopy)
1685 complexity += db->GetFolderFullPackSize(i);
1686 else
1687 {
1688 thereAreRepacks = true;
1689 complexity += repackSize;
1690 if (inSizeForReduce2 < repackSize)
1691 inSizeForReduce2 = repackSize;
1692 if (isEncrypted)
1693 needEncryptedRepack = true;
1694 }
1695 }
1696 }
1697
1698 UInt64 inSizeForReduce = 0;
1699 {
1700 bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0);
1701 FOR_VECTOR (i, updateItems)
1702 {
1703 const CUpdateItem &ui = updateItems[i];
1704 if (ui.NewData)
1705 {
1706 complexity += ui.Size;
1707 if (isSolid)
1708 inSizeForReduce += ui.Size;
1709 else if (inSizeForReduce < ui.Size)
1710 inSizeForReduce = ui.Size;
1711 }
1712 }
1713 }
1714
1715 if (inSizeForReduce < inSizeForReduce2)
1716 inSizeForReduce = inSizeForReduce2;
1717
1718 RINOK(updateCallback->SetTotal(complexity));
1719
1720 CLocalProgress *lps = new CLocalProgress;
1721 CMyComPtr<ICompressProgressInfo> progress = lps;
1722 lps->Init(updateCallback, true);
1723
1724 #ifndef _7ZIP_ST
1725
1726 CStreamBinder sb;
1727 if (options.MultiThreadMixer)
1728 {
1729 RINOK(sb.CreateEvents());
1730 }
1731
1732 #endif
1733
1734 CThreadDecoder threadDecoder(options.MultiThreadMixer);
1735
1736 #ifndef _7ZIP_ST
1737 if (options.MultiThreadMixer && thereAreRepacks)
1738 {
1739 #ifdef EXTERNAL_CODECS
1740 threadDecoder.__externalCodecs = __externalCodecs;
1741 #endif
1742 RINOK(threadDecoder.Create());
1743 }
1744 #endif
1745
1746 {
1747 CAnalysis analysis;
1748 if (options.AnalysisLevel == 0)
1749 {
1750 analysis.ParseWav = false;
1751 analysis.ParseExe = false;
1752 analysis.ParseAll = false;
1753 }
1754 else
1755 {
1756 analysis.Callback = opCallback;
1757 if (options.AnalysisLevel > 0)
1758 {
1759 analysis.ParseWav = true;
1760 if (options.AnalysisLevel >= 7)
1761 {
1762 analysis.ParseExe = true;
1763 if (options.AnalysisLevel >= 9)
1764 analysis.ParseAll = true;
1765 }
1766 }
1767 }
1768
1769 // ---------- Split files to groups ----------
1770
1771 const CCompressionMethodMode &method = *options.Method;
1772
1773 FOR_VECTOR (i, updateItems)
1774 {
1775 const CUpdateItem &ui = updateItems[i];
1776 if (!ui.NewData || !ui.HasStream())
1777 continue;
1778
1779 CFilterMode2 fm;
1780 if (useFilters)
1781 {
1782 RINOK(analysis.GetFilterGroup(i, ui, fm));
1783 }
1784 fm.Encrypted = method.PasswordIsDefined;
1785
1786 unsigned groupIndex = GetGroup(filters, fm);
1787 while (groupIndex >= groups.Size())
1788 groups.AddNew();
1789 groups[groupIndex].Indices.Add(i);
1790 }
1791 }
1792
1793
1794 #ifndef _NO_CRYPTO
1795
1796 CCryptoGetTextPassword *getPasswordSpec = NULL;
1797 CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1798 if (needEncryptedRepack)
1799 {
1800 getPasswordSpec = new CCryptoGetTextPassword;
1801 getTextPassword = getPasswordSpec;
1802
1803 #ifndef _7ZIP_ST
1804 threadDecoder.getTextPassword = getPasswordSpec;
1805 #endif
1806
1807 if (options.Method->PasswordIsDefined)
1808 getPasswordSpec->Password = options.Method->Password;
1809 else
1810 {
1811 if (!getDecoderPassword)
1812 return E_NOTIMPL;
1813 CMyComBSTR password;
1814 RINOK(getDecoderPassword->CryptoGetTextPassword(&password));
1815 if (password)
1816 getPasswordSpec->Password = password;
1817 }
1818 }
1819
1820 #endif
1821
1822
1823 // ---------- Compress ----------
1824
1825 RINOK(archive.Create(seqOutStream, false));
1826 RINOK(archive.SkipPrefixArchiveHeader());
1827
1828 /*
1829 CIntVector treeFolderToArcIndex;
1830 treeFolderToArcIndex.Reserve(treeFolders.Size());
1831 for (i = 0; i < treeFolders.Size(); i++)
1832 treeFolderToArcIndex.Add(-1);
1833 // ---------- Write Tree (only AUX dirs) ----------
1834 for (i = 1; i < treeFolders.Size(); i++)
1835 {
1836 const CTreeFolder &treeFolder = treeFolders[i];
1837 CFileItem file;
1838 CFileItem2 file2;
1839 file2.Init();
1840 int secureID = 0;
1841 if (treeFolder.UpdateItemIndex < 0)
1842 {
1843 // we can store virtual dir item wuthout attrib, but we want all items have attrib.
1844 file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY);
1845 file2.IsAux = true;
1846 }
1847 else
1848 {
1849 const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex];
1850 // if item is not dir, then it's parent for alt streams.
1851 // we will write such items later
1852 if (!ui.IsDir)
1853 continue;
1854 secureID = ui.SecureIndex;
1855 if (ui.NewProps)
1856 UpdateItem_To_FileItem(ui, file, file2);
1857 else
1858 GetFile(*db, ui.IndexInArchive, file, file2);
1859 }
1860 file.Size = 0;
1861 file.HasStream = false;
1862 file.IsDir = true;
1863 file.Parent = treeFolder.Parent;
1864
1865 treeFolderToArcIndex[i] = newDatabase.Files.Size();
1866 newDatabase.AddFile(file, file2, treeFolder.Name);
1867
1868 if (totalSecureDataSize != 0)
1869 newDatabase.SecureIDs.Add(secureID);
1870 }
1871 */
1872
1873 {
1874 /* ---------- Write non-AUX dirs and Empty files ---------- */
1875 CUIntVector emptyRefs;
1876
1877 unsigned i;
1878
1879 for (i = 0; i < updateItems.Size(); i++)
1880 {
1881 const CUpdateItem &ui = updateItems[i];
1882 if (ui.NewData)
1883 {
1884 if (ui.HasStream())
1885 continue;
1886 }
1887 else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream)
1888 continue;
1889 /*
1890 if (ui.TreeFolderIndex >= 0)
1891 continue;
1892 */
1893 emptyRefs.Add(i);
1894 }
1895
1896 emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
1897
1898 for (i = 0; i < emptyRefs.Size(); i++)
1899 {
1900 const CUpdateItem &ui = updateItems[emptyRefs[i]];
1901 CFileItem file;
1902 CFileItem2 file2;
1903 UString name;
1904 if (ui.NewProps)
1905 {
1906 UpdateItem_To_FileItem(ui, file, file2);
1907 file.CrcDefined = false;
1908 name = ui.Name;
1909 }
1910 else
1911 {
1912 GetFile(*db, ui.IndexInArchive, file, file2);
1913 db->GetPath(ui.IndexInArchive, name);
1914 }
1915
1916 /*
1917 if (totalSecureDataSize != 0)
1918 newDatabase.SecureIDs.Add(ui.SecureIndex);
1919 file.Parent = ui.ParentFolderIndex;
1920 */
1921 newDatabase.AddFile(file, file2, name);
1922 }
1923 }
1924
1925 lps->ProgressOffset = 0;
1926
1927 {
1928 // ---------- Sort Filters ----------
1929
1930 FOR_VECTOR (i, filters)
1931 {
1932 filters[i].GroupIndex = i;
1933 }
1934 filters.Sort2();
1935 }
1936
1937 for (unsigned groupIndex = 0; groupIndex < filters.Size(); groupIndex++)
1938 {
1939 const CFilterMode2 &filterMode = filters[groupIndex];
1940
1941 CCompressionMethodMode method = *options.Method;
1942 {
1943 HRESULT res = MakeExeMethod(method, filterMode,
1944 #ifdef _7ZIP_ST
1945 false
1946 #else
1947 options.MaxFilter && options.MultiThreadMixer
1948 #endif
1949 );
1950
1951 RINOK(res);
1952 }
1953
1954 if (filterMode.Encrypted)
1955 {
1956 if (!method.PasswordIsDefined)
1957 {
1958 #ifndef _NO_CRYPTO
1959 if (getPasswordSpec)
1960 method.Password = getPasswordSpec->Password;
1961 #endif
1962 method.PasswordIsDefined = true;
1963 }
1964 }
1965 else
1966 {
1967 method.PasswordIsDefined = false;
1968 method.Password.Empty();
1969 }
1970
1971 CEncoder encoder(method);
1972
1973 // ---------- Repack and copy old solid blocks ----------
1974
1975 const CSolidGroup &group = groups[filterMode.GroupIndex];
1976
1977 FOR_VECTOR(folderRefIndex, group.folderRefs)
1978 {
1979 const CFolderRepack &rep = group.folderRefs[folderRefIndex];
1980
1981 unsigned folderIndex = rep.FolderIndex;
1982
1983 CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
1984
1985 if (rep.NumCopyFiles == numUnpackStreams)
1986 {
1987 if (opCallback)
1988 {
1989 RINOK(opCallback->ReportOperation(
1990 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
1991 NUpdateNotifyOp::kReplicate));
1992
1993 // ---------- Copy old solid block ----------
1994 {
1995 CNum indexInFolder = 0;
1996 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
1997 {
1998 if (db->Files[fi].HasStream)
1999 {
2000 indexInFolder++;
2001 RINOK(opCallback->ReportOperation(
2002 NEventIndexType::kInArcIndex, (UInt32)fi,
2003 NUpdateNotifyOp::kReplicate));
2004 }
2005 }
2006 }
2007 }
2008
2009 UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
2010 RINOK(WriteRange(inStream, archive.SeqStream,
2011 db->GetFolderStreamPos(folderIndex, 0), packSize, progress));
2012 lps->ProgressOffset += packSize;
2013
2014 CFolder &folder = newDatabase.Folders.AddNew();
2015 db->ParseFolderInfo(folderIndex, folder);
2016 CNum startIndex = db->FoStartPackStreamIndex[folderIndex];
2017 FOR_VECTOR(j, folder.PackStreams)
2018 {
2019 newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j));
2020 // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
2021 // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
2022 }
2023
2024 size_t indexStart = db->FoToCoderUnpackSizes[folderIndex];
2025 size_t indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1];
2026 for (; indexStart < indexEnd; indexStart++)
2027 newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes[indexStart]);
2028 }
2029 else
2030 {
2031 // ---------- Repack old solid block ----------
2032
2033 CBoolVector extractStatuses;
2034
2035 CNum indexInFolder = 0;
2036
2037 if (opCallback)
2038 {
2039 RINOK(opCallback->ReportOperation(
2040 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2041 NUpdateNotifyOp::kRepack))
2042 }
2043
2044 /* We could reduce data size of decoded folder, if we don't need to repack
2045 last files in folder. But the gain in speed is small in most cases.
2046 So we unpack full folder. */
2047
2048 UInt64 sizeToEncode = 0;
2049
2050 /*
2051 UInt64 importantUnpackSize = 0;
2052 unsigned numImportantFiles = 0;
2053 UInt64 decodeSize = 0;
2054 */
2055
2056 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2057 {
2058 bool needExtract = false;
2059 const CFileItem &file = db->Files[fi];
2060
2061 if (file.HasStream)
2062 {
2063 indexInFolder++;
2064 int updateIndex = fileIndexToUpdateIndexMap[fi];
2065 if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
2066 needExtract = true;
2067 // decodeSize += file.Size;
2068 }
2069
2070 extractStatuses.Add(needExtract);
2071 if (needExtract)
2072 {
2073 sizeToEncode += file.Size;
2074 /*
2075 numImportantFiles = extractStatuses.Size();
2076 importantUnpackSize = decodeSize;
2077 */
2078 }
2079 }
2080
2081 // extractStatuses.DeleteFrom(numImportantFiles);
2082
2083 unsigned startPackIndex = newDatabase.PackSizes.Size();
2084 UInt64 curUnpackSize;
2085 {
2086
2087 CMyComPtr<ISequentialInStream> sbInStream;
2088 CRepackStreamBase *repackBase;
2089 CFolderInStream2 *FosSpec2 = NULL;
2090
2091 CRepackInStreamWithSizes *inStreamSizeCountSpec = new CRepackInStreamWithSizes;
2092 CMyComPtr<ISequentialInStream> inStreamSizeCount = inStreamSizeCountSpec;
2093 {
2094 #ifndef _7ZIP_ST
2095 if (options.MultiThreadMixer)
2096 {
2097 repackBase = threadDecoder.FosSpec;
2098 CMyComPtr<ISequentialOutStream> sbOutStream;
2099 sb.CreateStreams(&sbInStream, &sbOutStream);
2100 sb.ReInit();
2101
2102 threadDecoder.FosSpec->_stream = sbOutStream;
2103
2104 threadDecoder.InStream = inStream;
2105 threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0);
2106 threadDecoder.Folders = (const CFolders *)db;
2107 threadDecoder.FolderIndex = folderIndex;
2108
2109 // threadDecoder.UnpackSize = importantUnpackSize;
2110 // threadDecoder.send_UnpackSize = true;
2111 }
2112 else
2113 #endif
2114 {
2115 FosSpec2 = new CFolderInStream2;
2116 FosSpec2->Init();
2117 sbInStream = FosSpec2;
2118 repackBase = FosSpec2;
2119
2120 #ifndef _NO_CRYPTO
2121 bool isEncrypted = false;
2122 bool passwordIsDefined = false;
2123 UString password;
2124 #endif
2125
2126 CMyComPtr<ISequentialInStream> decodedStream;
2127 bool dataAfterEnd_Error = false;
2128
2129 HRESULT res = threadDecoder.Decoder.Decode(
2130 EXTERNAL_CODECS_LOC_VARS
2131 inStream,
2132 db->ArcInfo.DataStartPosition, // db->GetFolderStreamPos(folderIndex, 0);,
2133 *db, folderIndex,
2134 // &importantUnpackSize, // *unpackSize
2135 NULL, // *unpackSize : FULL unpack
2136
2137 NULL, // *outStream
2138 NULL, // *compressProgress
2139
2140 &decodedStream
2141 , dataAfterEnd_Error
2142
2143 _7Z_DECODER_CRYPRO_VARS
2144 #ifndef _7ZIP_ST
2145 , false // mtMode
2146 , 1 // numThreads
2147 , 0 // memUsage
2148 #endif
2149 );
2150
2151 RINOK(res);
2152 if (!decodedStream)
2153 return E_FAIL;
2154
2155 FosSpec2->_inStream = decodedStream;
2156 }
2157
2158 repackBase->_db = db;
2159 repackBase->_opCallback = opCallback;
2160 repackBase->_extractCallback = extractCallback;
2161
2162 UInt32 startIndex = db->FolderStartFileIndex[folderIndex];
2163 RINOK(repackBase->Init(startIndex, &extractStatuses));
2164
2165 inStreamSizeCountSpec->_db = db;
2166 inStreamSizeCountSpec->Init(sbInStream, startIndex, &extractStatuses);
2167
2168 #ifndef _7ZIP_ST
2169 if (options.MultiThreadMixer)
2170 {
2171 threadDecoder.Start();
2172 }
2173 #endif
2174 }
2175
2176 curUnpackSize = sizeToEncode;
2177
2178 HRESULT encodeRes = encoder.Encode(
2179 EXTERNAL_CODECS_LOC_VARS
2180 inStreamSizeCount,
2181 // NULL,
2182 &inSizeForReduce,
2183 newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curUnpackSize,
2184 archive.SeqStream, newDatabase.PackSizes, progress);
2185
2186 if (encodeRes == k_My_HRESULT_CRC_ERROR)
2187 return E_FAIL;
2188
2189 #ifndef _7ZIP_ST
2190 if (options.MultiThreadMixer)
2191 {
2192 // 16.00: hang was fixed : for case if decoding was not finished.
2193 // We close CBinderInStream and it calls CStreamBinder::CloseRead()
2194 inStreamSizeCount.Release();
2195 sbInStream.Release();
2196
2197 threadDecoder.WaitExecuteFinish();
2198
2199 HRESULT decodeRes = threadDecoder.Result;
2200 // if (res == k_My_HRESULT_CRC_ERROR)
2201 if (decodeRes == S_FALSE || threadDecoder.dataAfterEnd_Error)
2202 {
2203 if (extractCallback)
2204 {
2205 RINOK(extractCallback->ReportExtractResult(
2206 NEventIndexType::kInArcIndex, db->FolderStartFileIndex[folderIndex],
2207 // NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2208 (decodeRes != S_OK ?
2209 NExtract::NOperationResult::kDataError :
2210 NExtract::NOperationResult::kDataAfterEnd)));
2211 }
2212 if (decodeRes != S_OK)
2213 return E_FAIL;
2214 }
2215 RINOK(decodeRes);
2216 if (encodeRes == S_OK)
2217 if (sb.ProcessedSize != sizeToEncode)
2218 encodeRes = E_FAIL;
2219 }
2220 else
2221 #endif
2222 {
2223 if (FosSpec2->Result == S_FALSE)
2224 {
2225 if (extractCallback)
2226 {
2227 RINOK(extractCallback->ReportExtractResult(
2228 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2229 NExtract::NOperationResult::kDataError));
2230 }
2231 return E_FAIL;
2232 }
2233 RINOK(FosSpec2->Result);
2234 }
2235
2236 RINOK(encodeRes);
2237 RINOK(repackBase->CheckFinishedState());
2238
2239 if (curUnpackSize != sizeToEncode)
2240 return E_FAIL;
2241 }
2242
2243 for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2244 lps->OutSize += newDatabase.PackSizes[startPackIndex];
2245 lps->InSize += curUnpackSize;
2246 }
2247
2248 newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
2249
2250 CNum indexInFolder = 0;
2251 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2252 {
2253 if (db->Files[fi].HasStream)
2254 {
2255 indexInFolder++;
2256 int updateIndex = fileIndexToUpdateIndexMap[fi];
2257 if (updateIndex >= 0)
2258 {
2259 const CUpdateItem &ui = updateItems[updateIndex];
2260 if (ui.NewData)
2261 continue;
2262
2263 UString name;
2264 CFileItem file;
2265 CFileItem2 file2;
2266 GetFile(*db, fi, file, file2);
2267
2268 if (ui.NewProps)
2269 {
2270 UpdateItem_To_FileItem2(ui, file2);
2271 file.IsDir = ui.IsDir;
2272 name = ui.Name;
2273 }
2274 else
2275 db->GetPath(fi, name);
2276
2277 /*
2278 file.Parent = ui.ParentFolderIndex;
2279 if (ui.TreeFolderIndex >= 0)
2280 treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2281 if (totalSecureDataSize != 0)
2282 newDatabase.SecureIDs.Add(ui.SecureIndex);
2283 */
2284 newDatabase.AddFile(file, file2, name);
2285 }
2286 }
2287 }
2288 }
2289
2290
2291 // ---------- Compress files to new solid blocks ----------
2292
2293 unsigned numFiles = group.Indices.Size();
2294 if (numFiles == 0)
2295 continue;
2296 CRecordVector<CRefItem> refItems;
2297 refItems.ClearAndSetSize(numFiles);
2298 // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1
2299 bool sortByType = options.UseTypeSorting;
2300
2301 unsigned i;
2302
2303 for (i = 0; i < numFiles; i++)
2304 refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType);
2305
2306 CSortParam sortParam;
2307 // sortParam.TreeFolders = &treeFolders;
2308 sortParam.SortByType = sortByType;
2309 refItems.Sort(CompareUpdateItems, (void *)&sortParam);
2310
2311 CObjArray<UInt32> indices(numFiles);
2312
2313 for (i = 0; i < numFiles; i++)
2314 {
2315 UInt32 index = refItems[i].Index;
2316 indices[i] = index;
2317 /*
2318 const CUpdateItem &ui = updateItems[index];
2319 CFileItem file;
2320 if (ui.NewProps)
2321 UpdateItem_To_FileItem(ui, file);
2322 else
2323 file = db.Files[ui.IndexInArchive];
2324 if (file.IsAnti || file.IsDir)
2325 return E_FAIL;
2326 newDatabase.Files.Add(file);
2327 */
2328 }
2329
2330 for (i = 0; i < numFiles;)
2331 {
2332 UInt64 totalSize = 0;
2333 unsigned numSubFiles;
2334
2335 const wchar_t *prevExtension = NULL;
2336
2337 for (numSubFiles = 0; i + numSubFiles < numFiles && numSubFiles < numSolidFiles; numSubFiles++)
2338 {
2339 const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
2340 totalSize += ui.Size;
2341 if (totalSize > options.NumSolidBytes)
2342 break;
2343 if (options.SolidExtension)
2344 {
2345 int slashPos = ui.Name.ReverseFind_PathSepar();
2346 int dotPos = ui.Name.ReverseFind_Dot();
2347 const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : dotPos + 1);
2348 if (numSubFiles == 0)
2349 prevExtension = ext;
2350 else if (!StringsAreEqualNoCase(ext, prevExtension))
2351 break;
2352 }
2353 }
2354
2355 if (numSubFiles < 1)
2356 numSubFiles = 1;
2357
2358 RINOK(lps->SetCur());
2359
2360 CFolderInStream *inStreamSpec = new CFolderInStream;
2361 CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec);
2362 inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
2363
2364 unsigned startPackIndex = newDatabase.PackSizes.Size();
2365 UInt64 curFolderUnpackSize = totalSize;
2366 // curFolderUnpackSize = (UInt64)(Int64)-1;
2367
2368 RINOK(encoder.Encode(
2369 EXTERNAL_CODECS_LOC_VARS
2370 solidInStream,
2371 // NULL,
2372 &inSizeForReduce,
2373 newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curFolderUnpackSize,
2374 archive.SeqStream, newDatabase.PackSizes, progress));
2375
2376 if (!inStreamSpec->WasFinished())
2377 return E_FAIL;
2378
2379 for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2380 lps->OutSize += newDatabase.PackSizes[startPackIndex];
2381
2382 lps->InSize += curFolderUnpackSize;
2383 // for ()
2384 // newDatabase.PackCRCsDefined.Add(false);
2385 // newDatabase.PackCRCs.Add(0);
2386
2387 CNum numUnpackStreams = 0;
2388 UInt64 skippedSize = 0;
2389
2390 for (unsigned subIndex = 0; subIndex < numSubFiles; subIndex++)
2391 {
2392 const CUpdateItem &ui = updateItems[indices[i + subIndex]];
2393 CFileItem file;
2394 CFileItem2 file2;
2395 UString name;
2396 if (ui.NewProps)
2397 {
2398 UpdateItem_To_FileItem(ui, file, file2);
2399 name = ui.Name;
2400 }
2401 else
2402 {
2403 GetFile(*db, ui.IndexInArchive, file, file2);
2404 db->GetPath(ui.IndexInArchive, name);
2405 }
2406 if (file2.IsAnti || file.IsDir)
2407 return E_FAIL;
2408
2409 /*
2410 CFileItem &file = newDatabase.Files[
2411 startFileIndexInDatabase + i + subIndex];
2412 */
2413 if (!inStreamSpec->Processed[subIndex])
2414 {
2415 skippedSize += ui.Size;
2416 continue;
2417 // file.Name += ".locked";
2418 }
2419
2420 file.Crc = inStreamSpec->CRCs[subIndex];
2421 file.Size = inStreamSpec->Sizes[subIndex];
2422
2423 // if (file.Size >= 0) // test purposes
2424 if (file.Size != 0)
2425 {
2426 file.CrcDefined = true;
2427 file.HasStream = true;
2428 numUnpackStreams++;
2429 }
2430 else
2431 {
2432 file.CrcDefined = false;
2433 file.HasStream = false;
2434 }
2435
2436 /*
2437 file.Parent = ui.ParentFolderIndex;
2438 if (ui.TreeFolderIndex >= 0)
2439 treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2440 if (totalSecureDataSize != 0)
2441 newDatabase.SecureIDs.Add(ui.SecureIndex);
2442 */
2443 newDatabase.AddFile(file, file2, name);
2444 }
2445
2446 // numUnpackStreams = 0 is very bad case for locked files
2447 // v3.13 doesn't understand it.
2448 newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
2449 i += numSubFiles;
2450
2451 if (skippedSize != 0 && complexity >= skippedSize)
2452 {
2453 complexity -= skippedSize;
2454 RINOK(updateCallback->SetTotal(complexity));
2455 }
2456 }
2457 }
2458
2459 RINOK(lps->SetCur());
2460
2461 /*
2462 fileIndexToUpdateIndexMap.ClearAndFree();
2463 groups.ClearAndFree();
2464 */
2465
2466 /*
2467 for (i = 0; i < newDatabase.Files.Size(); i++)
2468 {
2469 CFileItem &file = newDatabase.Files[i];
2470 file.Parent = treeFolderToArcIndex[file.Parent];
2471 }
2472
2473 if (totalSecureDataSize != 0)
2474 {
2475 newDatabase.SecureBuf.SetCapacity(totalSecureDataSize);
2476 size_t pos = 0;
2477 newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size());
2478 for (i = 0; i < secureBlocks.Sorted.Size(); i++)
2479 {
2480 const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]];
2481 size_t size = buf.GetCapacity();
2482 if (size != 0)
2483 memcpy(newDatabase.SecureBuf + pos, buf, size);
2484 newDatabase.SecureSizes.Add((UInt32)size);
2485 pos += size;
2486 }
2487 }
2488 */
2489 newDatabase.ReserveDown();
2490
2491 if (opCallback)
2492 RINOK(opCallback->ReportOperation(NEventIndexType::kNoIndex, (UInt32)(Int32)-1, NUpdateNotifyOp::kHeader));
2493
2494 return S_OK;
2495 }
2496
2497 }}
2498