1 // 7zUpdate.cpp
2 
3 #include "StdAfx.h"
4 
5 #include "../../../../C/CpuArch.h"
6 
7 #include "../../../Common/Wildcard.h"
8 
9 #include "../../Common/CreateCoder.h"
10 #include "../../Common/LimitedStreams.h"
11 #include "../../Common/ProgressUtils.h"
12 
13 #include "../../Compress/CopyCoder.h"
14 
15 #include "../Common/ItemNameUtils.h"
16 
17 #include "7zDecode.h"
18 #include "7zEncode.h"
19 #include "7zFolderInStream.h"
20 #include "7zHandler.h"
21 #include "7zOut.h"
22 #include "7zUpdate.h"
23 
24 namespace NArchive {
25 namespace N7z {
26 
27 
28 #define k_X86 k_BCJ
29 
30 struct CFilterMode
31 {
32   UInt32 Id;
33   UInt32 Delta;
34 
CFilterModeNArchive::N7z::CFilterMode35   CFilterMode(): Id(0), Delta(0) {}
36 
SetDeltaNArchive::N7z::CFilterMode37   void SetDelta()
38   {
39     if (Id == k_IA64)
40       Delta = 16;
41     else if (Id == k_ARM || Id == k_PPC || Id == k_SPARC)
42       Delta = 4;
43     else if (Id == k_ARMT)
44       Delta = 2;
45     else
46       Delta = 0;
47   }
48 };
49 
50 
51 /* ---------- PE ---------- */
52 
53 #define MZ_SIG 0x5A4D
54 
55 #define PE_SIG 0x00004550
56 #define PE_OptHeader_Magic_32 0x10B
57 #define PE_OptHeader_Magic_64 0x20B
58 #define PE_SectHeaderSize 40
59 #define PE_SECT_EXECUTE 0x20000000
60 
Parse_EXE(const Byte * buf,size_t size,CFilterMode * filterMode)61 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode)
62 {
63   if (size < 512 || GetUi16(buf) != MZ_SIG)
64     return 0;
65 
66   const Byte *p;
67   UInt32 peOffset, optHeaderSize, filterId;
68 
69   peOffset = GetUi32(buf + 0x3C);
70   if (peOffset >= 0x1000 || peOffset + 512 > size || (peOffset & 7) != 0)
71     return 0;
72   p = buf + peOffset;
73   if (GetUi32(p) != PE_SIG)
74     return 0;
75   p += 4;
76 
77   switch (GetUi16(p))
78   {
79     case 0x014C:
80     case 0x8664:  filterId = k_X86; break;
81 
82     /*
83     IMAGE_FILE_MACHINE_ARM   0x01C0  // ARM LE
84     IMAGE_FILE_MACHINE_THUMB 0x01C2  // ARM Thumb / Thumb-2 LE
85     IMAGE_FILE_MACHINE_ARMNT 0x01C4  // ARM Thumb-2, LE
86     Note: We use ARM filter for 0x01C2. (WinCE 5 - 0x01C2) files mostly contain ARM code (not Thumb/Thumb-2).
87     */
88 
89     case 0x01C0:                            // WinCE old
90     case 0x01C2:  filterId = k_ARM; break;  // WinCE new
91     case 0x01C4:  filterId = k_ARMT; break; // WinRT
92 
93     case 0x0200:  filterId = k_IA64; break;
94     default:  return 0;
95   }
96 
97   optHeaderSize = GetUi16(p + 16);
98   if (optHeaderSize > (1 << 10))
99     return 0;
100 
101   p += 20; /* headerSize */
102 
103   switch (GetUi16(p))
104   {
105     case PE_OptHeader_Magic_32:
106     case PE_OptHeader_Magic_64:
107       break;
108     default:
109       return 0;
110   }
111 
112   filterMode->Id = filterId;
113   return 1;
114 }
115 
116 
117 /* ---------- ELF ---------- */
118 
119 #define ELF_SIG 0x464C457F
120 
121 #define ELF_CLASS_32  1
122 #define ELF_CLASS_64  2
123 
124 #define ELF_DATA_2LSB 1
125 #define ELF_DATA_2MSB 2
126 
Get16(const Byte * p,Bool be)127 static UInt16 Get16(const Byte *p, Bool be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); }
Get32(const Byte * p,Bool be)128 static UInt32 Get32(const Byte *p, Bool be) { if (be) return GetBe32(p); return GetUi32(p); }
129 // static UInt64 Get64(const Byte *p, Bool be) { if (be) return GetBe64(p); return GetUi64(p); }
130 
Parse_ELF(const Byte * buf,size_t size,CFilterMode * filterMode)131 static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode)
132 {
133   Bool /* is32, */ be;
134   UInt32 filterId;
135 
136   if (size < 512 || buf[6] != 1) /* ver */
137     return 0;
138 
139   if (GetUi32(buf) != ELF_SIG)
140     return 0;
141 
142   switch (buf[4])
143   {
144     case ELF_CLASS_32: /* is32 = True; */ break;
145     case ELF_CLASS_64: /* is32 = False; */ break;
146     default: return 0;
147   }
148 
149   switch (buf[5])
150   {
151     case ELF_DATA_2LSB: be = False; break;
152     case ELF_DATA_2MSB: be = True; break;
153     default: return 0;
154   }
155 
156   switch (Get16(buf + 0x12, be))
157   {
158     case 3:
159     case 6:
160     case 62: filterId = k_X86; break;
161     case 2:
162     case 18:
163     case 43: filterId = k_SPARC; break;
164     case 20:
165     case 21: if (!be) return 0; filterId = k_PPC; break;
166     case 40: if ( be) return 0; filterId = k_ARM; break;
167 
168     /* Some IA-64 ELF exacutable have size that is not aligned for 16 bytes.
169        So we don't use IA-64 filter for IA-64 ELF */
170     // case 50: if ( be) return 0; filterId = k_IA64; break;
171 
172     default: return 0;
173   }
174 
175   filterMode->Id = filterId;
176   return 1;
177 }
178 
179 
180 
181 /* ---------- Mach-O ---------- */
182 
183 #define MACH_SIG_BE_32 0xCEFAEDFE
184 #define MACH_SIG_BE_64 0xCFFAEDFE
185 #define MACH_SIG_LE_32 0xFEEDFACE
186 #define MACH_SIG_LE_64 0xFEEDFACF
187 
188 #define MACH_ARCH_ABI64 (1 << 24)
189 #define MACH_MACHINE_386 7
190 #define MACH_MACHINE_ARM 12
191 #define MACH_MACHINE_SPARC 14
192 #define MACH_MACHINE_PPC 18
193 #define MACH_MACHINE_PPC64 (MACH_ARCH_ABI64 | MACH_MACHINE_PPC)
194 #define MACH_MACHINE_AMD64 (MACH_ARCH_ABI64 | MACH_MACHINE_386)
195 
Parse_MACH(const Byte * buf,size_t size,CFilterMode * filterMode)196 static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode)
197 {
198   UInt32 filterId, numCommands, commandsSize;
199 
200   if (size < 512)
201     return 0;
202 
203   Bool /* mode64, */ be;
204   switch (GetUi32(buf))
205   {
206     case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break;
207     case MACH_SIG_BE_64: /* mode64 = True;  */ be = True; break;
208     case MACH_SIG_LE_32: /* mode64 = False; */ be = False; break;
209     case MACH_SIG_LE_64: /* mode64 = True;  */ be = False; break;
210     default: return 0;
211   }
212 
213   switch (Get32(buf + 4, be))
214   {
215     case MACH_MACHINE_386:
216     case MACH_MACHINE_AMD64: filterId = k_X86; break;
217     case MACH_MACHINE_ARM:   if ( be) return 0; filterId = k_ARM; break;
218     case MACH_MACHINE_SPARC: if (!be) return 0; filterId = k_SPARC; break;
219     case MACH_MACHINE_PPC:
220     case MACH_MACHINE_PPC64: if (!be) return 0; filterId = k_PPC; break;
221     default: return 0;
222   }
223 
224   numCommands = Get32(buf + 0x10, be);
225   commandsSize = Get32(buf + 0x14, be);
226 
227   if (commandsSize > (1 << 24) || numCommands > (1 << 18))
228     return 0;
229 
230   filterMode->Id = filterId;
231   return 1;
232 }
233 
234 
235 /* ---------- WAV ---------- */
236 
237 #define WAV_SUBCHUNK_fmt  0x20746D66
238 #define WAV_SUBCHUNK_data 0x61746164
239 
240 #define RIFF_SIG 0x46464952
241 
Parse_WAV(const Byte * buf,size_t size,CFilterMode * filterMode)242 static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode)
243 {
244   UInt32 subChunkSize, pos;
245   if (size < 0x2C)
246     return False;
247 
248   if (GetUi32(buf + 0) != RIFF_SIG ||
249       GetUi32(buf + 8) != 0x45564157 || // WAVE
250       GetUi32(buf + 0xC) != WAV_SUBCHUNK_fmt)
251     return False;
252   subChunkSize = GetUi32(buf + 0x10);
253   /* [0x14 = format] = 1 (PCM) */
254   if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1)
255     return False;
256 
257   unsigned numChannels = GetUi16(buf + 0x16);
258   unsigned bitsPerSample = GetUi16(buf + 0x22);
259 
260   if ((bitsPerSample & 0x7) != 0 || bitsPerSample >= 256 || numChannels >= 256)
261     return False;
262 
263   pos = 0x14 + subChunkSize;
264 
265   const int kNumSubChunksTests = 10;
266   // Do we need to scan more than 3 sub-chunks?
267   for (int i = 0; i < kNumSubChunksTests; i++)
268   {
269     if (pos + 8 > size)
270       return False;
271     subChunkSize = GetUi32(buf + pos + 4);
272     if (GetUi32(buf + pos) == WAV_SUBCHUNK_data)
273     {
274       unsigned delta = numChannels * (bitsPerSample >> 3);
275       if (delta >= 256)
276         return False;
277       filterMode->Id = k_Delta;
278       filterMode->Delta = delta;
279       return True;
280     }
281     if (subChunkSize > (1 << 16))
282       return False;
283     pos += subChunkSize + 8;
284   }
285   return False;
286 }
287 
ParseFile(const Byte * buf,size_t size,CFilterMode * filterMode)288 static Bool ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode)
289 {
290   filterMode->Id = 0;
291   filterMode->Delta = 0;
292 
293   if (Parse_EXE(buf, size, filterMode)) return True;
294   if (Parse_ELF(buf, size, filterMode)) return True;
295   if (Parse_MACH(buf, size, filterMode)) return True;
296   return Parse_WAV(buf, size, filterMode);
297 }
298 
299 
300 
301 
302 struct CFilterMode2: public CFilterMode
303 {
304   bool Encrypted;
305   unsigned GroupIndex;
306 
CFilterMode2NArchive::N7z::CFilterMode2307   CFilterMode2(): Encrypted(false) {}
308 
CompareNArchive::N7z::CFilterMode2309   int Compare(const CFilterMode2 &m) const
310   {
311     if (!Encrypted)
312     {
313       if (m.Encrypted)
314         return -1;
315     }
316     else if (!m.Encrypted)
317       return 1;
318 
319     if (Id < m.Id) return -1;
320     if (Id > m.Id) return 1;
321 
322     if (Delta < m.Delta) return -1;
323     if (Delta > m.Delta) return 1;
324 
325     return 0;
326   }
327 
operator ==NArchive::N7z::CFilterMode2328   bool operator ==(const CFilterMode2 &m) const
329   {
330     return Id == m.Id && Delta == m.Delta && Encrypted == m.Encrypted;
331   }
332 };
333 
GetGroup(CRecordVector<CFilterMode2> & filters,const CFilterMode2 & m)334 static unsigned GetGroup(CRecordVector<CFilterMode2> &filters, const CFilterMode2 &m)
335 {
336   unsigned i;
337   for (i = 0; i < filters.Size(); i++)
338   {
339     const CFilterMode2 &m2 = filters[i];
340     if (m == m2)
341       return i;
342     /*
343     if (m.Encrypted != m2.Encrypted)
344     {
345       if (!m.Encrypted)
346         break;
347       continue;
348     }
349 
350     if (m.Id < m2.Id)  break;
351     if (m.Id != m2.Id) continue;
352 
353     if (m.Delta < m2.Delta) break;
354     if (m.Delta != m2.Delta) continue;
355     */
356   }
357   // filters.Insert(i, m);
358   // return i;
359   return filters.Add(m);
360 }
361 
Is86Filter(CMethodId m)362 static inline bool Is86Filter(CMethodId m)
363 {
364   return (m == k_BCJ || m == k_BCJ2);
365 }
366 
IsExeFilter(CMethodId m)367 static inline bool IsExeFilter(CMethodId m)
368 {
369   switch (m)
370   {
371     case k_BCJ:
372     case k_BCJ2:
373     case k_ARM:
374     case k_ARMT:
375     case k_PPC:
376     case k_SPARC:
377     case k_IA64:
378       return true;
379   }
380   return false;
381 }
382 
Get_FilterGroup_for_Folder(CRecordVector<CFilterMode2> & filters,const CFolderEx & f,bool extractFilter)383 static unsigned Get_FilterGroup_for_Folder(
384     CRecordVector<CFilterMode2> &filters, const CFolderEx &f, bool extractFilter)
385 {
386   CFilterMode2 m;
387   m.Id = 0;
388   m.Delta = 0;
389   m.Encrypted = f.IsEncrypted();
390 
391   if (extractFilter)
392   {
393     const CCoderInfo &coder = f.Coders[f.UnpackCoder];
394 
395     if (coder.MethodID == k_Delta)
396     {
397       if (coder.Props.Size() == 1)
398       {
399         m.Delta = (unsigned)coder.Props[0] + 1;
400         m.Id = k_Delta;
401       }
402     }
403     else if (IsExeFilter(coder.MethodID))
404     {
405       m.Id = (UInt32)coder.MethodID;
406       if (m.Id == k_BCJ2)
407         m.Id = k_BCJ;
408       m.SetDelta();
409     }
410   }
411 
412   return GetGroup(filters, m);
413 }
414 
415 
416 
417 
WriteRange(IInStream * inStream,ISequentialOutStream * outStream,UInt64 position,UInt64 size,ICompressProgressInfo * progress)418 static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
419     UInt64 position, UInt64 size, ICompressProgressInfo *progress)
420 {
421   RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0));
422   CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream;
423   CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec);
424   streamSpec->SetStream(inStream);
425   streamSpec->Init(size);
426 
427   NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder;
428   CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec;
429   RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress));
430   return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL);
431 }
432 
433 /*
434 unsigned CUpdateItem::GetExtensionPos() const
435 {
436   int slashPos = Name.ReverseFind_PathSepar();
437   int dotPos = Name.ReverseFind_Dot();
438   if (dotPos <= slashPos)
439     return Name.Len();
440   return dotPos + 1;
441 }
442 
443 UString CUpdateItem::GetExtension() const
444 {
445   return Name.Ptr(GetExtensionPos());
446 }
447 */
448 
449 #define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
450 
451 #define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
452 
453 /*
454 static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
455 {
456   size_t c1 = a1.GetCapacity();
457   size_t c2 = a2.GetCapacity();
458   RINOZ_COMP(c1, c2);
459   for (size_t i = 0; i < c1; i++)
460     RINOZ_COMP(a1[i], a2[i]);
461   return 0;
462 }
463 
464 static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
465 {
466   RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
467   RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
468   RINOZ_COMP(c1.MethodID, c2.MethodID);
469   return CompareBuffers(c1.Props, c2.Props);
470 }
471 
472 static int CompareBonds(const CBond &b1, const CBond &b2)
473 {
474   RINOZ_COMP(b1.InIndex, b2.InIndex);
475   return MyCompare(b1.OutIndex, b2.OutIndex);
476 }
477 
478 static int CompareFolders(const CFolder &f1, const CFolder &f2)
479 {
480   int s1 = f1.Coders.Size();
481   int s2 = f2.Coders.Size();
482   RINOZ_COMP(s1, s2);
483   int i;
484   for (i = 0; i < s1; i++)
485     RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
486   s1 = f1.Bonds.Size();
487   s2 = f2.Bonds.Size();
488   RINOZ_COMP(s1, s2);
489   for (i = 0; i < s1; i++)
490     RINOZ(CompareBonds(f1.Bonds[i], f2.Bonds[i]));
491   return 0;
492 }
493 */
494 
495 /*
496 static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
497 {
498   return CompareFileNames(f1.Name, f2.Name);
499 }
500 */
501 
502 struct CFolderRepack
503 {
504   unsigned FolderIndex;
505   CNum NumCopyFiles;
506 };
507 
508 /*
509 static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *)
510 {
511   int i1 = p1->FolderIndex;
512   int i2 = p2->FolderIndex;
513   // In that version we don't want to parse folders here, so we don't compare folders
514   // probably it must be improved in future
515   // const CDbEx &db = *(const CDbEx *)param;
516   // RINOZ(CompareFolders(
517   //     db.Folders[i1],
518   //     db.Folders[i2]));
519 
520   return MyCompare(i1, i2);
521 
522   // RINOZ_COMP(
523   //     db.NumUnpackStreamsVector[i1],
524   //     db.NumUnpackStreamsVector[i2]);
525   // if (db.NumUnpackStreamsVector[i1] == 0)
526   //   return 0;
527   // return CompareFiles(
528   //     db.Files[db.FolderStartFileIndex[i1]],
529   //     db.Files[db.FolderStartFileIndex[i2]]);
530 }
531 */
532 
533 /*
534   we sort empty files and dirs in such order:
535   - Dir.NonAnti   (name sorted)
536   - File.NonAnti  (name sorted)
537   - File.Anti     (name sorted)
538   - Dir.Anti (reverse name sorted)
539 */
540 
CompareEmptyItems(const unsigned * p1,const unsigned * p2,void * param)541 static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param)
542 {
543   const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
544   const CUpdateItem &u1 = updateItems[*p1];
545   const CUpdateItem &u2 = updateItems[*p2];
546   // NonAnti < Anti
547   if (u1.IsAnti != u2.IsAnti)
548     return (u1.IsAnti ? 1 : -1);
549   if (u1.IsDir != u2.IsDir)
550   {
551     // Dir.NonAnti < File < Dir.Anti
552     if (u1.IsDir)
553       return (u1.IsAnti ? 1 : -1);
554     return (u2.IsAnti ? -1 : 1);
555   }
556   int n = CompareFileNames(u1.Name, u2.Name);
557   return (u1.IsDir && u1.IsAnti) ? -n : n;
558 }
559 
560 static const char *g_Exts =
561   " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
562   " zip jar ear war msi"
563   " 3gp avi mov mpeg mpg mpe wmv"
564   " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
565   " swf"
566   " chm hxi hxs"
567   " gif jpeg jpg jp2 png tiff  bmp ico psd psp"
568   " awg ps eps cgm dxf svg vrml wmf emf ai md"
569   " cad dwg pps key sxi"
570   " max 3ds"
571   " iso bin nrg mdf img pdi tar cpio xpi"
572   " vfd vhd vud vmc vsv"
573   " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
574   " inl inc idl acf asa"
575   " h hpp hxx c cpp cxx m mm go swift"
576   " rc java cs rs pas bas vb cls ctl frm dlg def"
577   " f77 f f90 f95"
578   " asm s"
579   " sql manifest dep"
580   " mak clw csproj vcproj sln dsp dsw"
581   " class"
582   " bat cmd bash sh"
583   " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
584   " awk sed hta js json php php3 php4 php5 phptml pl pm py pyo rb tcl ts vbs"
585   " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
586   " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
587   " abw afp cwk lwp wpd wps wpt wrf wri"
588   " abf afm bdf fon mgf otf pcf pfa snf ttf"
589   " dbf mdb nsf ntf wdb db fdb gdb"
590   " exe dll ocx vbx sfx sys tlb awx com obj lib out o so"
591   " pdb pch idb ncb opt";
592 
GetExtIndex(const char * ext)593 static unsigned GetExtIndex(const char *ext)
594 {
595   unsigned extIndex = 1;
596   const char *p = g_Exts;
597   for (;;)
598   {
599     char c = *p++;
600     if (c == 0)
601       return extIndex;
602     if (c == ' ')
603       continue;
604     unsigned pos = 0;
605     for (;;)
606     {
607       char c2 = ext[pos++];
608       if (c2 == 0 && (c == 0 || c == ' '))
609         return extIndex;
610       if (c != c2)
611         break;
612       c = *p++;
613     }
614     extIndex++;
615     for (;;)
616     {
617       if (c == 0)
618         return extIndex;
619       if (c == ' ')
620         break;
621       c = *p++;
622     }
623   }
624 }
625 
626 struct CRefItem
627 {
628   const CUpdateItem *UpdateItem;
629   UInt32 Index;
630   unsigned ExtensionPos;
631   unsigned NamePos;
632   unsigned ExtensionIndex;
633 
CRefItemNArchive::N7z::CRefItem634   CRefItem() {};
CRefItemNArchive::N7z::CRefItem635   CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
636     UpdateItem(&ui),
637     Index(index),
638     ExtensionPos(0),
639     NamePos(0),
640     ExtensionIndex(0)
641   {
642     if (sortByType)
643     {
644       int slashPos = ui.Name.ReverseFind_PathSepar();
645       NamePos = slashPos + 1;
646       int dotPos = ui.Name.ReverseFind_Dot();
647       if (dotPos <= slashPos)
648         ExtensionPos = ui.Name.Len();
649       else
650       {
651         ExtensionPos = dotPos + 1;
652         if (ExtensionPos != ui.Name.Len())
653         {
654           AString s;
655           for (unsigned pos = ExtensionPos;; pos++)
656           {
657             wchar_t c = ui.Name[pos];
658             if (c >= 0x80)
659               break;
660             if (c == 0)
661             {
662               ExtensionIndex = GetExtIndex(s);
663               break;
664             }
665             s += (char)MyCharLower_Ascii((char)c);
666           }
667         }
668       }
669     }
670   }
671 };
672 
673 struct CSortParam
674 {
675   // const CObjectVector<CTreeFolder> *TreeFolders;
676   bool SortByType;
677 };
678 
679 /*
680   we sort files in such order:
681   - Dir.NonAnti   (name sorted)
682   - alt streams
683   - Dirs
684   - Dir.Anti (reverse name sorted)
685 */
686 
687 
CompareUpdateItems(const CRefItem * p1,const CRefItem * p2,void * param)688 static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
689 {
690   const CRefItem &a1 = *p1;
691   const CRefItem &a2 = *p2;
692   const CUpdateItem &u1 = *a1.UpdateItem;
693   const CUpdateItem &u2 = *a2.UpdateItem;
694 
695   /*
696   if (u1.IsAltStream != u2.IsAltStream)
697     return u1.IsAltStream ? 1 : -1;
698   */
699 
700   // Actually there are no dirs that time. They were stored in other steps
701   // So that code is unused?
702   if (u1.IsDir != u2.IsDir)
703     return u1.IsDir ? 1 : -1;
704   if (u1.IsDir)
705   {
706     if (u1.IsAnti != u2.IsAnti)
707       return (u1.IsAnti ? 1 : -1);
708     int n = CompareFileNames(u1.Name, u2.Name);
709     return -n;
710   }
711 
712   // bool sortByType = *(bool *)param;
713   const CSortParam *sortParam = (const CSortParam *)param;
714   bool sortByType = sortParam->SortByType;
715   if (sortByType)
716   {
717     RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex);
718     RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos)));
719     RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos)));
720     if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
721     if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
722     if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime);
723     RINOZ_COMP(u1.Size, u2.Size);
724   }
725   /*
726   int par1 = a1.UpdateItem->ParentFolderIndex;
727   int par2 = a2.UpdateItem->ParentFolderIndex;
728   const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1];
729   const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2];
730 
731   int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd;
732   int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd;
733   if (b1 < b2)
734   {
735     if (e1 <= b2)
736       return -1;
737     // p2 in p1
738     int par = par2;
739     for (;;)
740     {
741       const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
742       par = tf.Parent;
743       if (par == par1)
744       {
745         RINOZ(CompareFileNames(u1.Name, tf.Name));
746         break;
747       }
748     }
749   }
750   else if (b2 < b1)
751   {
752     if (e2 <= b1)
753       return 1;
754     // p1 in p2
755     int par = par1;
756     for (;;)
757     {
758       const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
759       par = tf.Parent;
760       if (par == par2)
761       {
762         RINOZ(CompareFileNames(tf.Name, u2.Name));
763         break;
764       }
765     }
766   }
767   */
768   // RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex);
769   RINOK(CompareFileNames(u1.Name, u2.Name));
770   RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient);
771   RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive);
772   return 0;
773 }
774 
775 struct CSolidGroup
776 {
777   CRecordVector<UInt32> Indices;
778 
779   CRecordVector<CFolderRepack> folderRefs;
780 };
781 
782 static const char * const g_ExeExts[] =
783 {
784     "dll"
785   , "exe"
786   , "ocx"
787   , "sfx"
788   , "sys"
789 };
790 
IsExeExt(const wchar_t * ext)791 static bool IsExeExt(const wchar_t *ext)
792 {
793   for (unsigned i = 0; i < ARRAY_SIZE(g_ExeExts); i++)
794     if (StringsAreEqualNoCase_Ascii(ext, g_ExeExts[i]))
795       return true;
796   return false;
797 }
798 
799 struct CAnalysis
800 {
801   CMyComPtr<IArchiveUpdateCallbackFile> Callback;
802   CByteBuffer Buffer;
803 
804   bool ParseWav;
805   bool ParseExe;
806   bool ParseAll;
807 
CAnalysisNArchive::N7z::CAnalysis808   CAnalysis():
809       ParseWav(true),
810       ParseExe(false),
811       ParseAll(false)
812   {}
813 
814   HRESULT GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode);
815 };
816 
817 static const size_t kAnalysisBufSize = 1 << 14;
818 
GetFilterGroup(UInt32 index,const CUpdateItem & ui,CFilterMode & filterMode)819 HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode)
820 {
821   filterMode.Id = 0;
822   filterMode.Delta = 0;
823 
824   CFilterMode filterModeTemp = filterMode;
825 
826   int slashPos = ui.Name.ReverseFind_PathSepar();
827   int dotPos = ui.Name.ReverseFind_Dot();
828 
829   // if (dotPos > slashPos)
830   {
831     bool needReadFile = ParseAll;
832 
833     bool probablyIsSameIsa = false;
834 
835     if (!needReadFile || !Callback)
836     {
837       const wchar_t *ext;
838       if (dotPos > slashPos)
839         ext = ui.Name.Ptr(dotPos + 1);
840       else
841         ext = ui.Name.RightPtr(0);
842 
843       // p7zip uses the trick to store posix attributes in high 16 bits
844       if (ui.Attrib & 0x8000)
845       {
846         unsigned st_mode = ui.Attrib >> 16;
847         // st_mode = 00111;
848         if ((st_mode & 00111) && (ui.Size >= 2048))
849         {
850           #ifndef _WIN32
851           probablyIsSameIsa = true;
852           #endif
853           needReadFile = true;
854         }
855       }
856 
857       if (IsExeExt(ext))
858       {
859         needReadFile = true;
860         #ifdef _WIN32
861         probablyIsSameIsa = true;
862         needReadFile = ParseExe;
863         #endif
864       }
865       else if (StringsAreEqualNoCase_Ascii(ext, "wav"))
866       {
867         needReadFile = ParseWav;
868       }
869       /*
870       else if (!needReadFile && ParseUnixExt)
871       {
872         if (StringsAreEqualNoCase_Ascii(ext, "so")
873           || StringsAreEqualNoCase_Ascii(ext, ""))
874 
875           needReadFile = true;
876       }
877       */
878     }
879 
880     if (needReadFile && Callback)
881     {
882       if (Buffer.Size() != kAnalysisBufSize)
883       {
884         Buffer.Alloc(kAnalysisBufSize);
885       }
886       {
887         CMyComPtr<ISequentialInStream> stream;
888         HRESULT result = Callback->GetStream2(index, &stream, NUpdateNotifyOp::kAnalyze);
889         if (result == S_OK && stream)
890         {
891           size_t size = kAnalysisBufSize;
892           result = ReadStream(stream, Buffer, &size);
893           stream.Release();
894           // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK));
895           if (result == S_OK)
896           {
897             Bool parseRes = ParseFile(Buffer, size, &filterModeTemp);
898             if (parseRes && filterModeTemp.Delta == 0)
899             {
900               filterModeTemp.SetDelta();
901               if (filterModeTemp.Delta != 0 && filterModeTemp.Id != k_Delta)
902               {
903                 if (ui.Size % filterModeTemp.Delta != 0)
904                 {
905                   parseRes = false;
906                 }
907               }
908             }
909             if (!parseRes)
910             {
911               filterModeTemp.Id = 0;
912               filterModeTemp.Delta = 0;
913             }
914           }
915         }
916       }
917     }
918     else if ((needReadFile && !Callback) || probablyIsSameIsa)
919     {
920       #ifdef MY_CPU_X86_OR_AMD64
921       if (probablyIsSameIsa)
922         filterModeTemp.Id = k_X86;
923       #endif
924     }
925   }
926 
927   filterMode = filterModeTemp;
928   return S_OK;
929 }
930 
GetMethodFull(UInt64 methodID,UInt32 numStreams,CMethodFull & m)931 static inline void GetMethodFull(UInt64 methodID, UInt32 numStreams, CMethodFull &m)
932 {
933   m.Id = methodID;
934   m.NumStreams = numStreams;
935 }
936 
AddBondForFilter(CCompressionMethodMode & mode)937 static HRESULT AddBondForFilter(CCompressionMethodMode &mode)
938 {
939   for (unsigned c = 1; c < mode.Methods.Size(); c++)
940   {
941     if (!mode.IsThereBond_to_Coder(c))
942     {
943       CBond2 bond;
944       bond.OutCoder = 0;
945       bond.OutStream = 0;
946       bond.InCoder = c;
947       mode.Bonds.Add(bond);
948       return S_OK;
949     }
950   }
951   return E_INVALIDARG;
952 }
953 
AddFilterBond(CCompressionMethodMode & mode)954 static HRESULT AddFilterBond(CCompressionMethodMode &mode)
955 {
956   if (!mode.Bonds.IsEmpty())
957     return AddBondForFilter(mode);
958   return S_OK;
959 }
960 
AddBcj2Methods(CCompressionMethodMode & mode)961 static HRESULT AddBcj2Methods(CCompressionMethodMode &mode)
962 {
963   // mode.Methods[0] must be k_BCJ2 method !
964 
965   CMethodFull m;
966   GetMethodFull(k_LZMA, 1, m);
967 
968   m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20);
969   m.AddProp32(NCoderPropID::kNumFastBytes, 128);
970   m.AddProp32(NCoderPropID::kNumThreads, 1);
971   m.AddProp32(NCoderPropID::kLitPosBits, 2);
972   m.AddProp32(NCoderPropID::kLitContextBits, 0);
973   // m.AddProp_Ascii(NCoderPropID::kMatchFinder, "BT2");
974 
975   unsigned methodIndex = mode.Methods.Size();
976 
977   if (mode.Bonds.IsEmpty())
978   {
979     for (unsigned i = 1; i + 1 < mode.Methods.Size(); i++)
980     {
981       CBond2 bond;
982       bond.OutCoder = i;
983       bond.OutStream = 0;
984       bond.InCoder = i + 1;
985       mode.Bonds.Add(bond);
986     }
987   }
988 
989   mode.Methods.Add(m);
990   mode.Methods.Add(m);
991 
992   RINOK(AddBondForFilter(mode));
993   CBond2 bond;
994   bond.OutCoder = 0;
995   bond.InCoder = methodIndex;      bond.OutStream = 1;  mode.Bonds.Add(bond);
996   bond.InCoder = methodIndex + 1;  bond.OutStream = 2;  mode.Bonds.Add(bond);
997   return S_OK;
998 }
999 
MakeExeMethod(CCompressionMethodMode & mode,const CFilterMode & filterMode,bool bcj2Filter)1000 static HRESULT MakeExeMethod(CCompressionMethodMode &mode,
1001     const CFilterMode &filterMode, /* bool addFilter, */ bool bcj2Filter)
1002 {
1003   if (mode.Filter_was_Inserted)
1004   {
1005     const CMethodFull &m = mode.Methods[0];
1006     CMethodId id = m.Id;
1007     if (id == k_BCJ2)
1008       return AddBcj2Methods(mode);
1009     if (!m.IsSimpleCoder())
1010       return E_NOTIMPL;
1011     // if (Bonds.IsEmpty()) we can create bonds later
1012     return AddFilterBond(mode);
1013   }
1014 
1015   if (filterMode.Id == 0)
1016     return S_OK;
1017 
1018   CMethodFull &m = mode.Methods.InsertNew(0);
1019 
1020   {
1021     FOR_VECTOR(k, mode.Bonds)
1022     {
1023       CBond2 &bond = mode.Bonds[k];
1024       bond.InCoder++;
1025       bond.OutCoder++;
1026     }
1027   }
1028 
1029   HRESULT res;
1030 
1031   if (bcj2Filter && Is86Filter(filterMode.Id))
1032   {
1033     GetMethodFull(k_BCJ2, 4, m);
1034     res = AddBcj2Methods(mode);
1035   }
1036   else
1037   {
1038     GetMethodFull(filterMode.Id, 1, m);
1039     if (filterMode.Id == k_Delta)
1040       m.AddProp32(NCoderPropID::kDefaultProp, filterMode.Delta);
1041     res = AddFilterBond(mode);
1042 
1043     int alignBits = -1;
1044     if (filterMode.Id == k_Delta || filterMode.Delta != 0)
1045     {
1046            if (filterMode.Delta == 1) alignBits = 0;
1047       else if (filterMode.Delta == 2) alignBits = 1;
1048       else if (filterMode.Delta == 4) alignBits = 2;
1049       else if (filterMode.Delta == 8) alignBits = 3;
1050       else if (filterMode.Delta == 16) alignBits = 4;
1051     }
1052     else
1053     {
1054       // alignBits = GetAlignForFilterMethod(filterMode.Id);
1055     }
1056 
1057     if (res == S_OK && alignBits >= 0)
1058     {
1059       unsigned nextCoder = 1;
1060       if (!mode.Bonds.IsEmpty())
1061       {
1062         nextCoder = mode.Bonds.Back().InCoder;
1063       }
1064       if (nextCoder < mode.Methods.Size())
1065       {
1066         CMethodFull &nextMethod = mode.Methods[nextCoder];
1067         if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2)
1068         {
1069           if (!nextMethod.Are_Lzma_Model_Props_Defined())
1070           {
1071             if (alignBits != 0)
1072             {
1073               if (alignBits > 2 || filterMode.Id == k_Delta)
1074                 nextMethod.AddProp32(NCoderPropID::kPosStateBits, alignBits);
1075               unsigned lc = 0;
1076               if (alignBits < 3)
1077                 lc = 3 - alignBits;
1078               nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc);
1079               nextMethod.AddProp32(NCoderPropID::kLitPosBits, alignBits);
1080             }
1081           }
1082         }
1083       }
1084     }
1085   }
1086 
1087   return res;
1088 }
1089 
1090 
FromUpdateItemToFileItem(const CUpdateItem & ui,CFileItem & file,CFileItem2 & file2)1091 static void FromUpdateItemToFileItem(const CUpdateItem &ui,
1092     CFileItem &file, CFileItem2 &file2)
1093 {
1094   if (ui.AttribDefined)
1095     file.SetAttrib(ui.Attrib);
1096 
1097   file2.CTime = ui.CTime;  file2.CTimeDefined = ui.CTimeDefined;
1098   file2.ATime = ui.ATime;  file2.ATimeDefined = ui.ATimeDefined;
1099   file2.MTime = ui.MTime;  file2.MTimeDefined = ui.MTimeDefined;
1100   file2.IsAnti = ui.IsAnti;
1101   // file2.IsAux = false;
1102   file2.StartPosDefined = false;
1103 
1104   file.Size = ui.Size;
1105   file.IsDir = ui.IsDir;
1106   file.HasStream = ui.HasStream();
1107   // file.IsAltStream = ui.IsAltStream;
1108 }
1109 
1110 class CRepackInStreamWithSizes:
1111   public ISequentialInStream,
1112   public ICompressGetSubStreamSize,
1113   public CMyUnknownImp
1114 {
1115   CMyComPtr<ISequentialInStream> _stream;
1116   // UInt64 _size;
1117   const CBoolVector *_extractStatuses;
1118   UInt32 _startIndex;
1119 public:
1120   const CDbEx *_db;
1121 
Init(ISequentialInStream * stream,UInt32 startIndex,const CBoolVector * extractStatuses)1122   void Init(ISequentialInStream *stream, UInt32 startIndex, const CBoolVector *extractStatuses)
1123   {
1124     _startIndex = startIndex;
1125     _extractStatuses = extractStatuses;
1126     // _size = 0;
1127     _stream = stream;
1128   }
1129   // UInt64 GetSize() const { return _size; }
1130 
1131   MY_UNKNOWN_IMP2(ISequentialInStream, ICompressGetSubStreamSize)
1132 
1133   STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
1134 
1135   STDMETHOD(GetSubStreamSize)(UInt64 subStream, UInt64 *value);
1136 };
1137 
Read(void * data,UInt32 size,UInt32 * processedSize)1138 STDMETHODIMP CRepackInStreamWithSizes::Read(void *data, UInt32 size, UInt32 *processedSize)
1139 {
1140   return _stream->Read(data, size, processedSize);
1141   /*
1142   UInt32 realProcessedSize;
1143   HRESULT result = _stream->Read(data, size, &realProcessedSize);
1144   _size += realProcessedSize;
1145   if (processedSize)
1146     *processedSize = realProcessedSize;
1147   return result;
1148   */
1149 }
1150 
GetSubStreamSize(UInt64 subStream,UInt64 * value)1151 STDMETHODIMP CRepackInStreamWithSizes::GetSubStreamSize(UInt64 subStream, UInt64 *value)
1152 {
1153   *value = 0;
1154   if (subStream >= _extractStatuses->Size())
1155     return S_FALSE; // E_FAIL;
1156   unsigned index = (unsigned)subStream;
1157   if ((*_extractStatuses)[index])
1158   {
1159     const CFileItem &fi = _db->Files[_startIndex + index];
1160     if (fi.HasStream)
1161       *value = fi.Size;
1162   }
1163   return S_OK;
1164 }
1165 
1166 
1167 class CRepackStreamBase
1168 {
1169 protected:
1170   bool _needWrite;
1171   bool _fileIsOpen;
1172   bool _calcCrc;
1173   UInt32 _crc;
1174   UInt64 _rem;
1175 
1176   const CBoolVector *_extractStatuses;
1177   UInt32 _startIndex;
1178   unsigned _currentIndex;
1179 
1180   HRESULT OpenFile();
1181   HRESULT CloseFile();
1182   HRESULT ProcessEmptyFiles();
1183 
1184 public:
1185   const CDbEx *_db;
1186   CMyComPtr<IArchiveUpdateCallbackFile> _opCallback;
1187   CMyComPtr<IArchiveExtractCallbackMessage> _extractCallback;
1188 
1189   HRESULT Init(UInt32 startIndex, const CBoolVector *extractStatuses);
CheckFinishedState() const1190   HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
1191 };
1192 
Init(UInt32 startIndex,const CBoolVector * extractStatuses)1193 HRESULT CRepackStreamBase::Init(UInt32 startIndex, const CBoolVector *extractStatuses)
1194 {
1195   _startIndex = startIndex;
1196   _extractStatuses = extractStatuses;
1197 
1198   _currentIndex = 0;
1199   _fileIsOpen = false;
1200 
1201   return ProcessEmptyFiles();
1202 }
1203 
OpenFile()1204 HRESULT CRepackStreamBase::OpenFile()
1205 {
1206   UInt32 arcIndex = _startIndex + _currentIndex;
1207   const CFileItem &fi = _db->Files[arcIndex];
1208 
1209   _needWrite = (*_extractStatuses)[_currentIndex];
1210   if (_opCallback)
1211   {
1212     RINOK(_opCallback->ReportOperation(
1213         NEventIndexType::kInArcIndex, arcIndex,
1214         _needWrite ?
1215             NUpdateNotifyOp::kRepack :
1216             NUpdateNotifyOp::kSkip));
1217   }
1218 
1219   _crc = CRC_INIT_VAL;
1220   _calcCrc = (fi.CrcDefined && !fi.IsDir);
1221 
1222   _fileIsOpen = true;
1223   _rem = fi.Size;
1224   return S_OK;
1225 }
1226 
1227 const HRESULT k_My_HRESULT_CRC_ERROR = 0x20000002;
1228 
CloseFile()1229 HRESULT CRepackStreamBase::CloseFile()
1230 {
1231   UInt32 arcIndex = _startIndex + _currentIndex;
1232   const CFileItem &fi = _db->Files[arcIndex];
1233   _fileIsOpen = false;
1234   _currentIndex++;
1235   if (!_calcCrc || fi.Crc == CRC_GET_DIGEST(_crc))
1236     return S_OK;
1237 
1238   if (_extractCallback)
1239   {
1240     RINOK(_extractCallback->ReportExtractResult(
1241         NEventIndexType::kInArcIndex, arcIndex,
1242         NExtract::NOperationResult::kCRCError));
1243   }
1244   // return S_FALSE;
1245   return k_My_HRESULT_CRC_ERROR;
1246 }
1247 
ProcessEmptyFiles()1248 HRESULT CRepackStreamBase::ProcessEmptyFiles()
1249 {
1250   while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
1251   {
1252     RINOK(OpenFile());
1253     RINOK(CloseFile());
1254   }
1255   return S_OK;
1256 }
1257 
1258 
1259 
1260 #ifndef _7ZIP_ST
1261 
1262 class CFolderOutStream2:
1263   public CRepackStreamBase,
1264   public ISequentialOutStream,
1265   public CMyUnknownImp
1266 {
1267 public:
1268   CMyComPtr<ISequentialOutStream> _stream;
1269 
1270   MY_UNKNOWN_IMP
1271 
1272   STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
1273 };
1274 
Write(const void * data,UInt32 size,UInt32 * processedSize)1275 STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize)
1276 {
1277   if (processedSize)
1278     *processedSize = 0;
1279 
1280   while (size != 0)
1281   {
1282     if (_fileIsOpen)
1283     {
1284       UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1285       HRESULT result = S_OK;
1286       if (_needWrite)
1287         result = _stream->Write(data, cur, &cur);
1288       if (_calcCrc)
1289         _crc = CrcUpdate(_crc, data, cur);
1290       if (processedSize)
1291         *processedSize += cur;
1292       data = (const Byte *)data + cur;
1293       size -= cur;
1294       _rem -= cur;
1295       if (_rem == 0)
1296       {
1297         RINOK(CloseFile());
1298         RINOK(ProcessEmptyFiles());
1299       }
1300       RINOK(result);
1301       if (cur == 0)
1302         break;
1303       continue;
1304     }
1305 
1306     RINOK(ProcessEmptyFiles());
1307     if (_currentIndex == _extractStatuses->Size())
1308     {
1309       // we don't support write cut here
1310       return E_FAIL;
1311     }
1312     RINOK(OpenFile());
1313   }
1314 
1315   return S_OK;
1316 }
1317 
1318 #endif
1319 
1320 
1321 
1322 static const UInt32 kTempBufSize = 1 << 16;
1323 
1324 class CFolderInStream2:
1325   public CRepackStreamBase,
1326   public ISequentialInStream,
1327   public CMyUnknownImp
1328 {
1329   Byte *_buf;
1330 public:
1331   CMyComPtr<ISequentialInStream> _inStream;
1332   HRESULT Result;
1333 
1334   MY_UNKNOWN_IMP
1335 
CFolderInStream2()1336   CFolderInStream2():
1337       Result(S_OK)
1338   {
1339     _buf = new Byte[kTempBufSize];
1340   }
1341 
~CFolderInStream2()1342   ~CFolderInStream2()
1343   {
1344     delete []_buf;
1345   }
1346 
Init()1347   void Init() { Result = S_OK; }
1348   STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
1349 };
1350 
Read(void * data,UInt32 size,UInt32 * processedSize)1351 STDMETHODIMP CFolderInStream2::Read(void *data, UInt32 size, UInt32 *processedSize)
1352 {
1353   if (processedSize)
1354     *processedSize = 0;
1355 
1356   while (size != 0)
1357   {
1358     if (_fileIsOpen)
1359     {
1360       UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1361 
1362       void *buf;
1363       if (_needWrite)
1364         buf = data;
1365       else
1366       {
1367         buf = _buf;
1368         if (cur > kTempBufSize)
1369           cur = kTempBufSize;
1370       }
1371 
1372       HRESULT result = _inStream->Read(buf, cur, &cur);
1373       _crc = CrcUpdate(_crc, buf, cur);
1374       _rem -= cur;
1375 
1376       if (_needWrite)
1377       {
1378         data = (Byte *)data + cur;
1379         size -= cur;
1380         if (processedSize)
1381           *processedSize += cur;
1382       }
1383 
1384       if (result != S_OK)
1385         Result = result;
1386 
1387       if (_rem == 0)
1388       {
1389         RINOK(CloseFile());
1390         RINOK(ProcessEmptyFiles());
1391       }
1392 
1393       RINOK(result);
1394 
1395       if (cur == 0)
1396         return E_FAIL;
1397 
1398       continue;
1399     }
1400 
1401     RINOK(ProcessEmptyFiles());
1402     if (_currentIndex == _extractStatuses->Size())
1403     {
1404       return S_OK;
1405     }
1406     RINOK(OpenFile());
1407   }
1408 
1409   return S_OK;
1410 }
1411 
1412 
1413 class CThreadDecoder
1414   #ifndef _7ZIP_ST
1415     : public CVirtThread
1416   #endif
1417 {
1418 public:
1419   CDecoder Decoder;
1420 
CThreadDecoder(bool multiThreadMixer)1421   CThreadDecoder(bool multiThreadMixer):
1422       Decoder(multiThreadMixer)
1423   {
1424     #ifndef _7ZIP_ST
1425     if (multiThreadMixer)
1426     {
1427       MtMode = false;
1428       NumThreads = 1;
1429       FosSpec = new CFolderOutStream2;
1430       Fos = FosSpec;
1431       Result = E_FAIL;
1432     }
1433     #endif
1434     // UnpackSize = 0;
1435     // send_UnpackSize = false;
1436   }
1437 
1438   #ifndef _7ZIP_ST
1439 
1440   HRESULT Result;
1441   CMyComPtr<IInStream> InStream;
1442 
1443   CFolderOutStream2 *FosSpec;
1444   CMyComPtr<ISequentialOutStream> Fos;
1445 
1446   UInt64 StartPos;
1447   const CFolders *Folders;
1448   int FolderIndex;
1449 
1450   // bool send_UnpackSize;
1451   // UInt64 UnpackSize;
1452 
1453   #ifndef _NO_CRYPTO
1454   CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1455   #endif
1456 
1457   DECL_EXTERNAL_CODECS_LOC_VARS2;
1458 
1459   #ifndef _7ZIP_ST
1460   bool MtMode;
1461   UInt32 NumThreads;
1462   #endif
1463 
1464 
~CThreadDecoder()1465   ~CThreadDecoder() { CVirtThread::WaitThreadFinish(); }
1466   virtual void Execute();
1467 
1468   #endif
1469 };
1470 
1471 #ifndef _7ZIP_ST
1472 
Execute()1473 void CThreadDecoder::Execute()
1474 {
1475   try
1476   {
1477     #ifndef _NO_CRYPTO
1478       bool isEncrypted = false;
1479       bool passwordIsDefined = false;
1480       UString password;
1481     #endif
1482 
1483     Result = Decoder.Decode(
1484       EXTERNAL_CODECS_LOC_VARS
1485       InStream,
1486       StartPos,
1487       *Folders, FolderIndex,
1488 
1489       // send_UnpackSize ? &UnpackSize : NULL,
1490       NULL, // unpackSize : FULL unpack
1491 
1492       Fos,
1493       NULL, // compressProgress
1494       NULL  // *inStreamMainRes
1495 
1496       _7Z_DECODER_CRYPRO_VARS
1497       #ifndef _7ZIP_ST
1498         , MtMode, NumThreads
1499       #endif
1500       );
1501   }
1502   catch(...)
1503   {
1504     Result = E_FAIL;
1505   }
1506 
1507   /*
1508   if (Result == S_OK)
1509     Result = FosSpec->CheckFinishedState();
1510   */
1511   FosSpec->_stream.Release();
1512 }
1513 
1514 #endif
1515 
1516 #ifndef _NO_CRYPTO
1517 
1518 class CCryptoGetTextPassword:
1519   public ICryptoGetTextPassword,
1520   public CMyUnknownImp
1521 {
1522 public:
1523   UString Password;
1524 
1525   MY_UNKNOWN_IMP
1526   STDMETHOD(CryptoGetTextPassword)(BSTR *password);
1527 };
1528 
CryptoGetTextPassword(BSTR * password)1529 STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password)
1530 {
1531   return StringToBstr(Password, password);
1532 }
1533 
1534 #endif
1535 
1536 
GetFile(const CDatabase & inDb,unsigned index,CFileItem & file,CFileItem2 & file2)1537 static void GetFile(const CDatabase &inDb, unsigned index, CFileItem &file, CFileItem2 &file2)
1538 {
1539   file = inDb.Files[index];
1540   file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime);
1541   file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime);
1542   file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime);
1543   file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos);
1544   file2.IsAnti = inDb.IsItemAnti(index);
1545   // file2.IsAux = inDb.IsItemAux(index);
1546 }
1547 
Update(DECL_EXTERNAL_CODECS_LOC_VARS IInStream * inStream,const CDbEx * db,const CObjectVector<CUpdateItem> & updateItems,COutArchive & archive,CArchiveDatabaseOut & newDatabase,ISequentialOutStream * seqOutStream,IArchiveUpdateCallback * updateCallback,const CUpdateOptions & options,ICryptoGetTextPassword * getDecoderPassword)1548 HRESULT Update(
1549     DECL_EXTERNAL_CODECS_LOC_VARS
1550     IInStream *inStream,
1551     const CDbEx *db,
1552     const CObjectVector<CUpdateItem> &updateItems,
1553     // const CObjectVector<CTreeFolder> &treeFolders,
1554     // const CUniqBlocks &secureBlocks,
1555     COutArchive &archive,
1556     CArchiveDatabaseOut &newDatabase,
1557     ISequentialOutStream *seqOutStream,
1558     IArchiveUpdateCallback *updateCallback,
1559     const CUpdateOptions &options
1560     #ifndef _NO_CRYPTO
1561     , ICryptoGetTextPassword *getDecoderPassword
1562     #endif
1563     )
1564 {
1565   UInt64 numSolidFiles = options.NumSolidFiles;
1566   if (numSolidFiles == 0)
1567     numSolidFiles = 1;
1568 
1569   CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
1570   updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
1571 
1572   CMyComPtr<IArchiveExtractCallbackMessage> extractCallback;
1573   updateCallback->QueryInterface(IID_IArchiveExtractCallbackMessage, (void **)&extractCallback);
1574 
1575   // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes();
1576 
1577   /*
1578   CMyComPtr<IOutStream> outStream;
1579   RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream));
1580   if (!outStream)
1581     return E_NOTIMPL;
1582   */
1583 
1584   UInt64 startBlockSize = db ? db->ArcInfo.StartPosition: 0;
1585   if (startBlockSize > 0 && !options.RemoveSfxBlock)
1586   {
1587     RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL));
1588   }
1589 
1590   CIntArr fileIndexToUpdateIndexMap;
1591   UInt64 complexity = 0;
1592   UInt64 inSizeForReduce2 = 0;
1593   bool needEncryptedRepack = false;
1594 
1595   CRecordVector<CFilterMode2> filters;
1596   CObjectVector<CSolidGroup> groups;
1597   bool thereAreRepacks = false;
1598 
1599   bool useFilters = options.UseFilters;
1600   if (useFilters)
1601   {
1602     const CCompressionMethodMode &method = *options.Method;
1603 
1604     FOR_VECTOR (i, method.Methods)
1605       if (IsFilterMethod(method.Methods[i].Id))
1606       {
1607         useFilters = false;
1608         break;
1609       }
1610   }
1611 
1612   if (db)
1613   {
1614     fileIndexToUpdateIndexMap.Alloc(db->Files.Size());
1615     unsigned i;
1616 
1617     for (i = 0; i < db->Files.Size(); i++)
1618       fileIndexToUpdateIndexMap[i] = -1;
1619 
1620     for (i = 0; i < updateItems.Size(); i++)
1621     {
1622       int index = updateItems[i].IndexInArchive;
1623       if (index != -1)
1624         fileIndexToUpdateIndexMap[(unsigned)index] = i;
1625     }
1626 
1627     for (i = 0; i < db->NumFolders; i++)
1628     {
1629       CNum indexInFolder = 0;
1630       CNum numCopyItems = 0;
1631       CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
1632       UInt64 repackSize = 0;
1633 
1634       for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
1635       {
1636         const CFileItem &file = db->Files[fi];
1637         if (file.HasStream)
1638         {
1639           indexInFolder++;
1640           int updateIndex = fileIndexToUpdateIndexMap[fi];
1641           if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
1642           {
1643             numCopyItems++;
1644             repackSize += file.Size;
1645           }
1646         }
1647       }
1648 
1649       if (numCopyItems == 0)
1650         continue;
1651 
1652       CFolderRepack rep;
1653       rep.FolderIndex = i;
1654       rep.NumCopyFiles = numCopyItems;
1655       CFolderEx f;
1656       db->ParseFolderEx(i, f);
1657 
1658       const bool isEncrypted = f.IsEncrypted();
1659       const bool needCopy = (numCopyItems == numUnpackStreams);
1660       const bool extractFilter = (useFilters || needCopy);
1661 
1662       unsigned groupIndex = Get_FilterGroup_for_Folder(filters, f, extractFilter);
1663 
1664       while (groupIndex >= groups.Size())
1665         groups.AddNew();
1666 
1667       groups[groupIndex].folderRefs.Add(rep);
1668 
1669       if (needCopy)
1670         complexity += db->GetFolderFullPackSize(i);
1671       else
1672       {
1673         thereAreRepacks = true;
1674         complexity += repackSize;
1675         if (inSizeForReduce2 < repackSize)
1676           inSizeForReduce2 = repackSize;
1677         if (isEncrypted)
1678           needEncryptedRepack = true;
1679       }
1680     }
1681   }
1682 
1683   UInt64 inSizeForReduce = 0;
1684   {
1685     FOR_VECTOR (i, updateItems)
1686     {
1687       const CUpdateItem &ui = updateItems[i];
1688       if (ui.NewData)
1689       {
1690         complexity += ui.Size;
1691         if (numSolidFiles != 1)
1692           inSizeForReduce += ui.Size;
1693         else if (inSizeForReduce < ui.Size)
1694           inSizeForReduce = ui.Size;
1695       }
1696     }
1697   }
1698 
1699   if (inSizeForReduce < inSizeForReduce2)
1700     inSizeForReduce = inSizeForReduce2;
1701 
1702   RINOK(updateCallback->SetTotal(complexity));
1703 
1704   CLocalProgress *lps = new CLocalProgress;
1705   CMyComPtr<ICompressProgressInfo> progress = lps;
1706   lps->Init(updateCallback, true);
1707 
1708   #ifndef _7ZIP_ST
1709 
1710   CStreamBinder sb;
1711   if (options.MultiThreadMixer)
1712   {
1713     RINOK(sb.CreateEvents());
1714   }
1715 
1716   #endif
1717 
1718   CThreadDecoder threadDecoder(options.MultiThreadMixer);
1719 
1720   #ifndef _7ZIP_ST
1721   if (options.MultiThreadMixer && thereAreRepacks)
1722   {
1723     #ifdef EXTERNAL_CODECS
1724     threadDecoder.__externalCodecs = __externalCodecs;
1725     #endif
1726     RINOK(threadDecoder.Create());
1727   }
1728   #endif
1729 
1730   {
1731     CAnalysis analysis;
1732     if (options.AnalysisLevel == 0)
1733     {
1734       analysis.ParseWav = false;
1735       analysis.ParseExe = false;
1736       analysis.ParseAll = false;
1737     }
1738     else
1739     {
1740       analysis.Callback = opCallback;
1741       if (options.AnalysisLevel > 0)
1742       {
1743         analysis.ParseWav = true;
1744         if (options.AnalysisLevel >= 7)
1745         {
1746           analysis.ParseExe = true;
1747           if (options.AnalysisLevel >= 9)
1748             analysis.ParseAll = true;
1749         }
1750       }
1751     }
1752 
1753     // ---------- Split files to groups ----------
1754 
1755     const CCompressionMethodMode &method = *options.Method;
1756 
1757     FOR_VECTOR (i, updateItems)
1758     {
1759       const CUpdateItem &ui = updateItems[i];
1760       if (!ui.NewData || !ui.HasStream())
1761         continue;
1762 
1763       CFilterMode2 fm;
1764       if (useFilters)
1765       {
1766         RINOK(analysis.GetFilterGroup(i, ui, fm));
1767       }
1768       fm.Encrypted = method.PasswordIsDefined;
1769 
1770       unsigned groupIndex = GetGroup(filters, fm);
1771       while (groupIndex >= groups.Size())
1772         groups.AddNew();
1773       groups[groupIndex].Indices.Add(i);
1774     }
1775   }
1776 
1777 
1778   #ifndef _NO_CRYPTO
1779 
1780   CCryptoGetTextPassword *getPasswordSpec = NULL;
1781   CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1782   if (needEncryptedRepack)
1783   {
1784     getPasswordSpec = new CCryptoGetTextPassword;
1785     getTextPassword = getPasswordSpec;
1786 
1787     #ifndef _7ZIP_ST
1788     threadDecoder.getTextPassword = getPasswordSpec;
1789     #endif
1790 
1791     if (options.Method->PasswordIsDefined)
1792       getPasswordSpec->Password = options.Method->Password;
1793     else
1794     {
1795       if (!getDecoderPassword)
1796         return E_NOTIMPL;
1797       CMyComBSTR password;
1798       RINOK(getDecoderPassword->CryptoGetTextPassword(&password));
1799       if (password)
1800         getPasswordSpec->Password = password;
1801     }
1802   }
1803 
1804   #endif
1805 
1806 
1807   // ---------- Compress ----------
1808 
1809   RINOK(archive.Create(seqOutStream, false));
1810   RINOK(archive.SkipPrefixArchiveHeader());
1811 
1812   /*
1813   CIntVector treeFolderToArcIndex;
1814   treeFolderToArcIndex.Reserve(treeFolders.Size());
1815   for (i = 0; i < treeFolders.Size(); i++)
1816     treeFolderToArcIndex.Add(-1);
1817   // ---------- Write Tree (only AUX dirs) ----------
1818   for (i = 1; i < treeFolders.Size(); i++)
1819   {
1820     const CTreeFolder &treeFolder = treeFolders[i];
1821     CFileItem file;
1822     CFileItem2 file2;
1823     file2.Init();
1824     int secureID = 0;
1825     if (treeFolder.UpdateItemIndex < 0)
1826     {
1827       // we can store virtual dir item wuthout attrib, but we want all items have attrib.
1828       file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY);
1829       file2.IsAux = true;
1830     }
1831     else
1832     {
1833       const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex];
1834       // if item is not dir, then it's parent for alt streams.
1835       // we will write such items later
1836       if (!ui.IsDir)
1837         continue;
1838       secureID = ui.SecureIndex;
1839       if (ui.NewProps)
1840         FromUpdateItemToFileItem(ui, file, file2);
1841       else
1842         GetFile(*db, ui.IndexInArchive, file, file2);
1843     }
1844     file.Size = 0;
1845     file.HasStream = false;
1846     file.IsDir = true;
1847     file.Parent = treeFolder.Parent;
1848 
1849     treeFolderToArcIndex[i] = newDatabase.Files.Size();
1850     newDatabase.AddFile(file, file2, treeFolder.Name);
1851 
1852     if (totalSecureDataSize != 0)
1853       newDatabase.SecureIDs.Add(secureID);
1854   }
1855   */
1856 
1857   {
1858     /* ---------- Write non-AUX dirs and Empty files ---------- */
1859     CUIntVector emptyRefs;
1860 
1861     unsigned i;
1862 
1863     for (i = 0; i < updateItems.Size(); i++)
1864     {
1865       const CUpdateItem &ui = updateItems[i];
1866       if (ui.NewData)
1867       {
1868         if (ui.HasStream())
1869           continue;
1870       }
1871       else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream)
1872         continue;
1873       /*
1874       if (ui.TreeFolderIndex >= 0)
1875         continue;
1876       */
1877       emptyRefs.Add(i);
1878     }
1879 
1880     emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
1881 
1882     for (i = 0; i < emptyRefs.Size(); i++)
1883     {
1884       const CUpdateItem &ui = updateItems[emptyRefs[i]];
1885       CFileItem file;
1886       CFileItem2 file2;
1887       UString name;
1888       if (ui.NewProps)
1889       {
1890         FromUpdateItemToFileItem(ui, file, file2);
1891         name = ui.Name;
1892       }
1893       else
1894       {
1895         GetFile(*db, ui.IndexInArchive, file, file2);
1896         db->GetPath(ui.IndexInArchive, name);
1897       }
1898 
1899       /*
1900       if (totalSecureDataSize != 0)
1901         newDatabase.SecureIDs.Add(ui.SecureIndex);
1902       file.Parent = ui.ParentFolderIndex;
1903       */
1904       newDatabase.AddFile(file, file2, name);
1905     }
1906   }
1907 
1908   lps->ProgressOffset = 0;
1909 
1910   {
1911     // ---------- Sort Filters ----------
1912 
1913     FOR_VECTOR (i, filters)
1914     {
1915       filters[i].GroupIndex = i;
1916     }
1917     filters.Sort2();
1918   }
1919 
1920   for (unsigned groupIndex = 0; groupIndex < filters.Size(); groupIndex++)
1921   {
1922     const CFilterMode2 &filterMode = filters[groupIndex];
1923 
1924     CCompressionMethodMode method = *options.Method;
1925     {
1926       HRESULT res = MakeExeMethod(method, filterMode,
1927         #ifdef _7ZIP_ST
1928           false
1929         #else
1930           options.MaxFilter && options.MultiThreadMixer
1931         #endif
1932         );
1933 
1934       RINOK(res);
1935     }
1936 
1937     if (filterMode.Encrypted)
1938     {
1939       if (!method.PasswordIsDefined)
1940       {
1941         #ifndef _NO_CRYPTO
1942         if (getPasswordSpec)
1943           method.Password = getPasswordSpec->Password;
1944         #endif
1945         method.PasswordIsDefined = true;
1946       }
1947     }
1948     else
1949     {
1950       method.PasswordIsDefined = false;
1951       method.Password.Empty();
1952     }
1953 
1954     CEncoder encoder(method);
1955 
1956     // ---------- Repack and copy old solid blocks ----------
1957 
1958     const CSolidGroup &group = groups[filterMode.GroupIndex];
1959 
1960     FOR_VECTOR(folderRefIndex, group.folderRefs)
1961     {
1962       const CFolderRepack &rep = group.folderRefs[folderRefIndex];
1963 
1964       unsigned folderIndex = rep.FolderIndex;
1965 
1966       CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
1967 
1968       if (rep.NumCopyFiles == numUnpackStreams)
1969       {
1970         if (opCallback)
1971         {
1972           RINOK(opCallback->ReportOperation(
1973               NEventIndexType::kBlockIndex, (UInt32)folderIndex,
1974               NUpdateNotifyOp::kReplicate));
1975 
1976           // ---------- Copy old solid block ----------
1977           {
1978             CNum indexInFolder = 0;
1979             for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
1980             {
1981               if (db->Files[fi].HasStream)
1982               {
1983                 indexInFolder++;
1984                 RINOK(opCallback->ReportOperation(
1985                     NEventIndexType::kInArcIndex, (UInt32)fi,
1986                     NUpdateNotifyOp::kReplicate));
1987               }
1988             }
1989           }
1990         }
1991 
1992         UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
1993         RINOK(WriteRange(inStream, archive.SeqStream,
1994             db->GetFolderStreamPos(folderIndex, 0), packSize, progress));
1995         lps->ProgressOffset += packSize;
1996 
1997         CFolder &folder = newDatabase.Folders.AddNew();
1998         db->ParseFolderInfo(folderIndex, folder);
1999         CNum startIndex = db->FoStartPackStreamIndex[folderIndex];
2000         FOR_VECTOR(j, folder.PackStreams)
2001         {
2002           newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j));
2003           // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
2004           // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
2005         }
2006 
2007         size_t indexStart = db->FoToCoderUnpackSizes[folderIndex];
2008         size_t indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1];
2009         for (; indexStart < indexEnd; indexStart++)
2010           newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes[indexStart]);
2011       }
2012       else
2013       {
2014         // ---------- Repack old solid block ----------
2015 
2016         CBoolVector extractStatuses;
2017 
2018         CNum indexInFolder = 0;
2019 
2020         if (opCallback)
2021         {
2022           RINOK(opCallback->ReportOperation(
2023               NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2024               NUpdateNotifyOp::kRepack))
2025         }
2026 
2027         /* We could reduce data size of decoded folder, if we don't need to repack
2028            last files in folder. But the gain in speed is small in most cases.
2029            So we unpack full folder. */
2030 
2031         UInt64 sizeToEncode = 0;
2032 
2033         /*
2034         UInt64 importantUnpackSize = 0;
2035         unsigned numImportantFiles = 0;
2036         UInt64 decodeSize = 0;
2037         */
2038 
2039         for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2040         {
2041           bool needExtract = false;
2042           const CFileItem &file = db->Files[fi];
2043 
2044           if (file.HasStream)
2045           {
2046             indexInFolder++;
2047             int updateIndex = fileIndexToUpdateIndexMap[fi];
2048             if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
2049               needExtract = true;
2050             // decodeSize += file.Size;
2051           }
2052 
2053           extractStatuses.Add(needExtract);
2054           if (needExtract)
2055           {
2056             sizeToEncode += file.Size;
2057             /*
2058             numImportantFiles = extractStatuses.Size();
2059             importantUnpackSize = decodeSize;
2060             */
2061           }
2062         }
2063 
2064         // extractStatuses.DeleteFrom(numImportantFiles);
2065 
2066         unsigned startPackIndex = newDatabase.PackSizes.Size();
2067         UInt64 curUnpackSize;
2068         {
2069 
2070           CMyComPtr<ISequentialInStream> sbInStream;
2071           CRepackStreamBase *repackBase;
2072           CFolderInStream2 *FosSpec2 = NULL;
2073 
2074           CRepackInStreamWithSizes *inStreamSizeCountSpec = new CRepackInStreamWithSizes;
2075           CMyComPtr<ISequentialInStream> inStreamSizeCount = inStreamSizeCountSpec;
2076           {
2077             #ifndef _7ZIP_ST
2078             if (options.MultiThreadMixer)
2079             {
2080               repackBase = threadDecoder.FosSpec;
2081               CMyComPtr<ISequentialOutStream> sbOutStream;
2082               sb.CreateStreams(&sbInStream, &sbOutStream);
2083               sb.ReInit();
2084 
2085               threadDecoder.FosSpec->_stream = sbOutStream;
2086 
2087               threadDecoder.InStream = inStream;
2088               threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0);
2089               threadDecoder.Folders = (const CFolders *)db;
2090               threadDecoder.FolderIndex = folderIndex;
2091 
2092               // threadDecoder.UnpackSize = importantUnpackSize;
2093               // threadDecoder.send_UnpackSize = true;
2094             }
2095             else
2096             #endif
2097             {
2098               FosSpec2 = new CFolderInStream2;
2099               FosSpec2->Init();
2100               sbInStream = FosSpec2;
2101               repackBase = FosSpec2;
2102 
2103               #ifndef _NO_CRYPTO
2104               bool isEncrypted = false;
2105               bool passwordIsDefined = false;
2106               UString password;
2107               #endif
2108 
2109               CMyComPtr<ISequentialInStream> decodedStream;
2110               HRESULT res = threadDecoder.Decoder.Decode(
2111                   EXTERNAL_CODECS_LOC_VARS
2112                   inStream,
2113                   db->ArcInfo.DataStartPosition, // db->GetFolderStreamPos(folderIndex, 0);,
2114                   *db, folderIndex,
2115                   // &importantUnpackSize, // *unpackSize
2116                   NULL, // *unpackSize : FULL unpack
2117 
2118                   NULL, // *outStream
2119                   NULL, // *compressProgress
2120                   &decodedStream
2121 
2122                   _7Z_DECODER_CRYPRO_VARS
2123                   #ifndef _7ZIP_ST
2124                     , false // mtMode
2125                     , 1 // numThreads
2126                   #endif
2127                 );
2128 
2129               RINOK(res);
2130               if (!decodedStream)
2131                 return E_FAIL;
2132 
2133               FosSpec2->_inStream = decodedStream;
2134             }
2135 
2136             repackBase->_db = db;
2137             repackBase->_opCallback = opCallback;
2138             repackBase->_extractCallback = extractCallback;
2139 
2140             UInt32 startIndex = db->FolderStartFileIndex[folderIndex];
2141             RINOK(repackBase->Init(startIndex, &extractStatuses));
2142 
2143             inStreamSizeCountSpec->_db = db;
2144             inStreamSizeCountSpec->Init(sbInStream, startIndex, &extractStatuses);
2145 
2146             #ifndef _7ZIP_ST
2147             if (options.MultiThreadMixer)
2148             {
2149               threadDecoder.Start();
2150             }
2151             #endif
2152           }
2153 
2154 
2155           HRESULT encodeRes = encoder.Encode(
2156               EXTERNAL_CODECS_LOC_VARS
2157               inStreamSizeCount,
2158               // NULL,
2159               &inSizeForReduce,
2160               newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curUnpackSize,
2161               archive.SeqStream, newDatabase.PackSizes, progress);
2162 
2163           if (encodeRes == k_My_HRESULT_CRC_ERROR)
2164             return E_FAIL;
2165 
2166           #ifndef _7ZIP_ST
2167           if (options.MultiThreadMixer)
2168           {
2169             // 16.00: hang was fixed : for case if decoding was not finished.
2170             // We close CBinderInStream and it calls CStreamBinder::CloseRead()
2171             inStreamSizeCount.Release();
2172             sbInStream.Release();
2173 
2174             threadDecoder.WaitExecuteFinish();
2175 
2176             HRESULT decodeRes = threadDecoder.Result;
2177             // if (res == k_My_HRESULT_CRC_ERROR)
2178             if (decodeRes == S_FALSE)
2179             {
2180               if (extractCallback)
2181               {
2182                 RINOK(extractCallback->ReportExtractResult(
2183                     NEventIndexType::kInArcIndex, db->FolderStartFileIndex[folderIndex],
2184                     // NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2185                     NExtract::NOperationResult::kDataError));
2186               }
2187               return E_FAIL;
2188             }
2189             RINOK(decodeRes);
2190             if (encodeRes == S_OK)
2191               if (sb.ProcessedSize != sizeToEncode)
2192                 encodeRes = E_FAIL;
2193           }
2194           else
2195           #endif
2196           {
2197             if (FosSpec2->Result == S_FALSE)
2198             {
2199               if (extractCallback)
2200               {
2201                 RINOK(extractCallback->ReportExtractResult(
2202                     NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2203                     NExtract::NOperationResult::kDataError));
2204               }
2205               return E_FAIL;
2206             }
2207             RINOK(FosSpec2->Result);
2208           }
2209 
2210           RINOK(encodeRes);
2211           RINOK(repackBase->CheckFinishedState());
2212 
2213           if (curUnpackSize != sizeToEncode)
2214             return E_FAIL;
2215         }
2216 
2217         for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2218           lps->OutSize += newDatabase.PackSizes[startPackIndex];
2219         lps->InSize += curUnpackSize;
2220       }
2221 
2222       newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
2223 
2224       CNum indexInFolder = 0;
2225       for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2226       {
2227         CFileItem file;
2228         CFileItem2 file2;
2229         GetFile(*db, fi, file, file2);
2230         UString name;
2231         db->GetPath(fi, name);
2232         if (file.HasStream)
2233         {
2234           indexInFolder++;
2235           int updateIndex = fileIndexToUpdateIndexMap[fi];
2236           if (updateIndex >= 0)
2237           {
2238             const CUpdateItem &ui = updateItems[updateIndex];
2239             if (ui.NewData)
2240               continue;
2241             if (ui.NewProps)
2242             {
2243               CFileItem uf;
2244               FromUpdateItemToFileItem(ui, uf, file2);
2245               uf.Size = file.Size;
2246               uf.Crc = file.Crc;
2247               uf.CrcDefined = file.CrcDefined;
2248               uf.HasStream = file.HasStream;
2249               file = uf;
2250               name = ui.Name;
2251             }
2252             /*
2253             file.Parent = ui.ParentFolderIndex;
2254             if (ui.TreeFolderIndex >= 0)
2255               treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2256             if (totalSecureDataSize != 0)
2257               newDatabase.SecureIDs.Add(ui.SecureIndex);
2258             */
2259             newDatabase.AddFile(file, file2, name);
2260           }
2261         }
2262       }
2263     }
2264 
2265 
2266     // ---------- Compress files to new solid blocks ----------
2267 
2268     unsigned numFiles = group.Indices.Size();
2269     if (numFiles == 0)
2270       continue;
2271     CRecordVector<CRefItem> refItems;
2272     refItems.ClearAndSetSize(numFiles);
2273     bool sortByType = (options.UseTypeSorting && numSolidFiles > 1);
2274 
2275     unsigned i;
2276 
2277     for (i = 0; i < numFiles; i++)
2278       refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType);
2279 
2280     CSortParam sortParam;
2281     // sortParam.TreeFolders = &treeFolders;
2282     sortParam.SortByType = sortByType;
2283     refItems.Sort(CompareUpdateItems, (void *)&sortParam);
2284 
2285     CObjArray<UInt32> indices(numFiles);
2286 
2287     for (i = 0; i < numFiles; i++)
2288     {
2289       UInt32 index = refItems[i].Index;
2290       indices[i] = index;
2291       /*
2292       const CUpdateItem &ui = updateItems[index];
2293       CFileItem file;
2294       if (ui.NewProps)
2295         FromUpdateItemToFileItem(ui, file);
2296       else
2297         file = db.Files[ui.IndexInArchive];
2298       if (file.IsAnti || file.IsDir)
2299         return E_FAIL;
2300       newDatabase.Files.Add(file);
2301       */
2302     }
2303 
2304     for (i = 0; i < numFiles;)
2305     {
2306       UInt64 totalSize = 0;
2307       unsigned numSubFiles;
2308 
2309       const wchar_t *prevExtension = NULL;
2310 
2311       for (numSubFiles = 0; i + numSubFiles < numFiles && numSubFiles < numSolidFiles; numSubFiles++)
2312       {
2313         const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
2314         totalSize += ui.Size;
2315         if (totalSize > options.NumSolidBytes)
2316           break;
2317         if (options.SolidExtension)
2318         {
2319           int slashPos = ui.Name.ReverseFind_PathSepar();
2320           int dotPos = ui.Name.ReverseFind_Dot();
2321           const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : dotPos + 1);
2322           if (numSubFiles == 0)
2323             prevExtension = ext;
2324           else if (!StringsAreEqualNoCase(ext, prevExtension))
2325             break;
2326         }
2327       }
2328 
2329       if (numSubFiles < 1)
2330         numSubFiles = 1;
2331 
2332       RINOK(lps->SetCur());
2333 
2334       CFolderInStream *inStreamSpec = new CFolderInStream;
2335       CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec);
2336       inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
2337 
2338       unsigned startPackIndex = newDatabase.PackSizes.Size();
2339       UInt64 curFolderUnpackSize;
2340       RINOK(encoder.Encode(
2341           EXTERNAL_CODECS_LOC_VARS
2342           solidInStream,
2343           // NULL,
2344           &inSizeForReduce,
2345           newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curFolderUnpackSize,
2346           archive.SeqStream, newDatabase.PackSizes, progress));
2347 
2348       if (!inStreamSpec->WasFinished())
2349         return E_FAIL;
2350 
2351       for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2352         lps->OutSize += newDatabase.PackSizes[startPackIndex];
2353 
2354       lps->InSize += curFolderUnpackSize;
2355       // for ()
2356       // newDatabase.PackCRCsDefined.Add(false);
2357       // newDatabase.PackCRCs.Add(0);
2358 
2359       CNum numUnpackStreams = 0;
2360       UInt64 skippedSize = 0;
2361 
2362       for (unsigned subIndex = 0; subIndex < numSubFiles; subIndex++)
2363       {
2364         const CUpdateItem &ui = updateItems[indices[i + subIndex]];
2365         CFileItem file;
2366         CFileItem2 file2;
2367         UString name;
2368         if (ui.NewProps)
2369         {
2370           FromUpdateItemToFileItem(ui, file, file2);
2371           name = ui.Name;
2372         }
2373         else
2374         {
2375           GetFile(*db, ui.IndexInArchive, file, file2);
2376           db->GetPath(ui.IndexInArchive, name);
2377         }
2378         if (file2.IsAnti || file.IsDir)
2379           return E_FAIL;
2380 
2381         /*
2382         CFileItem &file = newDatabase.Files[
2383               startFileIndexInDatabase + i + subIndex];
2384         */
2385         if (!inStreamSpec->Processed[subIndex])
2386         {
2387           skippedSize += ui.Size;
2388           continue;
2389           // file.Name.AddAscii(".locked");
2390         }
2391 
2392         file.Crc = inStreamSpec->CRCs[subIndex];
2393         file.Size = inStreamSpec->Sizes[subIndex];
2394 
2395         // if (file.Size >= 0) // test purposes
2396         if (file.Size != 0)
2397         {
2398           file.CrcDefined = true;
2399           file.HasStream = true;
2400           numUnpackStreams++;
2401         }
2402         else
2403         {
2404           file.CrcDefined = false;
2405           file.HasStream = false;
2406         }
2407 
2408         /*
2409         file.Parent = ui.ParentFolderIndex;
2410         if (ui.TreeFolderIndex >= 0)
2411           treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2412         if (totalSecureDataSize != 0)
2413           newDatabase.SecureIDs.Add(ui.SecureIndex);
2414         */
2415         newDatabase.AddFile(file, file2, name);
2416       }
2417 
2418       // numUnpackStreams = 0 is very bad case for locked files
2419       // v3.13 doesn't understand it.
2420       newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
2421       i += numSubFiles;
2422 
2423       if (skippedSize != 0 && complexity >= skippedSize)
2424       {
2425         complexity -= skippedSize;
2426         RINOK(updateCallback->SetTotal(complexity));
2427       }
2428     }
2429   }
2430 
2431   RINOK(lps->SetCur());
2432 
2433   /*
2434   fileIndexToUpdateIndexMap.ClearAndFree();
2435   groups.ClearAndFree();
2436   */
2437 
2438   /*
2439   for (i = 0; i < newDatabase.Files.Size(); i++)
2440   {
2441     CFileItem &file = newDatabase.Files[i];
2442     file.Parent = treeFolderToArcIndex[file.Parent];
2443   }
2444 
2445   if (totalSecureDataSize != 0)
2446   {
2447     newDatabase.SecureBuf.SetCapacity(totalSecureDataSize);
2448     size_t pos = 0;
2449     newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size());
2450     for (i = 0; i < secureBlocks.Sorted.Size(); i++)
2451     {
2452       const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]];
2453       size_t size = buf.GetCapacity();
2454       if (size != 0)
2455         memcpy(newDatabase.SecureBuf + pos, buf, size);
2456       newDatabase.SecureSizes.Add((UInt32)size);
2457       pos += size;
2458     }
2459   }
2460   */
2461   newDatabase.ReserveDown();
2462 
2463   if (opCallback)
2464     RINOK(opCallback->ReportOperation(NEventIndexType::kNoIndex, (UInt32)(Int32)-1, NUpdateNotifyOp::kHeader));
2465 
2466   return S_OK;
2467 }
2468 
2469 }}
2470