1 // 7zUpdate.cpp
2 
3 #include "StdAfx.h"
4 
5 #include "../../../../C/CpuArch.h"
6 
7 #include "../../../Common/Wildcard.h"
8 
9 #include "../../Common/CreateCoder.h"
10 #include "../../Common/LimitedStreams.h"
11 #include "../../Common/ProgressUtils.h"
12 
13 #include "../../Compress/CopyCoder.h"
14 
15 #include "../Common/ItemNameUtils.h"
16 
17 #include "7zDecode.h"
18 #include "7zEncode.h"
19 #include "7zFolderInStream.h"
20 #include "7zHandler.h"
21 #include "7zOut.h"
22 #include "7zUpdate.h"
23 
24 namespace NArchive {
25 namespace N7z {
26 
27 
28 #define k_X86 k_BCJ
29 
30 struct CFilterMode
31 {
32   UInt32 Id;
33   UInt32 Delta;
34 
CFilterModeNArchive::N7z::CFilterMode35   CFilterMode(): Id(0), Delta(0) {}
36 
SetDeltaNArchive::N7z::CFilterMode37   void SetDelta()
38   {
39     if (Id == k_IA64)
40       Delta = 16;
41     else if (Id == k_ARM || Id == k_PPC || Id == k_SPARC)
42       Delta = 4;
43     else if (Id == k_ARMT)
44       Delta = 2;
45     else
46       Delta = 0;
47   }
48 };
49 
50 
51 /* ---------- PE ---------- */
52 
53 #define MZ_SIG 0x5A4D
54 
55 #define PE_SIG 0x00004550
56 #define PE_OptHeader_Magic_32 0x10B
57 #define PE_OptHeader_Magic_64 0x20B
58 // #define PE_SectHeaderSize 40
59 // #define PE_SECT_EXECUTE 0x20000000
60 
Parse_EXE(const Byte * buf,size_t size,CFilterMode * filterMode)61 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode)
62 {
63   if (size < 512 || GetUi16(buf) != MZ_SIG)
64     return 0;
65 
66   const Byte *p;
67   UInt32 peOffset, optHeaderSize, filterId;
68 
69   peOffset = GetUi32(buf + 0x3C);
70   if (peOffset >= 0x1000 || peOffset + 512 > size || (peOffset & 7) != 0)
71     return 0;
72   p = buf + peOffset;
73   if (GetUi32(p) != PE_SIG)
74     return 0;
75   p += 4;
76 
77   switch (GetUi16(p))
78   {
79     case 0x014C:
80     case 0x8664:  filterId = k_X86; break;
81 
82     /*
83     IMAGE_FILE_MACHINE_ARM   0x01C0  // ARM LE
84     IMAGE_FILE_MACHINE_THUMB 0x01C2  // ARM Thumb / Thumb-2 LE
85     IMAGE_FILE_MACHINE_ARMNT 0x01C4  // ARM Thumb-2, LE
86     Note: We use ARM filter for 0x01C2. (WinCE 5 - 0x01C2) files mostly contain ARM code (not Thumb/Thumb-2).
87     */
88 
89     case 0x01C0:                            // WinCE old
90     case 0x01C2:  filterId = k_ARM; break;  // WinCE new
91     case 0x01C4:  filterId = k_ARMT; break; // WinRT
92 
93     case 0x0200:  filterId = k_IA64; break;
94     default:  return 0;
95   }
96 
97   optHeaderSize = GetUi16(p + 16);
98   if (optHeaderSize > (1 << 10))
99     return 0;
100 
101   p += 20; /* headerSize */
102 
103   switch (GetUi16(p))
104   {
105     case PE_OptHeader_Magic_32:
106     case PE_OptHeader_Magic_64:
107       break;
108     default:
109       return 0;
110   }
111 
112   filterMode->Id = filterId;
113   return 1;
114 }
115 
116 
117 /* ---------- ELF ---------- */
118 
119 #define ELF_SIG 0x464C457F
120 
121 #define ELF_CLASS_32  1
122 #define ELF_CLASS_64  2
123 
124 #define ELF_DATA_2LSB 1
125 #define ELF_DATA_2MSB 2
126 
Get16(const Byte * p,BoolInt be)127 static UInt16 Get16(const Byte *p, BoolInt be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); }
Get32(const Byte * p,BoolInt be)128 static UInt32 Get32(const Byte *p, BoolInt be) { if (be) return GetBe32(p); return GetUi32(p); }
129 // static UInt64 Get64(const Byte *p, BoolInt be) { if (be) return GetBe64(p); return GetUi64(p); }
130 
Parse_ELF(const Byte * buf,size_t size,CFilterMode * filterMode)131 static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode)
132 {
133   BoolInt /* is32, */ be;
134   UInt32 filterId;
135 
136   if (size < 512 || buf[6] != 1) /* ver */
137     return 0;
138 
139   if (GetUi32(buf) != ELF_SIG)
140     return 0;
141 
142   switch (buf[4])
143   {
144     case ELF_CLASS_32: /* is32 = True; */ break;
145     case ELF_CLASS_64: /* is32 = False; */ break;
146     default: return 0;
147   }
148 
149   switch (buf[5])
150   {
151     case ELF_DATA_2LSB: be = False; break;
152     case ELF_DATA_2MSB: be = True; break;
153     default: return 0;
154   }
155 
156   switch (Get16(buf + 0x12, be))
157   {
158     case 3:
159     case 6:
160     case 62: filterId = k_X86; break;
161     case 2:
162     case 18:
163     case 43: filterId = k_SPARC; break;
164     case 20:
165     case 21: if (!be) return 0; filterId = k_PPC; break;
166     case 40: if ( be) return 0; filterId = k_ARM; break;
167 
168     /* Some IA-64 ELF exacutable have size that is not aligned for 16 bytes.
169        So we don't use IA-64 filter for IA-64 ELF */
170     // case 50: if ( be) return 0; filterId = k_IA64; break;
171 
172     default: return 0;
173   }
174 
175   filterMode->Id = filterId;
176   return 1;
177 }
178 
179 
180 
181 /* ---------- Mach-O ---------- */
182 
183 #define MACH_SIG_BE_32 0xCEFAEDFE
184 #define MACH_SIG_BE_64 0xCFFAEDFE
185 #define MACH_SIG_LE_32 0xFEEDFACE
186 #define MACH_SIG_LE_64 0xFEEDFACF
187 
188 #define MACH_ARCH_ABI64 (1 << 24)
189 #define MACH_MACHINE_386 7
190 #define MACH_MACHINE_ARM 12
191 #define MACH_MACHINE_SPARC 14
192 #define MACH_MACHINE_PPC 18
193 #define MACH_MACHINE_PPC64 (MACH_ARCH_ABI64 | MACH_MACHINE_PPC)
194 #define MACH_MACHINE_AMD64 (MACH_ARCH_ABI64 | MACH_MACHINE_386)
195 
Parse_MACH(const Byte * buf,size_t size,CFilterMode * filterMode)196 static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode)
197 {
198   UInt32 filterId, numCommands, commandsSize;
199 
200   if (size < 512)
201     return 0;
202 
203   BoolInt /* mode64, */ be;
204   switch (GetUi32(buf))
205   {
206     case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break;
207     case MACH_SIG_BE_64: /* mode64 = True;  */ be = True; break;
208     case MACH_SIG_LE_32: /* mode64 = False; */ be = False; break;
209     case MACH_SIG_LE_64: /* mode64 = True;  */ be = False; break;
210     default: return 0;
211   }
212 
213   switch (Get32(buf + 4, be))
214   {
215     case MACH_MACHINE_386:
216     case MACH_MACHINE_AMD64: filterId = k_X86; break;
217     case MACH_MACHINE_ARM:   if ( be) return 0; filterId = k_ARM; break;
218     case MACH_MACHINE_SPARC: if (!be) return 0; filterId = k_SPARC; break;
219     case MACH_MACHINE_PPC:
220     case MACH_MACHINE_PPC64: if (!be) return 0; filterId = k_PPC; break;
221     default: return 0;
222   }
223 
224   numCommands = Get32(buf + 0x10, be);
225   commandsSize = Get32(buf + 0x14, be);
226 
227   if (commandsSize > (1 << 24) || numCommands > (1 << 18))
228     return 0;
229 
230   filterMode->Id = filterId;
231   return 1;
232 }
233 
234 
235 /* ---------- WAV ---------- */
236 
237 #define WAV_SUBCHUNK_fmt  0x20746D66
238 #define WAV_SUBCHUNK_data 0x61746164
239 
240 #define RIFF_SIG 0x46464952
241 
Parse_WAV(const Byte * buf,size_t size,CFilterMode * filterMode)242 static BoolInt Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode)
243 {
244   UInt32 subChunkSize, pos;
245   if (size < 0x2C)
246     return False;
247 
248   if (GetUi32(buf + 0) != RIFF_SIG ||
249       GetUi32(buf + 8) != 0x45564157 || // WAVE
250       GetUi32(buf + 0xC) != WAV_SUBCHUNK_fmt)
251     return False;
252   subChunkSize = GetUi32(buf + 0x10);
253   /* [0x14 = format] = 1 (PCM) */
254   if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1)
255     return False;
256 
257   const unsigned numChannels = GetUi16(buf + 0x16);
258   const unsigned bitsPerSample = GetUi16(buf + 0x22);
259   if ((bitsPerSample & 0x7) != 0)
260     return False;
261   const UInt32 delta = (UInt32)numChannels * (bitsPerSample >> 3);
262   if (delta == 0 || delta > 256)
263     return False;
264 
265   pos = 0x14 + subChunkSize;
266 
267   const int kNumSubChunksTests = 10;
268   // Do we need to scan more than 3 sub-chunks?
269   for (int i = 0; i < kNumSubChunksTests; i++)
270   {
271     if (pos + 8 > size)
272       return False;
273     subChunkSize = GetUi32(buf + pos + 4);
274     if (GetUi32(buf + pos) == WAV_SUBCHUNK_data)
275     {
276       filterMode->Id = k_Delta;
277       filterMode->Delta = delta;
278       return True;
279     }
280     if (subChunkSize > (1 << 16))
281       return False;
282     pos += subChunkSize + 8;
283   }
284   return False;
285 }
286 
ParseFile(const Byte * buf,size_t size,CFilterMode * filterMode)287 static BoolInt ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode)
288 {
289   filterMode->Id = 0;
290   filterMode->Delta = 0;
291 
292   if (Parse_EXE(buf, size, filterMode)) return True;
293   if (Parse_ELF(buf, size, filterMode)) return True;
294   if (Parse_MACH(buf, size, filterMode)) return True;
295   return Parse_WAV(buf, size, filterMode);
296 }
297 
298 
299 
300 
301 struct CFilterMode2: public CFilterMode
302 {
303   bool Encrypted;
304   unsigned GroupIndex;
305 
CFilterMode2NArchive::N7z::CFilterMode2306   CFilterMode2(): Encrypted(false) {}
307 
CompareNArchive::N7z::CFilterMode2308   int Compare(const CFilterMode2 &m) const
309   {
310     if (!Encrypted)
311     {
312       if (m.Encrypted)
313         return -1;
314     }
315     else if (!m.Encrypted)
316       return 1;
317 
318     if (Id < m.Id) return -1;
319     if (Id > m.Id) return 1;
320 
321     if (Delta < m.Delta) return -1;
322     if (Delta > m.Delta) return 1;
323 
324     return 0;
325   }
326 
operator ==NArchive::N7z::CFilterMode2327   bool operator ==(const CFilterMode2 &m) const
328   {
329     return Id == m.Id && Delta == m.Delta && Encrypted == m.Encrypted;
330   }
331 };
332 
GetGroup(CRecordVector<CFilterMode2> & filters,const CFilterMode2 & m)333 static unsigned GetGroup(CRecordVector<CFilterMode2> &filters, const CFilterMode2 &m)
334 {
335   unsigned i;
336   for (i = 0; i < filters.Size(); i++)
337   {
338     const CFilterMode2 &m2 = filters[i];
339     if (m == m2)
340       return i;
341     /*
342     if (m.Encrypted != m2.Encrypted)
343     {
344       if (!m.Encrypted)
345         break;
346       continue;
347     }
348 
349     if (m.Id < m2.Id)  break;
350     if (m.Id != m2.Id) continue;
351 
352     if (m.Delta < m2.Delta) break;
353     if (m.Delta != m2.Delta) continue;
354     */
355   }
356   // filters.Insert(i, m);
357   // return i;
358   return filters.Add(m);
359 }
360 
Is86Filter(CMethodId m)361 static inline bool Is86Filter(CMethodId m)
362 {
363   return (m == k_BCJ || m == k_BCJ2);
364 }
365 
IsExeFilter(CMethodId m)366 static inline bool IsExeFilter(CMethodId m)
367 {
368   switch (m)
369   {
370     case k_BCJ:
371     case k_BCJ2:
372     case k_ARM:
373     case k_ARMT:
374     case k_PPC:
375     case k_SPARC:
376     case k_IA64:
377       return true;
378   }
379   return false;
380 }
381 
Get_FilterGroup_for_Folder(CRecordVector<CFilterMode2> & filters,const CFolderEx & f,bool extractFilter)382 static unsigned Get_FilterGroup_for_Folder(
383     CRecordVector<CFilterMode2> &filters, const CFolderEx &f, bool extractFilter)
384 {
385   CFilterMode2 m;
386   m.Id = 0;
387   m.Delta = 0;
388   m.Encrypted = f.IsEncrypted();
389 
390   if (extractFilter)
391   {
392     const CCoderInfo &coder = f.Coders[f.UnpackCoder];
393 
394     if (coder.MethodID == k_Delta)
395     {
396       if (coder.Props.Size() == 1)
397       {
398         m.Delta = (unsigned)coder.Props[0] + 1;
399         m.Id = k_Delta;
400       }
401     }
402     else if (IsExeFilter(coder.MethodID))
403     {
404       m.Id = (UInt32)coder.MethodID;
405       if (m.Id == k_BCJ2)
406         m.Id = k_BCJ;
407       m.SetDelta();
408     }
409   }
410 
411   return GetGroup(filters, m);
412 }
413 
414 
415 
416 
WriteRange(IInStream * inStream,ISequentialOutStream * outStream,UInt64 position,UInt64 size,ICompressProgressInfo * progress)417 static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
418     UInt64 position, UInt64 size, ICompressProgressInfo *progress)
419 {
420   RINOK(inStream->Seek((Int64)position, STREAM_SEEK_SET, 0));
421   CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream;
422   CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec);
423   streamSpec->SetStream(inStream);
424   streamSpec->Init(size);
425 
426   NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder;
427   CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec;
428   RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress));
429   return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL);
430 }
431 
432 /*
433 unsigned CUpdateItem::GetExtensionPos() const
434 {
435   int slashPos = Name.ReverseFind_PathSepar();
436   int dotPos = Name.ReverseFind_Dot();
437   if (dotPos <= slashPos)
438     return Name.Len();
439   return dotPos + 1;
440 }
441 
442 UString CUpdateItem::GetExtension() const
443 {
444   return Name.Ptr(GetExtensionPos());
445 }
446 */
447 
448 #define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
449 
450 #define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
451 
452 /*
453 static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
454 {
455   size_t c1 = a1.GetCapacity();
456   size_t c2 = a2.GetCapacity();
457   RINOZ_COMP(c1, c2);
458   for (size_t i = 0; i < c1; i++)
459     RINOZ_COMP(a1[i], a2[i]);
460   return 0;
461 }
462 
463 static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
464 {
465   RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
466   RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
467   RINOZ_COMP(c1.MethodID, c2.MethodID);
468   return CompareBuffers(c1.Props, c2.Props);
469 }
470 
471 static int CompareBonds(const CBond &b1, const CBond &b2)
472 {
473   RINOZ_COMP(b1.InIndex, b2.InIndex);
474   return MyCompare(b1.OutIndex, b2.OutIndex);
475 }
476 
477 static int CompareFolders(const CFolder &f1, const CFolder &f2)
478 {
479   int s1 = f1.Coders.Size();
480   int s2 = f2.Coders.Size();
481   RINOZ_COMP(s1, s2);
482   int i;
483   for (i = 0; i < s1; i++)
484     RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
485   s1 = f1.Bonds.Size();
486   s2 = f2.Bonds.Size();
487   RINOZ_COMP(s1, s2);
488   for (i = 0; i < s1; i++)
489     RINOZ(CompareBonds(f1.Bonds[i], f2.Bonds[i]));
490   return 0;
491 }
492 */
493 
494 /*
495 static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
496 {
497   return CompareFileNames(f1.Name, f2.Name);
498 }
499 */
500 
501 struct CFolderRepack
502 {
503   unsigned FolderIndex;
504   CNum NumCopyFiles;
505 };
506 
507 /*
508 static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *)
509 {
510   int i1 = p1->FolderIndex;
511   int i2 = p2->FolderIndex;
512   // In that version we don't want to parse folders here, so we don't compare folders
513   // probably it must be improved in future
514   // const CDbEx &db = *(const CDbEx *)param;
515   // RINOZ(CompareFolders(
516   //     db.Folders[i1],
517   //     db.Folders[i2]));
518 
519   return MyCompare(i1, i2);
520 
521   // RINOZ_COMP(
522   //     db.NumUnpackStreamsVector[i1],
523   //     db.NumUnpackStreamsVector[i2]);
524   // if (db.NumUnpackStreamsVector[i1] == 0)
525   //   return 0;
526   // return CompareFiles(
527   //     db.Files[db.FolderStartFileIndex[i1]],
528   //     db.Files[db.FolderStartFileIndex[i2]]);
529 }
530 */
531 
532 /*
533   we sort empty files and dirs in such order:
534   - Dir.NonAnti   (name sorted)
535   - File.NonAnti  (name sorted)
536   - File.Anti     (name sorted)
537   - Dir.Anti (reverse name sorted)
538 */
539 
CompareEmptyItems(const unsigned * p1,const unsigned * p2,void * param)540 static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param)
541 {
542   const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
543   const CUpdateItem &u1 = updateItems[*p1];
544   const CUpdateItem &u2 = updateItems[*p2];
545   // NonAnti < Anti
546   if (u1.IsAnti != u2.IsAnti)
547     return (u1.IsAnti ? 1 : -1);
548   if (u1.IsDir != u2.IsDir)
549   {
550     // Dir.NonAnti < File < Dir.Anti
551     if (u1.IsDir)
552       return (u1.IsAnti ? 1 : -1);
553     return (u2.IsAnti ? -1 : 1);
554   }
555   int n = CompareFileNames(u1.Name, u2.Name);
556   return (u1.IsDir && u1.IsAnti) ? -n : n;
557 }
558 
559 static const char *g_Exts =
560   " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
561   " zip jar ear war msi"
562   " 3gp avi mov mpeg mpg mpe wmv"
563   " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
564   " swf"
565   " chm hxi hxs"
566   " gif jpeg jpg jp2 png tiff  bmp ico psd psp"
567   " awg ps eps cgm dxf svg vrml wmf emf ai md"
568   " cad dwg pps key sxi"
569   " max 3ds"
570   " iso bin nrg mdf img pdi tar cpio xpi"
571   " vfd vhd vud vmc vsv"
572   " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
573   " inl inc idl acf asa"
574   " h hpp hxx c cpp cxx m mm go swift"
575   " rc java cs rs pas bas vb cls ctl frm dlg def"
576   " f77 f f90 f95"
577   " asm s"
578   " sql manifest dep"
579   " mak clw csproj vcproj sln dsp dsw"
580   " class"
581   " bat cmd bash sh"
582   " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
583   " awk sed hta js json php php3 php4 php5 phptml pl pm py pyo rb tcl ts vbs"
584   " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
585   " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
586   " abw afp cwk lwp wpd wps wpt wrf wri"
587   " abf afm bdf fon mgf otf pcf pfa snf ttf"
588   " dbf mdb nsf ntf wdb db fdb gdb"
589   " exe dll ocx vbx sfx sys tlb awx com obj lib out o so"
590   " pdb pch idb ncb opt";
591 
GetExtIndex(const char * ext)592 static unsigned GetExtIndex(const char *ext)
593 {
594   unsigned extIndex = 1;
595   const char *p = g_Exts;
596   for (;;)
597   {
598     char c = *p++;
599     if (c == 0)
600       return extIndex;
601     if (c == ' ')
602       continue;
603     unsigned pos = 0;
604     for (;;)
605     {
606       char c2 = ext[pos++];
607       if (c2 == 0 && (c == 0 || c == ' '))
608         return extIndex;
609       if (c != c2)
610         break;
611       c = *p++;
612     }
613     extIndex++;
614     for (;;)
615     {
616       if (c == 0)
617         return extIndex;
618       if (c == ' ')
619         break;
620       c = *p++;
621     }
622   }
623 }
624 
625 struct CRefItem
626 {
627   const CUpdateItem *UpdateItem;
628   UInt32 Index;
629   unsigned ExtensionPos;
630   unsigned NamePos;
631   unsigned ExtensionIndex;
632 
CRefItemNArchive::N7z::CRefItem633   CRefItem() {};
CRefItemNArchive::N7z::CRefItem634   CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
635     UpdateItem(&ui),
636     Index(index),
637     ExtensionPos(0),
638     NamePos(0),
639     ExtensionIndex(0)
640   {
641     if (sortByType)
642     {
643       int slashPos = ui.Name.ReverseFind_PathSepar();
644       NamePos = (unsigned)(slashPos + 1);
645       int dotPos = ui.Name.ReverseFind_Dot();
646       if (dotPos <= slashPos)
647         ExtensionPos = ui.Name.Len();
648       else
649       {
650         ExtensionPos = (unsigned)(dotPos + 1);
651         if (ExtensionPos != ui.Name.Len())
652         {
653           AString s;
654           for (unsigned pos = ExtensionPos;; pos++)
655           {
656             wchar_t c = ui.Name[pos];
657             if (c >= 0x80)
658               break;
659             if (c == 0)
660             {
661               ExtensionIndex = GetExtIndex(s);
662               break;
663             }
664             s += (char)MyCharLower_Ascii((char)c);
665           }
666         }
667       }
668     }
669   }
670 };
671 
672 struct CSortParam
673 {
674   // const CObjectVector<CTreeFolder> *TreeFolders;
675   bool SortByType;
676 };
677 
678 /*
679   we sort files in such order:
680   - Dir.NonAnti   (name sorted)
681   - alt streams
682   - Dirs
683   - Dir.Anti (reverse name sorted)
684 */
685 
686 
CompareUpdateItems(const CRefItem * p1,const CRefItem * p2,void * param)687 static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
688 {
689   const CRefItem &a1 = *p1;
690   const CRefItem &a2 = *p2;
691   const CUpdateItem &u1 = *a1.UpdateItem;
692   const CUpdateItem &u2 = *a2.UpdateItem;
693 
694   /*
695   if (u1.IsAltStream != u2.IsAltStream)
696     return u1.IsAltStream ? 1 : -1;
697   */
698 
699   // Actually there are no dirs that time. They were stored in other steps
700   // So that code is unused?
701   if (u1.IsDir != u2.IsDir)
702     return u1.IsDir ? 1 : -1;
703   if (u1.IsDir)
704   {
705     if (u1.IsAnti != u2.IsAnti)
706       return (u1.IsAnti ? 1 : -1);
707     int n = CompareFileNames(u1.Name, u2.Name);
708     return -n;
709   }
710 
711   // bool sortByType = *(bool *)param;
712   const CSortParam *sortParam = (const CSortParam *)param;
713   bool sortByType = sortParam->SortByType;
714   if (sortByType)
715   {
716     RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex);
717     RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos)));
718     RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos)));
719     if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
720     if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
721     if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime);
722     RINOZ_COMP(u1.Size, u2.Size);
723   }
724   /*
725   int par1 = a1.UpdateItem->ParentFolderIndex;
726   int par2 = a2.UpdateItem->ParentFolderIndex;
727   const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1];
728   const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2];
729 
730   int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd;
731   int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd;
732   if (b1 < b2)
733   {
734     if (e1 <= b2)
735       return -1;
736     // p2 in p1
737     int par = par2;
738     for (;;)
739     {
740       const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
741       par = tf.Parent;
742       if (par == par1)
743       {
744         RINOZ(CompareFileNames(u1.Name, tf.Name));
745         break;
746       }
747     }
748   }
749   else if (b2 < b1)
750   {
751     if (e2 <= b1)
752       return 1;
753     // p1 in p2
754     int par = par1;
755     for (;;)
756     {
757       const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
758       par = tf.Parent;
759       if (par == par2)
760       {
761         RINOZ(CompareFileNames(tf.Name, u2.Name));
762         break;
763       }
764     }
765   }
766   */
767   // RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex);
768   RINOK(CompareFileNames(u1.Name, u2.Name));
769   RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient);
770   RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive);
771   return 0;
772 }
773 
774 struct CSolidGroup
775 {
776   CRecordVector<UInt32> Indices;
777 
778   CRecordVector<CFolderRepack> folderRefs;
779 };
780 
781 static const char * const g_ExeExts[] =
782 {
783     "dll"
784   , "exe"
785   , "ocx"
786   , "sfx"
787   , "sys"
788 };
789 
IsExeExt(const wchar_t * ext)790 static bool IsExeExt(const wchar_t *ext)
791 {
792   for (unsigned i = 0; i < ARRAY_SIZE(g_ExeExts); i++)
793     if (StringsAreEqualNoCase_Ascii(ext, g_ExeExts[i]))
794       return true;
795   return false;
796 }
797 
798 struct CAnalysis
799 {
800   CMyComPtr<IArchiveUpdateCallbackFile> Callback;
801   CByteBuffer Buffer;
802 
803   bool ParseWav;
804   bool ParseExe;
805   bool ParseAll;
806 
CAnalysisNArchive::N7z::CAnalysis807   CAnalysis():
808       ParseWav(true),
809       ParseExe(false),
810       ParseAll(false)
811   {}
812 
813   HRESULT GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode);
814 };
815 
816 static const size_t kAnalysisBufSize = 1 << 14;
817 
GetFilterGroup(UInt32 index,const CUpdateItem & ui,CFilterMode & filterMode)818 HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode)
819 {
820   filterMode.Id = 0;
821   filterMode.Delta = 0;
822 
823   CFilterMode filterModeTemp = filterMode;
824 
825   int slashPos = ui.Name.ReverseFind_PathSepar();
826   int dotPos = ui.Name.ReverseFind_Dot();
827 
828   // if (dotPos > slashPos)
829   {
830     bool needReadFile = ParseAll;
831 
832     bool probablyIsSameIsa = false;
833 
834     if (!needReadFile || !Callback)
835     {
836       const wchar_t *ext;
837       if (dotPos > slashPos)
838         ext = ui.Name.Ptr((unsigned)(dotPos + 1));
839       else
840         ext = ui.Name.RightPtr(0);
841 
842       // p7zip uses the trick to store posix attributes in high 16 bits
843       if (ui.Attrib & 0x8000)
844       {
845         unsigned st_mode = ui.Attrib >> 16;
846         // st_mode = 00111;
847         if ((st_mode & 00111) && (ui.Size >= 2048))
848         {
849           #ifndef _WIN32
850           probablyIsSameIsa = true;
851           #endif
852           needReadFile = true;
853         }
854       }
855 
856       if (IsExeExt(ext))
857       {
858         needReadFile = true;
859         #ifdef _WIN32
860         probablyIsSameIsa = true;
861         needReadFile = ParseExe;
862         #endif
863       }
864       else if (StringsAreEqualNoCase_Ascii(ext, "wav"))
865       {
866         needReadFile = ParseWav;
867       }
868       /*
869       else if (!needReadFile && ParseUnixExt)
870       {
871         if (StringsAreEqualNoCase_Ascii(ext, "so")
872           || StringsAreEqualNoCase_Ascii(ext, ""))
873 
874           needReadFile = true;
875       }
876       */
877     }
878 
879     if (needReadFile && Callback)
880     {
881       if (Buffer.Size() != kAnalysisBufSize)
882       {
883         Buffer.Alloc(kAnalysisBufSize);
884       }
885       {
886         CMyComPtr<ISequentialInStream> stream;
887         HRESULT result = Callback->GetStream2(index, &stream, NUpdateNotifyOp::kAnalyze);
888         if (result == S_OK && stream)
889         {
890           size_t size = kAnalysisBufSize;
891           result = ReadStream(stream, Buffer, &size);
892           stream.Release();
893           // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK));
894           if (result == S_OK)
895           {
896             BoolInt parseRes = ParseFile(Buffer, size, &filterModeTemp);
897             if (parseRes && filterModeTemp.Delta == 0)
898             {
899               filterModeTemp.SetDelta();
900               if (filterModeTemp.Delta != 0 && filterModeTemp.Id != k_Delta)
901               {
902                 if (ui.Size % filterModeTemp.Delta != 0)
903                 {
904                   parseRes = false;
905                 }
906               }
907             }
908             if (!parseRes)
909             {
910               filterModeTemp.Id = 0;
911               filterModeTemp.Delta = 0;
912             }
913           }
914         }
915       }
916     }
917     else if ((needReadFile && !Callback) || probablyIsSameIsa)
918     {
919       #ifdef MY_CPU_X86_OR_AMD64
920       if (probablyIsSameIsa)
921         filterModeTemp.Id = k_X86;
922       #endif
923     }
924   }
925 
926   filterMode = filterModeTemp;
927   return S_OK;
928 }
929 
GetMethodFull(UInt64 methodID,UInt32 numStreams,CMethodFull & m)930 static inline void GetMethodFull(UInt64 methodID, UInt32 numStreams, CMethodFull &m)
931 {
932   m.Id = methodID;
933   m.NumStreams = numStreams;
934 }
935 
AddBondForFilter(CCompressionMethodMode & mode)936 static HRESULT AddBondForFilter(CCompressionMethodMode &mode)
937 {
938   for (unsigned c = 1; c < mode.Methods.Size(); c++)
939   {
940     if (!mode.IsThereBond_to_Coder(c))
941     {
942       CBond2 bond;
943       bond.OutCoder = 0;
944       bond.OutStream = 0;
945       bond.InCoder = c;
946       mode.Bonds.Add(bond);
947       return S_OK;
948     }
949   }
950   return E_INVALIDARG;
951 }
952 
AddFilterBond(CCompressionMethodMode & mode)953 static HRESULT AddFilterBond(CCompressionMethodMode &mode)
954 {
955   if (!mode.Bonds.IsEmpty())
956     return AddBondForFilter(mode);
957   return S_OK;
958 }
959 
AddBcj2Methods(CCompressionMethodMode & mode)960 static HRESULT AddBcj2Methods(CCompressionMethodMode &mode)
961 {
962   // mode.Methods[0] must be k_BCJ2 method !
963 
964   CMethodFull m;
965   GetMethodFull(k_LZMA, 1, m);
966 
967   m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20);
968   m.AddProp32(NCoderPropID::kNumFastBytes, 128);
969   m.AddProp32(NCoderPropID::kNumThreads, 1);
970   m.AddProp32(NCoderPropID::kLitPosBits, 2);
971   m.AddProp32(NCoderPropID::kLitContextBits, 0);
972   // m.AddProp_Ascii(NCoderPropID::kMatchFinder, "BT2");
973 
974   unsigned methodIndex = mode.Methods.Size();
975 
976   if (mode.Bonds.IsEmpty())
977   {
978     for (unsigned i = 1; i + 1 < mode.Methods.Size(); i++)
979     {
980       CBond2 bond;
981       bond.OutCoder = i;
982       bond.OutStream = 0;
983       bond.InCoder = i + 1;
984       mode.Bonds.Add(bond);
985     }
986   }
987 
988   mode.Methods.Add(m);
989   mode.Methods.Add(m);
990 
991   RINOK(AddBondForFilter(mode));
992   CBond2 bond;
993   bond.OutCoder = 0;
994   bond.InCoder = methodIndex;      bond.OutStream = 1;  mode.Bonds.Add(bond);
995   bond.InCoder = methodIndex + 1;  bond.OutStream = 2;  mode.Bonds.Add(bond);
996   return S_OK;
997 }
998 
MakeExeMethod(CCompressionMethodMode & mode,const CFilterMode & filterMode,bool bcj2Filter)999 static HRESULT MakeExeMethod(CCompressionMethodMode &mode,
1000     const CFilterMode &filterMode, /* bool addFilter, */ bool bcj2Filter)
1001 {
1002   if (mode.Filter_was_Inserted)
1003   {
1004     const CMethodFull &m = mode.Methods[0];
1005     CMethodId id = m.Id;
1006     if (id == k_BCJ2)
1007       return AddBcj2Methods(mode);
1008     if (!m.IsSimpleCoder())
1009       return E_NOTIMPL;
1010     // if (Bonds.IsEmpty()) we can create bonds later
1011     return AddFilterBond(mode);
1012   }
1013 
1014   if (filterMode.Id == 0)
1015     return S_OK;
1016 
1017   CMethodFull &m = mode.Methods.InsertNew(0);
1018 
1019   {
1020     FOR_VECTOR(k, mode.Bonds)
1021     {
1022       CBond2 &bond = mode.Bonds[k];
1023       bond.InCoder++;
1024       bond.OutCoder++;
1025     }
1026   }
1027 
1028   HRESULT res;
1029 
1030   if (bcj2Filter && Is86Filter(filterMode.Id))
1031   {
1032     GetMethodFull(k_BCJ2, 4, m);
1033     res = AddBcj2Methods(mode);
1034   }
1035   else
1036   {
1037     GetMethodFull(filterMode.Id, 1, m);
1038     if (filterMode.Id == k_Delta)
1039       m.AddProp32(NCoderPropID::kDefaultProp, filterMode.Delta);
1040     res = AddFilterBond(mode);
1041 
1042     int alignBits = -1;
1043     if (filterMode.Id == k_Delta || filterMode.Delta != 0)
1044     {
1045            if (filterMode.Delta == 1) alignBits = 0;
1046       else if (filterMode.Delta == 2) alignBits = 1;
1047       else if (filterMode.Delta == 4) alignBits = 2;
1048       else if (filterMode.Delta == 8) alignBits = 3;
1049       else if (filterMode.Delta == 16) alignBits = 4;
1050     }
1051     else
1052     {
1053       // alignBits = GetAlignForFilterMethod(filterMode.Id);
1054     }
1055 
1056     if (res == S_OK && alignBits >= 0)
1057     {
1058       unsigned nextCoder = 1;
1059       if (!mode.Bonds.IsEmpty())
1060       {
1061         nextCoder = mode.Bonds.Back().InCoder;
1062       }
1063       if (nextCoder < mode.Methods.Size())
1064       {
1065         CMethodFull &nextMethod = mode.Methods[nextCoder];
1066         if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2)
1067         {
1068           if (!nextMethod.Are_Lzma_Model_Props_Defined())
1069           {
1070             if (alignBits != 0)
1071             {
1072               if (alignBits > 2 || filterMode.Id == k_Delta)
1073                 nextMethod.AddProp32(NCoderPropID::kPosStateBits, (unsigned)alignBits);
1074               unsigned lc = 0;
1075               if (alignBits < 3)
1076                 lc = (unsigned)(3 - alignBits);
1077               nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc);
1078               nextMethod.AddProp32(NCoderPropID::kLitPosBits, (unsigned)alignBits);
1079             }
1080           }
1081         }
1082       }
1083     }
1084   }
1085 
1086   return res;
1087 }
1088 
1089 
UpdateItem_To_FileItem2(const CUpdateItem & ui,CFileItem2 & file2)1090 static void UpdateItem_To_FileItem2(const CUpdateItem &ui, CFileItem2 &file2)
1091 {
1092   file2.Attrib = ui.Attrib;  file2.AttribDefined = ui.AttribDefined;
1093   file2.CTime = ui.CTime;  file2.CTimeDefined = ui.CTimeDefined;
1094   file2.ATime = ui.ATime;  file2.ATimeDefined = ui.ATimeDefined;
1095   file2.MTime = ui.MTime;  file2.MTimeDefined = ui.MTimeDefined;
1096   file2.IsAnti = ui.IsAnti;
1097   // file2.IsAux = false;
1098   file2.StartPosDefined = false;
1099   // file2.StartPos = 0;
1100 }
1101 
1102 
UpdateItem_To_FileItem(const CUpdateItem & ui,CFileItem & file,CFileItem2 & file2)1103 static void UpdateItem_To_FileItem(const CUpdateItem &ui,
1104     CFileItem &file, CFileItem2 &file2)
1105 {
1106   UpdateItem_To_FileItem2(ui, file2);
1107 
1108   file.Size = ui.Size;
1109   file.IsDir = ui.IsDir;
1110   file.HasStream = ui.HasStream();
1111   // file.IsAltStream = ui.IsAltStream;
1112 }
1113 
1114 
1115 
1116 class CRepackInStreamWithSizes:
1117   public ISequentialInStream,
1118   public ICompressGetSubStreamSize,
1119   public CMyUnknownImp
1120 {
1121   CMyComPtr<ISequentialInStream> _stream;
1122   // UInt64 _size;
1123   const CBoolVector *_extractStatuses;
1124   UInt32 _startIndex;
1125 public:
1126   const CDbEx *_db;
1127 
Init(ISequentialInStream * stream,UInt32 startIndex,const CBoolVector * extractStatuses)1128   void Init(ISequentialInStream *stream, UInt32 startIndex, const CBoolVector *extractStatuses)
1129   {
1130     _startIndex = startIndex;
1131     _extractStatuses = extractStatuses;
1132     // _size = 0;
1133     _stream = stream;
1134   }
1135   // UInt64 GetSize() const { return _size; }
1136 
1137   MY_UNKNOWN_IMP2(ISequentialInStream, ICompressGetSubStreamSize)
1138 
1139   STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
1140 
1141   STDMETHOD(GetSubStreamSize)(UInt64 subStream, UInt64 *value);
1142 };
1143 
Read(void * data,UInt32 size,UInt32 * processedSize)1144 STDMETHODIMP CRepackInStreamWithSizes::Read(void *data, UInt32 size, UInt32 *processedSize)
1145 {
1146   return _stream->Read(data, size, processedSize);
1147   /*
1148   UInt32 realProcessedSize;
1149   HRESULT result = _stream->Read(data, size, &realProcessedSize);
1150   _size += realProcessedSize;
1151   if (processedSize)
1152     *processedSize = realProcessedSize;
1153   return result;
1154   */
1155 }
1156 
GetSubStreamSize(UInt64 subStream,UInt64 * value)1157 STDMETHODIMP CRepackInStreamWithSizes::GetSubStreamSize(UInt64 subStream, UInt64 *value)
1158 {
1159   *value = 0;
1160   if (subStream >= _extractStatuses->Size())
1161     return S_FALSE; // E_FAIL;
1162   unsigned index = (unsigned)subStream;
1163   if ((*_extractStatuses)[index])
1164   {
1165     const CFileItem &fi = _db->Files[_startIndex + index];
1166     if (fi.HasStream)
1167       *value = fi.Size;
1168   }
1169   return S_OK;
1170 }
1171 
1172 
1173 class CRepackStreamBase
1174 {
1175 protected:
1176   bool _needWrite;
1177   bool _fileIsOpen;
1178   bool _calcCrc;
1179   UInt32 _crc;
1180   UInt64 _rem;
1181 
1182   const CBoolVector *_extractStatuses;
1183   UInt32 _startIndex;
1184   unsigned _currentIndex;
1185 
1186   HRESULT OpenFile();
1187   HRESULT CloseFile();
1188   HRESULT ProcessEmptyFiles();
1189 
1190 public:
1191   const CDbEx *_db;
1192   CMyComPtr<IArchiveUpdateCallbackFile> _opCallback;
1193   CMyComPtr<IArchiveExtractCallbackMessage> _extractCallback;
1194 
1195   HRESULT Init(UInt32 startIndex, const CBoolVector *extractStatuses);
CheckFinishedState() const1196   HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
1197 };
1198 
Init(UInt32 startIndex,const CBoolVector * extractStatuses)1199 HRESULT CRepackStreamBase::Init(UInt32 startIndex, const CBoolVector *extractStatuses)
1200 {
1201   _startIndex = startIndex;
1202   _extractStatuses = extractStatuses;
1203 
1204   _currentIndex = 0;
1205   _fileIsOpen = false;
1206 
1207   return ProcessEmptyFiles();
1208 }
1209 
OpenFile()1210 HRESULT CRepackStreamBase::OpenFile()
1211 {
1212   UInt32 arcIndex = _startIndex + _currentIndex;
1213   const CFileItem &fi = _db->Files[arcIndex];
1214 
1215   _needWrite = (*_extractStatuses)[_currentIndex];
1216   if (_opCallback)
1217   {
1218     RINOK(_opCallback->ReportOperation(
1219         NEventIndexType::kInArcIndex, arcIndex,
1220         _needWrite ?
1221             NUpdateNotifyOp::kRepack :
1222             NUpdateNotifyOp::kSkip));
1223   }
1224 
1225   _crc = CRC_INIT_VAL;
1226   _calcCrc = (fi.CrcDefined && !fi.IsDir);
1227 
1228   _fileIsOpen = true;
1229   _rem = fi.Size;
1230   return S_OK;
1231 }
1232 
1233 const HRESULT k_My_HRESULT_CRC_ERROR = 0x20000002;
1234 
CloseFile()1235 HRESULT CRepackStreamBase::CloseFile()
1236 {
1237   UInt32 arcIndex = _startIndex + _currentIndex;
1238   const CFileItem &fi = _db->Files[arcIndex];
1239   _fileIsOpen = false;
1240   _currentIndex++;
1241   if (!_calcCrc || fi.Crc == CRC_GET_DIGEST(_crc))
1242     return S_OK;
1243 
1244   if (_extractCallback)
1245   {
1246     RINOK(_extractCallback->ReportExtractResult(
1247         NEventIndexType::kInArcIndex, arcIndex,
1248         NExtract::NOperationResult::kCRCError));
1249   }
1250   // return S_FALSE;
1251   return k_My_HRESULT_CRC_ERROR;
1252 }
1253 
ProcessEmptyFiles()1254 HRESULT CRepackStreamBase::ProcessEmptyFiles()
1255 {
1256   while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
1257   {
1258     RINOK(OpenFile());
1259     RINOK(CloseFile());
1260   }
1261   return S_OK;
1262 }
1263 
1264 
1265 
1266 #ifndef _7ZIP_ST
1267 
1268 class CFolderOutStream2:
1269   public CRepackStreamBase,
1270   public ISequentialOutStream,
1271   public CMyUnknownImp
1272 {
1273 public:
1274   CMyComPtr<ISequentialOutStream> _stream;
1275 
1276   MY_UNKNOWN_IMP
1277 
1278   STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
1279 };
1280 
Write(const void * data,UInt32 size,UInt32 * processedSize)1281 STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize)
1282 {
1283   if (processedSize)
1284     *processedSize = 0;
1285 
1286   while (size != 0)
1287   {
1288     if (_fileIsOpen)
1289     {
1290       UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1291       HRESULT result = S_OK;
1292       if (_needWrite)
1293         result = _stream->Write(data, cur, &cur);
1294       if (_calcCrc)
1295         _crc = CrcUpdate(_crc, data, cur);
1296       if (processedSize)
1297         *processedSize += cur;
1298       data = (const Byte *)data + cur;
1299       size -= cur;
1300       _rem -= cur;
1301       if (_rem == 0)
1302       {
1303         RINOK(CloseFile());
1304         RINOK(ProcessEmptyFiles());
1305       }
1306       RINOK(result);
1307       if (cur == 0)
1308         break;
1309       continue;
1310     }
1311 
1312     RINOK(ProcessEmptyFiles());
1313     if (_currentIndex == _extractStatuses->Size())
1314     {
1315       // we don't support write cut here
1316       return E_FAIL;
1317     }
1318     RINOK(OpenFile());
1319   }
1320 
1321   return S_OK;
1322 }
1323 
1324 #endif
1325 
1326 
1327 
1328 static const UInt32 kTempBufSize = 1 << 16;
1329 
1330 class CFolderInStream2:
1331   public CRepackStreamBase,
1332   public ISequentialInStream,
1333   public CMyUnknownImp
1334 {
1335   Byte *_buf;
1336 public:
1337   CMyComPtr<ISequentialInStream> _inStream;
1338   HRESULT Result;
1339 
1340   MY_UNKNOWN_IMP
1341 
CFolderInStream2()1342   CFolderInStream2():
1343       Result(S_OK)
1344   {
1345     _buf = new Byte[kTempBufSize];
1346   }
1347 
~CFolderInStream2()1348   ~CFolderInStream2()
1349   {
1350     delete []_buf;
1351   }
1352 
Init()1353   void Init() { Result = S_OK; }
1354   STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
1355 };
1356 
Read(void * data,UInt32 size,UInt32 * processedSize)1357 STDMETHODIMP CFolderInStream2::Read(void *data, UInt32 size, UInt32 *processedSize)
1358 {
1359   if (processedSize)
1360     *processedSize = 0;
1361 
1362   while (size != 0)
1363   {
1364     if (_fileIsOpen)
1365     {
1366       UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1367 
1368       void *buf;
1369       if (_needWrite)
1370         buf = data;
1371       else
1372       {
1373         buf = _buf;
1374         if (cur > kTempBufSize)
1375           cur = kTempBufSize;
1376       }
1377 
1378       HRESULT result = _inStream->Read(buf, cur, &cur);
1379       _crc = CrcUpdate(_crc, buf, cur);
1380       _rem -= cur;
1381 
1382       if (_needWrite)
1383       {
1384         data = (Byte *)data + cur;
1385         size -= cur;
1386         if (processedSize)
1387           *processedSize += cur;
1388       }
1389 
1390       if (result != S_OK)
1391         Result = result;
1392 
1393       if (_rem == 0)
1394       {
1395         RINOK(CloseFile());
1396         RINOK(ProcessEmptyFiles());
1397       }
1398 
1399       RINOK(result);
1400 
1401       if (cur == 0)
1402         return E_FAIL;
1403 
1404       continue;
1405     }
1406 
1407     RINOK(ProcessEmptyFiles());
1408     if (_currentIndex == _extractStatuses->Size())
1409     {
1410       return S_OK;
1411     }
1412     RINOK(OpenFile());
1413   }
1414 
1415   return S_OK;
1416 }
1417 
1418 
1419 class CThreadDecoder
1420   #ifndef _7ZIP_ST
1421     : public CVirtThread
1422   #endif
1423 {
1424 public:
1425   CDecoder Decoder;
1426 
CThreadDecoder(bool multiThreadMixer)1427   CThreadDecoder(bool multiThreadMixer):
1428       Decoder(multiThreadMixer)
1429   {
1430     #ifndef _7ZIP_ST
1431     if (multiThreadMixer)
1432     {
1433       MtMode = false;
1434       NumThreads = 1;
1435       FosSpec = new CFolderOutStream2;
1436       Fos = FosSpec;
1437       Result = E_FAIL;
1438     }
1439     #endif
1440     // UnpackSize = 0;
1441     // send_UnpackSize = false;
1442   }
1443 
1444   #ifndef _7ZIP_ST
1445 
1446   bool dataAfterEnd_Error;
1447   HRESULT Result;
1448   CMyComPtr<IInStream> InStream;
1449 
1450   CFolderOutStream2 *FosSpec;
1451   CMyComPtr<ISequentialOutStream> Fos;
1452 
1453   UInt64 StartPos;
1454   const CFolders *Folders;
1455   unsigned FolderIndex;
1456 
1457   // bool send_UnpackSize;
1458   // UInt64 UnpackSize;
1459 
1460   #ifndef _NO_CRYPTO
1461   CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1462   #endif
1463 
1464   DECL_EXTERNAL_CODECS_LOC_VARS2;
1465 
1466   #ifndef _7ZIP_ST
1467   bool MtMode;
1468   UInt32 NumThreads;
1469   #endif
1470 
1471 
~CThreadDecoder()1472   ~CThreadDecoder() { CVirtThread::WaitThreadFinish(); }
1473   virtual void Execute();
1474 
1475   #endif
1476 };
1477 
1478 #ifndef _7ZIP_ST
1479 
Execute()1480 void CThreadDecoder::Execute()
1481 {
1482   try
1483   {
1484     #ifndef _NO_CRYPTO
1485       bool isEncrypted = false;
1486       bool passwordIsDefined = false;
1487       UString password;
1488     #endif
1489 
1490     dataAfterEnd_Error = false;
1491 
1492     Result = Decoder.Decode(
1493       EXTERNAL_CODECS_LOC_VARS
1494       InStream,
1495       StartPos,
1496       *Folders, FolderIndex,
1497 
1498       // send_UnpackSize ? &UnpackSize : NULL,
1499       NULL, // unpackSize : FULL unpack
1500 
1501       Fos,
1502       NULL, // compressProgress
1503 
1504       NULL  // *inStreamMainRes
1505       , dataAfterEnd_Error
1506 
1507       _7Z_DECODER_CRYPRO_VARS
1508       #ifndef _7ZIP_ST
1509         , MtMode, NumThreads,
1510         0 // MemUsage
1511       #endif
1512 
1513       );
1514   }
1515   catch(...)
1516   {
1517     Result = E_FAIL;
1518   }
1519 
1520   /*
1521   if (Result == S_OK)
1522     Result = FosSpec->CheckFinishedState();
1523   */
1524   FosSpec->_stream.Release();
1525 }
1526 
1527 #endif
1528 
1529 #ifndef _NO_CRYPTO
1530 
1531 class CCryptoGetTextPassword:
1532   public ICryptoGetTextPassword,
1533   public CMyUnknownImp
1534 {
1535 public:
1536   UString Password;
1537 
1538   MY_UNKNOWN_IMP
1539   STDMETHOD(CryptoGetTextPassword)(BSTR *password);
1540 };
1541 
CryptoGetTextPassword(BSTR * password)1542 STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password)
1543 {
1544   return StringToBstr(Password, password);
1545 }
1546 
1547 #endif
1548 
1549 
GetFile(const CDatabase & inDb,unsigned index,CFileItem & file,CFileItem2 & file2)1550 static void GetFile(const CDatabase &inDb, unsigned index, CFileItem &file, CFileItem2 &file2)
1551 {
1552   file = inDb.Files[index];
1553   file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime);
1554   file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime);
1555   file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime);
1556   file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos);
1557   file2.AttribDefined = inDb.Attrib.GetItem(index, file2.Attrib);
1558   file2.IsAnti = inDb.IsItemAnti(index);
1559   // file2.IsAux = inDb.IsItemAux(index);
1560 }
1561 
Update(DECL_EXTERNAL_CODECS_LOC_VARS IInStream * inStream,const CDbEx * db,const CObjectVector<CUpdateItem> & updateItems,COutArchive & archive,CArchiveDatabaseOut & newDatabase,ISequentialOutStream * seqOutStream,IArchiveUpdateCallback * updateCallback,const CUpdateOptions & options,ICryptoGetTextPassword * getDecoderPassword)1562 HRESULT Update(
1563     DECL_EXTERNAL_CODECS_LOC_VARS
1564     IInStream *inStream,
1565     const CDbEx *db,
1566     const CObjectVector<CUpdateItem> &updateItems,
1567     // const CObjectVector<CTreeFolder> &treeFolders,
1568     // const CUniqBlocks &secureBlocks,
1569     COutArchive &archive,
1570     CArchiveDatabaseOut &newDatabase,
1571     ISequentialOutStream *seqOutStream,
1572     IArchiveUpdateCallback *updateCallback,
1573     const CUpdateOptions &options
1574     #ifndef _NO_CRYPTO
1575     , ICryptoGetTextPassword *getDecoderPassword
1576     #endif
1577     )
1578 {
1579   UInt64 numSolidFiles = options.NumSolidFiles;
1580   if (numSolidFiles == 0)
1581     numSolidFiles = 1;
1582 
1583   CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
1584   updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
1585 
1586   CMyComPtr<IArchiveExtractCallbackMessage> extractCallback;
1587   updateCallback->QueryInterface(IID_IArchiveExtractCallbackMessage, (void **)&extractCallback);
1588 
1589   // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes();
1590 
1591   /*
1592   CMyComPtr<IOutStream> outStream;
1593   RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream));
1594   if (!outStream)
1595     return E_NOTIMPL;
1596   */
1597 
1598   UInt64 startBlockSize = db ? db->ArcInfo.StartPosition: 0;
1599   if (startBlockSize > 0 && !options.RemoveSfxBlock)
1600   {
1601     RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL));
1602   }
1603 
1604   CIntArr fileIndexToUpdateIndexMap;
1605   UInt64 complexity = 0;
1606   UInt64 inSizeForReduce2 = 0;
1607   bool needEncryptedRepack = false;
1608 
1609   CRecordVector<CFilterMode2> filters;
1610   CObjectVector<CSolidGroup> groups;
1611 
1612   #ifndef _7ZIP_ST
1613   bool thereAreRepacks = false;
1614   #endif
1615 
1616   bool useFilters = options.UseFilters;
1617   if (useFilters)
1618   {
1619     const CCompressionMethodMode &method = *options.Method;
1620 
1621     FOR_VECTOR (i, method.Methods)
1622       if (IsFilterMethod(method.Methods[i].Id))
1623       {
1624         useFilters = false;
1625         break;
1626       }
1627   }
1628 
1629   if (db)
1630   {
1631     fileIndexToUpdateIndexMap.Alloc(db->Files.Size());
1632     unsigned i;
1633 
1634     for (i = 0; i < db->Files.Size(); i++)
1635       fileIndexToUpdateIndexMap[i] = -1;
1636 
1637     for (i = 0; i < updateItems.Size(); i++)
1638     {
1639       int index = updateItems[i].IndexInArchive;
1640       if (index != -1)
1641         fileIndexToUpdateIndexMap[(unsigned)index] = (int)i;
1642     }
1643 
1644     for (i = 0; i < db->NumFolders; i++)
1645     {
1646       CNum indexInFolder = 0;
1647       CNum numCopyItems = 0;
1648       CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
1649       UInt64 repackSize = 0;
1650 
1651       for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
1652       {
1653         if (fi >= db->Files.Size())
1654           return E_FAIL;
1655 
1656         const CFileItem &file = db->Files[fi];
1657         if (file.HasStream)
1658         {
1659           indexInFolder++;
1660           int updateIndex = fileIndexToUpdateIndexMap[fi];
1661           if (updateIndex >= 0 && !updateItems[(unsigned)updateIndex].NewData)
1662           {
1663             numCopyItems++;
1664             repackSize += file.Size;
1665           }
1666         }
1667       }
1668 
1669       if (numCopyItems == 0)
1670         continue;
1671 
1672       CFolderRepack rep;
1673       rep.FolderIndex = i;
1674       rep.NumCopyFiles = numCopyItems;
1675       CFolderEx f;
1676       db->ParseFolderEx(i, f);
1677 
1678       const bool isEncrypted = f.IsEncrypted();
1679       const bool needCopy = (numCopyItems == numUnpackStreams);
1680       const bool extractFilter = (useFilters || needCopy);
1681 
1682       unsigned groupIndex = Get_FilterGroup_for_Folder(filters, f, extractFilter);
1683 
1684       while (groupIndex >= groups.Size())
1685         groups.AddNew();
1686 
1687       groups[groupIndex].folderRefs.Add(rep);
1688 
1689       if (needCopy)
1690         complexity += db->GetFolderFullPackSize(i);
1691       else
1692       {
1693         #ifndef _7ZIP_ST
1694         thereAreRepacks = true;
1695         #endif
1696         complexity += repackSize;
1697         if (inSizeForReduce2 < repackSize)
1698           inSizeForReduce2 = repackSize;
1699         if (isEncrypted)
1700           needEncryptedRepack = true;
1701       }
1702     }
1703   }
1704 
1705   UInt64 inSizeForReduce = 0;
1706   {
1707     bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0);
1708     FOR_VECTOR (i, updateItems)
1709     {
1710       const CUpdateItem &ui = updateItems[i];
1711       if (ui.NewData)
1712       {
1713         complexity += ui.Size;
1714         if (isSolid)
1715           inSizeForReduce += ui.Size;
1716         else if (inSizeForReduce < ui.Size)
1717           inSizeForReduce = ui.Size;
1718       }
1719     }
1720   }
1721 
1722   if (inSizeForReduce < inSizeForReduce2)
1723     inSizeForReduce = inSizeForReduce2;
1724 
1725   RINOK(updateCallback->SetTotal(complexity));
1726 
1727   CLocalProgress *lps = new CLocalProgress;
1728   CMyComPtr<ICompressProgressInfo> progress = lps;
1729   lps->Init(updateCallback, true);
1730 
1731   #ifndef _7ZIP_ST
1732 
1733   CStreamBinder sb;
1734   /*
1735   if (options.MultiThreadMixer)
1736   {
1737     RINOK(sb.CreateEvents());
1738   }
1739   */
1740 
1741   #endif
1742 
1743   CThreadDecoder threadDecoder(options.MultiThreadMixer);
1744 
1745   #ifndef _7ZIP_ST
1746   if (options.MultiThreadMixer && thereAreRepacks)
1747   {
1748     #ifdef EXTERNAL_CODECS
1749     threadDecoder.__externalCodecs = __externalCodecs;
1750     #endif
1751     WRes wres = threadDecoder.Create();
1752     if (wres != 0)
1753       return HRESULT_FROM_WIN32(wres);
1754   }
1755   #endif
1756 
1757   {
1758     CAnalysis analysis;
1759     if (options.AnalysisLevel == 0)
1760     {
1761       analysis.ParseWav = false;
1762       analysis.ParseExe = false;
1763       analysis.ParseAll = false;
1764     }
1765     else
1766     {
1767       analysis.Callback = opCallback;
1768       if (options.AnalysisLevel > 0)
1769       {
1770         analysis.ParseWav = true;
1771         if (options.AnalysisLevel >= 7)
1772         {
1773           analysis.ParseExe = true;
1774           if (options.AnalysisLevel >= 9)
1775             analysis.ParseAll = true;
1776         }
1777       }
1778     }
1779 
1780     // ---------- Split files to groups ----------
1781 
1782     const CCompressionMethodMode &method = *options.Method;
1783 
1784     FOR_VECTOR (i, updateItems)
1785     {
1786       const CUpdateItem &ui = updateItems[i];
1787       if (!ui.NewData || !ui.HasStream())
1788         continue;
1789 
1790       CFilterMode2 fm;
1791       if (useFilters)
1792       {
1793         RINOK(analysis.GetFilterGroup(i, ui, fm));
1794       }
1795       fm.Encrypted = method.PasswordIsDefined;
1796 
1797       unsigned groupIndex = GetGroup(filters, fm);
1798       while (groupIndex >= groups.Size())
1799         groups.AddNew();
1800       groups[groupIndex].Indices.Add(i);
1801     }
1802   }
1803 
1804 
1805   #ifndef _NO_CRYPTO
1806 
1807   CCryptoGetTextPassword *getPasswordSpec = NULL;
1808   CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1809   if (needEncryptedRepack)
1810   {
1811     getPasswordSpec = new CCryptoGetTextPassword;
1812     getTextPassword = getPasswordSpec;
1813 
1814     #ifndef _7ZIP_ST
1815     threadDecoder.getTextPassword = getPasswordSpec;
1816     #endif
1817 
1818     if (options.Method->PasswordIsDefined)
1819       getPasswordSpec->Password = options.Method->Password;
1820     else
1821     {
1822       if (!getDecoderPassword)
1823         return E_NOTIMPL;
1824       CMyComBSTR password;
1825       RINOK(getDecoderPassword->CryptoGetTextPassword(&password));
1826       if (password)
1827         getPasswordSpec->Password = password;
1828     }
1829   }
1830 
1831   #endif
1832 
1833 
1834   // ---------- Compress ----------
1835 
1836   RINOK(archive.Create(seqOutStream, false));
1837   RINOK(archive.SkipPrefixArchiveHeader());
1838 
1839   /*
1840   CIntVector treeFolderToArcIndex;
1841   treeFolderToArcIndex.Reserve(treeFolders.Size());
1842   for (i = 0; i < treeFolders.Size(); i++)
1843     treeFolderToArcIndex.Add(-1);
1844   // ---------- Write Tree (only AUX dirs) ----------
1845   for (i = 1; i < treeFolders.Size(); i++)
1846   {
1847     const CTreeFolder &treeFolder = treeFolders[i];
1848     CFileItem file;
1849     CFileItem2 file2;
1850     file2.Init();
1851     int secureID = 0;
1852     if (treeFolder.UpdateItemIndex < 0)
1853     {
1854       // we can store virtual dir item wuthout attrib, but we want all items have attrib.
1855       file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY);
1856       file2.IsAux = true;
1857     }
1858     else
1859     {
1860       const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex];
1861       // if item is not dir, then it's parent for alt streams.
1862       // we will write such items later
1863       if (!ui.IsDir)
1864         continue;
1865       secureID = ui.SecureIndex;
1866       if (ui.NewProps)
1867         UpdateItem_To_FileItem(ui, file, file2);
1868       else
1869         GetFile(*db, ui.IndexInArchive, file, file2);
1870     }
1871     file.Size = 0;
1872     file.HasStream = false;
1873     file.IsDir = true;
1874     file.Parent = treeFolder.Parent;
1875 
1876     treeFolderToArcIndex[i] = newDatabase.Files.Size();
1877     newDatabase.AddFile(file, file2, treeFolder.Name);
1878 
1879     if (totalSecureDataSize != 0)
1880       newDatabase.SecureIDs.Add(secureID);
1881   }
1882   */
1883 
1884   {
1885     /* ---------- Write non-AUX dirs and Empty files ---------- */
1886     CUIntVector emptyRefs;
1887 
1888     unsigned i;
1889 
1890     for (i = 0; i < updateItems.Size(); i++)
1891     {
1892       const CUpdateItem &ui = updateItems[i];
1893       if (ui.NewData)
1894       {
1895         if (ui.HasStream())
1896           continue;
1897       }
1898       else if (ui.IndexInArchive != -1 && db->Files[(unsigned)ui.IndexInArchive].HasStream)
1899         continue;
1900       /*
1901       if (ui.TreeFolderIndex >= 0)
1902         continue;
1903       */
1904       emptyRefs.Add(i);
1905     }
1906 
1907     emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
1908 
1909     for (i = 0; i < emptyRefs.Size(); i++)
1910     {
1911       const CUpdateItem &ui = updateItems[emptyRefs[i]];
1912       CFileItem file;
1913       CFileItem2 file2;
1914       UString name;
1915       if (ui.NewProps)
1916       {
1917         UpdateItem_To_FileItem(ui, file, file2);
1918         file.CrcDefined = false;
1919         name = ui.Name;
1920       }
1921       else
1922       {
1923         GetFile(*db, (unsigned)ui.IndexInArchive, file, file2);
1924         db->GetPath((unsigned)ui.IndexInArchive, name);
1925       }
1926 
1927       /*
1928       if (totalSecureDataSize != 0)
1929         newDatabase.SecureIDs.Add(ui.SecureIndex);
1930       file.Parent = ui.ParentFolderIndex;
1931       */
1932       newDatabase.AddFile(file, file2, name);
1933     }
1934   }
1935 
1936   lps->ProgressOffset = 0;
1937 
1938   {
1939     // ---------- Sort Filters ----------
1940 
1941     FOR_VECTOR (i, filters)
1942     {
1943       filters[i].GroupIndex = i;
1944     }
1945     filters.Sort2();
1946   }
1947 
1948   for (unsigned groupIndex = 0; groupIndex < filters.Size(); groupIndex++)
1949   {
1950     const CFilterMode2 &filterMode = filters[groupIndex];
1951 
1952     CCompressionMethodMode method = *options.Method;
1953     {
1954       HRESULT res = MakeExeMethod(method, filterMode,
1955         #ifdef _7ZIP_ST
1956           false
1957         #else
1958           options.MaxFilter && options.MultiThreadMixer
1959         #endif
1960         );
1961 
1962       RINOK(res);
1963     }
1964 
1965     if (filterMode.Encrypted)
1966     {
1967       if (!method.PasswordIsDefined)
1968       {
1969         #ifndef _NO_CRYPTO
1970         if (getPasswordSpec)
1971           method.Password = getPasswordSpec->Password;
1972         #endif
1973         method.PasswordIsDefined = true;
1974       }
1975     }
1976     else
1977     {
1978       method.PasswordIsDefined = false;
1979       method.Password.Empty();
1980     }
1981 
1982     CEncoder encoder(method);
1983 
1984     // ---------- Repack and copy old solid blocks ----------
1985 
1986     const CSolidGroup &group = groups[filterMode.GroupIndex];
1987 
1988     FOR_VECTOR(folderRefIndex, group.folderRefs)
1989     {
1990       const CFolderRepack &rep = group.folderRefs[folderRefIndex];
1991 
1992       unsigned folderIndex = rep.FolderIndex;
1993 
1994       CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
1995 
1996       if (rep.NumCopyFiles == numUnpackStreams)
1997       {
1998         if (opCallback)
1999         {
2000           RINOK(opCallback->ReportOperation(
2001               NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2002               NUpdateNotifyOp::kReplicate));
2003 
2004           // ---------- Copy old solid block ----------
2005           {
2006             CNum indexInFolder = 0;
2007             for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2008             {
2009               if (db->Files[fi].HasStream)
2010               {
2011                 indexInFolder++;
2012                 RINOK(opCallback->ReportOperation(
2013                     NEventIndexType::kInArcIndex, (UInt32)fi,
2014                     NUpdateNotifyOp::kReplicate));
2015               }
2016             }
2017           }
2018         }
2019 
2020         UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
2021         RINOK(WriteRange(inStream, archive.SeqStream,
2022             db->GetFolderStreamPos(folderIndex, 0), packSize, progress));
2023         lps->ProgressOffset += packSize;
2024 
2025         CFolder &folder = newDatabase.Folders.AddNew();
2026         db->ParseFolderInfo(folderIndex, folder);
2027         CNum startIndex = db->FoStartPackStreamIndex[folderIndex];
2028         FOR_VECTOR(j, folder.PackStreams)
2029         {
2030           newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j));
2031           // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
2032           // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
2033         }
2034 
2035         size_t indexStart = db->FoToCoderUnpackSizes[folderIndex];
2036         size_t indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1];
2037         for (; indexStart < indexEnd; indexStart++)
2038           newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes[indexStart]);
2039       }
2040       else
2041       {
2042         // ---------- Repack old solid block ----------
2043 
2044         CBoolVector extractStatuses;
2045 
2046         CNum indexInFolder = 0;
2047 
2048         if (opCallback)
2049         {
2050           RINOK(opCallback->ReportOperation(
2051               NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2052               NUpdateNotifyOp::kRepack))
2053         }
2054 
2055         /* We could reduce data size of decoded folder, if we don't need to repack
2056            last files in folder. But the gain in speed is small in most cases.
2057            So we unpack full folder. */
2058 
2059         UInt64 sizeToEncode = 0;
2060 
2061         /*
2062         UInt64 importantUnpackSize = 0;
2063         unsigned numImportantFiles = 0;
2064         UInt64 decodeSize = 0;
2065         */
2066 
2067         for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2068         {
2069           bool needExtract = false;
2070           const CFileItem &file = db->Files[fi];
2071 
2072           if (file.HasStream)
2073           {
2074             indexInFolder++;
2075             int updateIndex = fileIndexToUpdateIndexMap[fi];
2076             if (updateIndex >= 0 && !updateItems[(unsigned)updateIndex].NewData)
2077               needExtract = true;
2078             // decodeSize += file.Size;
2079           }
2080 
2081           extractStatuses.Add(needExtract);
2082           if (needExtract)
2083           {
2084             sizeToEncode += file.Size;
2085             /*
2086             numImportantFiles = extractStatuses.Size();
2087             importantUnpackSize = decodeSize;
2088             */
2089           }
2090         }
2091 
2092         // extractStatuses.DeleteFrom(numImportantFiles);
2093 
2094         unsigned startPackIndex = newDatabase.PackSizes.Size();
2095         UInt64 curUnpackSize;
2096         {
2097 
2098           CMyComPtr<ISequentialInStream> sbInStream;
2099           CRepackStreamBase *repackBase;
2100           CFolderInStream2 *FosSpec2 = NULL;
2101 
2102           CRepackInStreamWithSizes *inStreamSizeCountSpec = new CRepackInStreamWithSizes;
2103           CMyComPtr<ISequentialInStream> inStreamSizeCount = inStreamSizeCountSpec;
2104           {
2105             #ifndef _7ZIP_ST
2106             if (options.MultiThreadMixer)
2107             {
2108               repackBase = threadDecoder.FosSpec;
2109               CMyComPtr<ISequentialOutStream> sbOutStream;
2110               sb.CreateStreams2(sbInStream, sbOutStream);
2111               RINOK(sb.Create_ReInit());
2112 
2113               threadDecoder.FosSpec->_stream = sbOutStream;
2114 
2115               threadDecoder.InStream = inStream;
2116               threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0);
2117               threadDecoder.Folders = (const CFolders *)db;
2118               threadDecoder.FolderIndex = folderIndex;
2119 
2120               // threadDecoder.UnpackSize = importantUnpackSize;
2121               // threadDecoder.send_UnpackSize = true;
2122             }
2123             else
2124             #endif
2125             {
2126               FosSpec2 = new CFolderInStream2;
2127               FosSpec2->Init();
2128               sbInStream = FosSpec2;
2129               repackBase = FosSpec2;
2130 
2131               #ifndef _NO_CRYPTO
2132               bool isEncrypted = false;
2133               bool passwordIsDefined = false;
2134               UString password;
2135               #endif
2136 
2137               CMyComPtr<ISequentialInStream> decodedStream;
2138               bool dataAfterEnd_Error = false;
2139 
2140               HRESULT res = threadDecoder.Decoder.Decode(
2141                   EXTERNAL_CODECS_LOC_VARS
2142                   inStream,
2143                   db->ArcInfo.DataStartPosition, // db->GetFolderStreamPos(folderIndex, 0);,
2144                   *db, folderIndex,
2145                   // &importantUnpackSize, // *unpackSize
2146                   NULL, // *unpackSize : FULL unpack
2147 
2148                   NULL, // *outStream
2149                   NULL, // *compressProgress
2150 
2151                   &decodedStream
2152                   , dataAfterEnd_Error
2153 
2154                   _7Z_DECODER_CRYPRO_VARS
2155                   #ifndef _7ZIP_ST
2156                     , false // mtMode
2157                     , 1 // numThreads
2158                     , 0 // memUsage
2159                   #endif
2160                 );
2161 
2162               RINOK(res);
2163               if (!decodedStream)
2164                 return E_FAIL;
2165 
2166               FosSpec2->_inStream = decodedStream;
2167             }
2168 
2169             repackBase->_db = db;
2170             repackBase->_opCallback = opCallback;
2171             repackBase->_extractCallback = extractCallback;
2172 
2173             UInt32 startIndex = db->FolderStartFileIndex[folderIndex];
2174             RINOK(repackBase->Init(startIndex, &extractStatuses));
2175 
2176             inStreamSizeCountSpec->_db = db;
2177             inStreamSizeCountSpec->Init(sbInStream, startIndex, &extractStatuses);
2178 
2179             #ifndef _7ZIP_ST
2180             if (options.MultiThreadMixer)
2181             {
2182               WRes wres = threadDecoder.Start();
2183               if (wres != 0)
2184                 return HRESULT_FROM_WIN32(wres);
2185             }
2186             #endif
2187           }
2188 
2189           curUnpackSize = sizeToEncode;
2190 
2191           HRESULT encodeRes = encoder.Encode(
2192               EXTERNAL_CODECS_LOC_VARS
2193               inStreamSizeCount,
2194               // NULL,
2195               &inSizeForReduce,
2196               newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curUnpackSize,
2197               archive.SeqStream, newDatabase.PackSizes, progress);
2198 
2199           if (encodeRes == k_My_HRESULT_CRC_ERROR)
2200             return E_FAIL;
2201 
2202           #ifndef _7ZIP_ST
2203           if (options.MultiThreadMixer)
2204           {
2205             // 16.00: hang was fixed : for case if decoding was not finished.
2206             // We close CBinderInStream and it calls CStreamBinder::CloseRead()
2207             inStreamSizeCount.Release();
2208             sbInStream.Release();
2209 
2210             {
2211               WRes wres = threadDecoder.WaitExecuteFinish();
2212               if (wres != 0)
2213                 return HRESULT_FROM_WIN32(wres);
2214             }
2215 
2216             HRESULT decodeRes = threadDecoder.Result;
2217             // if (res == k_My_HRESULT_CRC_ERROR)
2218             if (decodeRes == S_FALSE || threadDecoder.dataAfterEnd_Error)
2219             {
2220               if (extractCallback)
2221               {
2222                 RINOK(extractCallback->ReportExtractResult(
2223                     NEventIndexType::kInArcIndex, db->FolderStartFileIndex[folderIndex],
2224                     // NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2225                     (decodeRes != S_OK ?
2226                       NExtract::NOperationResult::kDataError :
2227                       NExtract::NOperationResult::kDataAfterEnd)));
2228               }
2229               if (decodeRes != S_OK)
2230                 return E_FAIL;
2231             }
2232             RINOK(decodeRes);
2233             if (encodeRes == S_OK)
2234               if (sb.ProcessedSize != sizeToEncode)
2235                 encodeRes = E_FAIL;
2236           }
2237           else
2238           #endif
2239           {
2240             if (FosSpec2->Result == S_FALSE)
2241             {
2242               if (extractCallback)
2243               {
2244                 RINOK(extractCallback->ReportExtractResult(
2245                     NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2246                     NExtract::NOperationResult::kDataError));
2247               }
2248               return E_FAIL;
2249             }
2250             RINOK(FosSpec2->Result);
2251           }
2252 
2253           RINOK(encodeRes);
2254           RINOK(repackBase->CheckFinishedState());
2255 
2256           if (curUnpackSize != sizeToEncode)
2257             return E_FAIL;
2258         }
2259 
2260         for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2261           lps->OutSize += newDatabase.PackSizes[startPackIndex];
2262         lps->InSize += curUnpackSize;
2263       }
2264 
2265       newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
2266 
2267       CNum indexInFolder = 0;
2268       for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2269       {
2270         if (db->Files[fi].HasStream)
2271         {
2272           indexInFolder++;
2273           int updateIndex = fileIndexToUpdateIndexMap[fi];
2274           if (updateIndex >= 0)
2275           {
2276             const CUpdateItem &ui = updateItems[(unsigned)updateIndex];
2277             if (ui.NewData)
2278               continue;
2279 
2280             UString name;
2281             CFileItem file;
2282             CFileItem2 file2;
2283             GetFile(*db, fi, file, file2);
2284 
2285             if (ui.NewProps)
2286             {
2287               UpdateItem_To_FileItem2(ui, file2);
2288               file.IsDir = ui.IsDir;
2289               name = ui.Name;
2290             }
2291             else
2292               db->GetPath(fi, name);
2293 
2294             /*
2295             file.Parent = ui.ParentFolderIndex;
2296             if (ui.TreeFolderIndex >= 0)
2297               treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2298             if (totalSecureDataSize != 0)
2299               newDatabase.SecureIDs.Add(ui.SecureIndex);
2300             */
2301             newDatabase.AddFile(file, file2, name);
2302           }
2303         }
2304       }
2305     }
2306 
2307 
2308     // ---------- Compress files to new solid blocks ----------
2309 
2310     unsigned numFiles = group.Indices.Size();
2311     if (numFiles == 0)
2312       continue;
2313     CRecordVector<CRefItem> refItems;
2314     refItems.ClearAndSetSize(numFiles);
2315     // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1
2316     bool sortByType = options.UseTypeSorting;
2317 
2318     unsigned i;
2319 
2320     for (i = 0; i < numFiles; i++)
2321       refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType);
2322 
2323     CSortParam sortParam;
2324     // sortParam.TreeFolders = &treeFolders;
2325     sortParam.SortByType = sortByType;
2326     refItems.Sort(CompareUpdateItems, (void *)&sortParam);
2327 
2328     CObjArray<UInt32> indices(numFiles);
2329 
2330     for (i = 0; i < numFiles; i++)
2331     {
2332       UInt32 index = refItems[i].Index;
2333       indices[i] = index;
2334       /*
2335       const CUpdateItem &ui = updateItems[index];
2336       CFileItem file;
2337       if (ui.NewProps)
2338         UpdateItem_To_FileItem(ui, file);
2339       else
2340         file = db.Files[ui.IndexInArchive];
2341       if (file.IsAnti || file.IsDir)
2342         return E_FAIL;
2343       newDatabase.Files.Add(file);
2344       */
2345     }
2346 
2347     for (i = 0; i < numFiles;)
2348     {
2349       UInt64 totalSize = 0;
2350       unsigned numSubFiles;
2351 
2352       const wchar_t *prevExtension = NULL;
2353 
2354       for (numSubFiles = 0; i + numSubFiles < numFiles && numSubFiles < numSolidFiles; numSubFiles++)
2355       {
2356         const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
2357         totalSize += ui.Size;
2358         if (totalSize > options.NumSolidBytes)
2359           break;
2360         if (options.SolidExtension)
2361         {
2362           int slashPos = ui.Name.ReverseFind_PathSepar();
2363           int dotPos = ui.Name.ReverseFind_Dot();
2364           const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : (unsigned)(dotPos + 1));
2365           if (numSubFiles == 0)
2366             prevExtension = ext;
2367           else if (!StringsAreEqualNoCase(ext, prevExtension))
2368             break;
2369         }
2370       }
2371 
2372       if (numSubFiles < 1)
2373         numSubFiles = 1;
2374 
2375       RINOK(lps->SetCur());
2376 
2377       CFolderInStream *inStreamSpec = new CFolderInStream;
2378       CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec);
2379       inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
2380 
2381       unsigned startPackIndex = newDatabase.PackSizes.Size();
2382       UInt64 curFolderUnpackSize = totalSize;
2383       // curFolderUnpackSize = (UInt64)(Int64)-1;
2384 
2385       RINOK(encoder.Encode(
2386           EXTERNAL_CODECS_LOC_VARS
2387           solidInStream,
2388           // NULL,
2389           &inSizeForReduce,
2390           newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curFolderUnpackSize,
2391           archive.SeqStream, newDatabase.PackSizes, progress));
2392 
2393       if (!inStreamSpec->WasFinished())
2394         return E_FAIL;
2395 
2396       for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2397         lps->OutSize += newDatabase.PackSizes[startPackIndex];
2398 
2399       lps->InSize += curFolderUnpackSize;
2400       // for ()
2401       // newDatabase.PackCRCsDefined.Add(false);
2402       // newDatabase.PackCRCs.Add(0);
2403 
2404       CNum numUnpackStreams = 0;
2405       UInt64 skippedSize = 0;
2406 
2407       for (unsigned subIndex = 0; subIndex < numSubFiles; subIndex++)
2408       {
2409         const CUpdateItem &ui = updateItems[indices[i + subIndex]];
2410         CFileItem file;
2411         CFileItem2 file2;
2412         UString name;
2413         if (ui.NewProps)
2414         {
2415           UpdateItem_To_FileItem(ui, file, file2);
2416           name = ui.Name;
2417         }
2418         else
2419         {
2420           GetFile(*db, (unsigned)ui.IndexInArchive, file, file2);
2421           db->GetPath((unsigned)ui.IndexInArchive, name);
2422         }
2423         if (file2.IsAnti || file.IsDir)
2424           return E_FAIL;
2425 
2426         /*
2427         CFileItem &file = newDatabase.Files[
2428               startFileIndexInDatabase + i + subIndex];
2429         */
2430         if (!inStreamSpec->Processed[subIndex])
2431         {
2432           skippedSize += ui.Size;
2433           continue;
2434           // file.Name += ".locked";
2435         }
2436 
2437         file.Crc = inStreamSpec->CRCs[subIndex];
2438         file.Size = inStreamSpec->Sizes[subIndex];
2439 
2440         // if (file.Size >= 0) // test purposes
2441         if (file.Size != 0)
2442         {
2443           file.CrcDefined = true;
2444           file.HasStream = true;
2445           numUnpackStreams++;
2446         }
2447         else
2448         {
2449           file.CrcDefined = false;
2450           file.HasStream = false;
2451         }
2452 
2453         /*
2454         file.Parent = ui.ParentFolderIndex;
2455         if (ui.TreeFolderIndex >= 0)
2456           treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2457         if (totalSecureDataSize != 0)
2458           newDatabase.SecureIDs.Add(ui.SecureIndex);
2459         */
2460         newDatabase.AddFile(file, file2, name);
2461       }
2462 
2463       // numUnpackStreams = 0 is very bad case for locked files
2464       // v3.13 doesn't understand it.
2465       newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
2466       i += numSubFiles;
2467 
2468       if (skippedSize != 0 && complexity >= skippedSize)
2469       {
2470         complexity -= skippedSize;
2471         RINOK(updateCallback->SetTotal(complexity));
2472       }
2473     }
2474   }
2475 
2476   RINOK(lps->SetCur());
2477 
2478   /*
2479   fileIndexToUpdateIndexMap.ClearAndFree();
2480   groups.ClearAndFree();
2481   */
2482 
2483   /*
2484   for (i = 0; i < newDatabase.Files.Size(); i++)
2485   {
2486     CFileItem &file = newDatabase.Files[i];
2487     file.Parent = treeFolderToArcIndex[file.Parent];
2488   }
2489 
2490   if (totalSecureDataSize != 0)
2491   {
2492     newDatabase.SecureBuf.SetCapacity(totalSecureDataSize);
2493     size_t pos = 0;
2494     newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size());
2495     for (i = 0; i < secureBlocks.Sorted.Size(); i++)
2496     {
2497       const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]];
2498       size_t size = buf.GetCapacity();
2499       if (size != 0)
2500         memcpy(newDatabase.SecureBuf + pos, buf, size);
2501       newDatabase.SecureSizes.Add((UInt32)size);
2502       pos += size;
2503     }
2504   }
2505   */
2506   newDatabase.ReserveDown();
2507 
2508   if (opCallback)
2509     RINOK(opCallback->ReportOperation(NEventIndexType::kNoIndex, (UInt32)(Int32)-1, NUpdateNotifyOp::kHeader));
2510 
2511   return S_OK;
2512 }
2513 
2514 }}
2515