1 // 7zUpdate.cpp
2 
3 #include "StdAfx.h"
4 
5 #include "../../../../C/CpuArch.h"
6 
7 #include "../../../Common/Wildcard.h"
8 
9 #include "../../Common/CreateCoder.h"
10 #include "../../Common/LimitedStreams.h"
11 #include "../../Common/ProgressUtils.h"
12 
13 #include "../../Compress/CopyCoder.h"
14 
15 #include "../Common/ItemNameUtils.h"
16 
17 #include "7zDecode.h"
18 #include "7zEncode.h"
19 #include "7zFolderInStream.h"
20 #include "7zHandler.h"
21 #include "7zOut.h"
22 #include "7zUpdate.h"
23 
24 namespace NArchive {
25 namespace N7z {
26 
27 
28 #define k_X86 k_BCJ
29 
30 struct CFilterMode
31 {
32   UInt32 Id;
33   UInt32 Delta;
34 
CFilterModeNArchive::N7z::CFilterMode35   CFilterMode(): Id(0), Delta(0) {}
36 
SetDeltaNArchive::N7z::CFilterMode37   void SetDelta()
38   {
39     if (Id == k_IA64)
40       Delta = 16;
41     else if (Id == k_ARM || Id == k_PPC || Id == k_SPARC)
42       Delta = 4;
43     else if (Id == k_ARMT)
44       Delta = 2;
45     else
46       Delta = 0;
47   }
48 };
49 
50 
51 /* ---------- PE ---------- */
52 
53 #define MZ_SIG 0x5A4D
54 
55 #define PE_SIG 0x00004550
56 #define PE_OptHeader_Magic_32 0x10B
57 #define PE_OptHeader_Magic_64 0x20B
58 #define PE_SectHeaderSize 40
59 #define PE_SECT_EXECUTE 0x20000000
60 
Parse_EXE(const Byte * buf,size_t size,CFilterMode * filterMode)61 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode)
62 {
63   if (size < 512 || GetUi16(buf) != MZ_SIG)
64     return 0;
65 
66   const Byte *p;
67   UInt32 peOffset, optHeaderSize, filterId;
68 
69   peOffset = GetUi32(buf + 0x3C);
70   if (peOffset >= 0x1000 || peOffset + 512 > size || (peOffset & 7) != 0)
71     return 0;
72   p = buf + peOffset;
73   if (GetUi32(p) != PE_SIG)
74     return 0;
75   p += 4;
76 
77   switch (GetUi16(p))
78   {
79     case 0x014C:
80     case 0x8664:  filterId = k_X86; break;
81 
82     /*
83     IMAGE_FILE_MACHINE_ARM   0x01C0  // ARM LE
84     IMAGE_FILE_MACHINE_THUMB 0x01C2  // ARM Thumb / Thumb-2 LE
85     IMAGE_FILE_MACHINE_ARMNT 0x01C4  // ARM Thumb-2, LE
86     Note: We use ARM filter for 0x01C2. (WinCE 5 - 0x01C2) files mostly contain ARM code (not Thumb/Thumb-2).
87     */
88 
89     case 0x01C0:                            // WinCE old
90     case 0x01C2:  filterId = k_ARM; break;  // WinCE new
91     case 0x01C4:  filterId = k_ARMT; break; // WinRT
92 
93     case 0x0200:  filterId = k_IA64; break;
94     default:  return 0;
95   }
96 
97   optHeaderSize = GetUi16(p + 16);
98   if (optHeaderSize > (1 << 10))
99     return 0;
100 
101   p += 20; /* headerSize */
102 
103   switch (GetUi16(p))
104   {
105     case PE_OptHeader_Magic_32:
106     case PE_OptHeader_Magic_64:
107       break;
108     default:
109       return 0;
110   }
111 
112   filterMode->Id = filterId;
113   return 1;
114 }
115 
116 
117 /* ---------- ELF ---------- */
118 
119 #define ELF_SIG 0x464C457F
120 
121 #define ELF_CLASS_32  1
122 #define ELF_CLASS_64  2
123 
124 #define ELF_DATA_2LSB 1
125 #define ELF_DATA_2MSB 2
126 
Get16(const Byte * p,Bool be)127 static UInt16 Get16(const Byte *p, Bool be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); }
Get32(const Byte * p,Bool be)128 static UInt32 Get32(const Byte *p, Bool be) { if (be) return GetBe32(p); return GetUi32(p); }
129 // static UInt64 Get64(const Byte *p, Bool be) { if (be) return GetBe64(p); return GetUi64(p); }
130 
Parse_ELF(const Byte * buf,size_t size,CFilterMode * filterMode)131 static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode)
132 {
133   Bool /* is32, */ be;
134   UInt32 filterId;
135 
136   if (size < 512 || buf[6] != 1) /* ver */
137     return 0;
138 
139   if (GetUi32(buf) != ELF_SIG)
140     return 0;
141 
142   switch (buf[4])
143   {
144     case ELF_CLASS_32: /* is32 = True; */ break;
145     case ELF_CLASS_64: /* is32 = False; */ break;
146     default: return 0;
147   }
148 
149   switch (buf[5])
150   {
151     case ELF_DATA_2LSB: be = False; break;
152     case ELF_DATA_2MSB: be = True; break;
153     default: return 0;
154   }
155 
156   switch (Get16(buf + 0x12, be))
157   {
158     case 3:
159     case 6:
160     case 62: filterId = k_X86; break;
161     case 2:
162     case 18:
163     case 43: filterId = k_SPARC; break;
164     case 20:
165     case 21: if (!be) return 0; filterId = k_PPC; break;
166     case 40: if ( be) return 0; filterId = k_ARM; break;
167 
168     /* Some IA-64 ELF exacutable have size that is not aligned for 16 bytes.
169        So we don't use IA-64 filter for IA-64 ELF */
170     // case 50: if ( be) return 0; filterId = k_IA64; break;
171 
172     default: return 0;
173   }
174 
175   filterMode->Id = filterId;
176   return 1;
177 }
178 
179 
180 
181 /* ---------- Mach-O ---------- */
182 
183 #define MACH_SIG_BE_32 0xCEFAEDFE
184 #define MACH_SIG_BE_64 0xCFFAEDFE
185 #define MACH_SIG_LE_32 0xFEEDFACE
186 #define MACH_SIG_LE_64 0xFEEDFACF
187 
188 #define MACH_ARCH_ABI64 (1 << 24)
189 #define MACH_MACHINE_386 7
190 #define MACH_MACHINE_ARM 12
191 #define MACH_MACHINE_SPARC 14
192 #define MACH_MACHINE_PPC 18
193 #define MACH_MACHINE_PPC64 (MACH_ARCH_ABI64 | MACH_MACHINE_PPC)
194 #define MACH_MACHINE_AMD64 (MACH_ARCH_ABI64 | MACH_MACHINE_386)
195 
Parse_MACH(const Byte * buf,size_t size,CFilterMode * filterMode)196 static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode)
197 {
198   UInt32 filterId, numCommands, commandsSize;
199 
200   if (size < 512)
201     return 0;
202 
203   Bool /* mode64, */ be;
204   switch (GetUi32(buf))
205   {
206     case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break;
207     case MACH_SIG_BE_64: /* mode64 = True;  */ be = True; break;
208     case MACH_SIG_LE_32: /* mode64 = False; */ be = False; break;
209     case MACH_SIG_LE_64: /* mode64 = True;  */ be = False; break;
210     default: return 0;
211   }
212 
213   switch (Get32(buf + 4, be))
214   {
215     case MACH_MACHINE_386:
216     case MACH_MACHINE_AMD64: filterId = k_X86; break;
217     case MACH_MACHINE_ARM:   if ( be) return 0; filterId = k_ARM; break;
218     case MACH_MACHINE_SPARC: if (!be) return 0; filterId = k_SPARC; break;
219     case MACH_MACHINE_PPC:
220     case MACH_MACHINE_PPC64: if (!be) return 0; filterId = k_PPC; break;
221     default: return 0;
222   }
223 
224   numCommands = Get32(buf + 0x10, be);
225   commandsSize = Get32(buf + 0x14, be);
226 
227   if (commandsSize > (1 << 24) || numCommands > (1 << 18))
228     return 0;
229 
230   filterMode->Id = filterId;
231   return 1;
232 }
233 
234 
235 /* ---------- WAV ---------- */
236 
237 #define WAV_SUBCHUNK_fmt  0x20746D66
238 #define WAV_SUBCHUNK_data 0x61746164
239 
240 #define RIFF_SIG 0x46464952
241 
Parse_WAV(const Byte * buf,size_t size,CFilterMode * filterMode)242 static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode)
243 {
244   UInt32 subChunkSize, pos;
245   if (size < 0x2C)
246     return False;
247 
248   if (GetUi32(buf + 0) != RIFF_SIG ||
249       GetUi32(buf + 8) != 0x45564157 || // WAVE
250       GetUi32(buf + 0xC) != WAV_SUBCHUNK_fmt)
251     return False;
252   subChunkSize = GetUi32(buf + 0x10);
253   /* [0x14 = format] = 1 (PCM) */
254   if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1)
255     return False;
256 
257   unsigned numChannels = GetUi16(buf + 0x16);
258   unsigned bitsPerSample = GetUi16(buf + 0x22);
259 
260   if ((bitsPerSample & 0x7) != 0 || bitsPerSample >= 256 || numChannels >= 256)
261     return False;
262 
263   pos = 0x14 + subChunkSize;
264 
265   const int kNumSubChunksTests = 10;
266   // Do we need to scan more than 3 sub-chunks?
267   for (int i = 0; i < kNumSubChunksTests; i++)
268   {
269     if (pos + 8 > size)
270       return False;
271     subChunkSize = GetUi32(buf + pos + 4);
272     if (GetUi32(buf + pos) == WAV_SUBCHUNK_data)
273     {
274       unsigned delta = numChannels * (bitsPerSample >> 3);
275       if (delta >= 256)
276         return False;
277       filterMode->Id = k_Delta;
278       filterMode->Delta = delta;
279       return True;
280     }
281     if (subChunkSize > (1 << 16))
282       return False;
283     pos += subChunkSize + 8;
284   }
285   return False;
286 }
287 
ParseFile(const Byte * buf,size_t size,CFilterMode * filterMode)288 static Bool ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode)
289 {
290   filterMode->Id = 0;
291   filterMode->Delta = 0;
292 
293   if (Parse_EXE(buf, size, filterMode)) return True;
294   if (Parse_ELF(buf, size, filterMode)) return True;
295   if (Parse_MACH(buf, size, filterMode)) return True;
296   return Parse_WAV(buf, size, filterMode);
297 }
298 
299 
300 
301 
302 struct CFilterMode2: public CFilterMode
303 {
304   bool Encrypted;
305   unsigned GroupIndex;
306 
CFilterMode2NArchive::N7z::CFilterMode2307   CFilterMode2(): Encrypted(false) {}
308 
CompareNArchive::N7z::CFilterMode2309   int Compare(const CFilterMode2 &m) const
310   {
311     if (!Encrypted)
312     {
313       if (m.Encrypted)
314         return -1;
315     }
316     else if (!m.Encrypted)
317       return 1;
318 
319     if (Id < m.Id) return -1;
320     if (Id > m.Id) return 1;
321 
322     if (Delta < m.Delta) return -1;
323     if (Delta > m.Delta) return 1;
324 
325     return 0;
326   }
327 
operator ==NArchive::N7z::CFilterMode2328   bool operator ==(const CFilterMode2 &m) const
329   {
330     return Id == m.Id && Delta == m.Delta && Encrypted == m.Encrypted;
331   }
332 };
333 
GetGroup(CRecordVector<CFilterMode2> & filters,const CFilterMode2 & m)334 static unsigned GetGroup(CRecordVector<CFilterMode2> &filters, const CFilterMode2 &m)
335 {
336   unsigned i;
337   for (i = 0; i < filters.Size(); i++)
338   {
339     const CFilterMode2 &m2 = filters[i];
340     if (m == m2)
341       return i;
342     /*
343     if (m.Encrypted != m2.Encrypted)
344     {
345       if (!m.Encrypted)
346         break;
347       continue;
348     }
349 
350     if (m.Id < m2.Id)  break;
351     if (m.Id != m2.Id) continue;
352 
353     if (m.Delta < m2.Delta) break;
354     if (m.Delta != m2.Delta) continue;
355     */
356   }
357   // filters.Insert(i, m);
358   // return i;
359   return filters.Add(m);
360 }
361 
Is86Filter(CMethodId m)362 static inline bool Is86Filter(CMethodId m)
363 {
364   return (m == k_BCJ || m == k_BCJ2);
365 }
366 
IsExeFilter(CMethodId m)367 static inline bool IsExeFilter(CMethodId m)
368 {
369   switch (m)
370   {
371     case k_BCJ:
372     case k_BCJ2:
373     case k_ARM:
374     case k_ARMT:
375     case k_PPC:
376     case k_SPARC:
377     case k_IA64:
378       return true;
379   }
380   return false;
381 }
382 
Get_FilterGroup_for_Folder(CRecordVector<CFilterMode2> & filters,const CFolderEx & f,bool extractFilter)383 static unsigned Get_FilterGroup_for_Folder(
384     CRecordVector<CFilterMode2> &filters, const CFolderEx &f, bool extractFilter)
385 {
386   CFilterMode2 m;
387   m.Id = 0;
388   m.Delta = 0;
389   m.Encrypted = f.IsEncrypted();
390 
391   if (extractFilter)
392   {
393     const CCoderInfo &coder = f.Coders[f.UnpackCoder];
394 
395     if (coder.MethodID == k_Delta)
396     {
397       if (coder.Props.Size() == 1)
398       {
399         m.Delta = (unsigned)coder.Props[0] + 1;
400         m.Id = k_Delta;
401       }
402     }
403     else if (IsExeFilter(coder.MethodID))
404     {
405       m.Id = (UInt32)coder.MethodID;
406       if (m.Id == k_BCJ2)
407         m.Id = k_BCJ;
408       m.SetDelta();
409     }
410   }
411 
412   return GetGroup(filters, m);
413 }
414 
415 
416 
417 
WriteRange(IInStream * inStream,ISequentialOutStream * outStream,UInt64 position,UInt64 size,ICompressProgressInfo * progress)418 static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
419     UInt64 position, UInt64 size, ICompressProgressInfo *progress)
420 {
421   RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0));
422   CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream;
423   CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec);
424   streamSpec->SetStream(inStream);
425   streamSpec->Init(size);
426 
427   NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder;
428   CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec;
429   RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress));
430   return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL);
431 }
432 
433 /*
434 unsigned CUpdateItem::GetExtensionPos() const
435 {
436   int slashPos = Name.ReverseFind_PathSepar();
437   int dotPos = Name.ReverseFind_Dot();
438   if (dotPos <= slashPos)
439     return Name.Len();
440   return dotPos + 1;
441 }
442 
443 UString CUpdateItem::GetExtension() const
444 {
445   return Name.Ptr(GetExtensionPos());
446 }
447 */
448 
449 #define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
450 
451 #define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
452 
453 /*
454 static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
455 {
456   size_t c1 = a1.GetCapacity();
457   size_t c2 = a2.GetCapacity();
458   RINOZ_COMP(c1, c2);
459   for (size_t i = 0; i < c1; i++)
460     RINOZ_COMP(a1[i], a2[i]);
461   return 0;
462 }
463 
464 static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
465 {
466   RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
467   RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
468   RINOZ_COMP(c1.MethodID, c2.MethodID);
469   return CompareBuffers(c1.Props, c2.Props);
470 }
471 
472 static int CompareBonds(const CBond &b1, const CBond &b2)
473 {
474   RINOZ_COMP(b1.InIndex, b2.InIndex);
475   return MyCompare(b1.OutIndex, b2.OutIndex);
476 }
477 
478 static int CompareFolders(const CFolder &f1, const CFolder &f2)
479 {
480   int s1 = f1.Coders.Size();
481   int s2 = f2.Coders.Size();
482   RINOZ_COMP(s1, s2);
483   int i;
484   for (i = 0; i < s1; i++)
485     RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
486   s1 = f1.Bonds.Size();
487   s2 = f2.Bonds.Size();
488   RINOZ_COMP(s1, s2);
489   for (i = 0; i < s1; i++)
490     RINOZ(CompareBonds(f1.Bonds[i], f2.Bonds[i]));
491   return 0;
492 }
493 */
494 
495 /*
496 static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
497 {
498   return CompareFileNames(f1.Name, f2.Name);
499 }
500 */
501 
502 struct CFolderRepack
503 {
504   unsigned FolderIndex;
505   CNum NumCopyFiles;
506 };
507 
508 /*
509 static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *)
510 {
511   int i1 = p1->FolderIndex;
512   int i2 = p2->FolderIndex;
513   // In that version we don't want to parse folders here, so we don't compare folders
514   // probably it must be improved in future
515   // const CDbEx &db = *(const CDbEx *)param;
516   // RINOZ(CompareFolders(
517   //     db.Folders[i1],
518   //     db.Folders[i2]));
519 
520   return MyCompare(i1, i2);
521 
522   // RINOZ_COMP(
523   //     db.NumUnpackStreamsVector[i1],
524   //     db.NumUnpackStreamsVector[i2]);
525   // if (db.NumUnpackStreamsVector[i1] == 0)
526   //   return 0;
527   // return CompareFiles(
528   //     db.Files[db.FolderStartFileIndex[i1]],
529   //     db.Files[db.FolderStartFileIndex[i2]]);
530 }
531 */
532 
533 /*
534   we sort empty files and dirs in such order:
535   - Dir.NonAnti   (name sorted)
536   - File.NonAnti  (name sorted)
537   - File.Anti     (name sorted)
538   - Dir.Anti (reverse name sorted)
539 */
540 
CompareEmptyItems(const unsigned * p1,const unsigned * p2,void * param)541 static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param)
542 {
543   const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
544   const CUpdateItem &u1 = updateItems[*p1];
545   const CUpdateItem &u2 = updateItems[*p2];
546   // NonAnti < Anti
547   if (u1.IsAnti != u2.IsAnti)
548     return (u1.IsAnti ? 1 : -1);
549   if (u1.IsDir != u2.IsDir)
550   {
551     // Dir.NonAnti < File < Dir.Anti
552     if (u1.IsDir)
553       return (u1.IsAnti ? 1 : -1);
554     return (u2.IsAnti ? -1 : 1);
555   }
556   int n = CompareFileNames(u1.Name, u2.Name);
557   return (u1.IsDir && u1.IsAnti) ? -n : n;
558 }
559 
560 static const char *g_Exts =
561   " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
562   " zip jar ear war msi"
563   " 3gp avi mov mpeg mpg mpe wmv"
564   " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
565   " swf"
566   " chm hxi hxs"
567   " gif jpeg jpg jp2 png tiff  bmp ico psd psp"
568   " awg ps eps cgm dxf svg vrml wmf emf ai md"
569   " cad dwg pps key sxi"
570   " max 3ds"
571   " iso bin nrg mdf img pdi tar cpio xpi"
572   " vfd vhd vud vmc vsv"
573   " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
574   " inl inc idl acf asa"
575   " h hpp hxx c cpp cxx m mm go swift"
576   " rc java cs rs pas bas vb cls ctl frm dlg def"
577   " f77 f f90 f95"
578   " asm s"
579   " sql manifest dep"
580   " mak clw csproj vcproj sln dsp dsw"
581   " class"
582   " bat cmd bash sh"
583   " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
584   " awk sed hta js json php php3 php4 php5 phptml pl pm py pyo rb tcl ts vbs"
585   " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
586   " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
587   " abw afp cwk lwp wpd wps wpt wrf wri"
588   " abf afm bdf fon mgf otf pcf pfa snf ttf"
589   " dbf mdb nsf ntf wdb db fdb gdb"
590   " exe dll ocx vbx sfx sys tlb awx com obj lib out o so"
591   " pdb pch idb ncb opt";
592 
GetExtIndex(const char * ext)593 static unsigned GetExtIndex(const char *ext)
594 {
595   unsigned extIndex = 1;
596   const char *p = g_Exts;
597   for (;;)
598   {
599     char c = *p++;
600     if (c == 0)
601       return extIndex;
602     if (c == ' ')
603       continue;
604     unsigned pos = 0;
605     for (;;)
606     {
607       char c2 = ext[pos++];
608       if (c2 == 0 && (c == 0 || c == ' '))
609         return extIndex;
610       if (c != c2)
611         break;
612       c = *p++;
613     }
614     extIndex++;
615     for (;;)
616     {
617       if (c == 0)
618         return extIndex;
619       if (c == ' ')
620         break;
621       c = *p++;
622     }
623   }
624 }
625 
626 struct CRefItem
627 {
628   const CUpdateItem *UpdateItem;
629   UInt32 Index;
630   unsigned ExtensionPos;
631   unsigned NamePos;
632   unsigned ExtensionIndex;
633 
CRefItemNArchive::N7z::CRefItem634   CRefItem() {};
CRefItemNArchive::N7z::CRefItem635   CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
636     UpdateItem(&ui),
637     Index(index),
638     ExtensionPos(0),
639     NamePos(0),
640     ExtensionIndex(0)
641   {
642     if (sortByType)
643     {
644       int slashPos = ui.Name.ReverseFind_PathSepar();
645       NamePos = slashPos + 1;
646       int dotPos = ui.Name.ReverseFind_Dot();
647       if (dotPos <= slashPos)
648         ExtensionPos = ui.Name.Len();
649       else
650       {
651         ExtensionPos = dotPos + 1;
652         if (ExtensionPos != ui.Name.Len())
653         {
654           AString s;
655           for (unsigned pos = ExtensionPos;; pos++)
656           {
657             wchar_t c = ui.Name[pos];
658             if (c >= 0x80)
659               break;
660             if (c == 0)
661             {
662               ExtensionIndex = GetExtIndex(s);
663               break;
664             }
665             s += (char)MyCharLower_Ascii((char)c);
666           }
667         }
668       }
669     }
670   }
671 };
672 
673 struct CSortParam
674 {
675   // const CObjectVector<CTreeFolder> *TreeFolders;
676   bool SortByType;
677 };
678 
679 /*
680   we sort files in such order:
681   - Dir.NonAnti   (name sorted)
682   - alt streams
683   - Dirs
684   - Dir.Anti (reverse name sorted)
685 */
686 
687 
CompareUpdateItems(const CRefItem * p1,const CRefItem * p2,void * param)688 static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
689 {
690   const CRefItem &a1 = *p1;
691   const CRefItem &a2 = *p2;
692   const CUpdateItem &u1 = *a1.UpdateItem;
693   const CUpdateItem &u2 = *a2.UpdateItem;
694 
695   /*
696   if (u1.IsAltStream != u2.IsAltStream)
697     return u1.IsAltStream ? 1 : -1;
698   */
699 
700   // Actually there are no dirs that time. They were stored in other steps
701   // So that code is unused?
702   if (u1.IsDir != u2.IsDir)
703     return u1.IsDir ? 1 : -1;
704   if (u1.IsDir)
705   {
706     if (u1.IsAnti != u2.IsAnti)
707       return (u1.IsAnti ? 1 : -1);
708     int n = CompareFileNames(u1.Name, u2.Name);
709     return -n;
710   }
711 
712   // bool sortByType = *(bool *)param;
713   const CSortParam *sortParam = (const CSortParam *)param;
714   bool sortByType = sortParam->SortByType;
715   if (sortByType)
716   {
717     RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex);
718     RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos)));
719     RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos)));
720     if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
721     if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
722     if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime);
723     RINOZ_COMP(u1.Size, u2.Size);
724   }
725   /*
726   int par1 = a1.UpdateItem->ParentFolderIndex;
727   int par2 = a2.UpdateItem->ParentFolderIndex;
728   const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1];
729   const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2];
730 
731   int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd;
732   int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd;
733   if (b1 < b2)
734   {
735     if (e1 <= b2)
736       return -1;
737     // p2 in p1
738     int par = par2;
739     for (;;)
740     {
741       const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
742       par = tf.Parent;
743       if (par == par1)
744       {
745         RINOZ(CompareFileNames(u1.Name, tf.Name));
746         break;
747       }
748     }
749   }
750   else if (b2 < b1)
751   {
752     if (e2 <= b1)
753       return 1;
754     // p1 in p2
755     int par = par1;
756     for (;;)
757     {
758       const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
759       par = tf.Parent;
760       if (par == par2)
761       {
762         RINOZ(CompareFileNames(tf.Name, u2.Name));
763         break;
764       }
765     }
766   }
767   */
768   // RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex);
769   RINOK(CompareFileNames(u1.Name, u2.Name));
770   RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient);
771   RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive);
772   return 0;
773 }
774 
775 struct CSolidGroup
776 {
777   CRecordVector<UInt32> Indices;
778 
779   CRecordVector<CFolderRepack> folderRefs;
780 };
781 
782 static const char * const g_ExeExts[] =
783 {
784     "dll"
785   , "exe"
786   , "ocx"
787   , "sfx"
788   , "sys"
789 };
790 
IsExeExt(const wchar_t * ext)791 static bool IsExeExt(const wchar_t *ext)
792 {
793   for (unsigned i = 0; i < ARRAY_SIZE(g_ExeExts); i++)
794     if (StringsAreEqualNoCase_Ascii(ext, g_ExeExts[i]))
795       return true;
796   return false;
797 }
798 
799 struct CAnalysis
800 {
801   CMyComPtr<IArchiveUpdateCallbackFile> Callback;
802   CByteBuffer Buffer;
803 
804   bool ParseWav;
805   bool ParseExe;
806   bool ParseAll;
807 
CAnalysisNArchive::N7z::CAnalysis808   CAnalysis():
809       ParseWav(true),
810       ParseExe(false),
811       ParseAll(false)
812   {}
813 
814   HRESULT GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode);
815 };
816 
817 static const size_t kAnalysisBufSize = 1 << 14;
818 
GetFilterGroup(UInt32 index,const CUpdateItem & ui,CFilterMode & filterMode)819 HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode)
820 {
821   filterMode.Id = 0;
822   filterMode.Delta = 0;
823 
824   CFilterMode filterModeTemp = filterMode;
825 
826   int slashPos = ui.Name.ReverseFind_PathSepar();
827   int dotPos = ui.Name.ReverseFind_Dot();
828 
829   // if (dotPos > slashPos)
830   {
831     bool needReadFile = ParseAll;
832 
833     bool probablyIsSameIsa = false;
834 
835     if (!needReadFile || !Callback)
836     {
837       const wchar_t *ext;
838       if (dotPos > slashPos)
839         ext = ui.Name.Ptr(dotPos + 1);
840       else
841         ext = ui.Name.RightPtr(0);
842 
843       // p7zip uses the trick to store posix attributes in high 16 bits
844       if (ui.Attrib & 0x8000)
845       {
846         unsigned st_mode = ui.Attrib >> 16;
847         // st_mode = 00111;
848         if ((st_mode & 00111) && (ui.Size >= 2048))
849         {
850           #ifndef _WIN32
851           probablyIsSameIsa = true;
852           #endif
853           needReadFile = true;
854         }
855       }
856 
857       if (IsExeExt(ext))
858       {
859         needReadFile = true;
860         #ifdef _WIN32
861         probablyIsSameIsa = true;
862         needReadFile = ParseExe;
863         #endif
864       }
865       else if (StringsAreEqualNoCase_Ascii(ext, "wav"))
866       {
867         needReadFile = ParseWav;
868       }
869       /*
870       else if (!needReadFile && ParseUnixExt)
871       {
872         if (StringsAreEqualNoCase_Ascii(ext, "so")
873           || StringsAreEqualNoCase_Ascii(ext, ""))
874 
875           needReadFile = true;
876       }
877       */
878     }
879 
880     if (needReadFile && Callback)
881     {
882       if (Buffer.Size() != kAnalysisBufSize)
883       {
884         Buffer.Alloc(kAnalysisBufSize);
885       }
886       {
887         CMyComPtr<ISequentialInStream> stream;
888         HRESULT result = Callback->GetStream2(index, &stream, NUpdateNotifyOp::kAnalyze);
889         if (result == S_OK && stream)
890         {
891           size_t size = kAnalysisBufSize;
892           result = ReadStream(stream, Buffer, &size);
893           stream.Release();
894           // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK));
895           if (result == S_OK)
896           {
897             Bool parseRes = ParseFile(Buffer, size, &filterModeTemp);
898             if (parseRes && filterModeTemp.Delta == 0)
899             {
900               filterModeTemp.SetDelta();
901               if (filterModeTemp.Delta != 0 && filterModeTemp.Id != k_Delta)
902               {
903                 if (ui.Size % filterModeTemp.Delta != 0)
904                 {
905                   parseRes = false;
906                 }
907               }
908             }
909             if (!parseRes)
910             {
911               filterModeTemp.Id = 0;
912               filterModeTemp.Delta = 0;
913             }
914           }
915         }
916       }
917     }
918     else if ((needReadFile && !Callback) || probablyIsSameIsa)
919     {
920       #ifdef MY_CPU_X86_OR_AMD64
921       if (probablyIsSameIsa)
922         filterModeTemp.Id = k_X86;
923       #endif
924     }
925   }
926 
927   filterMode = filterModeTemp;
928   return S_OK;
929 }
930 
GetMethodFull(UInt64 methodID,UInt32 numStreams,CMethodFull & m)931 static inline void GetMethodFull(UInt64 methodID, UInt32 numStreams, CMethodFull &m)
932 {
933   m.Id = methodID;
934   m.NumStreams = numStreams;
935 }
936 
AddBondForFilter(CCompressionMethodMode & mode)937 static HRESULT AddBondForFilter(CCompressionMethodMode &mode)
938 {
939   for (unsigned c = 1; c < mode.Methods.Size(); c++)
940   {
941     if (!mode.IsThereBond_to_Coder(c))
942     {
943       CBond2 bond;
944       bond.OutCoder = 0;
945       bond.OutStream = 0;
946       bond.InCoder = c;
947       mode.Bonds.Add(bond);
948       return S_OK;
949     }
950   }
951   return E_INVALIDARG;
952 }
953 
AddFilterBond(CCompressionMethodMode & mode)954 static HRESULT AddFilterBond(CCompressionMethodMode &mode)
955 {
956   if (!mode.Bonds.IsEmpty())
957     return AddBondForFilter(mode);
958   return S_OK;
959 }
960 
AddBcj2Methods(CCompressionMethodMode & mode)961 static HRESULT AddBcj2Methods(CCompressionMethodMode &mode)
962 {
963   // mode.Methods[0] must be k_BCJ2 method !
964 
965   CMethodFull m;
966   GetMethodFull(k_LZMA, 1, m);
967 
968   m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20);
969   m.AddProp32(NCoderPropID::kNumFastBytes, 128);
970   m.AddProp32(NCoderPropID::kNumThreads, 1);
971   m.AddProp32(NCoderPropID::kLitPosBits, 2);
972   m.AddProp32(NCoderPropID::kLitContextBits, 0);
973   // m.AddProp_Ascii(NCoderPropID::kMatchFinder, "BT2");
974 
975   unsigned methodIndex = mode.Methods.Size();
976 
977   if (mode.Bonds.IsEmpty())
978   {
979     for (unsigned i = 1; i + 1 < mode.Methods.Size(); i++)
980     {
981       CBond2 bond;
982       bond.OutCoder = i;
983       bond.OutStream = 0;
984       bond.InCoder = i + 1;
985       mode.Bonds.Add(bond);
986     }
987   }
988 
989   mode.Methods.Add(m);
990   mode.Methods.Add(m);
991 
992   RINOK(AddBondForFilter(mode));
993   CBond2 bond;
994   bond.OutCoder = 0;
995   bond.InCoder = methodIndex;      bond.OutStream = 1;  mode.Bonds.Add(bond);
996   bond.InCoder = methodIndex + 1;  bond.OutStream = 2;  mode.Bonds.Add(bond);
997   return S_OK;
998 }
999 
MakeExeMethod(CCompressionMethodMode & mode,const CFilterMode & filterMode,bool bcj2Filter)1000 static HRESULT MakeExeMethod(CCompressionMethodMode &mode,
1001     const CFilterMode &filterMode, /* bool addFilter, */ bool bcj2Filter)
1002 {
1003   if (mode.Filter_was_Inserted)
1004   {
1005     const CMethodFull &m = mode.Methods[0];
1006     CMethodId id = m.Id;
1007     if (id == k_BCJ2)
1008       return AddBcj2Methods(mode);
1009     if (!m.IsSimpleCoder())
1010       return E_NOTIMPL;
1011     // if (Bonds.IsEmpty()) we can create bonds later
1012     return AddFilterBond(mode);
1013   }
1014 
1015   if (filterMode.Id == 0)
1016     return S_OK;
1017 
1018   CMethodFull &m = mode.Methods.InsertNew(0);
1019 
1020   {
1021     FOR_VECTOR(k, mode.Bonds)
1022     {
1023       CBond2 &bond = mode.Bonds[k];
1024       bond.InCoder++;
1025       bond.OutCoder++;
1026     }
1027   }
1028 
1029   HRESULT res;
1030 
1031   if (bcj2Filter && Is86Filter(filterMode.Id))
1032   {
1033     GetMethodFull(k_BCJ2, 4, m);
1034     res = AddBcj2Methods(mode);
1035   }
1036   else
1037   {
1038     GetMethodFull(filterMode.Id, 1, m);
1039     if (filterMode.Id == k_Delta)
1040       m.AddProp32(NCoderPropID::kDefaultProp, filterMode.Delta);
1041     res = AddFilterBond(mode);
1042 
1043     int alignBits = -1;
1044     if (filterMode.Id == k_Delta || filterMode.Delta != 0)
1045     {
1046            if (filterMode.Delta == 1) alignBits = 0;
1047       else if (filterMode.Delta == 2) alignBits = 1;
1048       else if (filterMode.Delta == 4) alignBits = 2;
1049       else if (filterMode.Delta == 8) alignBits = 3;
1050       else if (filterMode.Delta == 16) alignBits = 4;
1051     }
1052     else
1053     {
1054       // alignBits = GetAlignForFilterMethod(filterMode.Id);
1055     }
1056 
1057     if (res == S_OK && alignBits >= 0)
1058     {
1059       unsigned nextCoder = 1;
1060       if (!mode.Bonds.IsEmpty())
1061       {
1062         nextCoder = mode.Bonds.Back().InCoder;
1063       }
1064       if (nextCoder < mode.Methods.Size())
1065       {
1066         CMethodFull &nextMethod = mode.Methods[nextCoder];
1067         if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2)
1068         {
1069           if (!nextMethod.Are_Lzma_Model_Props_Defined())
1070           {
1071             if (alignBits != 0)
1072             {
1073               if (alignBits > 2 || filterMode.Id == k_Delta)
1074                 nextMethod.AddProp32(NCoderPropID::kPosStateBits, alignBits);
1075               unsigned lc = 0;
1076               if (alignBits < 3)
1077                 lc = 3 - alignBits;
1078               nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc);
1079               nextMethod.AddProp32(NCoderPropID::kLitPosBits, alignBits);
1080             }
1081           }
1082         }
1083       }
1084     }
1085   }
1086 
1087   return res;
1088 }
1089 
1090 
UpdateItem_To_FileItem2(const CUpdateItem & ui,CFileItem2 & file2)1091 static void UpdateItem_To_FileItem2(const CUpdateItem &ui, CFileItem2 &file2)
1092 {
1093   file2.Attrib = ui.Attrib;  file2.AttribDefined = ui.AttribDefined;
1094   file2.CTime = ui.CTime;  file2.CTimeDefined = ui.CTimeDefined;
1095   file2.ATime = ui.ATime;  file2.ATimeDefined = ui.ATimeDefined;
1096   file2.MTime = ui.MTime;  file2.MTimeDefined = ui.MTimeDefined;
1097   file2.IsAnti = ui.IsAnti;
1098   // file2.IsAux = false;
1099   file2.StartPosDefined = false;
1100   // file2.StartPos = 0;
1101 }
1102 
1103 
UpdateItem_To_FileItem(const CUpdateItem & ui,CFileItem & file,CFileItem2 & file2)1104 static void UpdateItem_To_FileItem(const CUpdateItem &ui,
1105     CFileItem &file, CFileItem2 &file2)
1106 {
1107   UpdateItem_To_FileItem2(ui, file2);
1108 
1109   file.Size = ui.Size;
1110   file.IsDir = ui.IsDir;
1111   file.HasStream = ui.HasStream();
1112   // file.IsAltStream = ui.IsAltStream;
1113 }
1114 
1115 
1116 
1117 class CRepackInStreamWithSizes:
1118   public ISequentialInStream,
1119   public ICompressGetSubStreamSize,
1120   public CMyUnknownImp
1121 {
1122   CMyComPtr<ISequentialInStream> _stream;
1123   // UInt64 _size;
1124   const CBoolVector *_extractStatuses;
1125   UInt32 _startIndex;
1126 public:
1127   const CDbEx *_db;
1128 
Init(ISequentialInStream * stream,UInt32 startIndex,const CBoolVector * extractStatuses)1129   void Init(ISequentialInStream *stream, UInt32 startIndex, const CBoolVector *extractStatuses)
1130   {
1131     _startIndex = startIndex;
1132     _extractStatuses = extractStatuses;
1133     // _size = 0;
1134     _stream = stream;
1135   }
1136   // UInt64 GetSize() const { return _size; }
1137 
1138   MY_UNKNOWN_IMP2(ISequentialInStream, ICompressGetSubStreamSize)
1139 
1140   STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
1141 
1142   STDMETHOD(GetSubStreamSize)(UInt64 subStream, UInt64 *value);
1143 };
1144 
Read(void * data,UInt32 size,UInt32 * processedSize)1145 STDMETHODIMP CRepackInStreamWithSizes::Read(void *data, UInt32 size, UInt32 *processedSize)
1146 {
1147   return _stream->Read(data, size, processedSize);
1148   /*
1149   UInt32 realProcessedSize;
1150   HRESULT result = _stream->Read(data, size, &realProcessedSize);
1151   _size += realProcessedSize;
1152   if (processedSize)
1153     *processedSize = realProcessedSize;
1154   return result;
1155   */
1156 }
1157 
GetSubStreamSize(UInt64 subStream,UInt64 * value)1158 STDMETHODIMP CRepackInStreamWithSizes::GetSubStreamSize(UInt64 subStream, UInt64 *value)
1159 {
1160   *value = 0;
1161   if (subStream >= _extractStatuses->Size())
1162     return S_FALSE; // E_FAIL;
1163   unsigned index = (unsigned)subStream;
1164   if ((*_extractStatuses)[index])
1165   {
1166     const CFileItem &fi = _db->Files[_startIndex + index];
1167     if (fi.HasStream)
1168       *value = fi.Size;
1169   }
1170   return S_OK;
1171 }
1172 
1173 
1174 class CRepackStreamBase
1175 {
1176 protected:
1177   bool _needWrite;
1178   bool _fileIsOpen;
1179   bool _calcCrc;
1180   UInt32 _crc;
1181   UInt64 _rem;
1182 
1183   const CBoolVector *_extractStatuses;
1184   UInt32 _startIndex;
1185   unsigned _currentIndex;
1186 
1187   HRESULT OpenFile();
1188   HRESULT CloseFile();
1189   HRESULT ProcessEmptyFiles();
1190 
1191 public:
1192   const CDbEx *_db;
1193   CMyComPtr<IArchiveUpdateCallbackFile> _opCallback;
1194   CMyComPtr<IArchiveExtractCallbackMessage> _extractCallback;
1195 
1196   HRESULT Init(UInt32 startIndex, const CBoolVector *extractStatuses);
CheckFinishedState() const1197   HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
1198 };
1199 
Init(UInt32 startIndex,const CBoolVector * extractStatuses)1200 HRESULT CRepackStreamBase::Init(UInt32 startIndex, const CBoolVector *extractStatuses)
1201 {
1202   _startIndex = startIndex;
1203   _extractStatuses = extractStatuses;
1204 
1205   _currentIndex = 0;
1206   _fileIsOpen = false;
1207 
1208   return ProcessEmptyFiles();
1209 }
1210 
OpenFile()1211 HRESULT CRepackStreamBase::OpenFile()
1212 {
1213   UInt32 arcIndex = _startIndex + _currentIndex;
1214   const CFileItem &fi = _db->Files[arcIndex];
1215 
1216   _needWrite = (*_extractStatuses)[_currentIndex];
1217   if (_opCallback)
1218   {
1219     RINOK(_opCallback->ReportOperation(
1220         NEventIndexType::kInArcIndex, arcIndex,
1221         _needWrite ?
1222             NUpdateNotifyOp::kRepack :
1223             NUpdateNotifyOp::kSkip));
1224   }
1225 
1226   _crc = CRC_INIT_VAL;
1227   _calcCrc = (fi.CrcDefined && !fi.IsDir);
1228 
1229   _fileIsOpen = true;
1230   _rem = fi.Size;
1231   return S_OK;
1232 }
1233 
1234 const HRESULT k_My_HRESULT_CRC_ERROR = 0x20000002;
1235 
CloseFile()1236 HRESULT CRepackStreamBase::CloseFile()
1237 {
1238   UInt32 arcIndex = _startIndex + _currentIndex;
1239   const CFileItem &fi = _db->Files[arcIndex];
1240   _fileIsOpen = false;
1241   _currentIndex++;
1242   if (!_calcCrc || fi.Crc == CRC_GET_DIGEST(_crc))
1243     return S_OK;
1244 
1245   if (_extractCallback)
1246   {
1247     RINOK(_extractCallback->ReportExtractResult(
1248         NEventIndexType::kInArcIndex, arcIndex,
1249         NExtract::NOperationResult::kCRCError));
1250   }
1251   // return S_FALSE;
1252   return k_My_HRESULT_CRC_ERROR;
1253 }
1254 
ProcessEmptyFiles()1255 HRESULT CRepackStreamBase::ProcessEmptyFiles()
1256 {
1257   while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
1258   {
1259     RINOK(OpenFile());
1260     RINOK(CloseFile());
1261   }
1262   return S_OK;
1263 }
1264 
1265 
1266 
1267 #ifndef _7ZIP_ST
1268 
1269 class CFolderOutStream2:
1270   public CRepackStreamBase,
1271   public ISequentialOutStream,
1272   public CMyUnknownImp
1273 {
1274 public:
1275   CMyComPtr<ISequentialOutStream> _stream;
1276 
1277   MY_UNKNOWN_IMP
1278 
1279   STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
1280 };
1281 
Write(const void * data,UInt32 size,UInt32 * processedSize)1282 STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize)
1283 {
1284   if (processedSize)
1285     *processedSize = 0;
1286 
1287   while (size != 0)
1288   {
1289     if (_fileIsOpen)
1290     {
1291       UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1292       HRESULT result = S_OK;
1293       if (_needWrite)
1294         result = _stream->Write(data, cur, &cur);
1295       if (_calcCrc)
1296         _crc = CrcUpdate(_crc, data, cur);
1297       if (processedSize)
1298         *processedSize += cur;
1299       data = (const Byte *)data + cur;
1300       size -= cur;
1301       _rem -= cur;
1302       if (_rem == 0)
1303       {
1304         RINOK(CloseFile());
1305         RINOK(ProcessEmptyFiles());
1306       }
1307       RINOK(result);
1308       if (cur == 0)
1309         break;
1310       continue;
1311     }
1312 
1313     RINOK(ProcessEmptyFiles());
1314     if (_currentIndex == _extractStatuses->Size())
1315     {
1316       // we don't support write cut here
1317       return E_FAIL;
1318     }
1319     RINOK(OpenFile());
1320   }
1321 
1322   return S_OK;
1323 }
1324 
1325 #endif
1326 
1327 
1328 
1329 static const UInt32 kTempBufSize = 1 << 16;
1330 
1331 class CFolderInStream2:
1332   public CRepackStreamBase,
1333   public ISequentialInStream,
1334   public CMyUnknownImp
1335 {
1336   Byte *_buf;
1337 public:
1338   CMyComPtr<ISequentialInStream> _inStream;
1339   HRESULT Result;
1340 
1341   MY_UNKNOWN_IMP
1342 
CFolderInStream2()1343   CFolderInStream2():
1344       Result(S_OK)
1345   {
1346     _buf = new Byte[kTempBufSize];
1347   }
1348 
~CFolderInStream2()1349   ~CFolderInStream2()
1350   {
1351     delete []_buf;
1352   }
1353 
Init()1354   void Init() { Result = S_OK; }
1355   STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
1356 };
1357 
Read(void * data,UInt32 size,UInt32 * processedSize)1358 STDMETHODIMP CFolderInStream2::Read(void *data, UInt32 size, UInt32 *processedSize)
1359 {
1360   if (processedSize)
1361     *processedSize = 0;
1362 
1363   while (size != 0)
1364   {
1365     if (_fileIsOpen)
1366     {
1367       UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1368 
1369       void *buf;
1370       if (_needWrite)
1371         buf = data;
1372       else
1373       {
1374         buf = _buf;
1375         if (cur > kTempBufSize)
1376           cur = kTempBufSize;
1377       }
1378 
1379       HRESULT result = _inStream->Read(buf, cur, &cur);
1380       _crc = CrcUpdate(_crc, buf, cur);
1381       _rem -= cur;
1382 
1383       if (_needWrite)
1384       {
1385         data = (Byte *)data + cur;
1386         size -= cur;
1387         if (processedSize)
1388           *processedSize += cur;
1389       }
1390 
1391       if (result != S_OK)
1392         Result = result;
1393 
1394       if (_rem == 0)
1395       {
1396         RINOK(CloseFile());
1397         RINOK(ProcessEmptyFiles());
1398       }
1399 
1400       RINOK(result);
1401 
1402       if (cur == 0)
1403         return E_FAIL;
1404 
1405       continue;
1406     }
1407 
1408     RINOK(ProcessEmptyFiles());
1409     if (_currentIndex == _extractStatuses->Size())
1410     {
1411       return S_OK;
1412     }
1413     RINOK(OpenFile());
1414   }
1415 
1416   return S_OK;
1417 }
1418 
1419 
1420 class CThreadDecoder
1421   #ifndef _7ZIP_ST
1422     : public CVirtThread
1423   #endif
1424 {
1425 public:
1426   CDecoder Decoder;
1427 
CThreadDecoder(bool multiThreadMixer)1428   CThreadDecoder(bool multiThreadMixer):
1429       Decoder(multiThreadMixer)
1430   {
1431     #ifndef _7ZIP_ST
1432     if (multiThreadMixer)
1433     {
1434       MtMode = false;
1435       NumThreads = 1;
1436       FosSpec = new CFolderOutStream2;
1437       Fos = FosSpec;
1438       Result = E_FAIL;
1439     }
1440     #endif
1441     // UnpackSize = 0;
1442     // send_UnpackSize = false;
1443   }
1444 
1445   #ifndef _7ZIP_ST
1446 
1447   bool dataAfterEnd_Error;
1448   HRESULT Result;
1449   CMyComPtr<IInStream> InStream;
1450 
1451   CFolderOutStream2 *FosSpec;
1452   CMyComPtr<ISequentialOutStream> Fos;
1453 
1454   UInt64 StartPos;
1455   const CFolders *Folders;
1456   int FolderIndex;
1457 
1458   // bool send_UnpackSize;
1459   // UInt64 UnpackSize;
1460 
1461   #ifndef _NO_CRYPTO
1462   CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1463   #endif
1464 
1465   DECL_EXTERNAL_CODECS_LOC_VARS2;
1466 
1467   #ifndef _7ZIP_ST
1468   bool MtMode;
1469   UInt32 NumThreads;
1470   #endif
1471 
1472 
~CThreadDecoder()1473   ~CThreadDecoder() { CVirtThread::WaitThreadFinish(); }
1474   virtual void Execute();
1475 
1476   #endif
1477 };
1478 
1479 #ifndef _7ZIP_ST
1480 
Execute()1481 void CThreadDecoder::Execute()
1482 {
1483   try
1484   {
1485     #ifndef _NO_CRYPTO
1486       bool isEncrypted = false;
1487       bool passwordIsDefined = false;
1488       UString password;
1489     #endif
1490 
1491     dataAfterEnd_Error = false;
1492 
1493     Result = Decoder.Decode(
1494       EXTERNAL_CODECS_LOC_VARS
1495       InStream,
1496       StartPos,
1497       *Folders, FolderIndex,
1498 
1499       // send_UnpackSize ? &UnpackSize : NULL,
1500       NULL, // unpackSize : FULL unpack
1501 
1502       Fos,
1503       NULL, // compressProgress
1504 
1505       NULL  // *inStreamMainRes
1506       , dataAfterEnd_Error
1507 
1508       _7Z_DECODER_CRYPRO_VARS
1509       #ifndef _7ZIP_ST
1510         , MtMode, NumThreads,
1511         0 // MemUsage
1512       #endif
1513 
1514       );
1515   }
1516   catch(...)
1517   {
1518     Result = E_FAIL;
1519   }
1520 
1521   /*
1522   if (Result == S_OK)
1523     Result = FosSpec->CheckFinishedState();
1524   */
1525   FosSpec->_stream.Release();
1526 }
1527 
1528 #endif
1529 
1530 #ifndef _NO_CRYPTO
1531 
1532 class CCryptoGetTextPassword:
1533   public ICryptoGetTextPassword,
1534   public CMyUnknownImp
1535 {
1536 public:
1537   UString Password;
1538 
1539   MY_UNKNOWN_IMP
1540   STDMETHOD(CryptoGetTextPassword)(BSTR *password);
1541 };
1542 
CryptoGetTextPassword(BSTR * password)1543 STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password)
1544 {
1545   return StringToBstr(Password, password);
1546 }
1547 
1548 #endif
1549 
1550 
GetFile(const CDatabase & inDb,unsigned index,CFileItem & file,CFileItem2 & file2)1551 static void GetFile(const CDatabase &inDb, unsigned index, CFileItem &file, CFileItem2 &file2)
1552 {
1553   file = inDb.Files[index];
1554   file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime);
1555   file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime);
1556   file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime);
1557   file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos);
1558   file2.AttribDefined = inDb.Attrib.GetItem(index, file2.Attrib);
1559   file2.IsAnti = inDb.IsItemAnti(index);
1560   // file2.IsAux = inDb.IsItemAux(index);
1561 }
1562 
Update(DECL_EXTERNAL_CODECS_LOC_VARS IInStream * inStream,const CDbEx * db,const CObjectVector<CUpdateItem> & updateItems,COutArchive & archive,CArchiveDatabaseOut & newDatabase,ISequentialOutStream * seqOutStream,IArchiveUpdateCallback * updateCallback,const CUpdateOptions & options,ICryptoGetTextPassword * getDecoderPassword)1563 HRESULT Update(
1564     DECL_EXTERNAL_CODECS_LOC_VARS
1565     IInStream *inStream,
1566     const CDbEx *db,
1567     const CObjectVector<CUpdateItem> &updateItems,
1568     // const CObjectVector<CTreeFolder> &treeFolders,
1569     // const CUniqBlocks &secureBlocks,
1570     COutArchive &archive,
1571     CArchiveDatabaseOut &newDatabase,
1572     ISequentialOutStream *seqOutStream,
1573     IArchiveUpdateCallback *updateCallback,
1574     const CUpdateOptions &options
1575     #ifndef _NO_CRYPTO
1576     , ICryptoGetTextPassword *getDecoderPassword
1577     #endif
1578     )
1579 {
1580   UInt64 numSolidFiles = options.NumSolidFiles;
1581   if (numSolidFiles == 0)
1582     numSolidFiles = 1;
1583 
1584   CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
1585   updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
1586 
1587   CMyComPtr<IArchiveExtractCallbackMessage> extractCallback;
1588   updateCallback->QueryInterface(IID_IArchiveExtractCallbackMessage, (void **)&extractCallback);
1589 
1590   // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes();
1591 
1592   /*
1593   CMyComPtr<IOutStream> outStream;
1594   RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream));
1595   if (!outStream)
1596     return E_NOTIMPL;
1597   */
1598 
1599   UInt64 startBlockSize = db ? db->ArcInfo.StartPosition: 0;
1600   if (startBlockSize > 0 && !options.RemoveSfxBlock)
1601   {
1602     RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL));
1603   }
1604 
1605   CIntArr fileIndexToUpdateIndexMap;
1606   UInt64 complexity = 0;
1607   UInt64 inSizeForReduce2 = 0;
1608   bool needEncryptedRepack = false;
1609 
1610   CRecordVector<CFilterMode2> filters;
1611   CObjectVector<CSolidGroup> groups;
1612   bool thereAreRepacks = false;
1613 
1614   bool useFilters = options.UseFilters;
1615   if (useFilters)
1616   {
1617     const CCompressionMethodMode &method = *options.Method;
1618 
1619     FOR_VECTOR (i, method.Methods)
1620       if (IsFilterMethod(method.Methods[i].Id))
1621       {
1622         useFilters = false;
1623         break;
1624       }
1625   }
1626 
1627   if (db)
1628   {
1629     fileIndexToUpdateIndexMap.Alloc(db->Files.Size());
1630     unsigned i;
1631 
1632     for (i = 0; i < db->Files.Size(); i++)
1633       fileIndexToUpdateIndexMap[i] = -1;
1634 
1635     for (i = 0; i < updateItems.Size(); i++)
1636     {
1637       int index = updateItems[i].IndexInArchive;
1638       if (index != -1)
1639         fileIndexToUpdateIndexMap[(unsigned)index] = i;
1640     }
1641 
1642     for (i = 0; i < db->NumFolders; i++)
1643     {
1644       CNum indexInFolder = 0;
1645       CNum numCopyItems = 0;
1646       CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
1647       UInt64 repackSize = 0;
1648 
1649       for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
1650       {
1651         const CFileItem &file = db->Files[fi];
1652         if (file.HasStream)
1653         {
1654           indexInFolder++;
1655           int updateIndex = fileIndexToUpdateIndexMap[fi];
1656           if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
1657           {
1658             numCopyItems++;
1659             repackSize += file.Size;
1660           }
1661         }
1662       }
1663 
1664       if (numCopyItems == 0)
1665         continue;
1666 
1667       CFolderRepack rep;
1668       rep.FolderIndex = i;
1669       rep.NumCopyFiles = numCopyItems;
1670       CFolderEx f;
1671       db->ParseFolderEx(i, f);
1672 
1673       const bool isEncrypted = f.IsEncrypted();
1674       const bool needCopy = (numCopyItems == numUnpackStreams);
1675       const bool extractFilter = (useFilters || needCopy);
1676 
1677       unsigned groupIndex = Get_FilterGroup_for_Folder(filters, f, extractFilter);
1678 
1679       while (groupIndex >= groups.Size())
1680         groups.AddNew();
1681 
1682       groups[groupIndex].folderRefs.Add(rep);
1683 
1684       if (needCopy)
1685         complexity += db->GetFolderFullPackSize(i);
1686       else
1687       {
1688         thereAreRepacks = true;
1689         complexity += repackSize;
1690         if (inSizeForReduce2 < repackSize)
1691           inSizeForReduce2 = repackSize;
1692         if (isEncrypted)
1693           needEncryptedRepack = true;
1694       }
1695     }
1696   }
1697 
1698   UInt64 inSizeForReduce = 0;
1699   {
1700     bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0);
1701     FOR_VECTOR (i, updateItems)
1702     {
1703       const CUpdateItem &ui = updateItems[i];
1704       if (ui.NewData)
1705       {
1706         complexity += ui.Size;
1707         if (isSolid)
1708           inSizeForReduce += ui.Size;
1709         else if (inSizeForReduce < ui.Size)
1710           inSizeForReduce = ui.Size;
1711       }
1712     }
1713   }
1714 
1715   if (inSizeForReduce < inSizeForReduce2)
1716     inSizeForReduce = inSizeForReduce2;
1717 
1718   RINOK(updateCallback->SetTotal(complexity));
1719 
1720   CLocalProgress *lps = new CLocalProgress;
1721   CMyComPtr<ICompressProgressInfo> progress = lps;
1722   lps->Init(updateCallback, true);
1723 
1724   #ifndef _7ZIP_ST
1725 
1726   CStreamBinder sb;
1727   if (options.MultiThreadMixer)
1728   {
1729     RINOK(sb.CreateEvents());
1730   }
1731 
1732   #endif
1733 
1734   CThreadDecoder threadDecoder(options.MultiThreadMixer);
1735 
1736   #ifndef _7ZIP_ST
1737   if (options.MultiThreadMixer && thereAreRepacks)
1738   {
1739     #ifdef EXTERNAL_CODECS
1740     threadDecoder.__externalCodecs = __externalCodecs;
1741     #endif
1742     RINOK(threadDecoder.Create());
1743   }
1744   #endif
1745 
1746   {
1747     CAnalysis analysis;
1748     if (options.AnalysisLevel == 0)
1749     {
1750       analysis.ParseWav = false;
1751       analysis.ParseExe = false;
1752       analysis.ParseAll = false;
1753     }
1754     else
1755     {
1756       analysis.Callback = opCallback;
1757       if (options.AnalysisLevel > 0)
1758       {
1759         analysis.ParseWav = true;
1760         if (options.AnalysisLevel >= 7)
1761         {
1762           analysis.ParseExe = true;
1763           if (options.AnalysisLevel >= 9)
1764             analysis.ParseAll = true;
1765         }
1766       }
1767     }
1768 
1769     // ---------- Split files to groups ----------
1770 
1771     const CCompressionMethodMode &method = *options.Method;
1772 
1773     FOR_VECTOR (i, updateItems)
1774     {
1775       const CUpdateItem &ui = updateItems[i];
1776       if (!ui.NewData || !ui.HasStream())
1777         continue;
1778 
1779       CFilterMode2 fm;
1780       if (useFilters)
1781       {
1782         RINOK(analysis.GetFilterGroup(i, ui, fm));
1783       }
1784       fm.Encrypted = method.PasswordIsDefined;
1785 
1786       unsigned groupIndex = GetGroup(filters, fm);
1787       while (groupIndex >= groups.Size())
1788         groups.AddNew();
1789       groups[groupIndex].Indices.Add(i);
1790     }
1791   }
1792 
1793 
1794   #ifndef _NO_CRYPTO
1795 
1796   CCryptoGetTextPassword *getPasswordSpec = NULL;
1797   CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1798   if (needEncryptedRepack)
1799   {
1800     getPasswordSpec = new CCryptoGetTextPassword;
1801     getTextPassword = getPasswordSpec;
1802 
1803     #ifndef _7ZIP_ST
1804     threadDecoder.getTextPassword = getPasswordSpec;
1805     #endif
1806 
1807     if (options.Method->PasswordIsDefined)
1808       getPasswordSpec->Password = options.Method->Password;
1809     else
1810     {
1811       if (!getDecoderPassword)
1812         return E_NOTIMPL;
1813       CMyComBSTR password;
1814       RINOK(getDecoderPassword->CryptoGetTextPassword(&password));
1815       if (password)
1816         getPasswordSpec->Password = password;
1817     }
1818   }
1819 
1820   #endif
1821 
1822 
1823   // ---------- Compress ----------
1824 
1825   RINOK(archive.Create(seqOutStream, false));
1826   RINOK(archive.SkipPrefixArchiveHeader());
1827 
1828   /*
1829   CIntVector treeFolderToArcIndex;
1830   treeFolderToArcIndex.Reserve(treeFolders.Size());
1831   for (i = 0; i < treeFolders.Size(); i++)
1832     treeFolderToArcIndex.Add(-1);
1833   // ---------- Write Tree (only AUX dirs) ----------
1834   for (i = 1; i < treeFolders.Size(); i++)
1835   {
1836     const CTreeFolder &treeFolder = treeFolders[i];
1837     CFileItem file;
1838     CFileItem2 file2;
1839     file2.Init();
1840     int secureID = 0;
1841     if (treeFolder.UpdateItemIndex < 0)
1842     {
1843       // we can store virtual dir item wuthout attrib, but we want all items have attrib.
1844       file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY);
1845       file2.IsAux = true;
1846     }
1847     else
1848     {
1849       const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex];
1850       // if item is not dir, then it's parent for alt streams.
1851       // we will write such items later
1852       if (!ui.IsDir)
1853         continue;
1854       secureID = ui.SecureIndex;
1855       if (ui.NewProps)
1856         UpdateItem_To_FileItem(ui, file, file2);
1857       else
1858         GetFile(*db, ui.IndexInArchive, file, file2);
1859     }
1860     file.Size = 0;
1861     file.HasStream = false;
1862     file.IsDir = true;
1863     file.Parent = treeFolder.Parent;
1864 
1865     treeFolderToArcIndex[i] = newDatabase.Files.Size();
1866     newDatabase.AddFile(file, file2, treeFolder.Name);
1867 
1868     if (totalSecureDataSize != 0)
1869       newDatabase.SecureIDs.Add(secureID);
1870   }
1871   */
1872 
1873   {
1874     /* ---------- Write non-AUX dirs and Empty files ---------- */
1875     CUIntVector emptyRefs;
1876 
1877     unsigned i;
1878 
1879     for (i = 0; i < updateItems.Size(); i++)
1880     {
1881       const CUpdateItem &ui = updateItems[i];
1882       if (ui.NewData)
1883       {
1884         if (ui.HasStream())
1885           continue;
1886       }
1887       else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream)
1888         continue;
1889       /*
1890       if (ui.TreeFolderIndex >= 0)
1891         continue;
1892       */
1893       emptyRefs.Add(i);
1894     }
1895 
1896     emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
1897 
1898     for (i = 0; i < emptyRefs.Size(); i++)
1899     {
1900       const CUpdateItem &ui = updateItems[emptyRefs[i]];
1901       CFileItem file;
1902       CFileItem2 file2;
1903       UString name;
1904       if (ui.NewProps)
1905       {
1906         UpdateItem_To_FileItem(ui, file, file2);
1907         file.CrcDefined = false;
1908         name = ui.Name;
1909       }
1910       else
1911       {
1912         GetFile(*db, ui.IndexInArchive, file, file2);
1913         db->GetPath(ui.IndexInArchive, name);
1914       }
1915 
1916       /*
1917       if (totalSecureDataSize != 0)
1918         newDatabase.SecureIDs.Add(ui.SecureIndex);
1919       file.Parent = ui.ParentFolderIndex;
1920       */
1921       newDatabase.AddFile(file, file2, name);
1922     }
1923   }
1924 
1925   lps->ProgressOffset = 0;
1926 
1927   {
1928     // ---------- Sort Filters ----------
1929 
1930     FOR_VECTOR (i, filters)
1931     {
1932       filters[i].GroupIndex = i;
1933     }
1934     filters.Sort2();
1935   }
1936 
1937   for (unsigned groupIndex = 0; groupIndex < filters.Size(); groupIndex++)
1938   {
1939     const CFilterMode2 &filterMode = filters[groupIndex];
1940 
1941     CCompressionMethodMode method = *options.Method;
1942     {
1943       HRESULT res = MakeExeMethod(method, filterMode,
1944         #ifdef _7ZIP_ST
1945           false
1946         #else
1947           options.MaxFilter && options.MultiThreadMixer
1948         #endif
1949         );
1950 
1951       RINOK(res);
1952     }
1953 
1954     if (filterMode.Encrypted)
1955     {
1956       if (!method.PasswordIsDefined)
1957       {
1958         #ifndef _NO_CRYPTO
1959         if (getPasswordSpec)
1960           method.Password = getPasswordSpec->Password;
1961         #endif
1962         method.PasswordIsDefined = true;
1963       }
1964     }
1965     else
1966     {
1967       method.PasswordIsDefined = false;
1968       method.Password.Empty();
1969     }
1970 
1971     CEncoder encoder(method);
1972 
1973     // ---------- Repack and copy old solid blocks ----------
1974 
1975     const CSolidGroup &group = groups[filterMode.GroupIndex];
1976 
1977     FOR_VECTOR(folderRefIndex, group.folderRefs)
1978     {
1979       const CFolderRepack &rep = group.folderRefs[folderRefIndex];
1980 
1981       unsigned folderIndex = rep.FolderIndex;
1982 
1983       CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
1984 
1985       if (rep.NumCopyFiles == numUnpackStreams)
1986       {
1987         if (opCallback)
1988         {
1989           RINOK(opCallback->ReportOperation(
1990               NEventIndexType::kBlockIndex, (UInt32)folderIndex,
1991               NUpdateNotifyOp::kReplicate));
1992 
1993           // ---------- Copy old solid block ----------
1994           {
1995             CNum indexInFolder = 0;
1996             for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
1997             {
1998               if (db->Files[fi].HasStream)
1999               {
2000                 indexInFolder++;
2001                 RINOK(opCallback->ReportOperation(
2002                     NEventIndexType::kInArcIndex, (UInt32)fi,
2003                     NUpdateNotifyOp::kReplicate));
2004               }
2005             }
2006           }
2007         }
2008 
2009         UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
2010         RINOK(WriteRange(inStream, archive.SeqStream,
2011             db->GetFolderStreamPos(folderIndex, 0), packSize, progress));
2012         lps->ProgressOffset += packSize;
2013 
2014         CFolder &folder = newDatabase.Folders.AddNew();
2015         db->ParseFolderInfo(folderIndex, folder);
2016         CNum startIndex = db->FoStartPackStreamIndex[folderIndex];
2017         FOR_VECTOR(j, folder.PackStreams)
2018         {
2019           newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j));
2020           // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
2021           // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
2022         }
2023 
2024         size_t indexStart = db->FoToCoderUnpackSizes[folderIndex];
2025         size_t indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1];
2026         for (; indexStart < indexEnd; indexStart++)
2027           newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes[indexStart]);
2028       }
2029       else
2030       {
2031         // ---------- Repack old solid block ----------
2032 
2033         CBoolVector extractStatuses;
2034 
2035         CNum indexInFolder = 0;
2036 
2037         if (opCallback)
2038         {
2039           RINOK(opCallback->ReportOperation(
2040               NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2041               NUpdateNotifyOp::kRepack))
2042         }
2043 
2044         /* We could reduce data size of decoded folder, if we don't need to repack
2045            last files in folder. But the gain in speed is small in most cases.
2046            So we unpack full folder. */
2047 
2048         UInt64 sizeToEncode = 0;
2049 
2050         /*
2051         UInt64 importantUnpackSize = 0;
2052         unsigned numImportantFiles = 0;
2053         UInt64 decodeSize = 0;
2054         */
2055 
2056         for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2057         {
2058           bool needExtract = false;
2059           const CFileItem &file = db->Files[fi];
2060 
2061           if (file.HasStream)
2062           {
2063             indexInFolder++;
2064             int updateIndex = fileIndexToUpdateIndexMap[fi];
2065             if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
2066               needExtract = true;
2067             // decodeSize += file.Size;
2068           }
2069 
2070           extractStatuses.Add(needExtract);
2071           if (needExtract)
2072           {
2073             sizeToEncode += file.Size;
2074             /*
2075             numImportantFiles = extractStatuses.Size();
2076             importantUnpackSize = decodeSize;
2077             */
2078           }
2079         }
2080 
2081         // extractStatuses.DeleteFrom(numImportantFiles);
2082 
2083         unsigned startPackIndex = newDatabase.PackSizes.Size();
2084         UInt64 curUnpackSize;
2085         {
2086 
2087           CMyComPtr<ISequentialInStream> sbInStream;
2088           CRepackStreamBase *repackBase;
2089           CFolderInStream2 *FosSpec2 = NULL;
2090 
2091           CRepackInStreamWithSizes *inStreamSizeCountSpec = new CRepackInStreamWithSizes;
2092           CMyComPtr<ISequentialInStream> inStreamSizeCount = inStreamSizeCountSpec;
2093           {
2094             #ifndef _7ZIP_ST
2095             if (options.MultiThreadMixer)
2096             {
2097               repackBase = threadDecoder.FosSpec;
2098               CMyComPtr<ISequentialOutStream> sbOutStream;
2099               sb.CreateStreams(&sbInStream, &sbOutStream);
2100               sb.ReInit();
2101 
2102               threadDecoder.FosSpec->_stream = sbOutStream;
2103 
2104               threadDecoder.InStream = inStream;
2105               threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0);
2106               threadDecoder.Folders = (const CFolders *)db;
2107               threadDecoder.FolderIndex = folderIndex;
2108 
2109               // threadDecoder.UnpackSize = importantUnpackSize;
2110               // threadDecoder.send_UnpackSize = true;
2111             }
2112             else
2113             #endif
2114             {
2115               FosSpec2 = new CFolderInStream2;
2116               FosSpec2->Init();
2117               sbInStream = FosSpec2;
2118               repackBase = FosSpec2;
2119 
2120               #ifndef _NO_CRYPTO
2121               bool isEncrypted = false;
2122               bool passwordIsDefined = false;
2123               UString password;
2124               #endif
2125 
2126               CMyComPtr<ISequentialInStream> decodedStream;
2127               bool dataAfterEnd_Error = false;
2128 
2129               HRESULT res = threadDecoder.Decoder.Decode(
2130                   EXTERNAL_CODECS_LOC_VARS
2131                   inStream,
2132                   db->ArcInfo.DataStartPosition, // db->GetFolderStreamPos(folderIndex, 0);,
2133                   *db, folderIndex,
2134                   // &importantUnpackSize, // *unpackSize
2135                   NULL, // *unpackSize : FULL unpack
2136 
2137                   NULL, // *outStream
2138                   NULL, // *compressProgress
2139 
2140                   &decodedStream
2141                   , dataAfterEnd_Error
2142 
2143                   _7Z_DECODER_CRYPRO_VARS
2144                   #ifndef _7ZIP_ST
2145                     , false // mtMode
2146                     , 1 // numThreads
2147                     , 0 // memUsage
2148                   #endif
2149                 );
2150 
2151               RINOK(res);
2152               if (!decodedStream)
2153                 return E_FAIL;
2154 
2155               FosSpec2->_inStream = decodedStream;
2156             }
2157 
2158             repackBase->_db = db;
2159             repackBase->_opCallback = opCallback;
2160             repackBase->_extractCallback = extractCallback;
2161 
2162             UInt32 startIndex = db->FolderStartFileIndex[folderIndex];
2163             RINOK(repackBase->Init(startIndex, &extractStatuses));
2164 
2165             inStreamSizeCountSpec->_db = db;
2166             inStreamSizeCountSpec->Init(sbInStream, startIndex, &extractStatuses);
2167 
2168             #ifndef _7ZIP_ST
2169             if (options.MultiThreadMixer)
2170             {
2171               threadDecoder.Start();
2172             }
2173             #endif
2174           }
2175 
2176           curUnpackSize = sizeToEncode;
2177 
2178           HRESULT encodeRes = encoder.Encode(
2179               EXTERNAL_CODECS_LOC_VARS
2180               inStreamSizeCount,
2181               // NULL,
2182               &inSizeForReduce,
2183               newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curUnpackSize,
2184               archive.SeqStream, newDatabase.PackSizes, progress);
2185 
2186           if (encodeRes == k_My_HRESULT_CRC_ERROR)
2187             return E_FAIL;
2188 
2189           #ifndef _7ZIP_ST
2190           if (options.MultiThreadMixer)
2191           {
2192             // 16.00: hang was fixed : for case if decoding was not finished.
2193             // We close CBinderInStream and it calls CStreamBinder::CloseRead()
2194             inStreamSizeCount.Release();
2195             sbInStream.Release();
2196 
2197             threadDecoder.WaitExecuteFinish();
2198 
2199             HRESULT decodeRes = threadDecoder.Result;
2200             // if (res == k_My_HRESULT_CRC_ERROR)
2201             if (decodeRes == S_FALSE || threadDecoder.dataAfterEnd_Error)
2202             {
2203               if (extractCallback)
2204               {
2205                 RINOK(extractCallback->ReportExtractResult(
2206                     NEventIndexType::kInArcIndex, db->FolderStartFileIndex[folderIndex],
2207                     // NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2208                     (decodeRes != S_OK ?
2209                       NExtract::NOperationResult::kDataError :
2210                       NExtract::NOperationResult::kDataAfterEnd)));
2211               }
2212               if (decodeRes != S_OK)
2213                 return E_FAIL;
2214             }
2215             RINOK(decodeRes);
2216             if (encodeRes == S_OK)
2217               if (sb.ProcessedSize != sizeToEncode)
2218                 encodeRes = E_FAIL;
2219           }
2220           else
2221           #endif
2222           {
2223             if (FosSpec2->Result == S_FALSE)
2224             {
2225               if (extractCallback)
2226               {
2227                 RINOK(extractCallback->ReportExtractResult(
2228                     NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2229                     NExtract::NOperationResult::kDataError));
2230               }
2231               return E_FAIL;
2232             }
2233             RINOK(FosSpec2->Result);
2234           }
2235 
2236           RINOK(encodeRes);
2237           RINOK(repackBase->CheckFinishedState());
2238 
2239           if (curUnpackSize != sizeToEncode)
2240             return E_FAIL;
2241         }
2242 
2243         for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2244           lps->OutSize += newDatabase.PackSizes[startPackIndex];
2245         lps->InSize += curUnpackSize;
2246       }
2247 
2248       newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
2249 
2250       CNum indexInFolder = 0;
2251       for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2252       {
2253         if (db->Files[fi].HasStream)
2254         {
2255           indexInFolder++;
2256           int updateIndex = fileIndexToUpdateIndexMap[fi];
2257           if (updateIndex >= 0)
2258           {
2259             const CUpdateItem &ui = updateItems[updateIndex];
2260             if (ui.NewData)
2261               continue;
2262 
2263             UString name;
2264             CFileItem file;
2265             CFileItem2 file2;
2266             GetFile(*db, fi, file, file2);
2267 
2268             if (ui.NewProps)
2269             {
2270               UpdateItem_To_FileItem2(ui, file2);
2271               file.IsDir = ui.IsDir;
2272               name = ui.Name;
2273             }
2274             else
2275               db->GetPath(fi, name);
2276 
2277             /*
2278             file.Parent = ui.ParentFolderIndex;
2279             if (ui.TreeFolderIndex >= 0)
2280               treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2281             if (totalSecureDataSize != 0)
2282               newDatabase.SecureIDs.Add(ui.SecureIndex);
2283             */
2284             newDatabase.AddFile(file, file2, name);
2285           }
2286         }
2287       }
2288     }
2289 
2290 
2291     // ---------- Compress files to new solid blocks ----------
2292 
2293     unsigned numFiles = group.Indices.Size();
2294     if (numFiles == 0)
2295       continue;
2296     CRecordVector<CRefItem> refItems;
2297     refItems.ClearAndSetSize(numFiles);
2298     // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1
2299     bool sortByType = options.UseTypeSorting;
2300 
2301     unsigned i;
2302 
2303     for (i = 0; i < numFiles; i++)
2304       refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType);
2305 
2306     CSortParam sortParam;
2307     // sortParam.TreeFolders = &treeFolders;
2308     sortParam.SortByType = sortByType;
2309     refItems.Sort(CompareUpdateItems, (void *)&sortParam);
2310 
2311     CObjArray<UInt32> indices(numFiles);
2312 
2313     for (i = 0; i < numFiles; i++)
2314     {
2315       UInt32 index = refItems[i].Index;
2316       indices[i] = index;
2317       /*
2318       const CUpdateItem &ui = updateItems[index];
2319       CFileItem file;
2320       if (ui.NewProps)
2321         UpdateItem_To_FileItem(ui, file);
2322       else
2323         file = db.Files[ui.IndexInArchive];
2324       if (file.IsAnti || file.IsDir)
2325         return E_FAIL;
2326       newDatabase.Files.Add(file);
2327       */
2328     }
2329 
2330     for (i = 0; i < numFiles;)
2331     {
2332       UInt64 totalSize = 0;
2333       unsigned numSubFiles;
2334 
2335       const wchar_t *prevExtension = NULL;
2336 
2337       for (numSubFiles = 0; i + numSubFiles < numFiles && numSubFiles < numSolidFiles; numSubFiles++)
2338       {
2339         const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
2340         totalSize += ui.Size;
2341         if (totalSize > options.NumSolidBytes)
2342           break;
2343         if (options.SolidExtension)
2344         {
2345           int slashPos = ui.Name.ReverseFind_PathSepar();
2346           int dotPos = ui.Name.ReverseFind_Dot();
2347           const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : dotPos + 1);
2348           if (numSubFiles == 0)
2349             prevExtension = ext;
2350           else if (!StringsAreEqualNoCase(ext, prevExtension))
2351             break;
2352         }
2353       }
2354 
2355       if (numSubFiles < 1)
2356         numSubFiles = 1;
2357 
2358       RINOK(lps->SetCur());
2359 
2360       CFolderInStream *inStreamSpec = new CFolderInStream;
2361       CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec);
2362       inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
2363 
2364       unsigned startPackIndex = newDatabase.PackSizes.Size();
2365       UInt64 curFolderUnpackSize = totalSize;
2366       // curFolderUnpackSize = (UInt64)(Int64)-1;
2367 
2368       RINOK(encoder.Encode(
2369           EXTERNAL_CODECS_LOC_VARS
2370           solidInStream,
2371           // NULL,
2372           &inSizeForReduce,
2373           newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curFolderUnpackSize,
2374           archive.SeqStream, newDatabase.PackSizes, progress));
2375 
2376       if (!inStreamSpec->WasFinished())
2377         return E_FAIL;
2378 
2379       for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2380         lps->OutSize += newDatabase.PackSizes[startPackIndex];
2381 
2382       lps->InSize += curFolderUnpackSize;
2383       // for ()
2384       // newDatabase.PackCRCsDefined.Add(false);
2385       // newDatabase.PackCRCs.Add(0);
2386 
2387       CNum numUnpackStreams = 0;
2388       UInt64 skippedSize = 0;
2389 
2390       for (unsigned subIndex = 0; subIndex < numSubFiles; subIndex++)
2391       {
2392         const CUpdateItem &ui = updateItems[indices[i + subIndex]];
2393         CFileItem file;
2394         CFileItem2 file2;
2395         UString name;
2396         if (ui.NewProps)
2397         {
2398           UpdateItem_To_FileItem(ui, file, file2);
2399           name = ui.Name;
2400         }
2401         else
2402         {
2403           GetFile(*db, ui.IndexInArchive, file, file2);
2404           db->GetPath(ui.IndexInArchive, name);
2405         }
2406         if (file2.IsAnti || file.IsDir)
2407           return E_FAIL;
2408 
2409         /*
2410         CFileItem &file = newDatabase.Files[
2411               startFileIndexInDatabase + i + subIndex];
2412         */
2413         if (!inStreamSpec->Processed[subIndex])
2414         {
2415           skippedSize += ui.Size;
2416           continue;
2417           // file.Name += ".locked";
2418         }
2419 
2420         file.Crc = inStreamSpec->CRCs[subIndex];
2421         file.Size = inStreamSpec->Sizes[subIndex];
2422 
2423         // if (file.Size >= 0) // test purposes
2424         if (file.Size != 0)
2425         {
2426           file.CrcDefined = true;
2427           file.HasStream = true;
2428           numUnpackStreams++;
2429         }
2430         else
2431         {
2432           file.CrcDefined = false;
2433           file.HasStream = false;
2434         }
2435 
2436         /*
2437         file.Parent = ui.ParentFolderIndex;
2438         if (ui.TreeFolderIndex >= 0)
2439           treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2440         if (totalSecureDataSize != 0)
2441           newDatabase.SecureIDs.Add(ui.SecureIndex);
2442         */
2443         newDatabase.AddFile(file, file2, name);
2444       }
2445 
2446       // numUnpackStreams = 0 is very bad case for locked files
2447       // v3.13 doesn't understand it.
2448       newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
2449       i += numSubFiles;
2450 
2451       if (skippedSize != 0 && complexity >= skippedSize)
2452       {
2453         complexity -= skippedSize;
2454         RINOK(updateCallback->SetTotal(complexity));
2455       }
2456     }
2457   }
2458 
2459   RINOK(lps->SetCur());
2460 
2461   /*
2462   fileIndexToUpdateIndexMap.ClearAndFree();
2463   groups.ClearAndFree();
2464   */
2465 
2466   /*
2467   for (i = 0; i < newDatabase.Files.Size(); i++)
2468   {
2469     CFileItem &file = newDatabase.Files[i];
2470     file.Parent = treeFolderToArcIndex[file.Parent];
2471   }
2472 
2473   if (totalSecureDataSize != 0)
2474   {
2475     newDatabase.SecureBuf.SetCapacity(totalSecureDataSize);
2476     size_t pos = 0;
2477     newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size());
2478     for (i = 0; i < secureBlocks.Sorted.Size(); i++)
2479     {
2480       const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]];
2481       size_t size = buf.GetCapacity();
2482       if (size != 0)
2483         memcpy(newDatabase.SecureBuf + pos, buf, size);
2484       newDatabase.SecureSizes.Add((UInt32)size);
2485       pos += size;
2486     }
2487   }
2488   */
2489   newDatabase.ReserveDown();
2490 
2491   if (opCallback)
2492     RINOK(opCallback->ReportOperation(NEventIndexType::kNoIndex, (UInt32)(Int32)-1, NUpdateNotifyOp::kHeader));
2493 
2494   return S_OK;
2495 }
2496 
2497 }}
2498