1 // 7zUpdate.cpp
2 
3 #include "StdAfx.h"
4 
5 #include "../../../../C/CpuArch.h"
6 
7 #include "../../../Common/Wildcard.h"
8 
9 #include "../../Common/CreateCoder.h"
10 #include "../../Common/LimitedStreams.h"
11 #include "../../Common/ProgressUtils.h"
12 
13 #include "../../Compress/CopyCoder.h"
14 
15 #include "../Common/ItemNameUtils.h"
16 
17 #include "7zDecode.h"
18 #include "7zEncode.h"
19 #include "7zFolderInStream.h"
20 #include "7zHandler.h"
21 #include "7zOut.h"
22 #include "7zUpdate.h"
23 
24 namespace NArchive {
25 namespace N7z {
26 
27 
28 #define k_X86 k_BCJ
29 
30 struct CFilterMode
31 {
32   UInt32 Id;
33   UInt32 Delta;
34 
CFilterModeNArchive::N7z::CFilterMode35   CFilterMode(): Id(0), Delta(0) {}
36 
SetDeltaNArchive::N7z::CFilterMode37   void SetDelta()
38   {
39     if (Id == k_IA64)
40       Delta = 16;
41     else if (Id == k_ARM || Id == k_PPC || Id == k_SPARC)
42       Delta = 4;
43     else if (Id == k_ARMT)
44       Delta = 2;
45     else
46       Delta = 0;
47   }
48 };
49 
50 
51 /* ---------- PE ---------- */
52 
53 #define MZ_SIG 0x5A4D
54 
55 #define PE_SIG 0x00004550
56 #define PE_OptHeader_Magic_32 0x10B
57 #define PE_OptHeader_Magic_64 0x20B
58 #define PE_SectHeaderSize 40
59 #define PE_SECT_EXECUTE 0x20000000
60 
Parse_EXE(const Byte * buf,size_t size,CFilterMode * filterMode)61 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode)
62 {
63   if (size < 512 || GetUi16(buf) != MZ_SIG)
64     return 0;
65 
66   const Byte *p;
67   UInt32 peOffset, optHeaderSize, filterId;
68 
69   peOffset = GetUi32(buf + 0x3C);
70   if (peOffset >= 0x1000 || peOffset + 512 > size || (peOffset & 7) != 0)
71     return 0;
72   p = buf + peOffset;
73   if (GetUi32(p) != PE_SIG)
74     return 0;
75   p += 4;
76 
77   switch (GetUi16(p))
78   {
79     case 0x014C:
80     case 0x8664:  filterId = k_X86; break;
81 
82     /*
83     IMAGE_FILE_MACHINE_ARM   0x01C0  // ARM LE
84     IMAGE_FILE_MACHINE_THUMB 0x01C2  // ARM Thumb / Thumb-2 LE
85     IMAGE_FILE_MACHINE_ARMNT 0x01C4  // ARM Thumb-2, LE
86     Note: We use ARM filter for 0x01C2. (WinCE 5 - 0x01C2) files mostly contain ARM code (not Thumb/Thumb-2).
87     */
88 
89     case 0x01C0:                            // WinCE old
90     case 0x01C2:  filterId = k_ARM; break;  // WinCE new
91     case 0x01C4:  filterId = k_ARMT; break; // WinRT
92 
93     case 0x0200:  filterId = k_IA64; break;
94     default:  return 0;
95   }
96 
97   optHeaderSize = GetUi16(p + 16);
98   if (optHeaderSize > (1 << 10))
99     return 0;
100 
101   p += 20; /* headerSize */
102 
103   switch (GetUi16(p))
104   {
105     case PE_OptHeader_Magic_32:
106     case PE_OptHeader_Magic_64:
107       break;
108     default:
109       return 0;
110   }
111 
112   filterMode->Id = filterId;
113   return 1;
114 }
115 
116 
117 /* ---------- ELF ---------- */
118 
119 #define ELF_SIG 0x464C457F
120 
121 #define ELF_CLASS_32  1
122 #define ELF_CLASS_64  2
123 
124 #define ELF_DATA_2LSB 1
125 #define ELF_DATA_2MSB 2
126 
Get16(const Byte * p,Bool be)127 static UInt16 Get16(const Byte *p, Bool be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); }
Get32(const Byte * p,Bool be)128 static UInt32 Get32(const Byte *p, Bool be) { if (be) return GetBe32(p); return GetUi32(p); }
129 // static UInt64 Get64(const Byte *p, Bool be) { if (be) return GetBe64(p); return GetUi64(p); }
130 
Parse_ELF(const Byte * buf,size_t size,CFilterMode * filterMode)131 static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode)
132 {
133   Bool /* is32, */ be;
134   UInt32 filterId;
135 
136   if (size < 512 || buf[6] != 1) /* ver */
137     return 0;
138 
139   if (GetUi32(buf) != ELF_SIG)
140     return 0;
141 
142   switch (buf[4])
143   {
144     case ELF_CLASS_32: /* is32 = True; */ break;
145     case ELF_CLASS_64: /* is32 = False; */ break;
146     default: return 0;
147   }
148 
149   switch (buf[5])
150   {
151     case ELF_DATA_2LSB: be = False; break;
152     case ELF_DATA_2MSB: be = True; break;
153     default: return 0;
154   }
155 
156   switch (Get16(buf + 0x12, be))
157   {
158     case 3:
159     case 6:
160     case 62: filterId = k_X86; break;
161     case 2:
162     case 18:
163     case 43: filterId = k_SPARC; break;
164     case 20:
165     case 21: if (!be) return 0; filterId = k_PPC; break;
166     case 40: if ( be) return 0; filterId = k_ARM; break;
167 
168     /* Some IA-64 ELF exacutable have size that is not aligned for 16 bytes.
169        So we don't use IA-64 filter for IA-64 ELF */
170     // case 50: if ( be) return 0; filterId = k_IA64; break;
171 
172     default: return 0;
173   }
174 
175   filterMode->Id = filterId;
176   return 1;
177 }
178 
179 
180 
181 /* ---------- Mach-O ---------- */
182 
183 #define MACH_SIG_BE_32 0xCEFAEDFE
184 #define MACH_SIG_BE_64 0xCFFAEDFE
185 #define MACH_SIG_LE_32 0xFEEDFACE
186 #define MACH_SIG_LE_64 0xFEEDFACF
187 
188 #define MACH_ARCH_ABI64 (1 << 24)
189 #define MACH_MACHINE_386 7
190 #define MACH_MACHINE_ARM 12
191 #define MACH_MACHINE_SPARC 14
192 #define MACH_MACHINE_PPC 18
193 #define MACH_MACHINE_PPC64 (MACH_ARCH_ABI64 | MACH_MACHINE_PPC)
194 #define MACH_MACHINE_AMD64 (MACH_ARCH_ABI64 | MACH_MACHINE_386)
195 
Parse_MACH(const Byte * buf,size_t size,CFilterMode * filterMode)196 static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode)
197 {
198   UInt32 filterId, numCommands, commandsSize;
199 
200   if (size < 512)
201     return 0;
202 
203   Bool /* mode64, */ be;
204   switch (GetUi32(buf))
205   {
206     case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break;
207     case MACH_SIG_BE_64: /* mode64 = True;  */ be = True; break;
208     case MACH_SIG_LE_32: /* mode64 = False; */ be = False; break;
209     case MACH_SIG_LE_64: /* mode64 = True;  */ be = False; break;
210     default: return 0;
211   }
212 
213   switch (Get32(buf + 4, be))
214   {
215     case MACH_MACHINE_386:
216     case MACH_MACHINE_AMD64: filterId = k_X86; break;
217     case MACH_MACHINE_ARM:   if ( be) return 0; filterId = k_ARM; break;
218     case MACH_MACHINE_SPARC: if (!be) return 0; filterId = k_SPARC; break;
219     case MACH_MACHINE_PPC:
220     case MACH_MACHINE_PPC64: if (!be) return 0; filterId = k_PPC; break;
221     default: return 0;
222   }
223 
224   numCommands = Get32(buf + 0x10, be);
225   commandsSize = Get32(buf + 0x14, be);
226 
227   if (commandsSize > (1 << 24) || numCommands > (1 << 18))
228     return 0;
229 
230   filterMode->Id = filterId;
231   return 1;
232 }
233 
234 
235 /* ---------- WAV ---------- */
236 
237 #define WAV_SUBCHUNK_fmt  0x20746D66
238 #define WAV_SUBCHUNK_data 0x61746164
239 
240 #define RIFF_SIG 0x46464952
241 
Parse_WAV(const Byte * buf,size_t size,CFilterMode * filterMode)242 static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode)
243 {
244   UInt32 subChunkSize, pos;
245   if (size < 0x2C)
246     return False;
247 
248   if (GetUi32(buf + 0) != RIFF_SIG ||
249       GetUi32(buf + 8) != 0x45564157 || // WAVE
250       GetUi32(buf + 0xC) != WAV_SUBCHUNK_fmt)
251     return False;
252   subChunkSize = GetUi32(buf + 0x10);
253   /* [0x14 = format] = 1 (PCM) */
254   if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1)
255     return False;
256 
257   unsigned numChannels = GetUi16(buf + 0x16);
258   unsigned bitsPerSample = GetUi16(buf + 0x22);
259 
260   if ((bitsPerSample & 0x7) != 0 || bitsPerSample >= 256 || numChannels >= 256)
261     return False;
262 
263   pos = 0x14 + subChunkSize;
264 
265   const int kNumSubChunksTests = 10;
266   // Do we need to scan more than 3 sub-chunks?
267   for (int i = 0; i < kNumSubChunksTests; i++)
268   {
269     if (pos + 8 > size)
270       return False;
271     subChunkSize = GetUi32(buf + pos + 4);
272     if (GetUi32(buf + pos) == WAV_SUBCHUNK_data)
273     {
274       unsigned delta = numChannels * (bitsPerSample >> 3);
275       if (delta >= 256)
276         return False;
277       filterMode->Id = k_Delta;
278       filterMode->Delta = delta;
279       return True;
280     }
281     if (subChunkSize > (1 << 16))
282       return False;
283     pos += subChunkSize + 8;
284   }
285   return False;
286 }
287 
ParseFile(const Byte * buf,size_t size,CFilterMode * filterMode)288 static Bool ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode)
289 {
290   filterMode->Id = 0;
291   filterMode->Delta = 0;
292 
293   if (Parse_EXE(buf, size, filterMode)) return True;
294   if (Parse_ELF(buf, size, filterMode)) return True;
295   if (Parse_MACH(buf, size, filterMode)) return True;
296   return Parse_WAV(buf, size, filterMode);
297 }
298 
299 
300 
301 
302 struct CFilterMode2: public CFilterMode
303 {
304   bool Encrypted;
305   unsigned GroupIndex;
306 
CFilterMode2NArchive::N7z::CFilterMode2307   CFilterMode2(): Encrypted(false) {}
308 
CompareNArchive::N7z::CFilterMode2309   int Compare(const CFilterMode2 &m) const
310   {
311     if (!Encrypted)
312     {
313       if (m.Encrypted)
314         return -1;
315     }
316     else if (!m.Encrypted)
317       return 1;
318 
319     if (Id < m.Id) return -1;
320     if (Id > m.Id) return 1;
321 
322     if (Delta < m.Delta) return -1;
323     if (Delta > m.Delta) return 1;
324 
325     return 0;
326   }
327 
operator ==NArchive::N7z::CFilterMode2328   bool operator ==(const CFilterMode2 &m) const
329   {
330     return Id == m.Id && Delta == m.Delta && Encrypted == m.Encrypted;
331   }
332 };
333 
GetGroup(CRecordVector<CFilterMode2> & filters,const CFilterMode2 & m)334 static unsigned GetGroup(CRecordVector<CFilterMode2> &filters, const CFilterMode2 &m)
335 {
336   unsigned i;
337   for (i = 0; i < filters.Size(); i++)
338   {
339     const CFilterMode2 &m2 = filters[i];
340     if (m == m2)
341       return i;
342     /*
343     if (m.Encrypted != m2.Encrypted)
344     {
345       if (!m.Encrypted)
346         break;
347       continue;
348     }
349 
350     if (m.Id < m2.Id)  break;
351     if (m.Id != m2.Id) continue;
352 
353     if (m.Delta < m2.Delta) break;
354     if (m.Delta != m2.Delta) continue;
355     */
356   }
357   // filters.Insert(i, m);
358   // return i;
359   return filters.Add(m);
360 }
361 
Is86Filter(CMethodId m)362 static inline bool Is86Filter(CMethodId m)
363 {
364   return (m == k_BCJ || m == k_BCJ2);
365 }
366 
IsExeFilter(CMethodId m)367 static inline bool IsExeFilter(CMethodId m)
368 {
369   switch (m)
370   {
371     case k_BCJ:
372     case k_BCJ2:
373     case k_ARM:
374     case k_ARMT:
375     case k_PPC:
376     case k_SPARC:
377     case k_IA64:
378       return true;
379   }
380   return false;
381 }
382 
Get_FilterGroup_for_Folder(CRecordVector<CFilterMode2> & filters,const CFolderEx & f,bool extractFilter)383 static unsigned Get_FilterGroup_for_Folder(
384     CRecordVector<CFilterMode2> &filters, const CFolderEx &f, bool extractFilter)
385 {
386   CFilterMode2 m;
387   m.Id = 0;
388   m.Delta = 0;
389   m.Encrypted = f.IsEncrypted();
390 
391   if (extractFilter)
392   {
393     const CCoderInfo &coder = f.Coders[f.UnpackCoder];
394 
395     if (coder.MethodID == k_Delta)
396     {
397       if (coder.Props.Size() == 1)
398       {
399         m.Delta = (unsigned)coder.Props[0] + 1;
400         m.Id = k_Delta;
401       }
402     }
403     else if (IsExeFilter(coder.MethodID))
404     {
405       m.Id = (UInt32)coder.MethodID;
406       if (m.Id == k_BCJ2)
407         m.Id = k_BCJ;
408       m.SetDelta();
409     }
410   }
411 
412   return GetGroup(filters, m);
413 }
414 
415 
416 
417 
WriteRange(IInStream * inStream,ISequentialOutStream * outStream,UInt64 position,UInt64 size,ICompressProgressInfo * progress)418 static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
419     UInt64 position, UInt64 size, ICompressProgressInfo *progress)
420 {
421   RINOK(inStream->Seek(position, STREAM_SEEK_SET, 0));
422   CLimitedSequentialInStream *streamSpec = new CLimitedSequentialInStream;
423   CMyComPtr<CLimitedSequentialInStream> inStreamLimited(streamSpec);
424   streamSpec->SetStream(inStream);
425   streamSpec->Init(size);
426 
427   NCompress::CCopyCoder *copyCoderSpec = new NCompress::CCopyCoder;
428   CMyComPtr<ICompressCoder> copyCoder = copyCoderSpec;
429   RINOK(copyCoder->Code(inStreamLimited, outStream, NULL, NULL, progress));
430   return (copyCoderSpec->TotalSize == size ? S_OK : E_FAIL);
431 }
432 
433 /*
434 unsigned CUpdateItem::GetExtensionPos() const
435 {
436   int slashPos = Name.ReverseFind_PathSepar();
437   int dotPos = Name.ReverseFind_Dot();
438   if (dotPos <= slashPos)
439     return Name.Len();
440   return dotPos + 1;
441 }
442 
443 UString CUpdateItem::GetExtension() const
444 {
445   return Name.Ptr(GetExtensionPos());
446 }
447 */
448 
449 #define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
450 
451 #define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
452 
453 /*
454 static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
455 {
456   size_t c1 = a1.GetCapacity();
457   size_t c2 = a2.GetCapacity();
458   RINOZ_COMP(c1, c2);
459   for (size_t i = 0; i < c1; i++)
460     RINOZ_COMP(a1[i], a2[i]);
461   return 0;
462 }
463 
464 static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
465 {
466   RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
467   RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
468   RINOZ_COMP(c1.MethodID, c2.MethodID);
469   return CompareBuffers(c1.Props, c2.Props);
470 }
471 
472 static int CompareBonds(const CBond &b1, const CBond &b2)
473 {
474   RINOZ_COMP(b1.InIndex, b2.InIndex);
475   return MyCompare(b1.OutIndex, b2.OutIndex);
476 }
477 
478 static int CompareFolders(const CFolder &f1, const CFolder &f2)
479 {
480   int s1 = f1.Coders.Size();
481   int s2 = f2.Coders.Size();
482   RINOZ_COMP(s1, s2);
483   int i;
484   for (i = 0; i < s1; i++)
485     RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
486   s1 = f1.Bonds.Size();
487   s2 = f2.Bonds.Size();
488   RINOZ_COMP(s1, s2);
489   for (i = 0; i < s1; i++)
490     RINOZ(CompareBonds(f1.Bonds[i], f2.Bonds[i]));
491   return 0;
492 }
493 */
494 
495 /*
496 static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
497 {
498   return CompareFileNames(f1.Name, f2.Name);
499 }
500 */
501 
502 struct CFolderRepack
503 {
504   unsigned FolderIndex;
505   CNum NumCopyFiles;
506 };
507 
508 /*
509 static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *)
510 {
511   int i1 = p1->FolderIndex;
512   int i2 = p2->FolderIndex;
513   // In that version we don't want to parse folders here, so we don't compare folders
514   // probably it must be improved in future
515   // const CDbEx &db = *(const CDbEx *)param;
516   // RINOZ(CompareFolders(
517   //     db.Folders[i1],
518   //     db.Folders[i2]));
519 
520   return MyCompare(i1, i2);
521 
522   // RINOZ_COMP(
523   //     db.NumUnpackStreamsVector[i1],
524   //     db.NumUnpackStreamsVector[i2]);
525   // if (db.NumUnpackStreamsVector[i1] == 0)
526   //   return 0;
527   // return CompareFiles(
528   //     db.Files[db.FolderStartFileIndex[i1]],
529   //     db.Files[db.FolderStartFileIndex[i2]]);
530 }
531 */
532 
533 /*
534   we sort empty files and dirs in such order:
535   - Dir.NonAnti   (name sorted)
536   - File.NonAnti  (name sorted)
537   - File.Anti     (name sorted)
538   - Dir.Anti (reverse name sorted)
539 */
540 
CompareEmptyItems(const unsigned * p1,const unsigned * p2,void * param)541 static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param)
542 {
543   const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
544   const CUpdateItem &u1 = updateItems[*p1];
545   const CUpdateItem &u2 = updateItems[*p2];
546   // NonAnti < Anti
547   if (u1.IsAnti != u2.IsAnti)
548     return (u1.IsAnti ? 1 : -1);
549   if (u1.IsDir != u2.IsDir)
550   {
551     // Dir.NonAnti < File < Dir.Anti
552     if (u1.IsDir)
553       return (u1.IsAnti ? 1 : -1);
554     return (u2.IsAnti ? -1 : 1);
555   }
556   int n = CompareFileNames(u1.Name, u2.Name);
557   return (u1.IsDir && u1.IsAnti) ? -n : n;
558 }
559 
560 static const char *g_Exts =
561   " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
562   " zip jar ear war msi"
563   " 3gp avi mov mpeg mpg mpe wmv"
564   " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
565   " swf"
566   " chm hxi hxs"
567   " gif jpeg jpg jp2 png tiff  bmp ico psd psp"
568   " awg ps eps cgm dxf svg vrml wmf emf ai md"
569   " cad dwg pps key sxi"
570   " max 3ds"
571   " iso bin nrg mdf img pdi tar cpio xpi"
572   " vfd vhd vud vmc vsv"
573   " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
574   " inl inc idl acf asa"
575   " h hpp hxx c cpp cxx m mm go swift"
576   " rc java cs rs pas bas vb cls ctl frm dlg def"
577   " f77 f f90 f95"
578   " asm s"
579   " sql manifest dep"
580   " mak clw csproj vcproj sln dsp dsw"
581   " class"
582   " bat cmd bash sh"
583   " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
584   " awk sed hta js json php php3 php4 php5 phptml pl pm py pyo rb tcl ts vbs"
585   " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
586   " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
587   " abw afp cwk lwp wpd wps wpt wrf wri"
588   " abf afm bdf fon mgf otf pcf pfa snf ttf"
589   " dbf mdb nsf ntf wdb db fdb gdb"
590   " exe dll ocx vbx sfx sys tlb awx com obj lib out o so"
591   " pdb pch idb ncb opt";
592 
GetExtIndex(const char * ext)593 static unsigned GetExtIndex(const char *ext)
594 {
595   unsigned extIndex = 1;
596   const char *p = g_Exts;
597   for (;;)
598   {
599     char c = *p++;
600     if (c == 0)
601       return extIndex;
602     if (c == ' ')
603       continue;
604     unsigned pos = 0;
605     for (;;)
606     {
607       char c2 = ext[pos++];
608       if (c2 == 0 && (c == 0 || c == ' '))
609         return extIndex;
610       if (c != c2)
611         break;
612       c = *p++;
613     }
614     extIndex++;
615     for (;;)
616     {
617       if (c == 0)
618         return extIndex;
619       if (c == ' ')
620         break;
621       c = *p++;
622     }
623   }
624 }
625 
626 struct CRefItem
627 {
628   const CUpdateItem *UpdateItem;
629   UInt32 Index;
630   unsigned ExtensionPos;
631   unsigned NamePos;
632   unsigned ExtensionIndex;
633 
CRefItemNArchive::N7z::CRefItem634   CRefItem() {};
CRefItemNArchive::N7z::CRefItem635   CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
636     UpdateItem(&ui),
637     Index(index),
638     ExtensionPos(0),
639     NamePos(0),
640     ExtensionIndex(0)
641   {
642     if (sortByType)
643     {
644       int slashPos = ui.Name.ReverseFind_PathSepar();
645       NamePos = slashPos + 1;
646       int dotPos = ui.Name.ReverseFind_Dot();
647       if (dotPos <= slashPos)
648         ExtensionPos = ui.Name.Len();
649       else
650       {
651         ExtensionPos = dotPos + 1;
652         if (ExtensionPos != ui.Name.Len())
653         {
654           AString s;
655           for (unsigned pos = ExtensionPos;; pos++)
656           {
657             wchar_t c = ui.Name[pos];
658             if (c >= 0x80)
659               break;
660             if (c == 0)
661             {
662               ExtensionIndex = GetExtIndex(s);
663               break;
664             }
665             s += (char)MyCharLower_Ascii((char)c);
666           }
667         }
668       }
669     }
670   }
671 };
672 
673 struct CSortParam
674 {
675   // const CObjectVector<CTreeFolder> *TreeFolders;
676   bool SortByType;
677 };
678 
679 /*
680   we sort files in such order:
681   - Dir.NonAnti   (name sorted)
682   - alt streams
683   - Dirs
684   - Dir.Anti (reverse name sorted)
685 */
686 
687 
CompareUpdateItems(const CRefItem * p1,const CRefItem * p2,void * param)688 static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
689 {
690   const CRefItem &a1 = *p1;
691   const CRefItem &a2 = *p2;
692   const CUpdateItem &u1 = *a1.UpdateItem;
693   const CUpdateItem &u2 = *a2.UpdateItem;
694 
695   /*
696   if (u1.IsAltStream != u2.IsAltStream)
697     return u1.IsAltStream ? 1 : -1;
698   */
699 
700   // Actually there are no dirs that time. They were stored in other steps
701   // So that code is unused?
702   if (u1.IsDir != u2.IsDir)
703     return u1.IsDir ? 1 : -1;
704   if (u1.IsDir)
705   {
706     if (u1.IsAnti != u2.IsAnti)
707       return (u1.IsAnti ? 1 : -1);
708     int n = CompareFileNames(u1.Name, u2.Name);
709     return -n;
710   }
711 
712   // bool sortByType = *(bool *)param;
713   const CSortParam *sortParam = (const CSortParam *)param;
714   bool sortByType = sortParam->SortByType;
715   if (sortByType)
716   {
717     RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex);
718     RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos)));
719     RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos)));
720     if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
721     if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
722     if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime);
723     RINOZ_COMP(u1.Size, u2.Size);
724   }
725   /*
726   int par1 = a1.UpdateItem->ParentFolderIndex;
727   int par2 = a2.UpdateItem->ParentFolderIndex;
728   const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1];
729   const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2];
730 
731   int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd;
732   int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd;
733   if (b1 < b2)
734   {
735     if (e1 <= b2)
736       return -1;
737     // p2 in p1
738     int par = par2;
739     for (;;)
740     {
741       const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
742       par = tf.Parent;
743       if (par == par1)
744       {
745         RINOZ(CompareFileNames(u1.Name, tf.Name));
746         break;
747       }
748     }
749   }
750   else if (b2 < b1)
751   {
752     if (e2 <= b1)
753       return 1;
754     // p1 in p2
755     int par = par1;
756     for (;;)
757     {
758       const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
759       par = tf.Parent;
760       if (par == par2)
761       {
762         RINOZ(CompareFileNames(tf.Name, u2.Name));
763         break;
764       }
765     }
766   }
767   */
768   // RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex);
769   RINOK(CompareFileNames(u1.Name, u2.Name));
770   RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient);
771   RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive);
772   return 0;
773 }
774 
775 struct CSolidGroup
776 {
777   CRecordVector<UInt32> Indices;
778 
779   CRecordVector<CFolderRepack> folderRefs;
780 };
781 
782 static const char * const g_ExeExts[] =
783 {
784     "dll"
785   , "exe"
786   , "ocx"
787   , "sfx"
788   , "sys"
789 };
790 
IsExeExt(const wchar_t * ext)791 static bool IsExeExt(const wchar_t *ext)
792 {
793   for (unsigned i = 0; i < ARRAY_SIZE(g_ExeExts); i++)
794     if (StringsAreEqualNoCase_Ascii(ext, g_ExeExts[i]))
795       return true;
796   return false;
797 }
798 
799 struct CAnalysis
800 {
801   CMyComPtr<IArchiveUpdateCallbackFile> Callback;
802   CByteBuffer Buffer;
803 
804   bool ParseWav;
805   bool ParseExe;
806   bool ParseAll;
807 
CAnalysisNArchive::N7z::CAnalysis808   CAnalysis():
809       ParseWav(true),
810       ParseExe(false),
811       ParseAll(false)
812   {}
813 
814   HRESULT GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode);
815 };
816 
817 static const size_t kAnalysisBufSize = 1 << 14;
818 
GetFilterGroup(UInt32 index,const CUpdateItem & ui,CFilterMode & filterMode)819 HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode)
820 {
821   filterMode.Id = 0;
822   filterMode.Delta = 0;
823 
824   CFilterMode filterModeTemp = filterMode;
825 
826   int slashPos = ui.Name.ReverseFind_PathSepar();
827   int dotPos = ui.Name.ReverseFind_Dot();
828 
829   // if (dotPos > slashPos)
830   {
831     bool needReadFile = ParseAll;
832 
833     bool probablyIsSameIsa = false;
834 
835     if (!needReadFile || !Callback)
836     {
837       const wchar_t *ext;
838       if (dotPos > slashPos)
839         ext = ui.Name.Ptr(dotPos + 1);
840       else
841         ext = ui.Name.RightPtr(0);
842 
843       // p7zip uses the trick to store posix attributes in high 16 bits
844       if (ui.Attrib & 0x8000)
845       {
846         unsigned st_mode = ui.Attrib >> 16;
847         // st_mode = 00111;
848         if ((st_mode & 00111) && (ui.Size >= 2048))
849         {
850           #ifndef _WIN32
851           probablyIsSameIsa = true;
852           #endif
853           needReadFile = true;
854         }
855       }
856 
857       if (IsExeExt(ext))
858       {
859         needReadFile = true;
860         #ifdef _WIN32
861         probablyIsSameIsa = true;
862         needReadFile = ParseExe;
863         #endif
864       }
865       else if (StringsAreEqualNoCase_Ascii(ext, "wav"))
866       {
867         needReadFile = ParseWav;
868       }
869       /*
870       else if (!needReadFile && ParseUnixExt)
871       {
872         if (StringsAreEqualNoCase_Ascii(ext, "so")
873           || StringsAreEqualNoCase_Ascii(ext, ""))
874 
875           needReadFile = true;
876       }
877       */
878     }
879 
880     if (needReadFile && Callback)
881     {
882       if (Buffer.Size() != kAnalysisBufSize)
883       {
884         Buffer.Alloc(kAnalysisBufSize);
885       }
886       {
887         CMyComPtr<ISequentialInStream> stream;
888         HRESULT result = Callback->GetStream2(index, &stream, NUpdateNotifyOp::kAnalyze);
889         if (result == S_OK && stream)
890         {
891           size_t size = kAnalysisBufSize;
892           result = ReadStream(stream, Buffer, &size);
893           stream.Release();
894           // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK));
895           if (result == S_OK)
896           {
897             Bool parseRes = ParseFile(Buffer, size, &filterModeTemp);
898             if (parseRes && filterModeTemp.Delta == 0)
899             {
900               filterModeTemp.SetDelta();
901               if (filterModeTemp.Delta != 0 && filterModeTemp.Id != k_Delta)
902               {
903                 if (ui.Size % filterModeTemp.Delta != 0)
904                 {
905                   parseRes = false;
906                 }
907               }
908             }
909             if (!parseRes)
910             {
911               filterModeTemp.Id = 0;
912               filterModeTemp.Delta = 0;
913             }
914           }
915         }
916       }
917     }
918     else if ((needReadFile && !Callback) || probablyIsSameIsa)
919     {
920       #ifdef MY_CPU_X86_OR_AMD64
921       if (probablyIsSameIsa)
922         filterModeTemp.Id = k_X86;
923       #endif
924     }
925   }
926 
927   filterMode = filterModeTemp;
928   return S_OK;
929 }
930 
GetMethodFull(UInt64 methodID,UInt32 numStreams,CMethodFull & m)931 static inline void GetMethodFull(UInt64 methodID, UInt32 numStreams, CMethodFull &m)
932 {
933   m.Id = methodID;
934   m.NumStreams = numStreams;
935 }
936 
AddBondForFilter(CCompressionMethodMode & mode)937 static HRESULT AddBondForFilter(CCompressionMethodMode &mode)
938 {
939   for (unsigned c = 1; c < mode.Methods.Size(); c++)
940   {
941     if (!mode.IsThereBond_to_Coder(c))
942     {
943       CBond2 bond;
944       bond.OutCoder = 0;
945       bond.OutStream = 0;
946       bond.InCoder = c;
947       mode.Bonds.Add(bond);
948       return S_OK;
949     }
950   }
951   return E_INVALIDARG;
952 }
953 
AddFilterBond(CCompressionMethodMode & mode)954 static HRESULT AddFilterBond(CCompressionMethodMode &mode)
955 {
956   if (!mode.Bonds.IsEmpty())
957     return AddBondForFilter(mode);
958   return S_OK;
959 }
960 
AddBcj2Methods(CCompressionMethodMode & mode)961 static HRESULT AddBcj2Methods(CCompressionMethodMode &mode)
962 {
963   // mode.Methods[0] must be k_BCJ2 method !
964 
965   CMethodFull m;
966   GetMethodFull(k_LZMA, 1, m);
967 
968   m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20);
969   m.AddProp32(NCoderPropID::kNumFastBytes, 128);
970   m.AddProp32(NCoderPropID::kNumThreads, 1);
971   m.AddProp32(NCoderPropID::kLitPosBits, 2);
972   m.AddProp32(NCoderPropID::kLitContextBits, 0);
973   // m.AddProp_Ascii(NCoderPropID::kMatchFinder, "BT2");
974 
975   unsigned methodIndex = mode.Methods.Size();
976 
977   if (mode.Bonds.IsEmpty())
978   {
979     for (unsigned i = 1; i + 1 < mode.Methods.Size(); i++)
980     {
981       CBond2 bond;
982       bond.OutCoder = i;
983       bond.OutStream = 0;
984       bond.InCoder = i + 1;
985       mode.Bonds.Add(bond);
986     }
987   }
988 
989   mode.Methods.Add(m);
990   mode.Methods.Add(m);
991 
992   RINOK(AddBondForFilter(mode));
993   CBond2 bond;
994   bond.OutCoder = 0;
995   bond.InCoder = methodIndex;      bond.OutStream = 1;  mode.Bonds.Add(bond);
996   bond.InCoder = methodIndex + 1;  bond.OutStream = 2;  mode.Bonds.Add(bond);
997   return S_OK;
998 }
999 
MakeExeMethod(CCompressionMethodMode & mode,const CFilterMode & filterMode,bool bcj2Filter)1000 static HRESULT MakeExeMethod(CCompressionMethodMode &mode,
1001     const CFilterMode &filterMode, /* bool addFilter, */ bool bcj2Filter)
1002 {
1003   if (mode.Filter_was_Inserted)
1004   {
1005     const CMethodFull &m = mode.Methods[0];
1006     CMethodId id = m.Id;
1007     if (id == k_BCJ2)
1008       return AddBcj2Methods(mode);
1009     if (!m.IsSimpleCoder())
1010       return E_NOTIMPL;
1011     // if (Bonds.IsEmpty()) we can create bonds later
1012     return AddFilterBond(mode);
1013   }
1014 
1015   if (filterMode.Id == 0)
1016     return S_OK;
1017 
1018   CMethodFull &m = mode.Methods.InsertNew(0);
1019 
1020   {
1021     FOR_VECTOR(k, mode.Bonds)
1022     {
1023       CBond2 &bond = mode.Bonds[k];
1024       bond.InCoder++;
1025       bond.OutCoder++;
1026     }
1027   }
1028 
1029   HRESULT res;
1030 
1031   if (bcj2Filter && Is86Filter(filterMode.Id))
1032   {
1033     GetMethodFull(k_BCJ2, 4, m);
1034     res = AddBcj2Methods(mode);
1035   }
1036   else
1037   {
1038     GetMethodFull(filterMode.Id, 1, m);
1039     if (filterMode.Id == k_Delta)
1040       m.AddProp32(NCoderPropID::kDefaultProp, filterMode.Delta);
1041     res = AddFilterBond(mode);
1042 
1043     int alignBits = -1;
1044     if (filterMode.Id == k_Delta || filterMode.Delta != 0)
1045     {
1046            if (filterMode.Delta == 1) alignBits = 0;
1047       else if (filterMode.Delta == 2) alignBits = 1;
1048       else if (filterMode.Delta == 4) alignBits = 2;
1049       else if (filterMode.Delta == 8) alignBits = 3;
1050       else if (filterMode.Delta == 16) alignBits = 4;
1051     }
1052     else
1053     {
1054       // alignBits = GetAlignForFilterMethod(filterMode.Id);
1055     }
1056 
1057     if (res == S_OK && alignBits >= 0)
1058     {
1059       unsigned nextCoder = 1;
1060       if (!mode.Bonds.IsEmpty())
1061       {
1062         nextCoder = mode.Bonds.Back().InCoder;
1063       }
1064       if (nextCoder < mode.Methods.Size())
1065       {
1066         CMethodFull &nextMethod = mode.Methods[nextCoder];
1067         if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2)
1068         {
1069           if (!nextMethod.Are_Lzma_Model_Props_Defined())
1070           {
1071             if (alignBits != 0)
1072             {
1073               if (alignBits > 2 || filterMode.Id == k_Delta)
1074                 nextMethod.AddProp32(NCoderPropID::kPosStateBits, alignBits);
1075               unsigned lc = 0;
1076               if (alignBits < 3)
1077                 lc = 3 - alignBits;
1078               nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc);
1079               nextMethod.AddProp32(NCoderPropID::kLitPosBits, alignBits);
1080             }
1081           }
1082         }
1083       }
1084     }
1085   }
1086 
1087   return res;
1088 }
1089 
1090 
UpdateItem_To_FileItem2(const CUpdateItem & ui,CFileItem2 & file2)1091 static void UpdateItem_To_FileItem2(const CUpdateItem &ui, CFileItem2 &file2)
1092 {
1093   file2.Attrib = ui.Attrib;  file2.AttribDefined = ui.AttribDefined;
1094   file2.CTime = ui.CTime;  file2.CTimeDefined = ui.CTimeDefined;
1095   file2.ATime = ui.ATime;  file2.ATimeDefined = ui.ATimeDefined;
1096   file2.MTime = ui.MTime;  file2.MTimeDefined = ui.MTimeDefined;
1097   file2.IsAnti = ui.IsAnti;
1098   // file2.IsAux = false;
1099   file2.StartPosDefined = false;
1100   // file2.StartPos = 0;
1101 }
1102 
1103 
UpdateItem_To_FileItem(const CUpdateItem & ui,CFileItem & file,CFileItem2 & file2)1104 static void UpdateItem_To_FileItem(const CUpdateItem &ui,
1105     CFileItem &file, CFileItem2 &file2)
1106 {
1107   UpdateItem_To_FileItem2(ui, file2);
1108 
1109   file.Size = ui.Size;
1110   file.IsDir = ui.IsDir;
1111   file.HasStream = ui.HasStream();
1112   // file.IsAltStream = ui.IsAltStream;
1113 }
1114 
1115 
1116 
1117 class CRepackInStreamWithSizes:
1118   public ISequentialInStream,
1119   public ICompressGetSubStreamSize,
1120   public CMyUnknownImp
1121 {
1122   CMyComPtr<ISequentialInStream> _stream;
1123   // UInt64 _size;
1124   const CBoolVector *_extractStatuses;
1125   UInt32 _startIndex;
1126 public:
1127   const CDbEx *_db;
1128 
Init(ISequentialInStream * stream,UInt32 startIndex,const CBoolVector * extractStatuses)1129   void Init(ISequentialInStream *stream, UInt32 startIndex, const CBoolVector *extractStatuses)
1130   {
1131     _startIndex = startIndex;
1132     _extractStatuses = extractStatuses;
1133     // _size = 0;
1134     _stream = stream;
1135   }
1136   // UInt64 GetSize() const { return _size; }
1137 
1138   MY_UNKNOWN_IMP2(ISequentialInStream, ICompressGetSubStreamSize)
1139 
1140   STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
1141 
1142   STDMETHOD(GetSubStreamSize)(UInt64 subStream, UInt64 *value);
1143 };
1144 
Read(void * data,UInt32 size,UInt32 * processedSize)1145 STDMETHODIMP CRepackInStreamWithSizes::Read(void *data, UInt32 size, UInt32 *processedSize)
1146 {
1147   return _stream->Read(data, size, processedSize);
1148   /*
1149   UInt32 realProcessedSize;
1150   HRESULT result = _stream->Read(data, size, &realProcessedSize);
1151   _size += realProcessedSize;
1152   if (processedSize)
1153     *processedSize = realProcessedSize;
1154   return result;
1155   */
1156 }
1157 
GetSubStreamSize(UInt64 subStream,UInt64 * value)1158 STDMETHODIMP CRepackInStreamWithSizes::GetSubStreamSize(UInt64 subStream, UInt64 *value)
1159 {
1160   *value = 0;
1161   if (subStream >= _extractStatuses->Size())
1162     return S_FALSE; // E_FAIL;
1163   unsigned index = (unsigned)subStream;
1164   if ((*_extractStatuses)[index])
1165   {
1166     const CFileItem &fi = _db->Files[_startIndex + index];
1167     if (fi.HasStream)
1168       *value = fi.Size;
1169   }
1170   return S_OK;
1171 }
1172 
1173 
1174 class CRepackStreamBase
1175 {
1176 protected:
1177   bool _needWrite;
1178   bool _fileIsOpen;
1179   bool _calcCrc;
1180   UInt32 _crc;
1181   UInt64 _rem;
1182 
1183   const CBoolVector *_extractStatuses;
1184   UInt32 _startIndex;
1185   unsigned _currentIndex;
1186 
1187   HRESULT OpenFile();
1188   HRESULT CloseFile();
1189   HRESULT ProcessEmptyFiles();
1190 
1191 public:
1192   const CDbEx *_db;
1193   CMyComPtr<IArchiveUpdateCallbackFile> _opCallback;
1194   CMyComPtr<IArchiveExtractCallbackMessage> _extractCallback;
1195 
1196   HRESULT Init(UInt32 startIndex, const CBoolVector *extractStatuses);
CheckFinishedState() const1197   HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
1198 };
1199 
Init(UInt32 startIndex,const CBoolVector * extractStatuses)1200 HRESULT CRepackStreamBase::Init(UInt32 startIndex, const CBoolVector *extractStatuses)
1201 {
1202   _startIndex = startIndex;
1203   _extractStatuses = extractStatuses;
1204 
1205   _currentIndex = 0;
1206   _fileIsOpen = false;
1207 
1208   return ProcessEmptyFiles();
1209 }
1210 
OpenFile()1211 HRESULT CRepackStreamBase::OpenFile()
1212 {
1213   UInt32 arcIndex = _startIndex + _currentIndex;
1214   const CFileItem &fi = _db->Files[arcIndex];
1215 
1216   _needWrite = (*_extractStatuses)[_currentIndex];
1217   if (_opCallback)
1218   {
1219     RINOK(_opCallback->ReportOperation(
1220         NEventIndexType::kInArcIndex, arcIndex,
1221         _needWrite ?
1222             NUpdateNotifyOp::kRepack :
1223             NUpdateNotifyOp::kSkip));
1224   }
1225 
1226   _crc = CRC_INIT_VAL;
1227   _calcCrc = (fi.CrcDefined && !fi.IsDir);
1228 
1229   _fileIsOpen = true;
1230   _rem = fi.Size;
1231   return S_OK;
1232 }
1233 
1234 const HRESULT k_My_HRESULT_CRC_ERROR = 0x20000002;
1235 
CloseFile()1236 HRESULT CRepackStreamBase::CloseFile()
1237 {
1238   UInt32 arcIndex = _startIndex + _currentIndex;
1239   const CFileItem &fi = _db->Files[arcIndex];
1240   _fileIsOpen = false;
1241   _currentIndex++;
1242   if (!_calcCrc || fi.Crc == CRC_GET_DIGEST(_crc))
1243     return S_OK;
1244 
1245   if (_extractCallback)
1246   {
1247     RINOK(_extractCallback->ReportExtractResult(
1248         NEventIndexType::kInArcIndex, arcIndex,
1249         NExtract::NOperationResult::kCRCError));
1250   }
1251   // return S_FALSE;
1252   return k_My_HRESULT_CRC_ERROR;
1253 }
1254 
ProcessEmptyFiles()1255 HRESULT CRepackStreamBase::ProcessEmptyFiles()
1256 {
1257   while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
1258   {
1259     RINOK(OpenFile());
1260     RINOK(CloseFile());
1261   }
1262   return S_OK;
1263 }
1264 
1265 
1266 
1267 #ifndef _7ZIP_ST
1268 
1269 class CFolderOutStream2:
1270   public CRepackStreamBase,
1271   public ISequentialOutStream,
1272   public CMyUnknownImp
1273 {
1274 public:
1275   CMyComPtr<ISequentialOutStream> _stream;
1276 
1277   MY_UNKNOWN_IMP
1278 
1279   STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
1280 };
1281 
Write(const void * data,UInt32 size,UInt32 * processedSize)1282 STDMETHODIMP CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize)
1283 {
1284   if (processedSize)
1285     *processedSize = 0;
1286 
1287   while (size != 0)
1288   {
1289     if (_fileIsOpen)
1290     {
1291       UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1292       HRESULT result = S_OK;
1293       if (_needWrite)
1294         result = _stream->Write(data, cur, &cur);
1295       if (_calcCrc)
1296         _crc = CrcUpdate(_crc, data, cur);
1297       if (processedSize)
1298         *processedSize += cur;
1299       data = (const Byte *)data + cur;
1300       size -= cur;
1301       _rem -= cur;
1302       if (_rem == 0)
1303       {
1304         RINOK(CloseFile());
1305         RINOK(ProcessEmptyFiles());
1306       }
1307       RINOK(result);
1308       if (cur == 0)
1309         break;
1310       continue;
1311     }
1312 
1313     RINOK(ProcessEmptyFiles());
1314     if (_currentIndex == _extractStatuses->Size())
1315     {
1316       // we don't support write cut here
1317       return E_FAIL;
1318     }
1319     RINOK(OpenFile());
1320   }
1321 
1322   return S_OK;
1323 }
1324 
1325 #endif
1326 
1327 
1328 
1329 static const UInt32 kTempBufSize = 1 << 16;
1330 
1331 class CFolderInStream2:
1332   public CRepackStreamBase,
1333   public ISequentialInStream,
1334   public CMyUnknownImp
1335 {
1336   Byte *_buf;
1337 public:
1338   CMyComPtr<ISequentialInStream> _inStream;
1339   HRESULT Result;
1340 
1341   MY_UNKNOWN_IMP
1342 
CFolderInStream2()1343   CFolderInStream2():
1344       Result(S_OK)
1345   {
1346     _buf = new Byte[kTempBufSize];
1347   }
1348 
~CFolderInStream2()1349   ~CFolderInStream2()
1350   {
1351     delete []_buf;
1352   }
1353 
Init()1354   void Init() { Result = S_OK; }
1355   STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
1356 };
1357 
Read(void * data,UInt32 size,UInt32 * processedSize)1358 STDMETHODIMP CFolderInStream2::Read(void *data, UInt32 size, UInt32 *processedSize)
1359 {
1360   if (processedSize)
1361     *processedSize = 0;
1362 
1363   while (size != 0)
1364   {
1365     if (_fileIsOpen)
1366     {
1367       UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1368 
1369       void *buf;
1370       if (_needWrite)
1371         buf = data;
1372       else
1373       {
1374         buf = _buf;
1375         if (cur > kTempBufSize)
1376           cur = kTempBufSize;
1377       }
1378 
1379       HRESULT result = _inStream->Read(buf, cur, &cur);
1380       _crc = CrcUpdate(_crc, buf, cur);
1381       _rem -= cur;
1382 
1383       if (_needWrite)
1384       {
1385         data = (Byte *)data + cur;
1386         size -= cur;
1387         if (processedSize)
1388           *processedSize += cur;
1389       }
1390 
1391       if (result != S_OK)
1392         Result = result;
1393 
1394       if (_rem == 0)
1395       {
1396         RINOK(CloseFile());
1397         RINOK(ProcessEmptyFiles());
1398       }
1399 
1400       RINOK(result);
1401 
1402       if (cur == 0)
1403         return E_FAIL;
1404 
1405       continue;
1406     }
1407 
1408     RINOK(ProcessEmptyFiles());
1409     if (_currentIndex == _extractStatuses->Size())
1410     {
1411       return S_OK;
1412     }
1413     RINOK(OpenFile());
1414   }
1415 
1416   return S_OK;
1417 }
1418 
1419 
1420 class CThreadDecoder
1421   #ifndef _7ZIP_ST
1422     : public CVirtThread
1423   #endif
1424 {
1425 public:
1426   CDecoder Decoder;
1427 
CThreadDecoder(bool multiThreadMixer)1428   CThreadDecoder(bool multiThreadMixer):
1429       Decoder(multiThreadMixer)
1430   {
1431     #ifndef _7ZIP_ST
1432     if (multiThreadMixer)
1433     {
1434       MtMode = false;
1435       NumThreads = 1;
1436       FosSpec = new CFolderOutStream2;
1437       Fos = FosSpec;
1438       Result = E_FAIL;
1439     }
1440     #endif
1441     // UnpackSize = 0;
1442     // send_UnpackSize = false;
1443   }
1444 
1445   #ifndef _7ZIP_ST
1446 
1447   bool dataAfterEnd_Error;
1448   HRESULT Result;
1449   CMyComPtr<IInStream> InStream;
1450 
1451   CFolderOutStream2 *FosSpec;
1452   CMyComPtr<ISequentialOutStream> Fos;
1453 
1454   UInt64 StartPos;
1455   const CFolders *Folders;
1456   int FolderIndex;
1457 
1458   // bool send_UnpackSize;
1459   // UInt64 UnpackSize;
1460 
1461   #ifndef _NO_CRYPTO
1462   CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1463   #endif
1464 
1465   DECL_EXTERNAL_CODECS_LOC_VARS2;
1466 
1467   #ifndef _7ZIP_ST
1468   bool MtMode;
1469   UInt32 NumThreads;
1470   #endif
1471 
1472 
~CThreadDecoder()1473   ~CThreadDecoder() { CVirtThread::WaitThreadFinish(); }
1474   virtual void Execute();
1475 
1476   #endif
1477 };
1478 
1479 #ifndef _7ZIP_ST
1480 
Execute()1481 void CThreadDecoder::Execute()
1482 {
1483   try
1484   {
1485     #ifndef _NO_CRYPTO
1486       bool isEncrypted = false;
1487       bool passwordIsDefined = false;
1488       UString password;
1489     #endif
1490 
1491     dataAfterEnd_Error = false;
1492 
1493     Result = Decoder.Decode(
1494       EXTERNAL_CODECS_LOC_VARS
1495       InStream,
1496       StartPos,
1497       *Folders, FolderIndex,
1498 
1499       // send_UnpackSize ? &UnpackSize : NULL,
1500       NULL, // unpackSize : FULL unpack
1501 
1502       Fos,
1503       NULL, // compressProgress
1504 
1505       NULL  // *inStreamMainRes
1506       , dataAfterEnd_Error
1507 
1508       _7Z_DECODER_CRYPRO_VARS
1509       #ifndef _7ZIP_ST
1510         , MtMode, NumThreads
1511       #endif
1512 
1513       );
1514   }
1515   catch(...)
1516   {
1517     Result = E_FAIL;
1518   }
1519 
1520   /*
1521   if (Result == S_OK)
1522     Result = FosSpec->CheckFinishedState();
1523   */
1524   FosSpec->_stream.Release();
1525 }
1526 
1527 #endif
1528 
1529 #ifndef _NO_CRYPTO
1530 
1531 class CCryptoGetTextPassword:
1532   public ICryptoGetTextPassword,
1533   public CMyUnknownImp
1534 {
1535 public:
1536   UString Password;
1537 
1538   MY_UNKNOWN_IMP
1539   STDMETHOD(CryptoGetTextPassword)(BSTR *password);
1540 };
1541 
CryptoGetTextPassword(BSTR * password)1542 STDMETHODIMP CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password)
1543 {
1544   return StringToBstr(Password, password);
1545 }
1546 
1547 #endif
1548 
1549 
GetFile(const CDatabase & inDb,unsigned index,CFileItem & file,CFileItem2 & file2)1550 static void GetFile(const CDatabase &inDb, unsigned index, CFileItem &file, CFileItem2 &file2)
1551 {
1552   file = inDb.Files[index];
1553   file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime);
1554   file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime);
1555   file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime);
1556   file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos);
1557   file2.AttribDefined = inDb.Attrib.GetItem(index, file2.Attrib);
1558   file2.IsAnti = inDb.IsItemAnti(index);
1559   // file2.IsAux = inDb.IsItemAux(index);
1560 }
1561 
Update(DECL_EXTERNAL_CODECS_LOC_VARS IInStream * inStream,const CDbEx * db,const CObjectVector<CUpdateItem> & updateItems,COutArchive & archive,CArchiveDatabaseOut & newDatabase,ISequentialOutStream * seqOutStream,IArchiveUpdateCallback * updateCallback,const CUpdateOptions & options,ICryptoGetTextPassword * getDecoderPassword)1562 HRESULT Update(
1563     DECL_EXTERNAL_CODECS_LOC_VARS
1564     IInStream *inStream,
1565     const CDbEx *db,
1566     const CObjectVector<CUpdateItem> &updateItems,
1567     // const CObjectVector<CTreeFolder> &treeFolders,
1568     // const CUniqBlocks &secureBlocks,
1569     COutArchive &archive,
1570     CArchiveDatabaseOut &newDatabase,
1571     ISequentialOutStream *seqOutStream,
1572     IArchiveUpdateCallback *updateCallback,
1573     const CUpdateOptions &options
1574     #ifndef _NO_CRYPTO
1575     , ICryptoGetTextPassword *getDecoderPassword
1576     #endif
1577     )
1578 {
1579   UInt64 numSolidFiles = options.NumSolidFiles;
1580   if (numSolidFiles == 0)
1581     numSolidFiles = 1;
1582 
1583   CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
1584   updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
1585 
1586   CMyComPtr<IArchiveExtractCallbackMessage> extractCallback;
1587   updateCallback->QueryInterface(IID_IArchiveExtractCallbackMessage, (void **)&extractCallback);
1588 
1589   // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes();
1590 
1591   /*
1592   CMyComPtr<IOutStream> outStream;
1593   RINOK(seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStream));
1594   if (!outStream)
1595     return E_NOTIMPL;
1596   */
1597 
1598   UInt64 startBlockSize = db ? db->ArcInfo.StartPosition: 0;
1599   if (startBlockSize > 0 && !options.RemoveSfxBlock)
1600   {
1601     RINOK(WriteRange(inStream, seqOutStream, 0, startBlockSize, NULL));
1602   }
1603 
1604   CIntArr fileIndexToUpdateIndexMap;
1605   UInt64 complexity = 0;
1606   UInt64 inSizeForReduce2 = 0;
1607   bool needEncryptedRepack = false;
1608 
1609   CRecordVector<CFilterMode2> filters;
1610   CObjectVector<CSolidGroup> groups;
1611   bool thereAreRepacks = false;
1612 
1613   bool useFilters = options.UseFilters;
1614   if (useFilters)
1615   {
1616     const CCompressionMethodMode &method = *options.Method;
1617 
1618     FOR_VECTOR (i, method.Methods)
1619       if (IsFilterMethod(method.Methods[i].Id))
1620       {
1621         useFilters = false;
1622         break;
1623       }
1624   }
1625 
1626   if (db)
1627   {
1628     fileIndexToUpdateIndexMap.Alloc(db->Files.Size());
1629     unsigned i;
1630 
1631     for (i = 0; i < db->Files.Size(); i++)
1632       fileIndexToUpdateIndexMap[i] = -1;
1633 
1634     for (i = 0; i < updateItems.Size(); i++)
1635     {
1636       int index = updateItems[i].IndexInArchive;
1637       if (index != -1)
1638         fileIndexToUpdateIndexMap[(unsigned)index] = i;
1639     }
1640 
1641     for (i = 0; i < db->NumFolders; i++)
1642     {
1643       CNum indexInFolder = 0;
1644       CNum numCopyItems = 0;
1645       CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
1646       UInt64 repackSize = 0;
1647 
1648       for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
1649       {
1650         const CFileItem &file = db->Files[fi];
1651         if (file.HasStream)
1652         {
1653           indexInFolder++;
1654           int updateIndex = fileIndexToUpdateIndexMap[fi];
1655           if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
1656           {
1657             numCopyItems++;
1658             repackSize += file.Size;
1659           }
1660         }
1661       }
1662 
1663       if (numCopyItems == 0)
1664         continue;
1665 
1666       CFolderRepack rep;
1667       rep.FolderIndex = i;
1668       rep.NumCopyFiles = numCopyItems;
1669       CFolderEx f;
1670       db->ParseFolderEx(i, f);
1671 
1672       const bool isEncrypted = f.IsEncrypted();
1673       const bool needCopy = (numCopyItems == numUnpackStreams);
1674       const bool extractFilter = (useFilters || needCopy);
1675 
1676       unsigned groupIndex = Get_FilterGroup_for_Folder(filters, f, extractFilter);
1677 
1678       while (groupIndex >= groups.Size())
1679         groups.AddNew();
1680 
1681       groups[groupIndex].folderRefs.Add(rep);
1682 
1683       if (needCopy)
1684         complexity += db->GetFolderFullPackSize(i);
1685       else
1686       {
1687         thereAreRepacks = true;
1688         complexity += repackSize;
1689         if (inSizeForReduce2 < repackSize)
1690           inSizeForReduce2 = repackSize;
1691         if (isEncrypted)
1692           needEncryptedRepack = true;
1693       }
1694     }
1695   }
1696 
1697   UInt64 inSizeForReduce = 0;
1698   {
1699     FOR_VECTOR (i, updateItems)
1700     {
1701       const CUpdateItem &ui = updateItems[i];
1702       if (ui.NewData)
1703       {
1704         complexity += ui.Size;
1705         if (numSolidFiles != 1)
1706           inSizeForReduce += ui.Size;
1707         else if (inSizeForReduce < ui.Size)
1708           inSizeForReduce = ui.Size;
1709       }
1710     }
1711   }
1712 
1713   if (inSizeForReduce < inSizeForReduce2)
1714     inSizeForReduce = inSizeForReduce2;
1715 
1716   RINOK(updateCallback->SetTotal(complexity));
1717 
1718   CLocalProgress *lps = new CLocalProgress;
1719   CMyComPtr<ICompressProgressInfo> progress = lps;
1720   lps->Init(updateCallback, true);
1721 
1722   #ifndef _7ZIP_ST
1723 
1724   CStreamBinder sb;
1725   if (options.MultiThreadMixer)
1726   {
1727     RINOK(sb.CreateEvents());
1728   }
1729 
1730   #endif
1731 
1732   CThreadDecoder threadDecoder(options.MultiThreadMixer);
1733 
1734   #ifndef _7ZIP_ST
1735   if (options.MultiThreadMixer && thereAreRepacks)
1736   {
1737     #ifdef EXTERNAL_CODECS
1738     threadDecoder.__externalCodecs = __externalCodecs;
1739     #endif
1740     RINOK(threadDecoder.Create());
1741   }
1742   #endif
1743 
1744   {
1745     CAnalysis analysis;
1746     if (options.AnalysisLevel == 0)
1747     {
1748       analysis.ParseWav = false;
1749       analysis.ParseExe = false;
1750       analysis.ParseAll = false;
1751     }
1752     else
1753     {
1754       analysis.Callback = opCallback;
1755       if (options.AnalysisLevel > 0)
1756       {
1757         analysis.ParseWav = true;
1758         if (options.AnalysisLevel >= 7)
1759         {
1760           analysis.ParseExe = true;
1761           if (options.AnalysisLevel >= 9)
1762             analysis.ParseAll = true;
1763         }
1764       }
1765     }
1766 
1767     // ---------- Split files to groups ----------
1768 
1769     const CCompressionMethodMode &method = *options.Method;
1770 
1771     FOR_VECTOR (i, updateItems)
1772     {
1773       const CUpdateItem &ui = updateItems[i];
1774       if (!ui.NewData || !ui.HasStream())
1775         continue;
1776 
1777       CFilterMode2 fm;
1778       if (useFilters)
1779       {
1780         RINOK(analysis.GetFilterGroup(i, ui, fm));
1781       }
1782       fm.Encrypted = method.PasswordIsDefined;
1783 
1784       unsigned groupIndex = GetGroup(filters, fm);
1785       while (groupIndex >= groups.Size())
1786         groups.AddNew();
1787       groups[groupIndex].Indices.Add(i);
1788     }
1789   }
1790 
1791 
1792   #ifndef _NO_CRYPTO
1793 
1794   CCryptoGetTextPassword *getPasswordSpec = NULL;
1795   CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1796   if (needEncryptedRepack)
1797   {
1798     getPasswordSpec = new CCryptoGetTextPassword;
1799     getTextPassword = getPasswordSpec;
1800 
1801     #ifndef _7ZIP_ST
1802     threadDecoder.getTextPassword = getPasswordSpec;
1803     #endif
1804 
1805     if (options.Method->PasswordIsDefined)
1806       getPasswordSpec->Password = options.Method->Password;
1807     else
1808     {
1809       if (!getDecoderPassword)
1810         return E_NOTIMPL;
1811       CMyComBSTR password;
1812       RINOK(getDecoderPassword->CryptoGetTextPassword(&password));
1813       if (password)
1814         getPasswordSpec->Password = password;
1815     }
1816   }
1817 
1818   #endif
1819 
1820 
1821   // ---------- Compress ----------
1822 
1823   RINOK(archive.Create(seqOutStream, false));
1824   RINOK(archive.SkipPrefixArchiveHeader());
1825 
1826   /*
1827   CIntVector treeFolderToArcIndex;
1828   treeFolderToArcIndex.Reserve(treeFolders.Size());
1829   for (i = 0; i < treeFolders.Size(); i++)
1830     treeFolderToArcIndex.Add(-1);
1831   // ---------- Write Tree (only AUX dirs) ----------
1832   for (i = 1; i < treeFolders.Size(); i++)
1833   {
1834     const CTreeFolder &treeFolder = treeFolders[i];
1835     CFileItem file;
1836     CFileItem2 file2;
1837     file2.Init();
1838     int secureID = 0;
1839     if (treeFolder.UpdateItemIndex < 0)
1840     {
1841       // we can store virtual dir item wuthout attrib, but we want all items have attrib.
1842       file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY);
1843       file2.IsAux = true;
1844     }
1845     else
1846     {
1847       const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex];
1848       // if item is not dir, then it's parent for alt streams.
1849       // we will write such items later
1850       if (!ui.IsDir)
1851         continue;
1852       secureID = ui.SecureIndex;
1853       if (ui.NewProps)
1854         UpdateItem_To_FileItem(ui, file, file2);
1855       else
1856         GetFile(*db, ui.IndexInArchive, file, file2);
1857     }
1858     file.Size = 0;
1859     file.HasStream = false;
1860     file.IsDir = true;
1861     file.Parent = treeFolder.Parent;
1862 
1863     treeFolderToArcIndex[i] = newDatabase.Files.Size();
1864     newDatabase.AddFile(file, file2, treeFolder.Name);
1865 
1866     if (totalSecureDataSize != 0)
1867       newDatabase.SecureIDs.Add(secureID);
1868   }
1869   */
1870 
1871   {
1872     /* ---------- Write non-AUX dirs and Empty files ---------- */
1873     CUIntVector emptyRefs;
1874 
1875     unsigned i;
1876 
1877     for (i = 0; i < updateItems.Size(); i++)
1878     {
1879       const CUpdateItem &ui = updateItems[i];
1880       if (ui.NewData)
1881       {
1882         if (ui.HasStream())
1883           continue;
1884       }
1885       else if (ui.IndexInArchive != -1 && db->Files[ui.IndexInArchive].HasStream)
1886         continue;
1887       /*
1888       if (ui.TreeFolderIndex >= 0)
1889         continue;
1890       */
1891       emptyRefs.Add(i);
1892     }
1893 
1894     emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
1895 
1896     for (i = 0; i < emptyRefs.Size(); i++)
1897     {
1898       const CUpdateItem &ui = updateItems[emptyRefs[i]];
1899       CFileItem file;
1900       CFileItem2 file2;
1901       UString name;
1902       if (ui.NewProps)
1903       {
1904         UpdateItem_To_FileItem(ui, file, file2);
1905         file.CrcDefined = false;
1906         name = ui.Name;
1907       }
1908       else
1909       {
1910         GetFile(*db, ui.IndexInArchive, file, file2);
1911         db->GetPath(ui.IndexInArchive, name);
1912       }
1913 
1914       /*
1915       if (totalSecureDataSize != 0)
1916         newDatabase.SecureIDs.Add(ui.SecureIndex);
1917       file.Parent = ui.ParentFolderIndex;
1918       */
1919       newDatabase.AddFile(file, file2, name);
1920     }
1921   }
1922 
1923   lps->ProgressOffset = 0;
1924 
1925   {
1926     // ---------- Sort Filters ----------
1927 
1928     FOR_VECTOR (i, filters)
1929     {
1930       filters[i].GroupIndex = i;
1931     }
1932     filters.Sort2();
1933   }
1934 
1935   for (unsigned groupIndex = 0; groupIndex < filters.Size(); groupIndex++)
1936   {
1937     const CFilterMode2 &filterMode = filters[groupIndex];
1938 
1939     CCompressionMethodMode method = *options.Method;
1940     {
1941       HRESULT res = MakeExeMethod(method, filterMode,
1942         #ifdef _7ZIP_ST
1943           false
1944         #else
1945           options.MaxFilter && options.MultiThreadMixer
1946         #endif
1947         );
1948 
1949       RINOK(res);
1950     }
1951 
1952     if (filterMode.Encrypted)
1953     {
1954       if (!method.PasswordIsDefined)
1955       {
1956         #ifndef _NO_CRYPTO
1957         if (getPasswordSpec)
1958           method.Password = getPasswordSpec->Password;
1959         #endif
1960         method.PasswordIsDefined = true;
1961       }
1962     }
1963     else
1964     {
1965       method.PasswordIsDefined = false;
1966       method.Password.Empty();
1967     }
1968 
1969     CEncoder encoder(method);
1970 
1971     // ---------- Repack and copy old solid blocks ----------
1972 
1973     const CSolidGroup &group = groups[filterMode.GroupIndex];
1974 
1975     FOR_VECTOR(folderRefIndex, group.folderRefs)
1976     {
1977       const CFolderRepack &rep = group.folderRefs[folderRefIndex];
1978 
1979       unsigned folderIndex = rep.FolderIndex;
1980 
1981       CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
1982 
1983       if (rep.NumCopyFiles == numUnpackStreams)
1984       {
1985         if (opCallback)
1986         {
1987           RINOK(opCallback->ReportOperation(
1988               NEventIndexType::kBlockIndex, (UInt32)folderIndex,
1989               NUpdateNotifyOp::kReplicate));
1990 
1991           // ---------- Copy old solid block ----------
1992           {
1993             CNum indexInFolder = 0;
1994             for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
1995             {
1996               if (db->Files[fi].HasStream)
1997               {
1998                 indexInFolder++;
1999                 RINOK(opCallback->ReportOperation(
2000                     NEventIndexType::kInArcIndex, (UInt32)fi,
2001                     NUpdateNotifyOp::kReplicate));
2002               }
2003             }
2004           }
2005         }
2006 
2007         UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
2008         RINOK(WriteRange(inStream, archive.SeqStream,
2009             db->GetFolderStreamPos(folderIndex, 0), packSize, progress));
2010         lps->ProgressOffset += packSize;
2011 
2012         CFolder &folder = newDatabase.Folders.AddNew();
2013         db->ParseFolderInfo(folderIndex, folder);
2014         CNum startIndex = db->FoStartPackStreamIndex[folderIndex];
2015         FOR_VECTOR(j, folder.PackStreams)
2016         {
2017           newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j));
2018           // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
2019           // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
2020         }
2021 
2022         size_t indexStart = db->FoToCoderUnpackSizes[folderIndex];
2023         size_t indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1];
2024         for (; indexStart < indexEnd; indexStart++)
2025           newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes[indexStart]);
2026       }
2027       else
2028       {
2029         // ---------- Repack old solid block ----------
2030 
2031         CBoolVector extractStatuses;
2032 
2033         CNum indexInFolder = 0;
2034 
2035         if (opCallback)
2036         {
2037           RINOK(opCallback->ReportOperation(
2038               NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2039               NUpdateNotifyOp::kRepack))
2040         }
2041 
2042         /* We could reduce data size of decoded folder, if we don't need to repack
2043            last files in folder. But the gain in speed is small in most cases.
2044            So we unpack full folder. */
2045 
2046         UInt64 sizeToEncode = 0;
2047 
2048         /*
2049         UInt64 importantUnpackSize = 0;
2050         unsigned numImportantFiles = 0;
2051         UInt64 decodeSize = 0;
2052         */
2053 
2054         for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2055         {
2056           bool needExtract = false;
2057           const CFileItem &file = db->Files[fi];
2058 
2059           if (file.HasStream)
2060           {
2061             indexInFolder++;
2062             int updateIndex = fileIndexToUpdateIndexMap[fi];
2063             if (updateIndex >= 0 && !updateItems[updateIndex].NewData)
2064               needExtract = true;
2065             // decodeSize += file.Size;
2066           }
2067 
2068           extractStatuses.Add(needExtract);
2069           if (needExtract)
2070           {
2071             sizeToEncode += file.Size;
2072             /*
2073             numImportantFiles = extractStatuses.Size();
2074             importantUnpackSize = decodeSize;
2075             */
2076           }
2077         }
2078 
2079         // extractStatuses.DeleteFrom(numImportantFiles);
2080 
2081         unsigned startPackIndex = newDatabase.PackSizes.Size();
2082         UInt64 curUnpackSize;
2083         {
2084 
2085           CMyComPtr<ISequentialInStream> sbInStream;
2086           CRepackStreamBase *repackBase;
2087           CFolderInStream2 *FosSpec2 = NULL;
2088 
2089           CRepackInStreamWithSizes *inStreamSizeCountSpec = new CRepackInStreamWithSizes;
2090           CMyComPtr<ISequentialInStream> inStreamSizeCount = inStreamSizeCountSpec;
2091           {
2092             #ifndef _7ZIP_ST
2093             if (options.MultiThreadMixer)
2094             {
2095               repackBase = threadDecoder.FosSpec;
2096               CMyComPtr<ISequentialOutStream> sbOutStream;
2097               sb.CreateStreams(&sbInStream, &sbOutStream);
2098               sb.ReInit();
2099 
2100               threadDecoder.FosSpec->_stream = sbOutStream;
2101 
2102               threadDecoder.InStream = inStream;
2103               threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0);
2104               threadDecoder.Folders = (const CFolders *)db;
2105               threadDecoder.FolderIndex = folderIndex;
2106 
2107               // threadDecoder.UnpackSize = importantUnpackSize;
2108               // threadDecoder.send_UnpackSize = true;
2109             }
2110             else
2111             #endif
2112             {
2113               FosSpec2 = new CFolderInStream2;
2114               FosSpec2->Init();
2115               sbInStream = FosSpec2;
2116               repackBase = FosSpec2;
2117 
2118               #ifndef _NO_CRYPTO
2119               bool isEncrypted = false;
2120               bool passwordIsDefined = false;
2121               UString password;
2122               #endif
2123 
2124               CMyComPtr<ISequentialInStream> decodedStream;
2125               bool dataAfterEnd_Error = false;
2126 
2127               HRESULT res = threadDecoder.Decoder.Decode(
2128                   EXTERNAL_CODECS_LOC_VARS
2129                   inStream,
2130                   db->ArcInfo.DataStartPosition, // db->GetFolderStreamPos(folderIndex, 0);,
2131                   *db, folderIndex,
2132                   // &importantUnpackSize, // *unpackSize
2133                   NULL, // *unpackSize : FULL unpack
2134 
2135                   NULL, // *outStream
2136                   NULL, // *compressProgress
2137 
2138                   &decodedStream
2139                   , dataAfterEnd_Error
2140 
2141                   _7Z_DECODER_CRYPRO_VARS
2142                   #ifndef _7ZIP_ST
2143                     , false // mtMode
2144                     , 1 // numThreads
2145                   #endif
2146 
2147                 );
2148 
2149               RINOK(res);
2150               if (!decodedStream)
2151                 return E_FAIL;
2152 
2153               FosSpec2->_inStream = decodedStream;
2154             }
2155 
2156             repackBase->_db = db;
2157             repackBase->_opCallback = opCallback;
2158             repackBase->_extractCallback = extractCallback;
2159 
2160             UInt32 startIndex = db->FolderStartFileIndex[folderIndex];
2161             RINOK(repackBase->Init(startIndex, &extractStatuses));
2162 
2163             inStreamSizeCountSpec->_db = db;
2164             inStreamSizeCountSpec->Init(sbInStream, startIndex, &extractStatuses);
2165 
2166             #ifndef _7ZIP_ST
2167             if (options.MultiThreadMixer)
2168             {
2169               threadDecoder.Start();
2170             }
2171             #endif
2172           }
2173 
2174           curUnpackSize = sizeToEncode;
2175 
2176           HRESULT encodeRes = encoder.Encode(
2177               EXTERNAL_CODECS_LOC_VARS
2178               inStreamSizeCount,
2179               // NULL,
2180               &inSizeForReduce,
2181               newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curUnpackSize,
2182               archive.SeqStream, newDatabase.PackSizes, progress);
2183 
2184           if (encodeRes == k_My_HRESULT_CRC_ERROR)
2185             return E_FAIL;
2186 
2187           #ifndef _7ZIP_ST
2188           if (options.MultiThreadMixer)
2189           {
2190             // 16.00: hang was fixed : for case if decoding was not finished.
2191             // We close CBinderInStream and it calls CStreamBinder::CloseRead()
2192             inStreamSizeCount.Release();
2193             sbInStream.Release();
2194 
2195             threadDecoder.WaitExecuteFinish();
2196 
2197             HRESULT decodeRes = threadDecoder.Result;
2198             // if (res == k_My_HRESULT_CRC_ERROR)
2199             if (decodeRes == S_FALSE || threadDecoder.dataAfterEnd_Error)
2200             {
2201               if (extractCallback)
2202               {
2203                 RINOK(extractCallback->ReportExtractResult(
2204                     NEventIndexType::kInArcIndex, db->FolderStartFileIndex[folderIndex],
2205                     // NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2206                     (decodeRes != S_OK ?
2207                       NExtract::NOperationResult::kDataError :
2208                       NExtract::NOperationResult::kDataAfterEnd)));
2209               }
2210               if (decodeRes != S_OK)
2211                 return E_FAIL;
2212             }
2213             RINOK(decodeRes);
2214             if (encodeRes == S_OK)
2215               if (sb.ProcessedSize != sizeToEncode)
2216                 encodeRes = E_FAIL;
2217           }
2218           else
2219           #endif
2220           {
2221             if (FosSpec2->Result == S_FALSE)
2222             {
2223               if (extractCallback)
2224               {
2225                 RINOK(extractCallback->ReportExtractResult(
2226                     NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2227                     NExtract::NOperationResult::kDataError));
2228               }
2229               return E_FAIL;
2230             }
2231             RINOK(FosSpec2->Result);
2232           }
2233 
2234           RINOK(encodeRes);
2235           RINOK(repackBase->CheckFinishedState());
2236 
2237           if (curUnpackSize != sizeToEncode)
2238             return E_FAIL;
2239         }
2240 
2241         for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2242           lps->OutSize += newDatabase.PackSizes[startPackIndex];
2243         lps->InSize += curUnpackSize;
2244       }
2245 
2246       newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
2247 
2248       CNum indexInFolder = 0;
2249       for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2250       {
2251         if (db->Files[fi].HasStream)
2252         {
2253           indexInFolder++;
2254           int updateIndex = fileIndexToUpdateIndexMap[fi];
2255           if (updateIndex >= 0)
2256           {
2257             const CUpdateItem &ui = updateItems[updateIndex];
2258             if (ui.NewData)
2259               continue;
2260 
2261             UString name;
2262             CFileItem file;
2263             CFileItem2 file2;
2264             GetFile(*db, fi, file, file2);
2265 
2266             if (ui.NewProps)
2267             {
2268               UpdateItem_To_FileItem2(ui, file2);
2269               file.IsDir = ui.IsDir;
2270               name = ui.Name;
2271             }
2272             else
2273               db->GetPath(fi, name);
2274 
2275             /*
2276             file.Parent = ui.ParentFolderIndex;
2277             if (ui.TreeFolderIndex >= 0)
2278               treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2279             if (totalSecureDataSize != 0)
2280               newDatabase.SecureIDs.Add(ui.SecureIndex);
2281             */
2282             newDatabase.AddFile(file, file2, name);
2283           }
2284         }
2285       }
2286     }
2287 
2288 
2289     // ---------- Compress files to new solid blocks ----------
2290 
2291     unsigned numFiles = group.Indices.Size();
2292     if (numFiles == 0)
2293       continue;
2294     CRecordVector<CRefItem> refItems;
2295     refItems.ClearAndSetSize(numFiles);
2296     bool sortByType = (options.UseTypeSorting && numSolidFiles > 1);
2297 
2298     unsigned i;
2299 
2300     for (i = 0; i < numFiles; i++)
2301       refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType);
2302 
2303     CSortParam sortParam;
2304     // sortParam.TreeFolders = &treeFolders;
2305     sortParam.SortByType = sortByType;
2306     refItems.Sort(CompareUpdateItems, (void *)&sortParam);
2307 
2308     CObjArray<UInt32> indices(numFiles);
2309 
2310     for (i = 0; i < numFiles; i++)
2311     {
2312       UInt32 index = refItems[i].Index;
2313       indices[i] = index;
2314       /*
2315       const CUpdateItem &ui = updateItems[index];
2316       CFileItem file;
2317       if (ui.NewProps)
2318         UpdateItem_To_FileItem(ui, file);
2319       else
2320         file = db.Files[ui.IndexInArchive];
2321       if (file.IsAnti || file.IsDir)
2322         return E_FAIL;
2323       newDatabase.Files.Add(file);
2324       */
2325     }
2326 
2327     for (i = 0; i < numFiles;)
2328     {
2329       UInt64 totalSize = 0;
2330       unsigned numSubFiles;
2331 
2332       const wchar_t *prevExtension = NULL;
2333 
2334       for (numSubFiles = 0; i + numSubFiles < numFiles && numSubFiles < numSolidFiles; numSubFiles++)
2335       {
2336         const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
2337         totalSize += ui.Size;
2338         if (totalSize > options.NumSolidBytes)
2339           break;
2340         if (options.SolidExtension)
2341         {
2342           int slashPos = ui.Name.ReverseFind_PathSepar();
2343           int dotPos = ui.Name.ReverseFind_Dot();
2344           const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : dotPos + 1);
2345           if (numSubFiles == 0)
2346             prevExtension = ext;
2347           else if (!StringsAreEqualNoCase(ext, prevExtension))
2348             break;
2349         }
2350       }
2351 
2352       if (numSubFiles < 1)
2353         numSubFiles = 1;
2354 
2355       RINOK(lps->SetCur());
2356 
2357       CFolderInStream *inStreamSpec = new CFolderInStream;
2358       CMyComPtr<ISequentialInStream> solidInStream(inStreamSpec);
2359       inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
2360 
2361       unsigned startPackIndex = newDatabase.PackSizes.Size();
2362       UInt64 curFolderUnpackSize = totalSize;
2363       // curFolderUnpackSize = (UInt64)(Int64)-1;
2364 
2365       RINOK(encoder.Encode(
2366           EXTERNAL_CODECS_LOC_VARS
2367           solidInStream,
2368           // NULL,
2369           &inSizeForReduce,
2370           newDatabase.Folders.AddNew(), newDatabase.CoderUnpackSizes, curFolderUnpackSize,
2371           archive.SeqStream, newDatabase.PackSizes, progress));
2372 
2373       if (!inStreamSpec->WasFinished())
2374         return E_FAIL;
2375 
2376       for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2377         lps->OutSize += newDatabase.PackSizes[startPackIndex];
2378 
2379       lps->InSize += curFolderUnpackSize;
2380       // for ()
2381       // newDatabase.PackCRCsDefined.Add(false);
2382       // newDatabase.PackCRCs.Add(0);
2383 
2384       CNum numUnpackStreams = 0;
2385       UInt64 skippedSize = 0;
2386 
2387       for (unsigned subIndex = 0; subIndex < numSubFiles; subIndex++)
2388       {
2389         const CUpdateItem &ui = updateItems[indices[i + subIndex]];
2390         CFileItem file;
2391         CFileItem2 file2;
2392         UString name;
2393         if (ui.NewProps)
2394         {
2395           UpdateItem_To_FileItem(ui, file, file2);
2396           name = ui.Name;
2397         }
2398         else
2399         {
2400           GetFile(*db, ui.IndexInArchive, file, file2);
2401           db->GetPath(ui.IndexInArchive, name);
2402         }
2403         if (file2.IsAnti || file.IsDir)
2404           return E_FAIL;
2405 
2406         /*
2407         CFileItem &file = newDatabase.Files[
2408               startFileIndexInDatabase + i + subIndex];
2409         */
2410         if (!inStreamSpec->Processed[subIndex])
2411         {
2412           skippedSize += ui.Size;
2413           continue;
2414           // file.Name += ".locked";
2415         }
2416 
2417         file.Crc = inStreamSpec->CRCs[subIndex];
2418         file.Size = inStreamSpec->Sizes[subIndex];
2419 
2420         // if (file.Size >= 0) // test purposes
2421         if (file.Size != 0)
2422         {
2423           file.CrcDefined = true;
2424           file.HasStream = true;
2425           numUnpackStreams++;
2426         }
2427         else
2428         {
2429           file.CrcDefined = false;
2430           file.HasStream = false;
2431         }
2432 
2433         /*
2434         file.Parent = ui.ParentFolderIndex;
2435         if (ui.TreeFolderIndex >= 0)
2436           treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2437         if (totalSecureDataSize != 0)
2438           newDatabase.SecureIDs.Add(ui.SecureIndex);
2439         */
2440         newDatabase.AddFile(file, file2, name);
2441       }
2442 
2443       // numUnpackStreams = 0 is very bad case for locked files
2444       // v3.13 doesn't understand it.
2445       newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
2446       i += numSubFiles;
2447 
2448       if (skippedSize != 0 && complexity >= skippedSize)
2449       {
2450         complexity -= skippedSize;
2451         RINOK(updateCallback->SetTotal(complexity));
2452       }
2453     }
2454   }
2455 
2456   RINOK(lps->SetCur());
2457 
2458   /*
2459   fileIndexToUpdateIndexMap.ClearAndFree();
2460   groups.ClearAndFree();
2461   */
2462 
2463   /*
2464   for (i = 0; i < newDatabase.Files.Size(); i++)
2465   {
2466     CFileItem &file = newDatabase.Files[i];
2467     file.Parent = treeFolderToArcIndex[file.Parent];
2468   }
2469 
2470   if (totalSecureDataSize != 0)
2471   {
2472     newDatabase.SecureBuf.SetCapacity(totalSecureDataSize);
2473     size_t pos = 0;
2474     newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size());
2475     for (i = 0; i < secureBlocks.Sorted.Size(); i++)
2476     {
2477       const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]];
2478       size_t size = buf.GetCapacity();
2479       if (size != 0)
2480         memcpy(newDatabase.SecureBuf + pos, buf, size);
2481       newDatabase.SecureSizes.Add((UInt32)size);
2482       pos += size;
2483     }
2484   }
2485   */
2486   newDatabase.ReserveDown();
2487 
2488   if (opCallback)
2489     RINOK(opCallback->ReportOperation(NEventIndexType::kNoIndex, (UInt32)(Int32)-1, NUpdateNotifyOp::kHeader));
2490 
2491   return S_OK;
2492 }
2493 
2494 }}
2495