1 // LzmaEncoder.cpp
2
3 #include "StdAfx.h"
4
5 #include "../../../C/Alloc.h"
6
7 #include "../Common/CWrappers.h"
8 #include "../Common/StreamUtils.h"
9
10 #include "LzmaEncoder.h"
11
12 #include "../../Common/IntToString.h"
13 #include "../../Windows/TimeUtils.h"
14
15 // #define LOG_LZMA_THREADS
16
17 #ifdef LOG_LZMA_THREADS
18 #include <stdio.h>
19
20 EXTERN_C_BEGIN
21 void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]);
22 EXTERN_C_END
23
24 #endif
25
26 namespace NCompress {
27 namespace NLzma {
28
CEncoder()29 CEncoder::CEncoder()
30 {
31 _encoder = NULL;
32 _encoder = LzmaEnc_Create(&g_AlignedAlloc);
33 if (!_encoder)
34 throw 1;
35 }
36
~CEncoder()37 CEncoder::~CEncoder()
38 {
39 if (_encoder)
40 LzmaEnc_Destroy(_encoder, &g_AlignedAlloc, &g_BigAlloc);
41 }
42
GetLowCharFast(wchar_t c)43 static inline wchar_t GetLowCharFast(wchar_t c)
44 {
45 return c |= 0x20;
46 }
47
ParseMatchFinder(const wchar_t * s,int * btMode,int * numHashBytes)48 static int ParseMatchFinder(const wchar_t *s, int *btMode, int *numHashBytes)
49 {
50 wchar_t c = GetLowCharFast(*s++);
51 if (c == 'h')
52 {
53 if (GetLowCharFast(*s++) != 'c')
54 return 0;
55 int num = (int)(*s++ - L'0');
56 if (num < 4 || num > 5)
57 return 0;
58 if (*s != 0)
59 return 0;
60 *btMode = 0;
61 *numHashBytes = num;
62 return 1;
63 }
64
65 if (c != 'b')
66 return 0;
67 {
68 if (GetLowCharFast(*s++) != 't')
69 return 0;
70 int num = (int)(*s++ - L'0');
71 if (num < 2 || num > 5)
72 return 0;
73 if (*s != 0)
74 return 0;
75 *btMode = 1;
76 *numHashBytes = num;
77 return 1;
78 }
79 }
80
81 #define SET_PROP_32(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = (int)v; break;
82 #define SET_PROP_32U(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = v; break;
83
84 HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep);
SetLzmaProp(PROPID propID,const PROPVARIANT & prop,CLzmaEncProps & ep)85 HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep)
86 {
87 if (propID == NCoderPropID::kMatchFinder)
88 {
89 if (prop.vt != VT_BSTR)
90 return E_INVALIDARG;
91 return ParseMatchFinder(prop.bstrVal, &ep.btMode, &ep.numHashBytes) ? S_OK : E_INVALIDARG;
92 }
93
94 if (propID == NCoderPropID::kAffinity)
95 {
96 if (prop.vt == VT_UI8)
97 ep.affinity = prop.uhVal.QuadPart;
98 else
99 return E_INVALIDARG;
100 return S_OK;
101 }
102
103 if (propID > NCoderPropID::kReduceSize)
104 return S_OK;
105
106 if (propID == NCoderPropID::kReduceSize)
107 {
108 if (prop.vt == VT_UI8)
109 ep.reduceSize = prop.uhVal.QuadPart;
110 else
111 return E_INVALIDARG;
112 return S_OK;
113 }
114
115 if (propID == NCoderPropID::kDictionarySize)
116 {
117 if (prop.vt == VT_UI8)
118 {
119 // 21.03 : we support 64-bit VT_UI8 for dictionary and (dict == 4 GiB)
120 const UInt64 v = prop.uhVal.QuadPart;
121 if (v > ((UInt64)1 << 32))
122 return E_INVALIDARG;
123 UInt32 dict;
124 if (v == ((UInt64)1 << 32))
125 dict = (UInt32)(Int32)-1;
126 else
127 dict = (UInt32)v;
128 ep.dictSize = dict;
129 return S_OK;
130 }
131 }
132
133 if (prop.vt != VT_UI4)
134 return E_INVALIDARG;
135 UInt32 v = prop.ulVal;
136 switch (propID)
137 {
138 case NCoderPropID::kDefaultProp:
139 if (v > 32)
140 return E_INVALIDARG;
141 ep.dictSize = (v == 32) ? (UInt32)(Int32)-1 : (UInt32)1 << (unsigned)v;
142 break;
143 SET_PROP_32(kLevel, level)
144 SET_PROP_32(kNumFastBytes, fb)
145 SET_PROP_32U(kMatchFinderCycles, mc)
146 SET_PROP_32(kAlgorithm, algo)
147 SET_PROP_32U(kDictionarySize, dictSize)
148 SET_PROP_32(kPosStateBits, pb)
149 SET_PROP_32(kLitPosBits, lp)
150 SET_PROP_32(kLitContextBits, lc)
151 SET_PROP_32(kNumThreads, numThreads)
152 default: return E_INVALIDARG;
153 }
154 return S_OK;
155 }
156
SetCoderProperties(const PROPID * propIDs,const PROPVARIANT * coderProps,UInt32 numProps)157 STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
158 const PROPVARIANT *coderProps, UInt32 numProps)
159 {
160 CLzmaEncProps props;
161 LzmaEncProps_Init(&props);
162
163 for (UInt32 i = 0; i < numProps; i++)
164 {
165 const PROPVARIANT &prop = coderProps[i];
166 PROPID propID = propIDs[i];
167 switch (propID)
168 {
169 case NCoderPropID::kEndMarker:
170 if (prop.vt != VT_BOOL)
171 return E_INVALIDARG;
172 props.writeEndMark = (prop.boolVal != VARIANT_FALSE);
173 break;
174 default:
175 RINOK(SetLzmaProp(propID, prop, props));
176 }
177 }
178 return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props));
179 }
180
181
SetCoderPropertiesOpt(const PROPID * propIDs,const PROPVARIANT * coderProps,UInt32 numProps)182 STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs,
183 const PROPVARIANT *coderProps, UInt32 numProps)
184 {
185 for (UInt32 i = 0; i < numProps; i++)
186 {
187 const PROPVARIANT &prop = coderProps[i];
188 PROPID propID = propIDs[i];
189 if (propID == NCoderPropID::kExpectedDataSize)
190 if (prop.vt == VT_UI8)
191 LzmaEnc_SetDataSize(_encoder, prop.uhVal.QuadPart);
192 }
193 return S_OK;
194 }
195
196
WriteCoderProperties(ISequentialOutStream * outStream)197 STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream)
198 {
199 Byte props[LZMA_PROPS_SIZE];
200 size_t size = LZMA_PROPS_SIZE;
201 RINOK(LzmaEnc_WriteProperties(_encoder, props, &size));
202 return WriteStream(outStream, props, size);
203 }
204
205
206 #define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \
207 if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes;
208
209
210
211 #ifdef LOG_LZMA_THREADS
212
GetTime64(const FILETIME & t)213 static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; }
214
PrintNum(UInt64 val,unsigned numDigits,char c=' ')215 static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ')
216 {
217 char temp[64];
218 char *p = temp + 32;
219 ConvertUInt64ToString(val, p);
220 unsigned len = (unsigned)strlen(p);
221 for (; len < numDigits; len++)
222 *--p = c;
223 printf("%s", p);
224 }
225
PrintTime(const char * s,UInt64 val,UInt64 total)226 static void PrintTime(const char *s, UInt64 val, UInt64 total)
227 {
228 printf(" %s :", s);
229 const UInt32 kFreq = 10000000;
230 UInt64 sec = val / kFreq;
231 PrintNum(sec, 6);
232 printf(" .");
233 UInt32 ms = (UInt32)(val - (sec * kFreq)) / (kFreq / 1000);
234 PrintNum(ms, 3, '0');
235
236 while (val > ((UInt64)1 << 56))
237 {
238 val >>= 1;
239 total >>= 1;
240 }
241
242 UInt64 percent = 0;
243 if (total != 0)
244 percent = val * 100 / total;
245 printf(" =");
246 PrintNum(percent, 4);
247 printf("%%");
248 }
249
250
251 struct CBaseStat
252 {
253 UInt64 kernelTime, userTime;
254
GetNCompress::NLzma::CBaseStat255 BOOL Get(HANDLE thread, const CBaseStat *prevStat)
256 {
257 FILETIME creationTimeFT, exitTimeFT, kernelTimeFT, userTimeFT;
258 BOOL res = GetThreadTimes(thread
259 , &creationTimeFT, &exitTimeFT, &kernelTimeFT, &userTimeFT);
260 if (res)
261 {
262 kernelTime = GetTime64(kernelTimeFT);
263 userTime = GetTime64(userTimeFT);
264 if (prevStat)
265 {
266 kernelTime -= prevStat->kernelTime;
267 userTime -= prevStat->userTime;
268 }
269 }
270 return res;
271 }
272 };
273
274
PrintStat(HANDLE thread,UInt64 totalTime,const CBaseStat * prevStat)275 static void PrintStat(HANDLE thread, UInt64 totalTime, const CBaseStat *prevStat)
276 {
277 CBaseStat newStat;
278 if (!newStat.Get(thread, prevStat))
279 return;
280
281 PrintTime("K", newStat.kernelTime, totalTime);
282
283 const UInt64 processTime = newStat.kernelTime + newStat.userTime;
284
285 PrintTime("U", newStat.userTime, totalTime);
286 PrintTime("S", processTime, totalTime);
287 printf("\n");
288 // PrintTime("G ", totalTime, totalTime);
289 }
290
291 #endif
292
293
294
Code(ISequentialInStream * inStream,ISequentialOutStream * outStream,const UInt64 *,const UInt64 *,ICompressProgressInfo * progress)295 STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
296 const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress)
297 {
298 CSeqInStreamWrap inWrap;
299 CSeqOutStreamWrap outWrap;
300 CCompressProgressWrap progressWrap;
301
302 inWrap.Init(inStream);
303 outWrap.Init(outStream);
304 progressWrap.Init(progress);
305
306 #ifdef LOG_LZMA_THREADS
307
308 FILETIME startTimeFT;
309 NWindows::NTime::GetCurUtcFileTime(startTimeFT);
310 UInt64 totalTime = GetTime64(startTimeFT);
311 CBaseStat oldStat;
312 if (!oldStat.Get(GetCurrentThread(), NULL))
313 return E_FAIL;
314
315 #endif
316
317
318 SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt,
319 progress ? &progressWrap.vt : NULL, &g_AlignedAlloc, &g_BigAlloc);
320
321 _inputProcessed = inWrap.Processed;
322
323 RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ)
324 RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE)
325 RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS)
326
327
328 #ifdef LOG_LZMA_THREADS
329
330 NWindows::NTime::GetCurUtcFileTime(startTimeFT);
331 totalTime = GetTime64(startTimeFT) - totalTime;
332 HANDLE lz_threads[2];
333 LzmaEnc_GetLzThreads(_encoder, lz_threads);
334 printf("\n");
335 printf("Main: "); PrintStat(GetCurrentThread(), totalTime, &oldStat);
336 printf("Hash: "); PrintStat(lz_threads[0], totalTime, NULL);
337 printf("BinT: "); PrintStat(lz_threads[1], totalTime, NULL);
338 // PrintTime("Total: ", totalTime, totalTime);
339 printf("\n");
340
341 #endif
342
343 return SResToHRESULT(res);
344 }
345
346 }}
347