1 /* XzEnc.c -- Xz Encode
2 2015-09-16 : Igor Pavlov : Public domain */
3
4 #include "Precomp.h"
5
6 #include <stdlib.h>
7 #include <string.h>
8
9 #include "7zCrc.h"
10 #include "Alloc.h"
11 #include "Bra.h"
12 #include "CpuArch.h"
13
14 #ifdef USE_SUBBLOCK
15 #include "Bcj3Enc.c"
16 #include "SbFind.c"
17 #include "SbEnc.c"
18 #endif
19
20 #include "XzEnc.h"
21
22 #define XzBlock_ClearFlags(p) (p)->flags = 0;
23 #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);
24 #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE;
25 #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;
26
WriteBytes(ISeqOutStream * s,const void * buf,UInt32 size)27 static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size)
28 {
29 return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE;
30 }
31
WriteBytesAndCrc(ISeqOutStream * s,const void * buf,UInt32 size,UInt32 * crc)32 static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc)
33 {
34 *crc = CrcUpdate(*crc, buf, size);
35 return WriteBytes(s, buf, size);
36 }
37
Xz_WriteHeader(CXzStreamFlags f,ISeqOutStream * s)38 static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s)
39 {
40 UInt32 crc;
41 Byte header[XZ_STREAM_HEADER_SIZE];
42 memcpy(header, XZ_SIG, XZ_SIG_SIZE);
43 header[XZ_SIG_SIZE] = (Byte)(f >> 8);
44 header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF);
45 crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE);
46 SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc);
47 return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE);
48 }
49
50
XzBlock_WriteHeader(const CXzBlock * p,ISeqOutStream * s)51 static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s)
52 {
53 Byte header[XZ_BLOCK_HEADER_SIZE_MAX];
54
55 unsigned pos = 1;
56 unsigned numFilters, i;
57 header[pos++] = p->flags;
58
59 if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize);
60 if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize);
61 numFilters = XzBlock_GetNumFilters(p);
62
63 for (i = 0; i < numFilters; i++)
64 {
65 const CXzFilter *f = &p->filters[i];
66 pos += Xz_WriteVarInt(header + pos, f->id);
67 pos += Xz_WriteVarInt(header + pos, f->propsSize);
68 memcpy(header + pos, f->props, f->propsSize);
69 pos += f->propsSize;
70 }
71
72 while ((pos & 3) != 0)
73 header[pos++] = 0;
74
75 header[0] = (Byte)(pos >> 2);
76 SetUi32(header + pos, CrcCalc(header, pos));
77 return WriteBytes(s, header, pos + 4);
78 }
79
80
Xz_WriteFooter(CXzStream * p,ISeqOutStream * s)81 static SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s)
82 {
83 Byte buf[32];
84 UInt64 globalPos;
85 {
86 UInt32 crc = CRC_INIT_VAL;
87 unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks);
88 size_t i;
89
90 globalPos = pos;
91 buf[0] = 0;
92 RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
93
94 for (i = 0; i < p->numBlocks; i++)
95 {
96 const CXzBlockSizes *block = &p->blocks[i];
97 pos = Xz_WriteVarInt(buf, block->totalSize);
98 pos += Xz_WriteVarInt(buf + pos, block->unpackSize);
99 globalPos += pos;
100 RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
101 }
102
103 pos = ((unsigned)globalPos & 3);
104
105 if (pos != 0)
106 {
107 buf[0] = buf[1] = buf[2] = 0;
108 RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc));
109 globalPos += 4 - pos;
110 }
111 {
112 SetUi32(buf, CRC_GET_DIGEST(crc));
113 RINOK(WriteBytes(s, buf, 4));
114 globalPos += 4;
115 }
116 }
117
118 {
119 UInt32 indexSize = (UInt32)((globalPos >> 2) - 1);
120 SetUi32(buf + 4, indexSize);
121 buf[8] = (Byte)(p->flags >> 8);
122 buf[9] = (Byte)(p->flags & 0xFF);
123 SetUi32(buf, CrcCalc(buf + 4, 6));
124 memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE);
125 return WriteBytes(s, buf, 12);
126 }
127 }
128
129
Xz_AddIndexRecord(CXzStream * p,UInt64 unpackSize,UInt64 totalSize,ISzAlloc * alloc)130 static SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc)
131 {
132 if (!p->blocks || p->numBlocksAllocated == p->numBlocks)
133 {
134 size_t num = p->numBlocks * 2 + 1;
135 size_t newSize = sizeof(CXzBlockSizes) * num;
136 CXzBlockSizes *blocks;
137 if (newSize / sizeof(CXzBlockSizes) != num)
138 return SZ_ERROR_MEM;
139 blocks = (CXzBlockSizes *)alloc->Alloc(alloc, newSize);
140 if (!blocks)
141 return SZ_ERROR_MEM;
142 if (p->numBlocks != 0)
143 {
144 memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes));
145 alloc->Free(alloc, p->blocks);
146 }
147 p->blocks = blocks;
148 p->numBlocksAllocated = num;
149 }
150 {
151 CXzBlockSizes *block = &p->blocks[p->numBlocks++];
152 block->unpackSize = unpackSize;
153 block->totalSize = totalSize;
154 }
155 return SZ_OK;
156 }
157
158
159 /* ---------- CSeqCheckInStream ---------- */
160
161 typedef struct
162 {
163 ISeqInStream p;
164 ISeqInStream *realStream;
165 UInt64 processed;
166 CXzCheck check;
167 } CSeqCheckInStream;
168
SeqCheckInStream_Init(CSeqCheckInStream * p,unsigned mode)169 static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned mode)
170 {
171 p->processed = 0;
172 XzCheck_Init(&p->check, mode);
173 }
174
SeqCheckInStream_GetDigest(CSeqCheckInStream * p,Byte * digest)175 static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)
176 {
177 XzCheck_Final(&p->check, digest);
178 }
179
SeqCheckInStream_Read(void * pp,void * data,size_t * size)180 static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size)
181 {
182 CSeqCheckInStream *p = (CSeqCheckInStream *)pp;
183 SRes res = p->realStream->Read(p->realStream, data, size);
184 XzCheck_Update(&p->check, data, *size);
185 p->processed += *size;
186 return res;
187 }
188
189
190 /* ---------- CSeqSizeOutStream ---------- */
191
192 typedef struct
193 {
194 ISeqOutStream p;
195 ISeqOutStream *realStream;
196 UInt64 processed;
197 } CSeqSizeOutStream;
198
MyWrite(void * pp,const void * data,size_t size)199 static size_t MyWrite(void *pp, const void *data, size_t size)
200 {
201 CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp;
202 size = p->realStream->Write(p->realStream, data, size);
203 p->processed += size;
204 return size;
205 }
206
207
208 /* ---------- CSeqInFilter ---------- */
209
210 #define FILTER_BUF_SIZE (1 << 20)
211
212 typedef struct
213 {
214 ISeqInStream p;
215 ISeqInStream *realStream;
216 IStateCoder StateCoder;
217 Byte *buf;
218 size_t curPos;
219 size_t endPos;
220 int srcWasFinished;
221 } CSeqInFilter;
222
SeqInFilter_Read(void * pp,void * data,size_t * size)223 static SRes SeqInFilter_Read(void *pp, void *data, size_t *size)
224 {
225 CSeqInFilter *p = (CSeqInFilter *)pp;
226 size_t sizeOriginal = *size;
227 if (sizeOriginal == 0)
228 return SZ_OK;
229 *size = 0;
230
231 for (;;)
232 {
233 if (!p->srcWasFinished && p->curPos == p->endPos)
234 {
235 p->curPos = 0;
236 p->endPos = FILTER_BUF_SIZE;
237 RINOK(p->realStream->Read(p->realStream, p->buf, &p->endPos));
238 if (p->endPos == 0)
239 p->srcWasFinished = 1;
240 }
241 {
242 SizeT srcLen = p->endPos - p->curPos;
243 int wasFinished;
244 SRes res;
245 *size = sizeOriginal;
246 res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen,
247 p->srcWasFinished, CODER_FINISH_ANY, &wasFinished);
248 p->curPos += srcLen;
249 if (*size != 0 || srcLen == 0 || res != 0)
250 return res;
251 }
252 }
253 }
254
SeqInFilter_Construct(CSeqInFilter * p)255 static void SeqInFilter_Construct(CSeqInFilter *p)
256 {
257 p->buf = NULL;
258 p->p.Read = SeqInFilter_Read;
259 }
260
SeqInFilter_Free(CSeqInFilter * p)261 static void SeqInFilter_Free(CSeqInFilter *p)
262 {
263 if (p->buf)
264 {
265 g_Alloc.Free(&g_Alloc, p->buf);
266 p->buf = NULL;
267 }
268 }
269
270 SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc);
271
SeqInFilter_Init(CSeqInFilter * p,const CXzFilter * props)272 static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props)
273 {
274 if (!p->buf)
275 {
276 p->buf = g_Alloc.Alloc(&g_Alloc, FILTER_BUF_SIZE);
277 if (!p->buf)
278 return SZ_ERROR_MEM;
279 }
280 p->curPos = p->endPos = 0;
281 p->srcWasFinished = 0;
282 RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc));
283 RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc));
284 p->StateCoder.Init(p->StateCoder.p);
285 return SZ_OK;
286 }
287
288
289 /* ---------- CSbEncInStream ---------- */
290
291 #ifdef USE_SUBBLOCK
292
293 typedef struct
294 {
295 ISeqInStream p;
296 ISeqInStream *inStream;
297 CSbEnc enc;
298 } CSbEncInStream;
299
SbEncInStream_Read(void * pp,void * data,size_t * size)300 static SRes SbEncInStream_Read(void *pp, void *data, size_t *size)
301 {
302 CSbEncInStream *p = (CSbEncInStream *)pp;
303 size_t sizeOriginal = *size;
304 if (sizeOriginal == 0)
305 return S_OK;
306
307 for (;;)
308 {
309 if (p->enc.needRead && !p->enc.readWasFinished)
310 {
311 size_t processed = p->enc.needReadSizeMax;
312 RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed));
313 p->enc.readPos += processed;
314 if (processed == 0)
315 {
316 p->enc.readWasFinished = True;
317 p->enc.isFinalFinished = True;
318 }
319 p->enc.needRead = False;
320 }
321
322 *size = sizeOriginal;
323 RINOK(SbEnc_Read(&p->enc, data, size));
324 if (*size != 0 || !p->enc.needRead)
325 return S_OK;
326 }
327 }
328
SbEncInStream_Construct(CSbEncInStream * p,ISzAlloc * alloc)329 void SbEncInStream_Construct(CSbEncInStream *p, ISzAlloc *alloc)
330 {
331 SbEnc_Construct(&p->enc, alloc);
332 p->p.Read = SbEncInStream_Read;
333 }
334
SbEncInStream_Init(CSbEncInStream * p)335 SRes SbEncInStream_Init(CSbEncInStream *p)
336 {
337 return SbEnc_Init(&p->enc);
338 }
339
SbEncInStream_Free(CSbEncInStream * p)340 void SbEncInStream_Free(CSbEncInStream *p)
341 {
342 SbEnc_Free(&p->enc);
343 }
344
345 #endif
346
347
348 typedef struct
349 {
350 CLzma2EncHandle lzma2;
351 #ifdef USE_SUBBLOCK
352 CSbEncInStream sb;
353 #endif
354 CSeqInFilter filter;
355 ISzAlloc *alloc;
356 ISzAlloc *bigAlloc;
357 } CLzma2WithFilters;
358
359
Lzma2WithFilters_Construct(CLzma2WithFilters * p,ISzAlloc * alloc,ISzAlloc * bigAlloc)360 static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc)
361 {
362 p->alloc = alloc;
363 p->bigAlloc = bigAlloc;
364 p->lzma2 = NULL;
365 #ifdef USE_SUBBLOCK
366 SbEncInStream_Construct(&p->sb, alloc);
367 #endif
368 SeqInFilter_Construct(&p->filter);
369 }
370
Lzma2WithFilters_Create(CLzma2WithFilters * p)371 static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p)
372 {
373 p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc);
374 if (!p->lzma2)
375 return SZ_ERROR_MEM;
376 return SZ_OK;
377 }
378
Lzma2WithFilters_Free(CLzma2WithFilters * p)379 static void Lzma2WithFilters_Free(CLzma2WithFilters *p)
380 {
381 SeqInFilter_Free(&p->filter);
382 #ifdef USE_SUBBLOCK
383 SbEncInStream_Free(&p->sb);
384 #endif
385 if (p->lzma2)
386 {
387 Lzma2Enc_Destroy(p->lzma2);
388 p->lzma2 = NULL;
389 }
390 }
391
392
XzProps_Init(CXzProps * p)393 void XzProps_Init(CXzProps *p)
394 {
395 p->lzma2Props = NULL;
396 p->filterProps = NULL;
397 p->checkId = XZ_CHECK_CRC32;
398 }
399
XzFilterProps_Init(CXzFilterProps * p)400 void XzFilterProps_Init(CXzFilterProps *p)
401 {
402 p->id = 0;
403 p->delta = 0;
404 p->ip = 0;
405 p->ipDefined = False;
406 }
407
408
Xz_Compress(CXzStream * xz,CLzma2WithFilters * lzmaf,ISeqOutStream * outStream,ISeqInStream * inStream,const CXzProps * props,ICompressProgress * progress)409 static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf,
410 ISeqOutStream *outStream, ISeqInStream *inStream,
411 const CXzProps *props, ICompressProgress *progress)
412 {
413 xz->flags = (Byte)props->checkId;
414
415 RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props));
416 RINOK(Xz_WriteHeader(xz->flags, outStream));
417
418 {
419 CSeqCheckInStream checkInStream;
420 CSeqSizeOutStream seqSizeOutStream;
421 CXzBlock block;
422 unsigned filterIndex = 0;
423 CXzFilter *filter = NULL;
424 const CXzFilterProps *fp = props->filterProps;
425
426 XzBlock_ClearFlags(&block);
427 XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0));
428
429 if (fp)
430 {
431 filter = &block.filters[filterIndex++];
432 filter->id = fp->id;
433 filter->propsSize = 0;
434
435 if (fp->id == XZ_ID_Delta)
436 {
437 filter->props[0] = (Byte)(fp->delta - 1);
438 filter->propsSize = 1;
439 }
440 else if (fp->ipDefined)
441 {
442 SetUi32(filter->props, fp->ip);
443 filter->propsSize = 4;
444 }
445 }
446
447 {
448 CXzFilter *f = &block.filters[filterIndex++];
449 f->id = XZ_ID_LZMA2;
450 f->propsSize = 1;
451 f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2);
452 }
453
454 seqSizeOutStream.p.Write = MyWrite;
455 seqSizeOutStream.realStream = outStream;
456 seqSizeOutStream.processed = 0;
457
458 RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p));
459
460 checkInStream.p.Read = SeqCheckInStream_Read;
461 checkInStream.realStream = inStream;
462 SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags));
463
464 if (fp)
465 {
466 #ifdef USE_SUBBLOCK
467 if (fp->id == XZ_ID_Subblock)
468 {
469 lzmaf->sb.inStream = &checkInStream.p;
470 RINOK(SbEncInStream_Init(&lzmaf->sb));
471 }
472 else
473 #endif
474 {
475 lzmaf->filter.realStream = &checkInStream.p;
476 RINOK(SeqInFilter_Init(&lzmaf->filter, filter));
477 }
478 }
479
480 {
481 UInt64 packPos = seqSizeOutStream.processed;
482
483 SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p,
484 fp ?
485 #ifdef USE_SUBBLOCK
486 (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.p:
487 #endif
488 &lzmaf->filter.p:
489 &checkInStream.p,
490 progress);
491
492 RINOK(res);
493 block.unpackSize = checkInStream.processed;
494 block.packSize = seqSizeOutStream.processed - packPos;
495 }
496
497 {
498 unsigned padSize = 0;
499 Byte buf[128];
500 while ((((unsigned)block.packSize + padSize) & 3) != 0)
501 buf[padSize++] = 0;
502 SeqCheckInStream_GetDigest(&checkInStream, buf + padSize);
503 RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags)));
504 RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc));
505 }
506 }
507 return Xz_WriteFooter(xz, outStream);
508 }
509
510
Xz_Encode(ISeqOutStream * outStream,ISeqInStream * inStream,const CXzProps * props,ICompressProgress * progress)511 SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
512 const CXzProps *props, ICompressProgress *progress)
513 {
514 SRes res;
515 CXzStream xz;
516 CLzma2WithFilters lzmaf;
517 Xz_Construct(&xz);
518 Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc);
519 res = Lzma2WithFilters_Create(&lzmaf);
520 if (res == SZ_OK)
521 res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress);
522 Lzma2WithFilters_Free(&lzmaf);
523 Xz_Free(&xz, &g_Alloc);
524 return res;
525 }
526
527
Xz_EncodeEmpty(ISeqOutStream * outStream)528 SRes Xz_EncodeEmpty(ISeqOutStream *outStream)
529 {
530 SRes res;
531 CXzStream xz;
532 Xz_Construct(&xz);
533 res = Xz_WriteHeader(xz.flags, outStream);
534 if (res == SZ_OK)
535 res = Xz_WriteFooter(&xz, outStream);
536 Xz_Free(&xz, &g_Alloc);
537 return res;
538 }
539