1 /*
2 LZ4 auto-framing library
3 Copyright (C) 2011-2016, Yann Collet.
4
5 BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are
9 met:
10
11 * Redistributions of source code must retain the above copyright
12 notice, this list of conditions and the following disclaimer.
13 * Redistributions in binary form must reproduce the above
14 copyright notice, this list of conditions and the following disclaimer
15 in the documentation and/or other materials provided with the
16 distribution.
17
18 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 You can contact the author at :
31 - LZ4 homepage : http://www.lz4.org
32 - LZ4 source repository : https://github.com/lz4/lz4
33 */
34
35 /* LZ4F is a stand-alone API to create LZ4-compressed Frames
36 * in full conformance with specification v1.5.0
37 * All related operations, including memory management, are handled by the library.
38 * */
39
40
41 /*-************************************
42 * Compiler Options
43 **************************************/
44 #ifdef _MSC_VER /* Visual Studio */
45 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
46 #endif
47
48
49 /*-************************************
50 * Memory routines
51 **************************************/
52 #include <stdlib.h> /* malloc, calloc, free */
53 #define ALLOCATOR(s) calloc(1,s)
54 #define FREEMEM free
55 #include <string.h> /* memset, memcpy, memmove */
56 #define MEM_INIT memset
57
58
59 /*-************************************
60 * Includes
61 **************************************/
62 #include "lz4frame_static.h"
63 #include "lz4.h"
64 #define LZ4_HC_STATIC_LINKING_ONLY
65 #include "lz4hc.h"
66 #define XXH_STATIC_LINKING_ONLY
67 #include "xxhash.h"
68
69
70 /*-************************************
71 * Debug
72 **************************************/
73 #define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
74
75
76 /*-************************************
77 * Basic Types
78 **************************************/
79 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
80 # include <stdint.h>
81 typedef uint8_t BYTE;
82 typedef uint16_t U16;
83 typedef uint32_t U32;
84 typedef int32_t S32;
85 typedef uint64_t U64;
86 #else
87 typedef unsigned char BYTE;
88 typedef unsigned short U16;
89 typedef unsigned int U32;
90 typedef signed int S32;
91 typedef unsigned long long U64;
92 #endif
93
94
95 /* unoptimized version; solves endianess & alignment issues */
LZ4F_readLE32(const void * src)96 static U32 LZ4F_readLE32 (const void* src)
97 {
98 const BYTE* const srcPtr = (const BYTE*)src;
99 U32 value32 = srcPtr[0];
100 value32 += (srcPtr[1]<<8);
101 value32 += (srcPtr[2]<<16);
102 value32 += ((U32)srcPtr[3])<<24;
103 return value32;
104 }
105
LZ4F_writeLE32(void * dst,U32 value32)106 static void LZ4F_writeLE32 (void* dst, U32 value32)
107 {
108 BYTE* const dstPtr = (BYTE*)dst;
109 dstPtr[0] = (BYTE)value32;
110 dstPtr[1] = (BYTE)(value32 >> 8);
111 dstPtr[2] = (BYTE)(value32 >> 16);
112 dstPtr[3] = (BYTE)(value32 >> 24);
113 }
114
LZ4F_readLE64(const void * src)115 static U64 LZ4F_readLE64 (const void* src)
116 {
117 const BYTE* const srcPtr = (const BYTE*)src;
118 U64 value64 = srcPtr[0];
119 value64 += ((U64)srcPtr[1]<<8);
120 value64 += ((U64)srcPtr[2]<<16);
121 value64 += ((U64)srcPtr[3]<<24);
122 value64 += ((U64)srcPtr[4]<<32);
123 value64 += ((U64)srcPtr[5]<<40);
124 value64 += ((U64)srcPtr[6]<<48);
125 value64 += ((U64)srcPtr[7]<<56);
126 return value64;
127 }
128
LZ4F_writeLE64(void * dst,U64 value64)129 static void LZ4F_writeLE64 (void* dst, U64 value64)
130 {
131 BYTE* const dstPtr = (BYTE*)dst;
132 dstPtr[0] = (BYTE)value64;
133 dstPtr[1] = (BYTE)(value64 >> 8);
134 dstPtr[2] = (BYTE)(value64 >> 16);
135 dstPtr[3] = (BYTE)(value64 >> 24);
136 dstPtr[4] = (BYTE)(value64 >> 32);
137 dstPtr[5] = (BYTE)(value64 >> 40);
138 dstPtr[6] = (BYTE)(value64 >> 48);
139 dstPtr[7] = (BYTE)(value64 >> 56);
140 }
141
142
143 /*-************************************
144 * Constants
145 **************************************/
146 #define KB *(1<<10)
147 #define MB *(1<<20)
148 #define GB *(1<<30)
149
150 #define _1BIT 0x01
151 #define _2BITS 0x03
152 #define _3BITS 0x07
153 #define _4BITS 0x0F
154 #define _8BITS 0xFF
155
156 #define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
157 #define LZ4F_MAGICNUMBER 0x184D2204U
158 #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
159 #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
160
161 static const size_t minFHSize = 7;
162 static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */
163 static const size_t BHSize = 4;
164
165
166 /*-************************************
167 * Structures and local types
168 **************************************/
169 typedef struct LZ4F_cctx_s
170 {
171 LZ4F_preferences_t prefs;
172 U32 version;
173 U32 cStage;
174 const LZ4F_CDict* cdict;
175 size_t maxBlockSize;
176 size_t maxBufferSize;
177 BYTE* tmpBuff;
178 BYTE* tmpIn;
179 size_t tmpInSize;
180 U64 totalInSize;
181 XXH32_state_t xxh;
182 void* lz4CtxPtr;
183 U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */
184 } LZ4F_cctx_t;
185
186
187 /*-************************************
188 * Error management
189 **************************************/
190 #define LZ4F_GENERATE_STRING(STRING) #STRING,
191 static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) };
192
193
LZ4F_isError(LZ4F_errorCode_t code)194 unsigned LZ4F_isError(LZ4F_errorCode_t code)
195 {
196 return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode));
197 }
198
LZ4F_getErrorName(LZ4F_errorCode_t code)199 const char* LZ4F_getErrorName(LZ4F_errorCode_t code)
200 {
201 static const char* codeError = "Unspecified error code";
202 if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)];
203 return codeError;
204 }
205
LZ4F_getErrorCode(size_t functionResult)206 LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
207 {
208 if (!LZ4F_isError(functionResult)) return LZ4F_OK_NoError;
209 return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult);
210 }
211
err0r(LZ4F_errorCodes code)212 static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
213 {
214 /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
215 LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
216 return (LZ4F_errorCode_t)-(ptrdiff_t)code;
217 }
218
LZ4F_getVersion(void)219 unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
220
LZ4F_compressionLevel_max(void)221 int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
222
223
224 /*-************************************
225 * Private functions
226 **************************************/
227 #define MIN(a,b) ( (a) < (b) ? (a) : (b) )
228
LZ4F_getBlockSize(unsigned blockSizeID)229 static size_t LZ4F_getBlockSize(unsigned blockSizeID)
230 {
231 static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB };
232
233 if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
234 blockSizeID -= 4;
235 if (blockSizeID > 3) return err0r(LZ4F_ERROR_maxBlockSize_invalid);
236 return blockSizes[blockSizeID];
237 }
238
LZ4F_headerChecksum(const void * header,size_t length)239 static BYTE LZ4F_headerChecksum (const void* header, size_t length)
240 {
241 U32 const xxh = XXH32(header, length, 0);
242 return (BYTE)(xxh >> 8);
243 }
244
245
246 /*-************************************
247 * Simple-pass compression functions
248 **************************************/
LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,const size_t srcSize)249 static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,
250 const size_t srcSize)
251 {
252 LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB;
253 size_t maxBlockSize = 64 KB;
254 while (requestedBSID > proposedBSID) {
255 if (srcSize <= maxBlockSize)
256 return proposedBSID;
257 proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1);
258 maxBlockSize <<= 2;
259 }
260 return requestedBSID;
261 }
262
263 /*! LZ4F_compressBound_internal() :
264 * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
265 * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
266 * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
267 * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
268 */
LZ4F_compressBound_internal(size_t srcSize,const LZ4F_preferences_t * preferencesPtr,size_t alreadyBuffered)269 static size_t LZ4F_compressBound_internal(size_t srcSize,
270 const LZ4F_preferences_t* preferencesPtr,
271 size_t alreadyBuffered)
272 {
273 LZ4F_preferences_t prefsNull;
274 memset(&prefsNull, 0, sizeof(prefsNull));
275 prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */
276 { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
277 U32 const flush = prefsPtr->autoFlush | (srcSize==0);
278 LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID;
279 size_t const blockSize = LZ4F_getBlockSize(blockID);
280 size_t const maxBuffered = blockSize - 1;
281 size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered);
282 size_t const maxSrcSize = srcSize + bufferedSize;
283 unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize);
284 size_t const partialBlockSize = (srcSize - (srcSize==0)) & (blockSize-1); /* 0 => -1 == MAX => blockSize-1 */
285 size_t const lastBlockSize = flush ? partialBlockSize : 0;
286 unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);
287
288 size_t const blockHeaderSize = 4;
289 size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag;
290 size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
291
292 return ((blockHeaderSize + blockCRCSize) * nbBlocks) +
293 (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;
294 }
295 }
296
LZ4F_compressFrameBound(size_t srcSize,const LZ4F_preferences_t * preferencesPtr)297 size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
298 {
299 LZ4F_preferences_t prefs;
300 size_t const headerSize = maxFHSize; /* max header size, including optional fields */
301
302 if (preferencesPtr!=NULL) prefs = *preferencesPtr;
303 else memset(&prefs, 0, sizeof(prefs));
304 prefs.autoFlush = 1;
305
306 return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);;
307 }
308
309
310 /*! LZ4F_compressFrame_usingCDict() :
311 * Compress srcBuffer using a dictionary, in a single step.
312 * cdict can be NULL, in which case, no dictionary is used.
313 * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
314 * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
315 * however, it's the only way to provide a dictID, so it's not recommended.
316 * @return : number of bytes written into dstBuffer,
317 * or an error code if it fails (can be tested using LZ4F_isError())
318 */
LZ4F_compressFrame_usingCDict(void * dstBuffer,size_t dstCapacity,const void * srcBuffer,size_t srcSize,const LZ4F_CDict * cdict,const LZ4F_preferences_t * preferencesPtr)319 size_t LZ4F_compressFrame_usingCDict(void* dstBuffer, size_t dstCapacity,
320 const void* srcBuffer, size_t srcSize,
321 const LZ4F_CDict* cdict,
322 const LZ4F_preferences_t* preferencesPtr)
323 {
324 LZ4F_cctx_t cctxI;
325 LZ4_stream_t lz4ctx;
326 LZ4F_preferences_t prefs;
327 LZ4F_compressOptions_t options;
328 BYTE* const dstStart = (BYTE*) dstBuffer;
329 BYTE* dstPtr = dstStart;
330 BYTE* const dstEnd = dstStart + dstCapacity;
331
332 memset(&cctxI, 0, sizeof(cctxI));
333 cctxI.version = LZ4F_VERSION;
334 cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
335
336 if (preferencesPtr!=NULL)
337 prefs = *preferencesPtr;
338 else
339 memset(&prefs, 0, sizeof(prefs));
340 if (prefs.frameInfo.contentSize != 0)
341 prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */
342
343 prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
344 prefs.autoFlush = 1;
345 if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
346 prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */
347
348 if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
349 cctxI.lz4CtxPtr = &lz4ctx;
350 cctxI.lz4CtxLevel = 1;
351 } /* fast compression context pre-created on stack */
352
353 memset(&options, 0, sizeof(options));
354 options.stableSrc = 1;
355
356 if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */
357 return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
358
359 { size_t const headerSize = LZ4F_compressBegin_usingCDict(&cctxI, dstBuffer, dstCapacity, cdict, &prefs); /* write header */
360 if (LZ4F_isError(headerSize)) return headerSize;
361 dstPtr += headerSize; /* header size */ }
362
363 { size_t const cSize = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
364 if (LZ4F_isError(cSize)) return cSize;
365 dstPtr += cSize; }
366
367 { size_t const tailSize = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */
368 if (LZ4F_isError(tailSize)) return tailSize;
369 dstPtr += tailSize; }
370
371 if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* Ctx allocation only for lz4hc */
372 FREEMEM(cctxI.lz4CtxPtr);
373
374 return (dstPtr - dstStart);
375 }
376
377
378 /*! LZ4F_compressFrame() :
379 * Compress an entire srcBuffer into a valid LZ4 frame, in a single step.
380 * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
381 * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
382 * @return : number of bytes written into dstBuffer.
383 * or an error code if it fails (can be tested using LZ4F_isError())
384 */
LZ4F_compressFrame(void * dstBuffer,size_t dstCapacity,const void * srcBuffer,size_t srcSize,const LZ4F_preferences_t * preferencesPtr)385 size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
386 const void* srcBuffer, size_t srcSize,
387 const LZ4F_preferences_t* preferencesPtr)
388 {
389 return LZ4F_compressFrame_usingCDict(dstBuffer, dstCapacity,
390 srcBuffer, srcSize,
391 NULL, preferencesPtr);
392 }
393
394
395 /*-***************************************************
396 * Dictionary compression
397 *****************************************************/
398
399 struct LZ4F_CDict_s {
400 void* dictContent;
401 LZ4_stream_t* fastCtx;
402 LZ4_streamHC_t* HCCtx;
403 }; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
404
405 /*! LZ4F_createCDict() :
406 * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
407 * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
408 * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
409 * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
410 * @return : digested dictionary for compression, or NULL if failed */
LZ4F_createCDict(const void * dictBuffer,size_t dictSize)411 LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
412 {
413 const char* dictStart = (const char*)dictBuffer;
414 LZ4F_CDict* cdict = (LZ4F_CDict*) malloc(sizeof(*cdict));
415 if (!cdict) return NULL;
416 if (dictSize > 64 KB) {
417 dictStart += dictSize - 64 KB;
418 dictSize = 64 KB;
419 }
420 cdict->dictContent = ALLOCATOR(dictSize);
421 cdict->fastCtx = LZ4_createStream();
422 cdict->HCCtx = LZ4_createStreamHC();
423 if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
424 LZ4F_freeCDict(cdict);
425 return NULL;
426 }
427 memcpy(cdict->dictContent, dictStart, dictSize);
428 LZ4_resetStream(cdict->fastCtx);
429 LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
430 LZ4_resetStreamHC(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
431 LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
432 return cdict;
433 }
434
LZ4F_freeCDict(LZ4F_CDict * cdict)435 void LZ4F_freeCDict(LZ4F_CDict* cdict)
436 {
437 if (cdict==NULL) return; /* support free on NULL */
438 FREEMEM(cdict->dictContent);
439 LZ4_freeStream(cdict->fastCtx);
440 LZ4_freeStreamHC(cdict->HCCtx);
441 FREEMEM(cdict);
442 }
443
444
445 /*-*********************************
446 * Advanced compression functions
447 ***********************************/
448
449 /*! LZ4F_createCompressionContext() :
450 * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
451 * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
452 * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries.
453 * The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
454 * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
455 * Object can release its memory using LZ4F_freeCompressionContext();
456 */
LZ4F_createCompressionContext(LZ4F_compressionContext_t * LZ4F_compressionContextPtr,unsigned version)457 LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
458 {
459 LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t));
460 if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
461
462 cctxPtr->version = version;
463 cctxPtr->cStage = 0; /* Next stage : init stream */
464
465 *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr;
466
467 return LZ4F_OK_NoError;
468 }
469
470
LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext)471 LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext)
472 {
473 LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext;
474
475 if (cctxPtr != NULL) { /* support free on NULL */
476 FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */
477 FREEMEM(cctxPtr->tmpBuff);
478 FREEMEM(LZ4F_compressionContext);
479 }
480
481 return LZ4F_OK_NoError;
482 }
483
484
485 /*! LZ4F_compressBegin_usingCDict() :
486 * init streaming compression and writes frame header into dstBuffer.
487 * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
488 * @return : number of bytes written into dstBuffer for the header
489 * or an error code (can be tested using LZ4F_isError())
490 */
LZ4F_compressBegin_usingCDict(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const LZ4F_CDict * cdict,const LZ4F_preferences_t * preferencesPtr)491 size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
492 void* dstBuffer, size_t dstCapacity,
493 const LZ4F_CDict* cdict,
494 const LZ4F_preferences_t* preferencesPtr)
495 {
496 LZ4F_preferences_t prefNull;
497 BYTE* const dstStart = (BYTE*)dstBuffer;
498 BYTE* dstPtr = dstStart;
499 BYTE* headerStart;
500
501 if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
502 memset(&prefNull, 0, sizeof(prefNull));
503 if (preferencesPtr == NULL) preferencesPtr = &prefNull;
504 cctxPtr->prefs = *preferencesPtr;
505
506 /* Ctx Management */
507 { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
508 if (cctxPtr->lz4CtxLevel < tableID) {
509 FREEMEM(cctxPtr->lz4CtxPtr);
510 if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
511 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
512 else
513 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
514 if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
515 cctxPtr->lz4CtxLevel = tableID;
516 } }
517
518 /* Buffer Management */
519 if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
520 cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
521 cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID);
522
523 { size_t const requiredBuffSize = preferencesPtr->autoFlush ?
524 (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */
525 cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
526
527 if (cctxPtr->maxBufferSize < requiredBuffSize) {
528 cctxPtr->maxBufferSize = 0;
529 FREEMEM(cctxPtr->tmpBuff);
530 cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize);
531 if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
532 cctxPtr->maxBufferSize = requiredBuffSize;
533 } }
534 cctxPtr->tmpIn = cctxPtr->tmpBuff;
535 cctxPtr->tmpInSize = 0;
536 XXH32_reset(&(cctxPtr->xxh), 0);
537
538 /* context init */
539 cctxPtr->cdict = cdict;
540 if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
541 /* frame init only for blockLinked : blockIndependent will be init at each block */
542 if (cdict) {
543 if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
544 memcpy(cctxPtr->lz4CtxPtr, cdict->fastCtx, sizeof(*cdict->fastCtx));
545 } else {
546 memcpy(cctxPtr->lz4CtxPtr, cdict->HCCtx, sizeof(*cdict->HCCtx));
547 LZ4_setCompressionLevel((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
548 }
549 } else {
550 if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
551 LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
552 else
553 LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
554 }
555 }
556
557 /* Magic Number */
558 LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
559 dstPtr += 4;
560 headerStart = dstPtr;
561
562 /* FLG Byte */
563 *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
564 + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
565 + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
566 + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
567 + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
568 + (cctxPtr->prefs.frameInfo.dictID > 0) );
569 /* BD Byte */
570 *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
571 /* Optional Frame content size field */
572 if (cctxPtr->prefs.frameInfo.contentSize) {
573 LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize);
574 dstPtr += 8;
575 cctxPtr->totalInSize = 0;
576 }
577 /* Optional dictionary ID field */
578 if (cctxPtr->prefs.frameInfo.dictID) {
579 LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
580 dstPtr += 4;
581 }
582 /* Header CRC Byte */
583 *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
584 dstPtr++;
585
586 cctxPtr->cStage = 1; /* header written, now request input data block */
587 return (dstPtr - dstStart);
588 }
589
590
591 /*! LZ4F_compressBegin() :
592 * init streaming compression and writes frame header into dstBuffer.
593 * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
594 * preferencesPtr can be NULL, in which case default parameters are selected.
595 * @return : number of bytes written into dstBuffer for the header
596 * or an error code (can be tested using LZ4F_isError())
597 */
LZ4F_compressBegin(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const LZ4F_preferences_t * preferencesPtr)598 size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,
599 void* dstBuffer, size_t dstCapacity,
600 const LZ4F_preferences_t* preferencesPtr)
601 {
602 return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity,
603 NULL, preferencesPtr);
604 }
605
606
607 /* LZ4F_compressBound() :
608 * @ return size of Dst buffer given a srcSize to handle worst case situations.
609 * The LZ4F_frameInfo_t structure is optional : if NULL, preferences will be set to cover worst case situations.
610 * This function cannot fail.
611 */
LZ4F_compressBound(size_t srcSize,const LZ4F_preferences_t * preferencesPtr)612 size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
613 {
614 return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1);
615 }
616
617
618 typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict);
619
620
621 /*! LZ4F_makeBlock():
622 * compress a single block, add header and checksum
623 * assumption : dst buffer capacity is >= srcSize */
LZ4F_makeBlock(void * dst,const void * src,size_t srcSize,compressFunc_t compress,void * lz4ctx,int level,const LZ4F_CDict * cdict,LZ4F_blockChecksum_t crcFlag)624 static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
625 compressFunc_t compress, void* lz4ctx, int level,
626 const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag)
627 {
628 BYTE* const cSizePtr = (BYTE*)dst;
629 U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
630 (int)(srcSize), (int)(srcSize-1),
631 level, cdict);
632 LZ4F_writeLE32(cSizePtr, cSize);
633 if (cSize == 0) { /* compression failed */
634 cSize = (U32)srcSize;
635 LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
636 memcpy(cSizePtr+4, src, srcSize);
637 }
638 if (crcFlag) {
639 U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */
640 LZ4F_writeLE32(cSizePtr+4+cSize, crc32);
641 }
642 return 4 + cSize + ((U32)crcFlag)*4;
643 }
644
645
LZ4F_compressBlock(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)646 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
647 {
648 int const acceleration = (level < -1) ? -level : 1;
649 if (cdict) {
650 memcpy(ctx, cdict->fastCtx, sizeof(*cdict->fastCtx));
651 return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
652 }
653 return LZ4_compress_fast_extState(ctx, src, dst, srcSize, dstCapacity, acceleration);
654 }
655
LZ4F_compressBlock_continue(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)656 static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
657 {
658 int const acceleration = (level < -1) ? -level : 1;
659 (void)cdict; /* init once at beginning of frame */
660 return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
661 }
662
LZ4F_compressBlockHC(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)663 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
664 {
665 if (cdict) {
666 memcpy(ctx, cdict->HCCtx, sizeof(*cdict->HCCtx));
667 LZ4_setCompressionLevel((LZ4_streamHC_t*)ctx, level);
668 return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
669 }
670 return LZ4_compress_HC_extStateHC(ctx, src, dst, srcSize, dstCapacity, level);
671 }
672
LZ4F_compressBlockHC_continue(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)673 static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
674 {
675 (void)level; (void)cdict; /* init once at beginning of frame */
676 return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
677 }
678
LZ4F_selectCompression(LZ4F_blockMode_t blockMode,int level)679 static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
680 {
681 if (level < LZ4HC_CLEVEL_MIN) {
682 if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
683 return LZ4F_compressBlock_continue;
684 }
685 if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC;
686 return LZ4F_compressBlockHC_continue;
687 }
688
LZ4F_localSaveDict(LZ4F_cctx_t * cctxPtr)689 static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
690 {
691 if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
692 return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
693 return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
694 }
695
696 typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
697
698 /*! LZ4F_compressUpdate() :
699 * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
700 * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
701 * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
702 * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
703 * or an error code if it fails (which can be tested using LZ4F_isError())
704 */
LZ4F_compressUpdate(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const void * srcBuffer,size_t srcSize,const LZ4F_compressOptions_t * compressOptionsPtr)705 size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
706 void* dstBuffer, size_t dstCapacity,
707 const void* srcBuffer, size_t srcSize,
708 const LZ4F_compressOptions_t* compressOptionsPtr)
709 {
710 LZ4F_compressOptions_t cOptionsNull;
711 size_t const blockSize = cctxPtr->maxBlockSize;
712 const BYTE* srcPtr = (const BYTE*)srcBuffer;
713 const BYTE* const srcEnd = srcPtr + srcSize;
714 BYTE* const dstStart = (BYTE*)dstBuffer;
715 BYTE* dstPtr = dstStart;
716 LZ4F_lastBlockStatus lastBlockCompressed = notDone;
717 compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
718
719
720 if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
721 if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
722 memset(&cOptionsNull, 0, sizeof(cOptionsNull));
723 if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
724
725 /* complete tmp buffer */
726 if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */
727 size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
728 if (sizeToCopy > srcSize) {
729 /* add src to tmpIn buffer */
730 memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
731 srcPtr = srcEnd;
732 cctxPtr->tmpInSize += srcSize;
733 /* still needs some CRC */
734 } else {
735 /* complete tmpIn block and then compress it */
736 lastBlockCompressed = fromTmpBuffer;
737 memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
738 srcPtr += sizeToCopy;
739
740 dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize,
741 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
742 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
743
744 if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
745 cctxPtr->tmpInSize = 0;
746 }
747 }
748
749 while ((size_t)(srcEnd - srcPtr) >= blockSize) {
750 /* compress full blocks */
751 lastBlockCompressed = fromSrcBuffer;
752 dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize,
753 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
754 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
755 srcPtr += blockSize;
756 }
757
758 if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
759 /* compress remaining input < blockSize */
760 lastBlockCompressed = fromSrcBuffer;
761 dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr,
762 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
763 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
764 srcPtr = srcEnd;
765 }
766
767 /* preserve dictionary if necessary */
768 if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
769 if (compressOptionsPtr->stableSrc) {
770 cctxPtr->tmpIn = cctxPtr->tmpBuff;
771 } else {
772 int const realDictSize = LZ4F_localSaveDict(cctxPtr);
773 if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC);
774 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
775 }
776 }
777
778 /* keep tmpIn within limits */
779 if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */
780 && !(cctxPtr->prefs.autoFlush))
781 {
782 int const realDictSize = LZ4F_localSaveDict(cctxPtr);
783 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
784 }
785
786 /* some input data left, necessarily < blockSize */
787 if (srcPtr < srcEnd) {
788 /* fill tmp buffer */
789 size_t const sizeToCopy = srcEnd - srcPtr;
790 memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
791 cctxPtr->tmpInSize = sizeToCopy;
792 }
793
794 if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
795 XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
796
797 cctxPtr->totalInSize += srcSize;
798 return dstPtr - dstStart;
799 }
800
801
802 /*! LZ4F_flush() :
803 * Should you need to create compressed data immediately, without waiting for a block to be filled,
804 * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
805 * The result of the function is the number of bytes written into dstBuffer
806 * (it can be zero, this means there was no data left within compressionContext)
807 * The function outputs an error code if it fails (can be tested using LZ4F_isError())
808 * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
809 */
LZ4F_flush(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const LZ4F_compressOptions_t * compressOptionsPtr)810 size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr)
811 {
812 BYTE* const dstStart = (BYTE*)dstBuffer;
813 BYTE* dstPtr = dstStart;
814 compressFunc_t compress;
815
816 if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
817 if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
818 if (dstCapacity < (cctxPtr->tmpInSize + 4)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +4 : block header(4) */
819 (void)compressOptionsPtr; /* not yet useful */
820
821 /* select compression function */
822 compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
823
824 /* compress tmp buffer */
825 dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize,
826 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
827 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
828 if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
829 cctxPtr->tmpInSize = 0;
830
831 /* keep tmpIn within limits */
832 if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */
833 int realDictSize = LZ4F_localSaveDict(cctxPtr);
834 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
835 }
836
837 return dstPtr - dstStart;
838 }
839
840
841 /*! LZ4F_compressEnd() :
842 * When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
843 * It will flush whatever data remained within compressionContext (like LZ4_flush())
844 * but also properly finalize the frame, with an endMark and a checksum.
845 * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
846 * The function outputs an error code if it fails (can be tested using LZ4F_isError())
847 * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
848 * compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same.
849 */
LZ4F_compressEnd(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstMaxSize,const LZ4F_compressOptions_t * compressOptionsPtr)850 size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr)
851 {
852 BYTE* const dstStart = (BYTE*)dstBuffer;
853 BYTE* dstPtr = dstStart;
854
855 size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstMaxSize, compressOptionsPtr);
856 if (LZ4F_isError(flushSize)) return flushSize;
857 dstPtr += flushSize;
858
859 LZ4F_writeLE32(dstPtr, 0);
860 dstPtr+=4; /* endMark */
861
862 if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {
863 U32 const xxh = XXH32_digest(&(cctxPtr->xxh));
864 LZ4F_writeLE32(dstPtr, xxh);
865 dstPtr+=4; /* content Checksum */
866 }
867
868 cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */
869 cctxPtr->maxBufferSize = 0; /* reuse HC context */
870
871 if (cctxPtr->prefs.frameInfo.contentSize) {
872 if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize)
873 return err0r(LZ4F_ERROR_frameSize_wrong);
874 }
875
876 return dstPtr - dstStart;
877 }
878
879
880 /*-***************************************************
881 * Frame Decompression
882 *****************************************************/
883
884 typedef enum {
885 dstage_getFrameHeader=0, dstage_storeFrameHeader,
886 dstage_init,
887 dstage_getBlockHeader, dstage_storeBlockHeader,
888 dstage_copyDirect, dstage_getBlockChecksum,
889 dstage_getCBlock, dstage_storeCBlock,
890 dstage_decodeCBlock, dstage_decodeCBlock_intoDst,
891 dstage_decodeCBlock_intoTmp, dstage_flushOut,
892 dstage_getSuffix, dstage_storeSuffix,
893 dstage_getSFrameSize, dstage_storeSFrameSize,
894 dstage_skipSkippable
895 } dStage_t;
896
897 struct LZ4F_dctx_s {
898 LZ4F_frameInfo_t frameInfo;
899 U32 version;
900 dStage_t dStage;
901 U64 frameRemainingSize;
902 size_t maxBlockSize;
903 size_t maxBufferSize;
904 BYTE* tmpIn;
905 size_t tmpInSize;
906 size_t tmpInTarget;
907 BYTE* tmpOutBuffer;
908 const BYTE* dict;
909 size_t dictSize;
910 BYTE* tmpOut;
911 size_t tmpOutSize;
912 size_t tmpOutStart;
913 XXH32_state_t xxh;
914 XXH32_state_t blockChecksum;
915 BYTE header[LZ4F_HEADER_SIZE_MAX];
916 }; /* typedef'd to LZ4F_dctx in lz4frame.h */
917
918
919 /*! LZ4F_createDecompressionContext() :
920 * Create a decompressionContext object, which will track all decompression operations.
921 * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
922 * Object can later be released using LZ4F_freeDecompressionContext().
923 * @return : if != 0, there was an error during context creation.
924 */
LZ4F_createDecompressionContext(LZ4F_dctx ** LZ4F_decompressionContextPtr,unsigned versionNumber)925 LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
926 {
927 LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOCATOR(sizeof(LZ4F_dctx));
928 if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
929
930 dctx->version = versionNumber;
931 *LZ4F_decompressionContextPtr = dctx;
932 return LZ4F_OK_NoError;
933 }
934
LZ4F_freeDecompressionContext(LZ4F_dctx * dctx)935 LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx)
936 {
937 LZ4F_errorCode_t result = LZ4F_OK_NoError;
938 if (dctx != NULL) { /* can accept NULL input, like free() */
939 result = (LZ4F_errorCode_t)dctx->dStage;
940 FREEMEM(dctx->tmpIn);
941 FREEMEM(dctx->tmpOutBuffer);
942 FREEMEM(dctx);
943 }
944 return result;
945 }
946
947
948 /*==--- Streaming Decompression operations ---==*/
949
LZ4F_resetDecompressionContext(LZ4F_dctx * dctx)950 void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
951 {
952 dctx->dStage = dstage_getFrameHeader;
953 dctx->dict = NULL;
954 dctx->dictSize = 0;
955 }
956
957
958 /*! LZ4F_headerSize() :
959 * @return : size of frame header
960 * or an error code, which can be tested using LZ4F_isError()
961 */
LZ4F_headerSize(const void * src,size_t srcSize)962 static size_t LZ4F_headerSize(const void* src, size_t srcSize)
963 {
964 /* minimal srcSize to determine header size */
965 if (srcSize < 5) return err0r(LZ4F_ERROR_frameHeader_incomplete);
966
967 /* special case : skippable frames */
968 if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8;
969
970 /* control magic number */
971 if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
972 return err0r(LZ4F_ERROR_frameType_unknown);
973
974 /* Frame Header Size */
975 { BYTE const FLG = ((const BYTE*)src)[4];
976 U32 const contentSizeFlag = (FLG>>3) & _1BIT;
977 U32 const dictIDFlag = FLG & _1BIT;
978 return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
979 }
980 }
981
982
983 /*! LZ4F_decodeHeader() :
984 * input : `src` points at the **beginning of the frame**
985 * output : set internal values of dctx, such as
986 * dctx->frameInfo and dctx->dStage.
987 * Also allocates internal buffers.
988 * @return : nb Bytes read from src (necessarily <= srcSize)
989 * or an error code (testable with LZ4F_isError())
990 */
LZ4F_decodeHeader(LZ4F_dctx * dctx,const void * src,size_t srcSize)991 static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize)
992 {
993 unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID;
994 size_t frameHeaderSize;
995 const BYTE* srcPtr = (const BYTE*)src;
996
997 /* need to decode header to get frameInfo */
998 if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */
999 memset(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
1000
1001 /* special case : skippable frames */
1002 if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {
1003 dctx->frameInfo.frameType = LZ4F_skippableFrame;
1004 if (src == (void*)(dctx->header)) {
1005 dctx->tmpInSize = srcSize;
1006 dctx->tmpInTarget = 8;
1007 dctx->dStage = dstage_storeSFrameSize;
1008 return srcSize;
1009 } else {
1010 dctx->dStage = dstage_getSFrameSize;
1011 return 4;
1012 }
1013 }
1014
1015 /* control magic number */
1016 if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)
1017 return err0r(LZ4F_ERROR_frameType_unknown);
1018 dctx->frameInfo.frameType = LZ4F_frame;
1019
1020 /* Flags */
1021 { U32 const FLG = srcPtr[4];
1022 U32 const version = (FLG>>6) & _2BITS;
1023 blockChecksumFlag = (FLG>>4) & _1BIT;
1024 blockMode = (FLG>>5) & _1BIT;
1025 contentSizeFlag = (FLG>>3) & _1BIT;
1026 contentChecksumFlag = (FLG>>2) & _1BIT;
1027 dictIDFlag = FLG & _1BIT;
1028 /* validate */
1029 if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
1030 if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */
1031 }
1032
1033 /* Frame Header Size */
1034 frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
1035
1036 if (srcSize < frameHeaderSize) {
1037 /* not enough input to fully decode frame header */
1038 if (srcPtr != dctx->header)
1039 memcpy(dctx->header, srcPtr, srcSize);
1040 dctx->tmpInSize = srcSize;
1041 dctx->tmpInTarget = frameHeaderSize;
1042 dctx->dStage = dstage_storeFrameHeader;
1043 return srcSize;
1044 }
1045
1046 { U32 const BD = srcPtr[5];
1047 blockSizeID = (BD>>4) & _3BITS;
1048 /* validate */
1049 if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
1050 if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */
1051 if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
1052 }
1053
1054 /* check header */
1055 { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
1056 if (HC != srcPtr[frameHeaderSize-1])
1057 return err0r(LZ4F_ERROR_headerChecksum_invalid);
1058 }
1059
1060 /* save */
1061 dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
1062 dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;
1063 dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
1064 dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
1065 dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
1066 if (contentSizeFlag)
1067 dctx->frameRemainingSize =
1068 dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
1069 if (dictIDFlag)
1070 dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
1071
1072 dctx->dStage = dstage_init;
1073
1074 return frameHeaderSize;
1075 }
1076
1077
1078 /*! LZ4F_getFrameInfo() :
1079 * This function extracts frame parameters (max blockSize, frame checksum, etc.).
1080 * Usage is optional. Objective is to provide relevant information for allocation purposes.
1081 * This function works in 2 situations :
1082 * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
1083 * Amount of input data provided must be large enough to successfully decode the frame header.
1084 * A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum.
1085 * - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx.
1086 * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
1087 * Decompression must resume from (srcBuffer + *srcSizePtr).
1088 * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
1089 * or an error code which can be tested using LZ4F_isError()
1090 * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
1091 * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
1092 */
LZ4F_getFrameInfo(LZ4F_dctx * dctx,LZ4F_frameInfo_t * frameInfoPtr,const void * srcBuffer,size_t * srcSizePtr)1093 LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr,
1094 const void* srcBuffer, size_t* srcSizePtr)
1095 {
1096 if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */
1097 /* frameInfo already decoded */
1098 size_t o=0, i=0;
1099 *srcSizePtr = 0;
1100 *frameInfoPtr = dctx->frameInfo;
1101 /* returns : recommended nb of bytes for LZ4F_decompress() */
1102 return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL);
1103 } else {
1104 if (dctx->dStage == dstage_storeFrameHeader) {
1105 /* frame decoding already started, in the middle of header => automatic fail */
1106 *srcSizePtr = 0;
1107 return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
1108 } else {
1109 size_t decodeResult;
1110 size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
1111 if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
1112 if (*srcSizePtr < hSize) {
1113 *srcSizePtr=0;
1114 return err0r(LZ4F_ERROR_frameHeader_incomplete);
1115 }
1116
1117 decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
1118 if (LZ4F_isError(decodeResult)) {
1119 *srcSizePtr = 0;
1120 } else {
1121 *srcSizePtr = decodeResult;
1122 decodeResult = BHSize; /* block header size */
1123 }
1124 *frameInfoPtr = dctx->frameInfo;
1125 return decodeResult;
1126 } }
1127 }
1128
1129
1130 /* LZ4F_updateDict() :
1131 * only used for LZ4F_blockLinked mode */
LZ4F_updateDict(LZ4F_dctx * dctx,const BYTE * dstPtr,size_t dstSize,const BYTE * dstPtr0,unsigned withinTmp)1132 static void LZ4F_updateDict(LZ4F_dctx* dctx, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
1133 {
1134 if (dctx->dictSize==0)
1135 dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */
1136
1137 if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity */
1138 dctx->dictSize += dstSize;
1139 return;
1140 }
1141
1142 if (dstPtr - dstPtr0 + dstSize >= 64 KB) { /* dstBuffer large enough to become dictionary */
1143 dctx->dict = (const BYTE*)dstPtr0;
1144 dctx->dictSize = dstPtr - dstPtr0 + dstSize;
1145 return;
1146 }
1147
1148 if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) {
1149 /* assumption : dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart */
1150 dctx->dictSize += dstSize;
1151 return;
1152 }
1153
1154 if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
1155 size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
1156 size_t copySize = 64 KB - dctx->tmpOutSize;
1157 const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
1158 if (dctx->tmpOutSize > 64 KB) copySize = 0;
1159 if (copySize > preserveSize) copySize = preserveSize;
1160
1161 memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
1162
1163 dctx->dict = dctx->tmpOutBuffer;
1164 dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize;
1165 return;
1166 }
1167
1168 if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */
1169 if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */
1170 size_t const preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */
1171 memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
1172 dctx->dictSize = preserveSize;
1173 }
1174 memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize);
1175 dctx->dictSize += dstSize;
1176 return;
1177 }
1178
1179 /* join dict & dest into tmp */
1180 { size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */
1181 if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize;
1182 memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
1183 memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize);
1184 dctx->dict = dctx->tmpOutBuffer;
1185 dctx->dictSize = preserveSize + dstSize;
1186 }
1187 }
1188
1189
1190
1191 /*! LZ4F_decompress() :
1192 * Call this function repetitively to regenerate compressed data in srcBuffer.
1193 * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer
1194 * into dstBuffer of capacity *dstSizePtr.
1195 *
1196 * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
1197 *
1198 * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
1199 * If number of bytes read is < number of bytes provided, then decompression operation is not complete.
1200 * Remaining data will have to be presented again in a subsequent invocation.
1201 *
1202 * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
1203 * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
1204 * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling.
1205 * Note that this is just a hint, and it's always possible to any srcSize value.
1206 * When a frame is fully decoded, @return will be 0.
1207 * If decompression failed, @return is an error code which can be tested using LZ4F_isError().
1208 */
LZ4F_decompress(LZ4F_dctx * dctx,void * dstBuffer,size_t * dstSizePtr,const void * srcBuffer,size_t * srcSizePtr,const LZ4F_decompressOptions_t * decompressOptionsPtr)1209 size_t LZ4F_decompress(LZ4F_dctx* dctx,
1210 void* dstBuffer, size_t* dstSizePtr,
1211 const void* srcBuffer, size_t* srcSizePtr,
1212 const LZ4F_decompressOptions_t* decompressOptionsPtr)
1213 {
1214 LZ4F_decompressOptions_t optionsNull;
1215 const BYTE* const srcStart = (const BYTE*)srcBuffer;
1216 const BYTE* const srcEnd = srcStart + *srcSizePtr;
1217 const BYTE* srcPtr = srcStart;
1218 BYTE* const dstStart = (BYTE*)dstBuffer;
1219 BYTE* const dstEnd = dstStart + *dstSizePtr;
1220 BYTE* dstPtr = dstStart;
1221 const BYTE* selectedIn = NULL;
1222 unsigned doAnotherStage = 1;
1223 size_t nextSrcSizeHint = 1;
1224
1225
1226 memset(&optionsNull, 0, sizeof(optionsNull));
1227 if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;
1228 *srcSizePtr = 0;
1229 *dstSizePtr = 0;
1230
1231 /* behaves as a state machine */
1232
1233 while (doAnotherStage) {
1234
1235 switch(dctx->dStage)
1236 {
1237
1238 case dstage_getFrameHeader:
1239 if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
1240 size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */
1241 if (LZ4F_isError(hSize)) return hSize;
1242 srcPtr += hSize;
1243 break;
1244 }
1245 dctx->tmpInSize = 0;
1246 if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */
1247 dctx->tmpInTarget = minFHSize; /* minimum to attempt decode */
1248 dctx->dStage = dstage_storeFrameHeader;
1249 /* fall-through */
1250
1251 case dstage_storeFrameHeader:
1252 { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr));
1253 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
1254 dctx->tmpInSize += sizeToCopy;
1255 srcPtr += sizeToCopy;
1256 if (dctx->tmpInSize < dctx->tmpInTarget) {
1257 nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */
1258 doAnotherStage = 0; /* not enough src data, ask for some more */
1259 break;
1260 }
1261 { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */
1262 if (LZ4F_isError(hSize)) return hSize;
1263 }
1264 break;
1265 }
1266
1267 case dstage_init:
1268 if (dctx->frameInfo.contentChecksumFlag) XXH32_reset(&(dctx->xxh), 0);
1269 /* internal buffers allocation */
1270 { size_t const bufferNeeded = dctx->maxBlockSize + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB) + 4 /* block checksum */;
1271 if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
1272 dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
1273 FREEMEM(dctx->tmpIn);
1274 dctx->tmpIn = (BYTE*)ALLOCATOR(dctx->maxBlockSize);
1275 if (dctx->tmpIn == NULL) return err0r(LZ4F_ERROR_allocation_failed);
1276 FREEMEM(dctx->tmpOutBuffer);
1277 dctx->tmpOutBuffer= (BYTE*)ALLOCATOR(bufferNeeded);
1278 if (dctx->tmpOutBuffer== NULL) return err0r(LZ4F_ERROR_allocation_failed);
1279 dctx->maxBufferSize = bufferNeeded;
1280 } }
1281 dctx->tmpInSize = 0;
1282 dctx->tmpInTarget = 0;
1283 dctx->tmpOut = dctx->tmpOutBuffer;
1284 dctx->tmpOutStart = 0;
1285 dctx->tmpOutSize = 0;
1286
1287 dctx->dStage = dstage_getBlockHeader;
1288 /* fall-through */
1289
1290 case dstage_getBlockHeader:
1291 if ((size_t)(srcEnd - srcPtr) >= BHSize) {
1292 selectedIn = srcPtr;
1293 srcPtr += BHSize;
1294 } else {
1295 /* not enough input to read cBlockSize field */
1296 dctx->tmpInSize = 0;
1297 dctx->dStage = dstage_storeBlockHeader;
1298 }
1299
1300 if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */
1301 case dstage_storeBlockHeader:
1302 { size_t sizeToCopy = BHSize - dctx->tmpInSize;
1303 if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
1304 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
1305 srcPtr += sizeToCopy;
1306 dctx->tmpInSize += sizeToCopy;
1307 if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */
1308 nextSrcSizeHint = BHSize - dctx->tmpInSize;
1309 doAnotherStage = 0;
1310 break;
1311 }
1312 selectedIn = dctx->tmpIn;
1313 }
1314
1315 /* decode block header */
1316 { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
1317 size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4;
1318 if (nextCBlockSize==0) { /* frameEnd signal, no more block */
1319 dctx->dStage = dstage_getSuffix;
1320 break;
1321 }
1322 if (nextCBlockSize > dctx->maxBlockSize)
1323 return err0r(LZ4F_ERROR_maxBlockSize_invalid);
1324 if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
1325 /* next block is uncompressed */
1326 dctx->tmpInTarget = nextCBlockSize;
1327 if (dctx->frameInfo.blockChecksumFlag) {
1328 XXH32_reset(&dctx->blockChecksum, 0);
1329 }
1330 dctx->dStage = dstage_copyDirect;
1331 break;
1332 }
1333 /* next block is a compressed block */
1334 dctx->tmpInTarget = nextCBlockSize + crcSize;
1335 dctx->dStage = dstage_getCBlock;
1336 if (dstPtr==dstEnd) {
1337 nextSrcSizeHint = nextCBlockSize + crcSize + BHSize;
1338 doAnotherStage = 0;
1339 }
1340 break;
1341 }
1342
1343 case dstage_copyDirect: /* uncompressed block */
1344 { size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
1345 size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
1346 memcpy(dstPtr, srcPtr, sizeToCopy);
1347 if (dctx->frameInfo.blockChecksumFlag) {
1348 XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
1349 }
1350 if (dctx->frameInfo.contentChecksumFlag)
1351 XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
1352 if (dctx->frameInfo.contentSize)
1353 dctx->frameRemainingSize -= sizeToCopy;
1354
1355 /* history management (linked blocks only)*/
1356 if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
1357 LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0);
1358
1359 srcPtr += sizeToCopy;
1360 dstPtr += sizeToCopy;
1361 if (sizeToCopy == dctx->tmpInTarget) { /* all done */
1362 if (dctx->frameInfo.blockChecksumFlag) {
1363 dctx->tmpInSize = 0;
1364 dctx->dStage = dstage_getBlockChecksum;
1365 } else
1366 dctx->dStage = dstage_getBlockHeader; /* new block */
1367 break;
1368 }
1369 dctx->tmpInTarget -= sizeToCopy; /* need to copy more */
1370 nextSrcSizeHint = dctx->tmpInTarget +
1371 + dctx->frameInfo.contentChecksumFlag * 4 /* block checksum */
1372 + BHSize /* next header size */;
1373 doAnotherStage = 0;
1374 break;
1375 }
1376
1377 /* check block checksum for recently transferred uncompressed block */
1378 case dstage_getBlockChecksum:
1379 { const void* crcSrc;
1380 if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) {
1381 crcSrc = srcPtr;
1382 srcPtr += 4;
1383 } else {
1384 size_t const stillToCopy = 4 - dctx->tmpInSize;
1385 size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr));
1386 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
1387 dctx->tmpInSize += sizeToCopy;
1388 srcPtr += sizeToCopy;
1389 if (dctx->tmpInSize < 4) { /* all input consumed */
1390 doAnotherStage = 0;
1391 break;
1392 }
1393 crcSrc = dctx->header;
1394 }
1395 { U32 const readCRC = LZ4F_readLE32(crcSrc);
1396 U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
1397 if (readCRC != calcCRC)
1398 return err0r(LZ4F_ERROR_blockChecksum_invalid);
1399 }
1400 }
1401 dctx->dStage = dstage_getBlockHeader; /* new block */
1402 break;
1403
1404 case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */
1405 if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) {
1406 dctx->tmpInSize = 0;
1407 dctx->dStage = dstage_storeCBlock;
1408 break;
1409 }
1410 /* input large enough to read full block directly */
1411 selectedIn = srcPtr;
1412 srcPtr += dctx->tmpInTarget;
1413 dctx->dStage = dstage_decodeCBlock;
1414 break;
1415
1416 case dstage_storeCBlock:
1417 { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd-srcPtr));
1418 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
1419 dctx->tmpInSize += sizeToCopy;
1420 srcPtr += sizeToCopy;
1421 if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */
1422 nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize;
1423 doAnotherStage=0;
1424 break;
1425 }
1426 selectedIn = dctx->tmpIn;
1427 dctx->dStage = dstage_decodeCBlock;
1428 }
1429 /* fall-through */
1430
1431 /* At this stage, input is large enough to decode a block */
1432 case dstage_decodeCBlock:
1433 if (dctx->frameInfo.blockChecksumFlag) {
1434 dctx->tmpInTarget -= 4;
1435 { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
1436 U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
1437 if (readBlockCrc != calcBlockCrc)
1438 return err0r(LZ4F_ERROR_blockChecksum_invalid);
1439 } }
1440 if ((size_t)(dstEnd-dstPtr) < dctx->maxBlockSize) /* not enough place into dst : decode into tmpOut */
1441 dctx->dStage = dstage_decodeCBlock_intoTmp;
1442 else
1443 dctx->dStage = dstage_decodeCBlock_intoDst;
1444 break;
1445
1446 case dstage_decodeCBlock_intoDst:
1447 { int const decodedSize = LZ4_decompress_safe_usingDict(
1448 (const char*)selectedIn, (char*)dstPtr,
1449 (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
1450 (const char*)dctx->dict, (int)dctx->dictSize);
1451 if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
1452 if (dctx->frameInfo.contentChecksumFlag)
1453 XXH32_update(&(dctx->xxh), dstPtr, decodedSize);
1454 if (dctx->frameInfo.contentSize)
1455 dctx->frameRemainingSize -= decodedSize;
1456
1457 /* dictionary management */
1458 if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
1459 LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0);
1460
1461 dstPtr += decodedSize;
1462 dctx->dStage = dstage_getBlockHeader;
1463 break;
1464 }
1465
1466 case dstage_decodeCBlock_intoTmp:
1467 /* not enough place into dst : decode into tmpOut */
1468
1469 /* ensure enough place for tmpOut */
1470 if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
1471 if (dctx->dict == dctx->tmpOutBuffer) {
1472 if (dctx->dictSize > 128 KB) {
1473 memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);
1474 dctx->dictSize = 64 KB;
1475 }
1476 dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;
1477 } else { /* dict not within tmp */
1478 size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
1479 dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
1480 }
1481 }
1482
1483 /* Decode block */
1484 { int const decodedSize = LZ4_decompress_safe_usingDict(
1485 (const char*)selectedIn, (char*)dctx->tmpOut,
1486 (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
1487 (const char*)dctx->dict, (int)dctx->dictSize);
1488 if (decodedSize < 0) /* decompression failed */
1489 return err0r(LZ4F_ERROR_decompressionFailed);
1490 if (dctx->frameInfo.contentChecksumFlag)
1491 XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize);
1492 if (dctx->frameInfo.contentSize)
1493 dctx->frameRemainingSize -= decodedSize;
1494 dctx->tmpOutSize = decodedSize;
1495 dctx->tmpOutStart = 0;
1496 dctx->dStage = dstage_flushOut;
1497 }
1498 /* fall-through */
1499
1500 case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */
1501 { size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr));
1502 memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);
1503
1504 /* dictionary management */
1505 if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
1506 LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1);
1507
1508 dctx->tmpOutStart += sizeToCopy;
1509 dstPtr += sizeToCopy;
1510
1511 if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */
1512 dctx->dStage = dstage_getBlockHeader; /* get next block */
1513 break;
1514 }
1515 nextSrcSizeHint = BHSize;
1516 doAnotherStage = 0; /* still some data to flush */
1517 break;
1518 }
1519
1520 case dstage_getSuffix:
1521 { size_t const suffixSize = dctx->frameInfo.contentChecksumFlag * 4;
1522 if (dctx->frameRemainingSize)
1523 return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
1524 if (suffixSize == 0) { /* frame completed */
1525 nextSrcSizeHint = 0;
1526 LZ4F_resetDecompressionContext(dctx);
1527 doAnotherStage = 0;
1528 break;
1529 }
1530 if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */
1531 dctx->tmpInSize = 0;
1532 dctx->dStage = dstage_storeSuffix;
1533 } else {
1534 selectedIn = srcPtr;
1535 srcPtr += 4;
1536 }
1537 }
1538
1539 if (dctx->dStage == dstage_storeSuffix) /* can be skipped */
1540 case dstage_storeSuffix:
1541 {
1542 size_t sizeToCopy = 4 - dctx->tmpInSize;
1543 if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
1544 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
1545 srcPtr += sizeToCopy;
1546 dctx->tmpInSize += sizeToCopy;
1547 if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */
1548 nextSrcSizeHint = 4 - dctx->tmpInSize;
1549 doAnotherStage=0;
1550 break;
1551 }
1552 selectedIn = dctx->tmpIn;
1553 }
1554
1555 /* case dstage_checkSuffix: */ /* no direct call, avoid scan-build warning */
1556 { U32 const readCRC = LZ4F_readLE32(selectedIn);
1557 U32 const resultCRC = XXH32_digest(&(dctx->xxh));
1558 if (readCRC != resultCRC)
1559 return err0r(LZ4F_ERROR_contentChecksum_invalid);
1560 nextSrcSizeHint = 0;
1561 LZ4F_resetDecompressionContext(dctx);
1562 doAnotherStage = 0;
1563 break;
1564 }
1565
1566 case dstage_getSFrameSize:
1567 if ((srcEnd - srcPtr) >= 4) {
1568 selectedIn = srcPtr;
1569 srcPtr += 4;
1570 } else {
1571 /* not enough input to read cBlockSize field */
1572 dctx->tmpInSize = 4;
1573 dctx->tmpInTarget = 8;
1574 dctx->dStage = dstage_storeSFrameSize;
1575 }
1576
1577 if (dctx->dStage == dstage_storeSFrameSize)
1578 case dstage_storeSFrameSize:
1579 {
1580 size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
1581 (size_t)(srcEnd - srcPtr) );
1582 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
1583 srcPtr += sizeToCopy;
1584 dctx->tmpInSize += sizeToCopy;
1585 if (dctx->tmpInSize < dctx->tmpInTarget) {
1586 /* not enough input to get full sBlockSize; wait for more */
1587 nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize;
1588 doAnotherStage = 0;
1589 break;
1590 }
1591 selectedIn = dctx->header + 4;
1592 }
1593
1594 /* case dstage_decodeSFrameSize: */ /* no direct access */
1595 { size_t const SFrameSize = LZ4F_readLE32(selectedIn);
1596 dctx->frameInfo.contentSize = SFrameSize;
1597 dctx->tmpInTarget = SFrameSize;
1598 dctx->dStage = dstage_skipSkippable;
1599 break;
1600 }
1601
1602 case dstage_skipSkippable:
1603 { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr));
1604 srcPtr += skipSize;
1605 dctx->tmpInTarget -= skipSize;
1606 doAnotherStage = 0;
1607 nextSrcSizeHint = dctx->tmpInTarget;
1608 if (nextSrcSizeHint) break; /* still more to skip */
1609 LZ4F_resetDecompressionContext(dctx);
1610 break;
1611 }
1612 }
1613 }
1614
1615 /* preserve history within tmp if necessary */
1616 if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked)
1617 && (dctx->dict != dctx->tmpOutBuffer)
1618 && (dctx->dStage != dstage_getFrameHeader)
1619 && (!decompressOptionsPtr->stableDst)
1620 && ((unsigned)(dctx->dStage-1) < (unsigned)(dstage_getSuffix-1)) )
1621 {
1622 if (dctx->dStage == dstage_flushOut) {
1623 size_t preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
1624 size_t copySize = 64 KB - dctx->tmpOutSize;
1625 const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
1626 if (dctx->tmpOutSize > 64 KB) copySize = 0;
1627 if (copySize > preserveSize) copySize = preserveSize;
1628
1629 memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
1630
1631 dctx->dict = dctx->tmpOutBuffer;
1632 dctx->dictSize = preserveSize + dctx->tmpOutStart;
1633 } else {
1634 size_t newDictSize = dctx->dictSize;
1635 const BYTE* oldDictEnd = dctx->dict + dctx->dictSize;
1636 if ((newDictSize) > 64 KB) newDictSize = 64 KB;
1637
1638 memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
1639
1640 dctx->dict = dctx->tmpOutBuffer;
1641 dctx->dictSize = newDictSize;
1642 dctx->tmpOut = dctx->tmpOutBuffer + newDictSize;
1643 }
1644 }
1645
1646 *srcSizePtr = (srcPtr - srcStart);
1647 *dstSizePtr = (dstPtr - dstStart);
1648 return nextSrcSizeHint;
1649 }
1650
1651 /*! LZ4F_decompress_usingDict() :
1652 * Same as LZ4F_decompress(), using a predefined dictionary.
1653 * Dictionary is used "in place", without any preprocessing.
1654 * It must remain accessible throughout the entire frame decoding.
1655 */
LZ4F_decompress_usingDict(LZ4F_dctx * dctx,void * dstBuffer,size_t * dstSizePtr,const void * srcBuffer,size_t * srcSizePtr,const void * dict,size_t dictSize,const LZ4F_decompressOptions_t * decompressOptionsPtr)1656 size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx,
1657 void* dstBuffer, size_t* dstSizePtr,
1658 const void* srcBuffer, size_t* srcSizePtr,
1659 const void* dict, size_t dictSize,
1660 const LZ4F_decompressOptions_t* decompressOptionsPtr)
1661 {
1662 if (dctx->dStage <= dstage_init) {
1663 dctx->dict = (const BYTE*)dict;
1664 dctx->dictSize = dictSize;
1665 }
1666 return LZ4F_decompress(dctx, dstBuffer, dstSizePtr,
1667 srcBuffer, srcSizePtr,
1668 decompressOptionsPtr);
1669 }
1670