1 /*
2 LZ4 auto-framing library
3 Copyright (C) 2011-2016, Yann Collet.
4
5 BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are
9 met:
10
11 * Redistributions of source code must retain the above copyright
12 notice, this list of conditions and the following disclaimer.
13 * Redistributions in binary form must reproduce the above
14 copyright notice, this list of conditions and the following disclaimer
15 in the documentation and/or other materials provided with the
16 distribution.
17
18 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 You can contact the author at :
31 - LZ4 homepage : http://www.lz4.org
32 - LZ4 source repository : https://github.com/lz4/lz4
33 */
34
35 /* LZ4F is a stand-alone API to create LZ4-compressed Frames
36 * in full conformance with specification v1.6.1 .
37 * This library rely upon memory management capabilities.
38 * */
39
40
41 /*-************************************
42 * Compiler Options
43 **************************************/
44 #ifdef _MSC_VER /* Visual Studio */
45 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
46 #endif
47
48
49 /*-************************************
50 * Tuning parameters
51 **************************************/
52 /*
53 * LZ4F_HEAPMODE :
54 * Select how default compression functions will allocate memory for their hash table,
55 * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
56 */
57 #ifndef LZ4F_HEAPMODE
58 # define LZ4F_HEAPMODE 0
59 #endif
60
61
62 /*-************************************
63 * Memory routines
64 **************************************/
65 #include <stdlib.h> /* malloc, calloc, free */
66 #define ALLOC(s) malloc(s)
67 #ifndef LZ4_SRC_INCLUDED
68 #define ALLOC_AND_ZERO(s) calloc(1,(s))
69 #endif
70 #define FREEMEM(p) free(p)
71 #include <string.h> /* memset, memcpy, memmove */
72 #ifndef LZ4_SRC_INCLUDED
73 #define MEM_INIT memset
74 #endif
75
76
77
78 /*-************************************
79 * Includes
80 **************************************/
81 #define LZ4F_STATIC_LINKING_ONLY
82 #include "lz4frame.h"
83 #define LZ4_STATIC_LINKING_ONLY
84 #include "lz4.h"
85 #define LZ4_HC_STATIC_LINKING_ONLY
86 #include "lz4hc.h"
87 #define XXH_STATIC_LINKING_ONLY
88 #include "xxhash.h"
89
90
91 /*-************************************
92 * Debug
93 **************************************/
94 #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1)
95 # include <assert.h>
96 #else
97 # ifndef assert
98 # define assert(condition) ((void)0)
99 # endif
100 #endif
101
102 #define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
103
104 #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) && !defined(DEBUGLOG)
105 # include <stdio.h>
106 static int g_debuglog_enable = 1;
107 # define DEBUGLOG(l, ...) { \
108 if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \
109 fprintf(stderr, __FILE__ ": "); \
110 fprintf(stderr, __VA_ARGS__); \
111 fprintf(stderr, " \n"); \
112 } }
113 #else
114 # define DEBUGLOG(l, ...) {} /* disabled */
115 #endif
116
117
118 /*-************************************
119 * Basic Types
120 **************************************/
121 #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
122 # include <stdint.h>
123 typedef uint8_t BYTE;
124 typedef uint16_t U16;
125 typedef uint32_t U32;
126 typedef int32_t S32;
127 typedef uint64_t U64;
128 #else
129 typedef unsigned char BYTE;
130 typedef unsigned short U16;
131 typedef unsigned int U32;
132 typedef signed int S32;
133 typedef unsigned long long U64;
134 #endif
135
136
137 /* unoptimized version; solves endianess & alignment issues */
LZ4F_readLE32(const void * src)138 static U32 LZ4F_readLE32 (const void* src)
139 {
140 const BYTE* const srcPtr = (const BYTE*)src;
141 U32 value32 = srcPtr[0];
142 value32 += (srcPtr[1]<<8);
143 value32 += (srcPtr[2]<<16);
144 value32 += ((U32)srcPtr[3])<<24;
145 return value32;
146 }
147
LZ4F_writeLE32(void * dst,U32 value32)148 static void LZ4F_writeLE32 (void* dst, U32 value32)
149 {
150 BYTE* const dstPtr = (BYTE*)dst;
151 dstPtr[0] = (BYTE)value32;
152 dstPtr[1] = (BYTE)(value32 >> 8);
153 dstPtr[2] = (BYTE)(value32 >> 16);
154 dstPtr[3] = (BYTE)(value32 >> 24);
155 }
156
LZ4F_readLE64(const void * src)157 static U64 LZ4F_readLE64 (const void* src)
158 {
159 const BYTE* const srcPtr = (const BYTE*)src;
160 U64 value64 = srcPtr[0];
161 value64 += ((U64)srcPtr[1]<<8);
162 value64 += ((U64)srcPtr[2]<<16);
163 value64 += ((U64)srcPtr[3]<<24);
164 value64 += ((U64)srcPtr[4]<<32);
165 value64 += ((U64)srcPtr[5]<<40);
166 value64 += ((U64)srcPtr[6]<<48);
167 value64 += ((U64)srcPtr[7]<<56);
168 return value64;
169 }
170
LZ4F_writeLE64(void * dst,U64 value64)171 static void LZ4F_writeLE64 (void* dst, U64 value64)
172 {
173 BYTE* const dstPtr = (BYTE*)dst;
174 dstPtr[0] = (BYTE)value64;
175 dstPtr[1] = (BYTE)(value64 >> 8);
176 dstPtr[2] = (BYTE)(value64 >> 16);
177 dstPtr[3] = (BYTE)(value64 >> 24);
178 dstPtr[4] = (BYTE)(value64 >> 32);
179 dstPtr[5] = (BYTE)(value64 >> 40);
180 dstPtr[6] = (BYTE)(value64 >> 48);
181 dstPtr[7] = (BYTE)(value64 >> 56);
182 }
183
184
185 /*-************************************
186 * Constants
187 **************************************/
188 #ifndef LZ4_SRC_INCLUDED
189 #define KB *(1<<10)
190 #define MB *(1<<20)
191 #define GB *(1<<30)
192 #endif
193
194 #define _1BIT 0x01
195 #define _2BITS 0x03
196 #define _3BITS 0x07
197 #define _4BITS 0x0F
198 #define _8BITS 0xFF
199
200 #define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
201 #define LZ4F_MAGICNUMBER 0x184D2204U
202 #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
203 #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
204
205 static const size_t minFHSize = 7;
206 static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */
207 static const size_t BHSize = 4;
208
209
210 /*-************************************
211 * Structures and local types
212 **************************************/
213 typedef struct LZ4F_cctx_s
214 {
215 LZ4F_preferences_t prefs;
216 U32 version;
217 U32 cStage;
218 const LZ4F_CDict* cdict;
219 size_t maxBlockSize;
220 size_t maxBufferSize;
221 BYTE* tmpBuff;
222 BYTE* tmpIn;
223 size_t tmpInSize;
224 U64 totalInSize;
225 XXH32_state_t xxh;
226 void* lz4CtxPtr;
227 U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
228 U16 lz4CtxState; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */
229 } LZ4F_cctx_t;
230
231
232 /*-************************************
233 * Error management
234 **************************************/
235 #define LZ4F_GENERATE_STRING(STRING) #STRING,
236 static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) };
237
238
LZ4F_isError(LZ4F_errorCode_t code)239 unsigned LZ4F_isError(LZ4F_errorCode_t code)
240 {
241 return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode));
242 }
243
LZ4F_getErrorName(LZ4F_errorCode_t code)244 const char* LZ4F_getErrorName(LZ4F_errorCode_t code)
245 {
246 static const char* codeError = "Unspecified error code";
247 if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)];
248 return codeError;
249 }
250
LZ4F_getErrorCode(size_t functionResult)251 LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult)
252 {
253 if (!LZ4F_isError(functionResult)) return LZ4F_OK_NoError;
254 return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult);
255 }
256
err0r(LZ4F_errorCodes code)257 static LZ4F_errorCode_t err0r(LZ4F_errorCodes code)
258 {
259 /* A compilation error here means sizeof(ptrdiff_t) is not large enough */
260 LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t));
261 return (LZ4F_errorCode_t)-(ptrdiff_t)code;
262 }
263
LZ4F_getVersion(void)264 unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; }
265
LZ4F_compressionLevel_max(void)266 int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; }
267
LZ4F_getBlockSize(unsigned blockSizeID)268 size_t LZ4F_getBlockSize(unsigned blockSizeID)
269 {
270 static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB };
271
272 if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
273 if (blockSizeID < 4 || blockSizeID > 7) return err0r(LZ4F_ERROR_maxBlockSize_invalid);
274 blockSizeID -= 4;
275 return blockSizes[blockSizeID];
276 }
277
278 /*-************************************
279 * Private functions
280 **************************************/
281 #define MIN(a,b) ( (a) < (b) ? (a) : (b) )
282
LZ4F_headerChecksum(const void * header,size_t length)283 static BYTE LZ4F_headerChecksum (const void* header, size_t length)
284 {
285 U32 const xxh = XXH32(header, length, 0);
286 return (BYTE)(xxh >> 8);
287 }
288
289
290 /*-************************************
291 * Simple-pass compression functions
292 **************************************/
LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,const size_t srcSize)293 static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID,
294 const size_t srcSize)
295 {
296 LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB;
297 size_t maxBlockSize = 64 KB;
298 while (requestedBSID > proposedBSID) {
299 if (srcSize <= maxBlockSize)
300 return proposedBSID;
301 proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1);
302 maxBlockSize <<= 2;
303 }
304 return requestedBSID;
305 }
306
307 /*! LZ4F_compressBound_internal() :
308 * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
309 * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario.
310 * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers.
311 * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
312 */
LZ4F_compressBound_internal(size_t srcSize,const LZ4F_preferences_t * preferencesPtr,size_t alreadyBuffered)313 static size_t LZ4F_compressBound_internal(size_t srcSize,
314 const LZ4F_preferences_t* preferencesPtr,
315 size_t alreadyBuffered)
316 {
317 LZ4F_preferences_t prefsNull;
318 MEM_INIT(&prefsNull, 0, sizeof(prefsNull));
319 prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */
320 { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
321 U32 const flush = prefsPtr->autoFlush | (srcSize==0);
322 LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID;
323 size_t const blockSize = LZ4F_getBlockSize(blockID);
324 size_t const maxBuffered = blockSize - 1;
325 size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered);
326 size_t const maxSrcSize = srcSize + bufferedSize;
327 unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize);
328 size_t const partialBlockSize = maxSrcSize & (blockSize-1);
329 size_t const lastBlockSize = flush ? partialBlockSize : 0;
330 unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);
331
332 size_t const blockHeaderSize = 4;
333 size_t const blockCRCSize = 4 * prefsPtr->frameInfo.blockChecksumFlag;
334 size_t const frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4);
335
336 return ((blockHeaderSize + blockCRCSize) * nbBlocks) +
337 (blockSize * nbFullBlocks) + lastBlockSize + frameEnd;
338 }
339 }
340
LZ4F_compressFrameBound(size_t srcSize,const LZ4F_preferences_t * preferencesPtr)341 size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
342 {
343 LZ4F_preferences_t prefs;
344 size_t const headerSize = maxFHSize; /* max header size, including optional fields */
345
346 if (preferencesPtr!=NULL) prefs = *preferencesPtr;
347 else MEM_INIT(&prefs, 0, sizeof(prefs));
348 prefs.autoFlush = 1;
349
350 return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);;
351 }
352
353
354 /*! LZ4F_compressFrame_usingCDict() :
355 * Compress srcBuffer using a dictionary, in a single step.
356 * cdict can be NULL, in which case, no dictionary is used.
357 * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
358 * The LZ4F_preferences_t structure is optional : you may provide NULL as argument,
359 * however, it's the only way to provide a dictID, so it's not recommended.
360 * @return : number of bytes written into dstBuffer,
361 * or an error code if it fails (can be tested using LZ4F_isError())
362 */
LZ4F_compressFrame_usingCDict(LZ4F_cctx * cctx,void * dstBuffer,size_t dstCapacity,const void * srcBuffer,size_t srcSize,const LZ4F_CDict * cdict,const LZ4F_preferences_t * preferencesPtr)363 size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx,
364 void* dstBuffer, size_t dstCapacity,
365 const void* srcBuffer, size_t srcSize,
366 const LZ4F_CDict* cdict,
367 const LZ4F_preferences_t* preferencesPtr)
368 {
369 LZ4F_preferences_t prefs;
370 LZ4F_compressOptions_t options;
371 BYTE* const dstStart = (BYTE*) dstBuffer;
372 BYTE* dstPtr = dstStart;
373 BYTE* const dstEnd = dstStart + dstCapacity;
374
375 if (preferencesPtr!=NULL)
376 prefs = *preferencesPtr;
377 else
378 MEM_INIT(&prefs, 0, sizeof(prefs));
379 if (prefs.frameInfo.contentSize != 0)
380 prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */
381
382 prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
383 prefs.autoFlush = 1;
384 if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID))
385 prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */
386
387 MEM_INIT(&options, 0, sizeof(options));
388 options.stableSrc = 1;
389
390 if (dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs)) /* condition to guarantee success */
391 return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
392
393 { size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */
394 if (LZ4F_isError(headerSize)) return headerSize;
395 dstPtr += headerSize; /* header size */ }
396
397 { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options);
398 if (LZ4F_isError(cSize)) return cSize;
399 dstPtr += cSize; }
400
401 { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */
402 if (LZ4F_isError(tailSize)) return tailSize;
403 dstPtr += tailSize; }
404
405 return (dstPtr - dstStart);
406 }
407
408
409 /*! LZ4F_compressFrame() :
410 * Compress an entire srcBuffer into a valid LZ4 frame, in a single step.
411 * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr).
412 * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
413 * @return : number of bytes written into dstBuffer.
414 * or an error code if it fails (can be tested using LZ4F_isError())
415 */
LZ4F_compressFrame(void * dstBuffer,size_t dstCapacity,const void * srcBuffer,size_t srcSize,const LZ4F_preferences_t * preferencesPtr)416 size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity,
417 const void* srcBuffer, size_t srcSize,
418 const LZ4F_preferences_t* preferencesPtr)
419 {
420 size_t result;
421 #if (LZ4F_HEAPMODE)
422 LZ4F_cctx_t *cctxPtr;
423 result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION);
424 if (LZ4F_isError(result)) return result;
425 #else
426 LZ4F_cctx_t cctx;
427 LZ4_stream_t lz4ctx;
428 LZ4F_cctx_t *cctxPtr = &cctx;
429
430 DEBUGLOG(4, "LZ4F_compressFrame");
431 MEM_INIT(&cctx, 0, sizeof(cctx));
432 cctx.version = LZ4F_VERSION;
433 cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */
434 if (preferencesPtr == NULL ||
435 preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN)
436 {
437 LZ4_resetStream(&lz4ctx);
438 cctxPtr->lz4CtxPtr = &lz4ctx;
439 cctxPtr->lz4CtxAlloc = 1;
440 cctxPtr->lz4CtxState = 1;
441 }
442 #endif
443
444 result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity,
445 srcBuffer, srcSize,
446 NULL, preferencesPtr);
447
448 #if (LZ4F_HEAPMODE)
449 LZ4F_freeCompressionContext(cctxPtr);
450 #else
451 if (preferencesPtr != NULL &&
452 preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN)
453 {
454 FREEMEM(cctxPtr->lz4CtxPtr);
455 }
456 #endif
457 return result;
458 }
459
460
461 /*-***************************************************
462 * Dictionary compression
463 *****************************************************/
464
465 struct LZ4F_CDict_s {
466 void* dictContent;
467 LZ4_stream_t* fastCtx;
468 LZ4_streamHC_t* HCCtx;
469 }; /* typedef'd to LZ4F_CDict within lz4frame_static.h */
470
471 /*! LZ4F_createCDict() :
472 * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
473 * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
474 * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
475 * `dictBuffer` can be released after LZ4F_CDict creation, since its content is copied within CDict
476 * @return : digested dictionary for compression, or NULL if failed */
LZ4F_createCDict(const void * dictBuffer,size_t dictSize)477 LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize)
478 {
479 const char* dictStart = (const char*)dictBuffer;
480 LZ4F_CDict* cdict = (LZ4F_CDict*) ALLOC(sizeof(*cdict));
481 DEBUGLOG(4, "LZ4F_createCDict");
482 if (!cdict) return NULL;
483 if (dictSize > 64 KB) {
484 dictStart += dictSize - 64 KB;
485 dictSize = 64 KB;
486 }
487 cdict->dictContent = ALLOC(dictSize);
488 cdict->fastCtx = LZ4_createStream();
489 cdict->HCCtx = LZ4_createStreamHC();
490 if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) {
491 LZ4F_freeCDict(cdict);
492 return NULL;
493 }
494 memcpy(cdict->dictContent, dictStart, dictSize);
495 LZ4_loadDict (cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize);
496 LZ4_setCompressionLevel(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT);
497 LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize);
498 return cdict;
499 }
500
LZ4F_freeCDict(LZ4F_CDict * cdict)501 void LZ4F_freeCDict(LZ4F_CDict* cdict)
502 {
503 if (cdict==NULL) return; /* support free on NULL */
504 FREEMEM(cdict->dictContent);
505 LZ4_freeStream(cdict->fastCtx);
506 LZ4_freeStreamHC(cdict->HCCtx);
507 FREEMEM(cdict);
508 }
509
510
511 /*-*********************************
512 * Advanced compression functions
513 ***********************************/
514
515 /*! LZ4F_createCompressionContext() :
516 * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
517 * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
518 * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries.
519 * The function will provide a pointer to an allocated LZ4F_compressionContext_t object.
520 * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation.
521 * Object can release its memory using LZ4F_freeCompressionContext();
522 */
LZ4F_createCompressionContext(LZ4F_compressionContext_t * LZ4F_compressionContextPtr,unsigned version)523 LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version)
524 {
525 LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)ALLOC_AND_ZERO(sizeof(LZ4F_cctx_t));
526 if (cctxPtr==NULL) return err0r(LZ4F_ERROR_allocation_failed);
527
528 cctxPtr->version = version;
529 cctxPtr->cStage = 0; /* Next stage : init stream */
530
531 *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr;
532
533 return LZ4F_OK_NoError;
534 }
535
536
LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext)537 LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext)
538 {
539 LZ4F_cctx_t* const cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext;
540
541 if (cctxPtr != NULL) { /* support free on NULL */
542 FREEMEM(cctxPtr->lz4CtxPtr); /* works because LZ4_streamHC_t and LZ4_stream_t are simple POD types */
543 FREEMEM(cctxPtr->tmpBuff);
544 FREEMEM(LZ4F_compressionContext);
545 }
546
547 return LZ4F_OK_NoError;
548 }
549
550
551 /**
552 * This function prepares the internal LZ4(HC) stream for a new compression,
553 * resetting the context and attaching the dictionary, if there is one.
554 *
555 * It needs to be called at the beginning of each independent compression
556 * stream (i.e., at the beginning of a frame in blockLinked mode, or at the
557 * beginning of each block in blockIndependent mode).
558 */
LZ4F_initStream(void * ctx,const LZ4F_CDict * cdict,int level,LZ4F_blockMode_t blockMode)559 static void LZ4F_initStream(void* ctx,
560 const LZ4F_CDict* cdict,
561 int level,
562 LZ4F_blockMode_t blockMode) {
563 if (level < LZ4HC_CLEVEL_MIN) {
564 if (cdict != NULL || blockMode == LZ4F_blockLinked) {
565 /* In these cases, we will call LZ4_compress_fast_continue(),
566 * which needs an already reset context. Otherwise, we'll call a
567 * one-shot API. The non-continued APIs internally perform their own
568 * resets at the beginning of their calls, where they know what
569 * tableType they need the context to be in. So in that case this
570 * would be misguided / wasted work. */
571 LZ4_resetStream_fast((LZ4_stream_t*)ctx);
572 }
573 LZ4_attach_dictionary((LZ4_stream_t *)ctx, cdict ? cdict->fastCtx : NULL);
574 } else {
575 LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level);
576 LZ4_attach_HC_dictionary((LZ4_streamHC_t *)ctx, cdict ? cdict->HCCtx : NULL);
577 }
578 }
579
580
581 /*! LZ4F_compressBegin_usingCDict() :
582 * init streaming compression and writes frame header into dstBuffer.
583 * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
584 * @return : number of bytes written into dstBuffer for the header
585 * or an error code (can be tested using LZ4F_isError())
586 */
LZ4F_compressBegin_usingCDict(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const LZ4F_CDict * cdict,const LZ4F_preferences_t * preferencesPtr)587 size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctxPtr,
588 void* dstBuffer, size_t dstCapacity,
589 const LZ4F_CDict* cdict,
590 const LZ4F_preferences_t* preferencesPtr)
591 {
592 LZ4F_preferences_t prefNull;
593 BYTE* const dstStart = (BYTE*)dstBuffer;
594 BYTE* dstPtr = dstStart;
595 BYTE* headerStart;
596
597 if (dstCapacity < maxFHSize) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
598 MEM_INIT(&prefNull, 0, sizeof(prefNull));
599 if (preferencesPtr == NULL) preferencesPtr = &prefNull;
600 cctxPtr->prefs = *preferencesPtr;
601
602 /* Ctx Management */
603 { U16 const ctxTypeID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
604 if (cctxPtr->lz4CtxAlloc < ctxTypeID) {
605 FREEMEM(cctxPtr->lz4CtxPtr);
606 if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
607 cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
608 } else {
609 cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
610 }
611 if (cctxPtr->lz4CtxPtr == NULL) return err0r(LZ4F_ERROR_allocation_failed);
612 cctxPtr->lz4CtxAlloc = ctxTypeID;
613 cctxPtr->lz4CtxState = ctxTypeID;
614 } else if (cctxPtr->lz4CtxState != ctxTypeID) {
615 /* otherwise, a sufficient buffer is allocated, but we need to
616 * reset it to the correct context type */
617 if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
618 LZ4_resetStream((LZ4_stream_t *) cctxPtr->lz4CtxPtr);
619 } else {
620 LZ4_resetStreamHC((LZ4_streamHC_t *) cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel);
621 }
622 cctxPtr->lz4CtxState = ctxTypeID;
623 }
624 }
625
626 /* Buffer Management */
627 if (cctxPtr->prefs.frameInfo.blockSizeID == 0)
628 cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT;
629 cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID);
630
631 { size_t const requiredBuffSize = preferencesPtr->autoFlush ?
632 (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB : /* only needs windows size */
633 cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB);
634
635 if (cctxPtr->maxBufferSize < requiredBuffSize) {
636 cctxPtr->maxBufferSize = 0;
637 FREEMEM(cctxPtr->tmpBuff);
638 cctxPtr->tmpBuff = (BYTE*)ALLOC_AND_ZERO(requiredBuffSize);
639 if (cctxPtr->tmpBuff == NULL) return err0r(LZ4F_ERROR_allocation_failed);
640 cctxPtr->maxBufferSize = requiredBuffSize;
641 } }
642 cctxPtr->tmpIn = cctxPtr->tmpBuff;
643 cctxPtr->tmpInSize = 0;
644 (void)XXH32_reset(&(cctxPtr->xxh), 0);
645
646 /* context init */
647 cctxPtr->cdict = cdict;
648 if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) {
649 /* frame init only for blockLinked : blockIndependent will be init at each block */
650 LZ4F_initStream(cctxPtr->lz4CtxPtr, cdict, cctxPtr->prefs.compressionLevel, LZ4F_blockLinked);
651 }
652 if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) {
653 LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctxPtr->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed);
654 }
655
656 /* Magic Number */
657 LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER);
658 dstPtr += 4;
659 headerStart = dstPtr;
660
661 /* FLG Byte */
662 *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */
663 + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5)
664 + ((cctxPtr->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4)
665 + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)
666 + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2)
667 + (cctxPtr->prefs.frameInfo.dictID > 0) );
668 /* BD Byte */
669 *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
670 /* Optional Frame content size field */
671 if (cctxPtr->prefs.frameInfo.contentSize) {
672 LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize);
673 dstPtr += 8;
674 cctxPtr->totalInSize = 0;
675 }
676 /* Optional dictionary ID field */
677 if (cctxPtr->prefs.frameInfo.dictID) {
678 LZ4F_writeLE32(dstPtr, cctxPtr->prefs.frameInfo.dictID);
679 dstPtr += 4;
680 }
681 /* Header CRC Byte */
682 *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
683 dstPtr++;
684
685 cctxPtr->cStage = 1; /* header written, now request input data block */
686 return (dstPtr - dstStart);
687 }
688
689
690 /*! LZ4F_compressBegin() :
691 * init streaming compression and writes frame header into dstBuffer.
692 * dstBuffer must be >= LZ4F_HEADER_SIZE_MAX bytes.
693 * preferencesPtr can be NULL, in which case default parameters are selected.
694 * @return : number of bytes written into dstBuffer for the header
695 * or an error code (can be tested using LZ4F_isError())
696 */
LZ4F_compressBegin(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const LZ4F_preferences_t * preferencesPtr)697 size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr,
698 void* dstBuffer, size_t dstCapacity,
699 const LZ4F_preferences_t* preferencesPtr)
700 {
701 return LZ4F_compressBegin_usingCDict(cctxPtr, dstBuffer, dstCapacity,
702 NULL, preferencesPtr);
703 }
704
705
706 /* LZ4F_compressBound() :
707 * @return minimum capacity of dstBuffer for a given srcSize to handle worst case scenario.
708 * LZ4F_preferences_t structure is optional : if NULL, preferences will be set to cover worst case scenario.
709 * This function cannot fail.
710 */
LZ4F_compressBound(size_t srcSize,const LZ4F_preferences_t * preferencesPtr)711 size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr)
712 {
713 return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1);
714 }
715
716
717 typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict);
718
719
720 /*! LZ4F_makeBlock():
721 * compress a single block, add header and checksum
722 * assumption : dst buffer capacity is >= srcSize */
LZ4F_makeBlock(void * dst,const void * src,size_t srcSize,compressFunc_t compress,void * lz4ctx,int level,const LZ4F_CDict * cdict,LZ4F_blockChecksum_t crcFlag)723 static size_t LZ4F_makeBlock(void* dst, const void* src, size_t srcSize,
724 compressFunc_t compress, void* lz4ctx, int level,
725 const LZ4F_CDict* cdict, LZ4F_blockChecksum_t crcFlag)
726 {
727 BYTE* const cSizePtr = (BYTE*)dst;
728 U32 cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4),
729 (int)(srcSize), (int)(srcSize-1),
730 level, cdict);
731 LZ4F_writeLE32(cSizePtr, cSize);
732 if (cSize == 0) { /* compression failed */
733 cSize = (U32)srcSize;
734 LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG);
735 memcpy(cSizePtr+4, src, srcSize);
736 }
737 if (crcFlag) {
738 U32 const crc32 = XXH32(cSizePtr+4, cSize, 0); /* checksum of compressed data */
739 LZ4F_writeLE32(cSizePtr+4+cSize, crc32);
740 }
741 return 4 + cSize + ((U32)crcFlag)*4;
742 }
743
744
LZ4F_compressBlock(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)745 static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
746 {
747 int const acceleration = (level < 0) ? -level + 1 : 1;
748 LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
749 if (cdict) {
750 return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
751 } else {
752 return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration);
753 }
754 }
755
LZ4F_compressBlock_continue(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)756 static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
757 {
758 int const acceleration = (level < 0) ? -level + 1 : 1;
759 (void)cdict; /* init once at beginning of frame */
760 return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration);
761 }
762
LZ4F_compressBlockHC(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)763 static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
764 {
765 LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent);
766 if (cdict) {
767 return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
768 }
769 return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level);
770 }
771
LZ4F_compressBlockHC_continue(void * ctx,const char * src,char * dst,int srcSize,int dstCapacity,int level,const LZ4F_CDict * cdict)772 static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict)
773 {
774 (void)level; (void)cdict; /* init once at beginning of frame */
775 return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity);
776 }
777
LZ4F_selectCompression(LZ4F_blockMode_t blockMode,int level)778 static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
779 {
780 if (level < LZ4HC_CLEVEL_MIN) {
781 if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock;
782 return LZ4F_compressBlock_continue;
783 }
784 if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC;
785 return LZ4F_compressBlockHC_continue;
786 }
787
LZ4F_localSaveDict(LZ4F_cctx_t * cctxPtr)788 static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
789 {
790 if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
791 return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
792 return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
793 }
794
795 typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
796
797 /*! LZ4F_compressUpdate() :
798 * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
799 * dstBuffer MUST be >= LZ4F_compressBound(srcSize, preferencesPtr).
800 * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
801 * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered.
802 * or an error code if it fails (which can be tested using LZ4F_isError())
803 */
LZ4F_compressUpdate(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const void * srcBuffer,size_t srcSize,const LZ4F_compressOptions_t * compressOptionsPtr)804 size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
805 void* dstBuffer, size_t dstCapacity,
806 const void* srcBuffer, size_t srcSize,
807 const LZ4F_compressOptions_t* compressOptionsPtr)
808 {
809 LZ4F_compressOptions_t cOptionsNull;
810 size_t const blockSize = cctxPtr->maxBlockSize;
811 const BYTE* srcPtr = (const BYTE*)srcBuffer;
812 const BYTE* const srcEnd = srcPtr + srcSize;
813 BYTE* const dstStart = (BYTE*)dstBuffer;
814 BYTE* dstPtr = dstStart;
815 LZ4F_lastBlockStatus lastBlockCompressed = notDone;
816 compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
817
818 DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize);
819
820 if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
821 if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
822 return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
823 MEM_INIT(&cOptionsNull, 0, sizeof(cOptionsNull));
824 if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
825
826 /* complete tmp buffer */
827 if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */
828 size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize;
829 if (sizeToCopy > srcSize) {
830 /* add src to tmpIn buffer */
831 memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize);
832 srcPtr = srcEnd;
833 cctxPtr->tmpInSize += srcSize;
834 /* still needs some CRC */
835 } else {
836 /* complete tmpIn block and then compress it */
837 lastBlockCompressed = fromTmpBuffer;
838 memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy);
839 srcPtr += sizeToCopy;
840
841 dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, blockSize,
842 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
843 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
844
845 if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize;
846 cctxPtr->tmpInSize = 0;
847 }
848 }
849
850 while ((size_t)(srcEnd - srcPtr) >= blockSize) {
851 /* compress full blocks */
852 lastBlockCompressed = fromSrcBuffer;
853 dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, blockSize,
854 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
855 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
856 srcPtr += blockSize;
857 }
858
859 if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) {
860 /* compress remaining input < blockSize */
861 lastBlockCompressed = fromSrcBuffer;
862 dstPtr += LZ4F_makeBlock(dstPtr, srcPtr, srcEnd - srcPtr,
863 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
864 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
865 srcPtr = srcEnd;
866 }
867
868 /* preserve dictionary if necessary */
869 if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) {
870 if (compressOptionsPtr->stableSrc) {
871 cctxPtr->tmpIn = cctxPtr->tmpBuff;
872 } else {
873 int const realDictSize = LZ4F_localSaveDict(cctxPtr);
874 if (realDictSize==0) return err0r(LZ4F_ERROR_GENERIC);
875 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
876 }
877 }
878
879 /* keep tmpIn within limits */
880 if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */
881 && !(cctxPtr->prefs.autoFlush))
882 {
883 int const realDictSize = LZ4F_localSaveDict(cctxPtr);
884 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
885 }
886
887 /* some input data left, necessarily < blockSize */
888 if (srcPtr < srcEnd) {
889 /* fill tmp buffer */
890 size_t const sizeToCopy = srcEnd - srcPtr;
891 memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy);
892 cctxPtr->tmpInSize = sizeToCopy;
893 }
894
895 if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled)
896 (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
897
898 cctxPtr->totalInSize += srcSize;
899 return dstPtr - dstStart;
900 }
901
902
903 /*! LZ4F_flush() :
904 * Should you need to create compressed data immediately, without waiting for a block to be filled,
905 * you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext.
906 * The result of the function is the number of bytes written into dstBuffer
907 * (it can be zero, this means there was no data left within compressionContext)
908 * The function outputs an error code if it fails (can be tested using LZ4F_isError())
909 * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
910 */
LZ4F_flush(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const LZ4F_compressOptions_t * compressOptionsPtr)911 size_t LZ4F_flush(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* compressOptionsPtr)
912 {
913 BYTE* const dstStart = (BYTE*)dstBuffer;
914 BYTE* dstPtr = dstStart;
915 compressFunc_t compress;
916
917 if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
918 if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
919 if (dstCapacity < (cctxPtr->tmpInSize + 4)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall); /* +4 : block header(4) */
920 (void)compressOptionsPtr; /* not yet useful */
921
922 /* select compression function */
923 compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel);
924
925 /* compress tmp buffer */
926 dstPtr += LZ4F_makeBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize,
927 compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel,
928 cctxPtr->cdict, cctxPtr->prefs.frameInfo.blockChecksumFlag);
929 if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize;
930 cctxPtr->tmpInSize = 0;
931
932 /* keep tmpIn within limits */
933 if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */
934 int realDictSize = LZ4F_localSaveDict(cctxPtr);
935 cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
936 }
937
938 return dstPtr - dstStart;
939 }
940
941
942 /*! LZ4F_compressEnd() :
943 * When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
944 * It will flush whatever data remained within compressionContext (like LZ4_flush())
945 * but also properly finalize the frame, with an endMark and an (optional) checksum.
946 * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
947 * @return: the number of bytes written into dstBuffer (necessarily >= 4 (endMark size))
948 * or an error code if it fails (can be tested using LZ4F_isError())
949 * The context can then be used again to compress a new frame, starting with LZ4F_compressBegin().
950 */
LZ4F_compressEnd(LZ4F_cctx * cctxPtr,void * dstBuffer,size_t dstCapacity,const LZ4F_compressOptions_t * compressOptionsPtr)951 size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr,
952 void* dstBuffer, size_t dstCapacity,
953 const LZ4F_compressOptions_t* compressOptionsPtr)
954 {
955 BYTE* const dstStart = (BYTE*)dstBuffer;
956 BYTE* dstPtr = dstStart;
957
958 size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr);
959 if (LZ4F_isError(flushSize)) return flushSize;
960 dstPtr += flushSize;
961
962 assert(flushSize <= dstCapacity);
963 dstCapacity -= flushSize;
964
965 if (dstCapacity < 4) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
966 LZ4F_writeLE32(dstPtr, 0);
967 dstPtr += 4; /* endMark */
968
969 if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) {
970 U32 const xxh = XXH32_digest(&(cctxPtr->xxh));
971 if (dstCapacity < 8) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
972 LZ4F_writeLE32(dstPtr, xxh);
973 dstPtr+=4; /* content Checksum */
974 }
975
976 cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */
977 cctxPtr->maxBufferSize = 0; /* reuse HC context */
978
979 if (cctxPtr->prefs.frameInfo.contentSize) {
980 if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize)
981 return err0r(LZ4F_ERROR_frameSize_wrong);
982 }
983
984 return dstPtr - dstStart;
985 }
986
987
988 /*-***************************************************
989 * Frame Decompression
990 *****************************************************/
991
992 typedef enum {
993 dstage_getFrameHeader=0, dstage_storeFrameHeader,
994 dstage_init,
995 dstage_getBlockHeader, dstage_storeBlockHeader,
996 dstage_copyDirect, dstage_getBlockChecksum,
997 dstage_getCBlock, dstage_storeCBlock,
998 dstage_flushOut,
999 dstage_getSuffix, dstage_storeSuffix,
1000 dstage_getSFrameSize, dstage_storeSFrameSize,
1001 dstage_skipSkippable
1002 } dStage_t;
1003
1004 struct LZ4F_dctx_s {
1005 LZ4F_frameInfo_t frameInfo;
1006 U32 version;
1007 dStage_t dStage;
1008 U64 frameRemainingSize;
1009 size_t maxBlockSize;
1010 size_t maxBufferSize;
1011 BYTE* tmpIn;
1012 size_t tmpInSize;
1013 size_t tmpInTarget;
1014 BYTE* tmpOutBuffer;
1015 const BYTE* dict;
1016 size_t dictSize;
1017 BYTE* tmpOut;
1018 size_t tmpOutSize;
1019 size_t tmpOutStart;
1020 XXH32_state_t xxh;
1021 XXH32_state_t blockChecksum;
1022 BYTE header[LZ4F_HEADER_SIZE_MAX];
1023 }; /* typedef'd to LZ4F_dctx in lz4frame.h */
1024
1025
1026 /*! LZ4F_createDecompressionContext() :
1027 * Create a decompressionContext object, which will track all decompression operations.
1028 * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object.
1029 * Object can later be released using LZ4F_freeDecompressionContext().
1030 * @return : if != 0, there was an error during context creation.
1031 */
LZ4F_createDecompressionContext(LZ4F_dctx ** LZ4F_decompressionContextPtr,unsigned versionNumber)1032 LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber)
1033 {
1034 LZ4F_dctx* const dctx = (LZ4F_dctx*)ALLOC_AND_ZERO(sizeof(LZ4F_dctx));
1035 if (dctx==NULL) return err0r(LZ4F_ERROR_GENERIC);
1036
1037 dctx->version = versionNumber;
1038 *LZ4F_decompressionContextPtr = dctx;
1039 return LZ4F_OK_NoError;
1040 }
1041
LZ4F_freeDecompressionContext(LZ4F_dctx * dctx)1042 LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx)
1043 {
1044 LZ4F_errorCode_t result = LZ4F_OK_NoError;
1045 if (dctx != NULL) { /* can accept NULL input, like free() */
1046 result = (LZ4F_errorCode_t)dctx->dStage;
1047 FREEMEM(dctx->tmpIn);
1048 FREEMEM(dctx->tmpOutBuffer);
1049 FREEMEM(dctx);
1050 }
1051 return result;
1052 }
1053
1054
1055 /*==--- Streaming Decompression operations ---==*/
1056
LZ4F_resetDecompressionContext(LZ4F_dctx * dctx)1057 void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx)
1058 {
1059 dctx->dStage = dstage_getFrameHeader;
1060 dctx->dict = NULL;
1061 dctx->dictSize = 0;
1062 }
1063
1064
1065 /*! LZ4F_headerSize() :
1066 * @return : size of frame header
1067 * or an error code, which can be tested using LZ4F_isError()
1068 */
LZ4F_headerSize(const void * src,size_t srcSize)1069 static size_t LZ4F_headerSize(const void* src, size_t srcSize)
1070 {
1071 /* minimal srcSize to determine header size */
1072 if (srcSize < 5) return err0r(LZ4F_ERROR_frameHeader_incomplete);
1073
1074 /* special case : skippable frames */
1075 if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) return 8;
1076
1077 /* control magic number */
1078 if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER)
1079 return err0r(LZ4F_ERROR_frameType_unknown);
1080
1081 /* Frame Header Size */
1082 { BYTE const FLG = ((const BYTE*)src)[4];
1083 U32 const contentSizeFlag = (FLG>>3) & _1BIT;
1084 U32 const dictIDFlag = FLG & _1BIT;
1085 return minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
1086 }
1087 }
1088
1089
1090 /*! LZ4F_decodeHeader() :
1091 * input : `src` points at the **beginning of the frame**
1092 * output : set internal values of dctx, such as
1093 * dctx->frameInfo and dctx->dStage.
1094 * Also allocates internal buffers.
1095 * @return : nb Bytes read from src (necessarily <= srcSize)
1096 * or an error code (testable with LZ4F_isError())
1097 */
LZ4F_decodeHeader(LZ4F_dctx * dctx,const void * src,size_t srcSize)1098 static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize)
1099 {
1100 unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID;
1101 size_t frameHeaderSize;
1102 const BYTE* srcPtr = (const BYTE*)src;
1103
1104 /* need to decode header to get frameInfo */
1105 if (srcSize < minFHSize) return err0r(LZ4F_ERROR_frameHeader_incomplete); /* minimal frame header size */
1106 MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo));
1107
1108 /* special case : skippable frames */
1109 if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) {
1110 dctx->frameInfo.frameType = LZ4F_skippableFrame;
1111 if (src == (void*)(dctx->header)) {
1112 dctx->tmpInSize = srcSize;
1113 dctx->tmpInTarget = 8;
1114 dctx->dStage = dstage_storeSFrameSize;
1115 return srcSize;
1116 } else {
1117 dctx->dStage = dstage_getSFrameSize;
1118 return 4;
1119 }
1120 }
1121
1122 /* control magic number */
1123 if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER)
1124 return err0r(LZ4F_ERROR_frameType_unknown);
1125 dctx->frameInfo.frameType = LZ4F_frame;
1126
1127 /* Flags */
1128 { U32 const FLG = srcPtr[4];
1129 U32 const version = (FLG>>6) & _2BITS;
1130 blockChecksumFlag = (FLG>>4) & _1BIT;
1131 blockMode = (FLG>>5) & _1BIT;
1132 contentSizeFlag = (FLG>>3) & _1BIT;
1133 contentChecksumFlag = (FLG>>2) & _1BIT;
1134 dictIDFlag = FLG & _1BIT;
1135 /* validate */
1136 if (((FLG>>1)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
1137 if (version != 1) return err0r(LZ4F_ERROR_headerVersion_wrong); /* Version Number, only supported value */
1138 }
1139
1140 /* Frame Header Size */
1141 frameHeaderSize = minFHSize + (contentSizeFlag*8) + (dictIDFlag*4);
1142
1143 if (srcSize < frameHeaderSize) {
1144 /* not enough input to fully decode frame header */
1145 if (srcPtr != dctx->header)
1146 memcpy(dctx->header, srcPtr, srcSize);
1147 dctx->tmpInSize = srcSize;
1148 dctx->tmpInTarget = frameHeaderSize;
1149 dctx->dStage = dstage_storeFrameHeader;
1150 return srcSize;
1151 }
1152
1153 { U32 const BD = srcPtr[5];
1154 blockSizeID = (BD>>4) & _3BITS;
1155 /* validate */
1156 if (((BD>>7)&_1BIT) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bit */
1157 if (blockSizeID < 4) return err0r(LZ4F_ERROR_maxBlockSize_invalid); /* 4-7 only supported values for the time being */
1158 if (((BD>>0)&_4BITS) != 0) return err0r(LZ4F_ERROR_reservedFlag_set); /* Reserved bits */
1159 }
1160
1161 /* check header */
1162 { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
1163 if (HC != srcPtr[frameHeaderSize-1])
1164 return err0r(LZ4F_ERROR_headerChecksum_invalid);
1165 }
1166
1167 /* save */
1168 dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode;
1169 dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag;
1170 dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag;
1171 dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID;
1172 dctx->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
1173 if (contentSizeFlag)
1174 dctx->frameRemainingSize =
1175 dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6);
1176 if (dictIDFlag)
1177 dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5);
1178
1179 dctx->dStage = dstage_init;
1180
1181 return frameHeaderSize;
1182 }
1183
1184
1185 /*! LZ4F_getFrameInfo() :
1186 * This function extracts frame parameters (max blockSize, frame checksum, etc.).
1187 * Usage is optional. Objective is to provide relevant information for allocation purposes.
1188 * This function works in 2 situations :
1189 * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process.
1190 * Amount of input data provided must be large enough to successfully decode the frame header.
1191 * A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum.
1192 * - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx.
1193 * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
1194 * Decompression must resume from (srcBuffer + *srcSizePtr).
1195 * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call,
1196 * or an error code which can be tested using LZ4F_isError()
1197 * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped.
1198 * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure.
1199 */
LZ4F_getFrameInfo(LZ4F_dctx * dctx,LZ4F_frameInfo_t * frameInfoPtr,const void * srcBuffer,size_t * srcSizePtr)1200 LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, LZ4F_frameInfo_t* frameInfoPtr,
1201 const void* srcBuffer, size_t* srcSizePtr)
1202 {
1203 if (dctx->dStage > dstage_storeFrameHeader) { /* assumption : dstage_* header enum at beginning of range */
1204 /* frameInfo already decoded */
1205 size_t o=0, i=0;
1206 *srcSizePtr = 0;
1207 *frameInfoPtr = dctx->frameInfo;
1208 /* returns : recommended nb of bytes for LZ4F_decompress() */
1209 return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL);
1210 } else {
1211 if (dctx->dStage == dstage_storeFrameHeader) {
1212 /* frame decoding already started, in the middle of header => automatic fail */
1213 *srcSizePtr = 0;
1214 return err0r(LZ4F_ERROR_frameDecoding_alreadyStarted);
1215 } else {
1216 size_t decodeResult;
1217 size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr);
1218 if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; }
1219 if (*srcSizePtr < hSize) {
1220 *srcSizePtr=0;
1221 return err0r(LZ4F_ERROR_frameHeader_incomplete);
1222 }
1223
1224 decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize);
1225 if (LZ4F_isError(decodeResult)) {
1226 *srcSizePtr = 0;
1227 } else {
1228 *srcSizePtr = decodeResult;
1229 decodeResult = BHSize; /* block header size */
1230 }
1231 *frameInfoPtr = dctx->frameInfo;
1232 return decodeResult;
1233 } }
1234 }
1235
1236
1237 /* LZ4F_updateDict() :
1238 * only used for LZ4F_blockLinked mode */
LZ4F_updateDict(LZ4F_dctx * dctx,const BYTE * dstPtr,size_t dstSize,const BYTE * dstBufferStart,unsigned withinTmp)1239 static void LZ4F_updateDict(LZ4F_dctx* dctx,
1240 const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart,
1241 unsigned withinTmp)
1242 {
1243 if (dctx->dictSize==0)
1244 dctx->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */
1245
1246 if (dctx->dict + dctx->dictSize == dstPtr) { /* dictionary continuity, directly within dstBuffer */
1247 dctx->dictSize += dstSize;
1248 return;
1249 }
1250
1251 if (dstPtr - dstBufferStart + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */
1252 dctx->dict = (const BYTE*)dstBufferStart;
1253 dctx->dictSize = dstPtr - dstBufferStart + dstSize;
1254 return;
1255 }
1256
1257 assert(dstSize < 64 KB); /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */
1258
1259 /* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOut */
1260
1261 if ((withinTmp) && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */
1262 /* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */
1263 assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart);
1264 dctx->dictSize += dstSize;
1265 return;
1266 }
1267
1268 if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */
1269 size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
1270 size_t copySize = 64 KB - dctx->tmpOutSize;
1271 const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
1272 if (dctx->tmpOutSize > 64 KB) copySize = 0;
1273 if (copySize > preserveSize) copySize = preserveSize;
1274
1275 memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
1276
1277 dctx->dict = dctx->tmpOutBuffer;
1278 dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize;
1279 return;
1280 }
1281
1282 if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */
1283 if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */
1284 size_t const preserveSize = 64 KB - dstSize;
1285 memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
1286 dctx->dictSize = preserveSize;
1287 }
1288 memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize);
1289 dctx->dictSize += dstSize;
1290 return;
1291 }
1292
1293 /* join dict & dest into tmp */
1294 { size_t preserveSize = 64 KB - dstSize;
1295 if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize;
1296 memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize);
1297 memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize);
1298 dctx->dict = dctx->tmpOutBuffer;
1299 dctx->dictSize = preserveSize + dstSize;
1300 }
1301 }
1302
1303
1304
1305 /*! LZ4F_decompress() :
1306 * Call this function repetitively to regenerate compressed data in srcBuffer.
1307 * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer
1308 * into dstBuffer of capacity *dstSizePtr.
1309 *
1310 * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
1311 *
1312 * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
1313 * If number of bytes read is < number of bytes provided, then decompression operation is not complete.
1314 * Remaining data will have to be presented again in a subsequent invocation.
1315 *
1316 * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress.
1317 * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
1318 * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling.
1319 * Note that this is just a hint, and it's always possible to any srcSize value.
1320 * When a frame is fully decoded, @return will be 0.
1321 * If decompression failed, @return is an error code which can be tested using LZ4F_isError().
1322 */
LZ4F_decompress(LZ4F_dctx * dctx,void * dstBuffer,size_t * dstSizePtr,const void * srcBuffer,size_t * srcSizePtr,const LZ4F_decompressOptions_t * decompressOptionsPtr)1323 size_t LZ4F_decompress(LZ4F_dctx* dctx,
1324 void* dstBuffer, size_t* dstSizePtr,
1325 const void* srcBuffer, size_t* srcSizePtr,
1326 const LZ4F_decompressOptions_t* decompressOptionsPtr)
1327 {
1328 LZ4F_decompressOptions_t optionsNull;
1329 const BYTE* const srcStart = (const BYTE*)srcBuffer;
1330 const BYTE* const srcEnd = srcStart + *srcSizePtr;
1331 const BYTE* srcPtr = srcStart;
1332 BYTE* const dstStart = (BYTE*)dstBuffer;
1333 BYTE* const dstEnd = dstStart + *dstSizePtr;
1334 BYTE* dstPtr = dstStart;
1335 const BYTE* selectedIn = NULL;
1336 unsigned doAnotherStage = 1;
1337 size_t nextSrcSizeHint = 1;
1338
1339
1340 MEM_INIT(&optionsNull, 0, sizeof(optionsNull));
1341 if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull;
1342 *srcSizePtr = 0;
1343 *dstSizePtr = 0;
1344
1345 /* behaves as a state machine */
1346
1347 while (doAnotherStage) {
1348
1349 switch(dctx->dStage)
1350 {
1351
1352 case dstage_getFrameHeader:
1353 if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */
1354 size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, srcEnd-srcPtr); /* will update dStage appropriately */
1355 if (LZ4F_isError(hSize)) return hSize;
1356 srcPtr += hSize;
1357 break;
1358 }
1359 dctx->tmpInSize = 0;
1360 if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */
1361 dctx->tmpInTarget = minFHSize; /* minimum size to decode header */
1362 dctx->dStage = dstage_storeFrameHeader;
1363 /* fall-through */
1364
1365 case dstage_storeFrameHeader:
1366 { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr));
1367 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
1368 dctx->tmpInSize += sizeToCopy;
1369 srcPtr += sizeToCopy;
1370 }
1371 if (dctx->tmpInSize < dctx->tmpInTarget) {
1372 nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */
1373 doAnotherStage = 0; /* not enough src data, ask for some more */
1374 break;
1375 }
1376 { size_t const hSize = LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget); /* will update dStage appropriately */
1377 if (LZ4F_isError(hSize)) return hSize;
1378 }
1379 break;
1380
1381 case dstage_init:
1382 if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0);
1383 /* internal buffers allocation */
1384 { size_t const bufferNeeded = dctx->maxBlockSize
1385 + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB);
1386 if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */
1387 dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/
1388 FREEMEM(dctx->tmpIn);
1389 dctx->tmpIn = (BYTE*)ALLOC(dctx->maxBlockSize + 4 /* block checksum */);
1390 if (dctx->tmpIn == NULL)
1391 return err0r(LZ4F_ERROR_allocation_failed);
1392 FREEMEM(dctx->tmpOutBuffer);
1393 dctx->tmpOutBuffer= (BYTE*)ALLOC(bufferNeeded);
1394 if (dctx->tmpOutBuffer== NULL)
1395 return err0r(LZ4F_ERROR_allocation_failed);
1396 dctx->maxBufferSize = bufferNeeded;
1397 } }
1398 dctx->tmpInSize = 0;
1399 dctx->tmpInTarget = 0;
1400 dctx->tmpOut = dctx->tmpOutBuffer;
1401 dctx->tmpOutStart = 0;
1402 dctx->tmpOutSize = 0;
1403
1404 dctx->dStage = dstage_getBlockHeader;
1405 /* fall-through */
1406
1407 case dstage_getBlockHeader:
1408 if ((size_t)(srcEnd - srcPtr) >= BHSize) {
1409 selectedIn = srcPtr;
1410 srcPtr += BHSize;
1411 } else {
1412 /* not enough input to read cBlockSize field */
1413 dctx->tmpInSize = 0;
1414 dctx->dStage = dstage_storeBlockHeader;
1415 }
1416
1417 if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */
1418 case dstage_storeBlockHeader:
1419 { size_t const remainingInput = (size_t)(srcEnd - srcPtr);
1420 size_t const wantedData = BHSize - dctx->tmpInSize;
1421 size_t const sizeToCopy = MIN(wantedData, remainingInput);
1422 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
1423 srcPtr += sizeToCopy;
1424 dctx->tmpInSize += sizeToCopy;
1425
1426 if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */
1427 nextSrcSizeHint = BHSize - dctx->tmpInSize;
1428 doAnotherStage = 0;
1429 break;
1430 }
1431 selectedIn = dctx->tmpIn;
1432 } /* if (dctx->dStage == dstage_storeBlockHeader) */
1433
1434 /* decode block header */
1435 { size_t const nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU;
1436 size_t const crcSize = dctx->frameInfo.blockChecksumFlag * 4;
1437 if (nextCBlockSize==0) { /* frameEnd signal, no more block */
1438 dctx->dStage = dstage_getSuffix;
1439 break;
1440 }
1441 if (nextCBlockSize > dctx->maxBlockSize)
1442 return err0r(LZ4F_ERROR_maxBlockSize_invalid);
1443 if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) {
1444 /* next block is uncompressed */
1445 dctx->tmpInTarget = nextCBlockSize;
1446 if (dctx->frameInfo.blockChecksumFlag) {
1447 (void)XXH32_reset(&dctx->blockChecksum, 0);
1448 }
1449 dctx->dStage = dstage_copyDirect;
1450 break;
1451 }
1452 /* next block is a compressed block */
1453 dctx->tmpInTarget = nextCBlockSize + crcSize;
1454 dctx->dStage = dstage_getCBlock;
1455 if (dstPtr==dstEnd) {
1456 nextSrcSizeHint = nextCBlockSize + crcSize + BHSize;
1457 doAnotherStage = 0;
1458 }
1459 break;
1460 }
1461
1462 case dstage_copyDirect: /* uncompressed block */
1463 { size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr));
1464 size_t const sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize);
1465 memcpy(dstPtr, srcPtr, sizeToCopy);
1466 if (dctx->frameInfo.blockChecksumFlag) {
1467 (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy);
1468 }
1469 if (dctx->frameInfo.contentChecksumFlag)
1470 (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy);
1471 if (dctx->frameInfo.contentSize)
1472 dctx->frameRemainingSize -= sizeToCopy;
1473
1474 /* history management (linked blocks only)*/
1475 if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
1476 LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0);
1477
1478 srcPtr += sizeToCopy;
1479 dstPtr += sizeToCopy;
1480 if (sizeToCopy == dctx->tmpInTarget) { /* all done */
1481 if (dctx->frameInfo.blockChecksumFlag) {
1482 dctx->tmpInSize = 0;
1483 dctx->dStage = dstage_getBlockChecksum;
1484 } else
1485 dctx->dStage = dstage_getBlockHeader; /* new block */
1486 break;
1487 }
1488 dctx->tmpInTarget -= sizeToCopy; /* need to copy more */
1489 nextSrcSizeHint = dctx->tmpInTarget +
1490 + dctx->frameInfo.blockChecksumFlag * 4 /* size for block checksum */
1491 + BHSize /* next header size */;
1492 doAnotherStage = 0;
1493 break;
1494 }
1495
1496 /* check block checksum for recently transferred uncompressed block */
1497 case dstage_getBlockChecksum:
1498 { const void* crcSrc;
1499 if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) {
1500 crcSrc = srcPtr;
1501 srcPtr += 4;
1502 } else {
1503 size_t const stillToCopy = 4 - dctx->tmpInSize;
1504 size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr));
1505 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
1506 dctx->tmpInSize += sizeToCopy;
1507 srcPtr += sizeToCopy;
1508 if (dctx->tmpInSize < 4) { /* all input consumed */
1509 doAnotherStage = 0;
1510 break;
1511 }
1512 crcSrc = dctx->header;
1513 }
1514 { U32 const readCRC = LZ4F_readLE32(crcSrc);
1515 U32 const calcCRC = XXH32_digest(&dctx->blockChecksum);
1516 if (readCRC != calcCRC)
1517 return err0r(LZ4F_ERROR_blockChecksum_invalid);
1518 } }
1519 dctx->dStage = dstage_getBlockHeader; /* new block */
1520 break;
1521
1522 case dstage_getCBlock:
1523 if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) {
1524 dctx->tmpInSize = 0;
1525 dctx->dStage = dstage_storeCBlock;
1526 break;
1527 }
1528 /* input large enough to read full block directly */
1529 selectedIn = srcPtr;
1530 srcPtr += dctx->tmpInTarget;
1531
1532 if (0) /* jump over next block */
1533 case dstage_storeCBlock:
1534 { size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize;
1535 size_t const inputLeft = (size_t)(srcEnd-srcPtr);
1536 size_t const sizeToCopy = MIN(wantedData, inputLeft);
1537 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
1538 dctx->tmpInSize += sizeToCopy;
1539 srcPtr += sizeToCopy;
1540 if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */
1541 nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize)
1542 + dctx->frameInfo.blockChecksumFlag * 4 /* size for block checksum */
1543 + BHSize /* next header size */;
1544 doAnotherStage = 0;
1545 break;
1546 }
1547 selectedIn = dctx->tmpIn;
1548 }
1549
1550 /* At this stage, input is large enough to decode a block */
1551 if (dctx->frameInfo.blockChecksumFlag) {
1552 dctx->tmpInTarget -= 4;
1553 assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */
1554 { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget);
1555 U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0);
1556 if (readBlockCrc != calcBlockCrc)
1557 return err0r(LZ4F_ERROR_blockChecksum_invalid);
1558 } }
1559
1560 if ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) {
1561 const char* dict = (const char*)dctx->dict;
1562 size_t dictSize = dctx->dictSize;
1563 int decodedSize;
1564 if (dict && dictSize > 1 GB) {
1565 /* the dictSize param is an int, avoid truncation / sign issues */
1566 dict += dictSize - 64 KB;
1567 dictSize = 64 KB;
1568 }
1569 /* enough capacity in `dst` to decompress directly there */
1570 decodedSize = LZ4_decompress_safe_usingDict(
1571 (const char*)selectedIn, (char*)dstPtr,
1572 (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
1573 dict, (int)dictSize);
1574 if (decodedSize < 0) return err0r(LZ4F_ERROR_GENERIC); /* decompression failed */
1575 if (dctx->frameInfo.contentChecksumFlag)
1576 XXH32_update(&(dctx->xxh), dstPtr, decodedSize);
1577 if (dctx->frameInfo.contentSize)
1578 dctx->frameRemainingSize -= decodedSize;
1579
1580 /* dictionary management */
1581 if (dctx->frameInfo.blockMode==LZ4F_blockLinked)
1582 LZ4F_updateDict(dctx, dstPtr, decodedSize, dstStart, 0);
1583
1584 dstPtr += decodedSize;
1585 dctx->dStage = dstage_getBlockHeader;
1586 break;
1587 }
1588
1589 /* not enough place into dst : decode into tmpOut */
1590 /* ensure enough place for tmpOut */
1591 if (dctx->frameInfo.blockMode == LZ4F_blockLinked) {
1592 if (dctx->dict == dctx->tmpOutBuffer) {
1593 if (dctx->dictSize > 128 KB) {
1594 memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB);
1595 dctx->dictSize = 64 KB;
1596 }
1597 dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize;
1598 } else { /* dict not within tmp */
1599 size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB);
1600 dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace;
1601 } }
1602
1603 /* Decode block */
1604 { const char* dict = (const char*)dctx->dict;
1605 size_t dictSize = dctx->dictSize;
1606 int decodedSize;
1607 if (dict && dictSize > 1 GB) {
1608 /* the dictSize param is an int, avoid truncation / sign issues */
1609 dict += dictSize - 64 KB;
1610 dictSize = 64 KB;
1611 }
1612 decodedSize = LZ4_decompress_safe_usingDict(
1613 (const char*)selectedIn, (char*)dctx->tmpOut,
1614 (int)dctx->tmpInTarget, (int)dctx->maxBlockSize,
1615 dict, (int)dictSize);
1616 if (decodedSize < 0) /* decompression failed */
1617 return err0r(LZ4F_ERROR_decompressionFailed);
1618 if (dctx->frameInfo.contentChecksumFlag)
1619 XXH32_update(&(dctx->xxh), dctx->tmpOut, decodedSize);
1620 if (dctx->frameInfo.contentSize)
1621 dctx->frameRemainingSize -= decodedSize;
1622 dctx->tmpOutSize = decodedSize;
1623 dctx->tmpOutStart = 0;
1624 dctx->dStage = dstage_flushOut;
1625 }
1626 /* fall-through */
1627
1628 case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */
1629 { size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr));
1630 memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy);
1631
1632 /* dictionary management */
1633 if (dctx->frameInfo.blockMode == LZ4F_blockLinked)
1634 LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1 /*withinTmp*/);
1635
1636 dctx->tmpOutStart += sizeToCopy;
1637 dstPtr += sizeToCopy;
1638
1639 if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */
1640 dctx->dStage = dstage_getBlockHeader; /* get next block */
1641 break;
1642 }
1643 /* could not flush everything : stop there, just request a block header */
1644 doAnotherStage = 0;
1645 nextSrcSizeHint = BHSize;
1646 break;
1647 }
1648
1649 case dstage_getSuffix:
1650 if (dctx->frameRemainingSize)
1651 return err0r(LZ4F_ERROR_frameSize_wrong); /* incorrect frame size decoded */
1652 if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */
1653 nextSrcSizeHint = 0;
1654 LZ4F_resetDecompressionContext(dctx);
1655 doAnotherStage = 0;
1656 break;
1657 }
1658 if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */
1659 dctx->tmpInSize = 0;
1660 dctx->dStage = dstage_storeSuffix;
1661 } else {
1662 selectedIn = srcPtr;
1663 srcPtr += 4;
1664 }
1665
1666 if (dctx->dStage == dstage_storeSuffix) /* can be skipped */
1667 case dstage_storeSuffix:
1668 { size_t const remainingInput = (size_t)(srcEnd - srcPtr);
1669 size_t const wantedData = 4 - dctx->tmpInSize;
1670 size_t const sizeToCopy = MIN(wantedData, remainingInput);
1671 memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy);
1672 srcPtr += sizeToCopy;
1673 dctx->tmpInSize += sizeToCopy;
1674 if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */
1675 nextSrcSizeHint = 4 - dctx->tmpInSize;
1676 doAnotherStage=0;
1677 break;
1678 }
1679 selectedIn = dctx->tmpIn;
1680 } /* if (dctx->dStage == dstage_storeSuffix) */
1681
1682 /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */
1683 { U32 const readCRC = LZ4F_readLE32(selectedIn);
1684 U32 const resultCRC = XXH32_digest(&(dctx->xxh));
1685 if (readCRC != resultCRC)
1686 return err0r(LZ4F_ERROR_contentChecksum_invalid);
1687 nextSrcSizeHint = 0;
1688 LZ4F_resetDecompressionContext(dctx);
1689 doAnotherStage = 0;
1690 break;
1691 }
1692
1693 case dstage_getSFrameSize:
1694 if ((srcEnd - srcPtr) >= 4) {
1695 selectedIn = srcPtr;
1696 srcPtr += 4;
1697 } else {
1698 /* not enough input to read cBlockSize field */
1699 dctx->tmpInSize = 4;
1700 dctx->tmpInTarget = 8;
1701 dctx->dStage = dstage_storeSFrameSize;
1702 }
1703
1704 if (dctx->dStage == dstage_storeSFrameSize)
1705 case dstage_storeSFrameSize:
1706 { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize,
1707 (size_t)(srcEnd - srcPtr) );
1708 memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy);
1709 srcPtr += sizeToCopy;
1710 dctx->tmpInSize += sizeToCopy;
1711 if (dctx->tmpInSize < dctx->tmpInTarget) {
1712 /* not enough input to get full sBlockSize; wait for more */
1713 nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize;
1714 doAnotherStage = 0;
1715 break;
1716 }
1717 selectedIn = dctx->header + 4;
1718 } /* if (dctx->dStage == dstage_storeSFrameSize) */
1719
1720 /* case dstage_decodeSFrameSize: */ /* no direct entry */
1721 { size_t const SFrameSize = LZ4F_readLE32(selectedIn);
1722 dctx->frameInfo.contentSize = SFrameSize;
1723 dctx->tmpInTarget = SFrameSize;
1724 dctx->dStage = dstage_skipSkippable;
1725 break;
1726 }
1727
1728 case dstage_skipSkippable:
1729 { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr));
1730 srcPtr += skipSize;
1731 dctx->tmpInTarget -= skipSize;
1732 doAnotherStage = 0;
1733 nextSrcSizeHint = dctx->tmpInTarget;
1734 if (nextSrcSizeHint) break; /* still more to skip */
1735 /* frame fully skipped : prepare context for a new frame */
1736 LZ4F_resetDecompressionContext(dctx);
1737 break;
1738 }
1739 } /* switch (dctx->dStage) */
1740 } /* while (doAnotherStage) */
1741
1742 /* preserve history within tmp whenever necessary */
1743 LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2);
1744 if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */
1745 && (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */
1746 && (!decompressOptionsPtr->stableDst) /* cannot rely on dst data to remain there for next call */
1747 && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */
1748 {
1749 if (dctx->dStage == dstage_flushOut) {
1750 size_t const preserveSize = dctx->tmpOut - dctx->tmpOutBuffer;
1751 size_t copySize = 64 KB - dctx->tmpOutSize;
1752 const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart;
1753 if (dctx->tmpOutSize > 64 KB) copySize = 0;
1754 if (copySize > preserveSize) copySize = preserveSize;
1755
1756 if (copySize > 0)
1757 memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize);
1758
1759 dctx->dict = dctx->tmpOutBuffer;
1760 dctx->dictSize = preserveSize + dctx->tmpOutStart;
1761 } else {
1762 const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize;
1763 size_t const newDictSize = MIN(dctx->dictSize, 64 KB);
1764
1765 if (newDictSize > 0)
1766 memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize);
1767
1768 dctx->dict = dctx->tmpOutBuffer;
1769 dctx->dictSize = newDictSize;
1770 dctx->tmpOut = dctx->tmpOutBuffer + newDictSize;
1771 }
1772 }
1773
1774 *srcSizePtr = (srcPtr - srcStart);
1775 *dstSizePtr = (dstPtr - dstStart);
1776 return nextSrcSizeHint;
1777 }
1778
1779 /*! LZ4F_decompress_usingDict() :
1780 * Same as LZ4F_decompress(), using a predefined dictionary.
1781 * Dictionary is used "in place", without any preprocessing.
1782 * It must remain accessible throughout the entire frame decoding.
1783 */
LZ4F_decompress_usingDict(LZ4F_dctx * dctx,void * dstBuffer,size_t * dstSizePtr,const void * srcBuffer,size_t * srcSizePtr,const void * dict,size_t dictSize,const LZ4F_decompressOptions_t * decompressOptionsPtr)1784 size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx,
1785 void* dstBuffer, size_t* dstSizePtr,
1786 const void* srcBuffer, size_t* srcSizePtr,
1787 const void* dict, size_t dictSize,
1788 const LZ4F_decompressOptions_t* decompressOptionsPtr)
1789 {
1790 if (dctx->dStage <= dstage_init) {
1791 dctx->dict = (const BYTE*)dict;
1792 dctx->dictSize = dictSize;
1793 }
1794 return LZ4F_decompress(dctx, dstBuffer, dstSizePtr,
1795 srcBuffer, srcSizePtr,
1796 decompressOptionsPtr);
1797 }
1798