1 /*
2 LZ4 HC - High Compression Mode of LZ4
3 Copyright (C) 2011-2015, Yann Collet.
4
5 BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are
9 met:
10
11 * Redistributions of source code must retain the above copyright
12 notice, this list of conditions and the following disclaimer.
13 * Redistributions in binary form must reproduce the above
14 copyright notice, this list of conditions and the following disclaimer
15 in the documentation and/or other materials provided with the
16 distribution.
17
18 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 You can contact the author at :
31 - LZ4 source repository : https://github.com/Cyan4973/lz4
32 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
33 */
34
35
36
37 /**************************************
38 Tuning Parameter
39 **************************************/
40 static const int LZ4HC_compressionLevel_default = 9;
41
42
43 /**************************************
44 Includes
45 **************************************/
46 #include "fastlz/lz4hc.h"
47
48
49 /**************************************
50 Local Compiler Options
51 **************************************/
52 #if defined(__GNUC__)
53 # pragma GCC diagnostic ignored "-Wunused-function"
54 #endif
55
56 #if defined (__clang__)
57 # pragma clang diagnostic ignored "-Wunused-function"
58 #endif
59
60
61 /**************************************
62 Common LZ4 definition
63 **************************************/
64 #define LZ4_COMMONDEFS_ONLY
65 #include "lz4.c"
66
67
68 /**************************************
69 Local Constants
70 **************************************/
71 #define DICTIONARY_LOGSIZE 16
72 #define MAXD (1<<DICTIONARY_LOGSIZE)
73 #define MAXD_MASK ((U32)(MAXD - 1))
74
75 #define HASH_LOG (DICTIONARY_LOGSIZE-1)
76 #define HASHTABLESIZE (1 << HASH_LOG)
77 #define HASH_MASK (HASHTABLESIZE - 1)
78
79 #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
80
81 static const int g_maxCompressionLevel = 16;
82
83
84 /**************************************
85 Local Types
86 **************************************/
87 typedef struct
88 {
89 U32 hashTable[HASHTABLESIZE];
90 U16 chainTable[MAXD];
91 const BYTE* end; /* next block here to continue on current prefix */
92 const BYTE* base; /* All index relative to this position */
93 const BYTE* dictBase; /* alternate base for extDict */
94 const BYTE* inputBuffer;/* deprecated */
95 U32 dictLimit; /* below that point, need extDict */
96 U32 lowLimit; /* below that point, no more dict */
97 U32 nextToUpdate;
98 U32 compressionLevel;
99 } LZ4HC_Data_Structure;
100
101
102 /**************************************
103 Local Macros
104 **************************************/
105 #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
106 #define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK]
107 #define GETNEXT(p) ((p) - (size_t)DELTANEXT(p))
108
LZ4HC_hashPtr(const void * ptr)109 static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
110
111
112
113 /**************************************
114 HC Compression
115 **************************************/
LZ4HC_init(LZ4HC_Data_Structure * hc4,const BYTE * start)116 static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start)
117 {
118 MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
119 MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
120 hc4->nextToUpdate = 64 KB;
121 hc4->base = start - 64 KB;
122 hc4->inputBuffer = start;
123 hc4->end = start;
124 hc4->dictBase = start - 64 KB;
125 hc4->dictLimit = 64 KB;
126 hc4->lowLimit = 64 KB;
127 }
128
129
130 /* Update chains up to ip (excluded) */
LZ4HC_Insert(LZ4HC_Data_Structure * hc4,const BYTE * ip)131 FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
132 {
133 U16* chainTable = hc4->chainTable;
134 U32* HashTable = hc4->hashTable;
135 const BYTE* const base = hc4->base;
136 const U32 target = (U32)(ip - base);
137 U32 idx = hc4->nextToUpdate;
138
139 while(idx < target)
140 {
141 U32 h = LZ4HC_hashPtr(base+idx);
142 size_t delta = idx - HashTable[h];
143 if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
144 chainTable[idx & 0xFFFF] = (U16)delta;
145 HashTable[h] = idx;
146 idx++;
147 }
148
149 hc4->nextToUpdate = target;
150 }
151
152
LZ4HC_InsertAndFindBestMatch(LZ4HC_Data_Structure * hc4,const BYTE * ip,const BYTE * const iLimit,const BYTE ** matchpos,const int maxNbAttempts)153 FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* Index table will be updated */
154 const BYTE* ip, const BYTE* const iLimit,
155 const BYTE** matchpos,
156 const int maxNbAttempts)
157 {
158 U16* const chainTable = hc4->chainTable;
159 U32* const HashTable = hc4->hashTable;
160 const BYTE* const base = hc4->base;
161 const BYTE* const dictBase = hc4->dictBase;
162 const U32 dictLimit = hc4->dictLimit;
163 const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
164 U32 matchIndex;
165 const BYTE* match;
166 int nbAttempts=maxNbAttempts;
167 size_t ml=0;
168
169 /* HC4 match finder */
170 LZ4HC_Insert(hc4, ip);
171 matchIndex = HashTable[LZ4HC_hashPtr(ip)];
172
173 while ((matchIndex>=lowLimit) && (nbAttempts))
174 {
175 nbAttempts--;
176 if (matchIndex >= dictLimit)
177 {
178 match = base + matchIndex;
179 if (*(match+ml) == *(ip+ml)
180 && (LZ4_read32(match) == LZ4_read32(ip)))
181 {
182 size_t mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
183 if (mlt > ml) { ml = mlt; *matchpos = match; }
184 }
185 }
186 else
187 {
188 match = dictBase + matchIndex;
189 if (LZ4_read32(match) == LZ4_read32(ip))
190 {
191 size_t mlt;
192 const BYTE* vLimit = ip + (dictLimit - matchIndex);
193 if (vLimit > iLimit) vLimit = iLimit;
194 mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
195 if ((ip+mlt == vLimit) && (vLimit < iLimit))
196 mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit);
197 if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
198 }
199 }
200 matchIndex -= chainTable[matchIndex & 0xFFFF];
201 }
202
203 return (int)ml;
204 }
205
206
LZ4HC_InsertAndGetWiderMatch(LZ4HC_Data_Structure * hc4,const BYTE * const ip,const BYTE * const iLowLimit,const BYTE * const iHighLimit,int longest,const BYTE ** matchpos,const BYTE ** startpos,const int maxNbAttempts)207 FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
208 LZ4HC_Data_Structure* hc4,
209 const BYTE* const ip,
210 const BYTE* const iLowLimit,
211 const BYTE* const iHighLimit,
212 int longest,
213 const BYTE** matchpos,
214 const BYTE** startpos,
215 const int maxNbAttempts)
216 {
217 U16* const chainTable = hc4->chainTable;
218 U32* const HashTable = hc4->hashTable;
219 const BYTE* const base = hc4->base;
220 const U32 dictLimit = hc4->dictLimit;
221 const BYTE* const lowPrefixPtr = base + dictLimit;
222 const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
223 const BYTE* const dictBase = hc4->dictBase;
224 U32 matchIndex;
225 int nbAttempts = maxNbAttempts;
226 int delta = (int)(ip-iLowLimit);
227
228
229 /* First Match */
230 LZ4HC_Insert(hc4, ip);
231 matchIndex = HashTable[LZ4HC_hashPtr(ip)];
232
233 while ((matchIndex>=lowLimit) && (nbAttempts))
234 {
235 nbAttempts--;
236 if (matchIndex >= dictLimit)
237 {
238 const BYTE* matchPtr = base + matchIndex;
239 if (*(iLowLimit + longest) == *(matchPtr - delta + longest))
240 if (LZ4_read32(matchPtr) == LZ4_read32(ip))
241 {
242 int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
243 int back = 0;
244
245 while ((ip+back>iLowLimit)
246 && (matchPtr+back > lowPrefixPtr)
247 && (ip[back-1] == matchPtr[back-1]))
248 back--;
249
250 mlt -= back;
251
252 if (mlt > longest)
253 {
254 longest = (int)mlt;
255 *matchpos = matchPtr+back;
256 *startpos = ip+back;
257 }
258 }
259 }
260 else
261 {
262 const BYTE* matchPtr = dictBase + matchIndex;
263 if (LZ4_read32(matchPtr) == LZ4_read32(ip))
264 {
265 size_t mlt;
266 int back=0;
267 const BYTE* vLimit = ip + (dictLimit - matchIndex);
268 if (vLimit > iHighLimit) vLimit = iHighLimit;
269 mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
270 if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
271 mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
272 while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
273 mlt -= back;
274 if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
275 }
276 }
277 matchIndex -= chainTable[matchIndex & 0xFFFF];
278 }
279
280 return longest;
281 }
282
283
284 typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
285
286 #define LZ4HC_DEBUG 0
287 #if LZ4HC_DEBUG
288 static unsigned debug = 0;
289 #endif
290
LZ4HC_encodeSequence(const BYTE ** ip,BYTE ** op,const BYTE ** anchor,int matchLength,const BYTE * const match,limitedOutput_directive limitedOutputBuffer,BYTE * oend)291 FORCE_INLINE int LZ4HC_encodeSequence (
292 const BYTE** ip,
293 BYTE** op,
294 const BYTE** anchor,
295 int matchLength,
296 const BYTE* const match,
297 limitedOutput_directive limitedOutputBuffer,
298 BYTE* oend)
299 {
300 int length;
301 BYTE* token;
302
303 #if LZ4HC_DEBUG
304 if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
305 #endif
306
307 /* Encode Literal length */
308 length = (int)(*ip - *anchor);
309 token = (*op)++;
310 if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
311 if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; }
312 else *token = (BYTE)(length<<ML_BITS);
313
314 /* Copy Literals */
315 LZ4_wildCopy(*op, *anchor, (*op) + length);
316 *op += length;
317
318 /* Encode Offset */
319 LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
320
321 /* Encode MatchLength */
322 length = (int)(matchLength-MINMATCH);
323 if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
324 if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; }
325 else *token += (BYTE)(length);
326
327 /* Prepare next loop */
328 *ip += matchLength;
329 *anchor = *ip;
330
331 return 0;
332 }
333
334
LZ4HC_compress_generic(void * ctxvoid,const char * source,char * dest,int inputSize,int maxOutputSize,int compressionLevel,limitedOutput_directive limit)335 static int LZ4HC_compress_generic (
336 void* ctxvoid,
337 const char* source,
338 char* dest,
339 int inputSize,
340 int maxOutputSize,
341 int compressionLevel,
342 limitedOutput_directive limit
343 )
344 {
345 LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid;
346 const BYTE* ip = (const BYTE*) source;
347 const BYTE* anchor = ip;
348 const BYTE* const iend = ip + inputSize;
349 const BYTE* const mflimit = iend - MFLIMIT;
350 const BYTE* const matchlimit = (iend - LASTLITERALS);
351
352 BYTE* op = (BYTE*) dest;
353 BYTE* const oend = op + maxOutputSize;
354
355 unsigned maxNbAttempts;
356 int ml, ml2, ml3, ml0;
357 const BYTE* ref=NULL;
358 const BYTE* start2=NULL;
359 const BYTE* ref2=NULL;
360 const BYTE* start3=NULL;
361 const BYTE* ref3=NULL;
362 const BYTE* start0;
363 const BYTE* ref0;
364
365
366 /* init */
367 if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel;
368 if (compressionLevel < 1) compressionLevel = LZ4HC_compressionLevel_default;
369 maxNbAttempts = 1 << (compressionLevel-1);
370 ctx->end += inputSize;
371
372 ip++;
373
374 /* Main Loop */
375 while (ip < mflimit)
376 {
377 ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts);
378 if (!ml) { ip++; continue; }
379
380 /* saved, in case we would skip too much */
381 start0 = ip;
382 ref0 = ref;
383 ml0 = ml;
384
385 _Search2:
386 if (ip+ml < mflimit)
387 ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts);
388 else ml2 = ml;
389
390 if (ml2 == ml) /* No better match */
391 {
392 if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
393 continue;
394 }
395
396 if (start0 < ip)
397 {
398 if (start2 < ip + ml0) /* empirical */
399 {
400 ip = start0;
401 ref = ref0;
402 ml = ml0;
403 }
404 }
405
406 /* Here, start0==ip */
407 if ((start2 - ip) < 3) /* First Match too small : removed */
408 {
409 ml = ml2;
410 ip = start2;
411 ref =ref2;
412 goto _Search2;
413 }
414
415 _Search3:
416 /*
417 * Currently we have :
418 * ml2 > ml1, and
419 * ip1+3 <= ip2 (usually < ip1+ml1)
420 */
421 if ((start2 - ip) < OPTIMAL_ML)
422 {
423 int correction;
424 int new_ml = ml;
425 if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;
426 if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
427 correction = new_ml - (int)(start2 - ip);
428 if (correction > 0)
429 {
430 start2 += correction;
431 ref2 += correction;
432 ml2 -= correction;
433 }
434 }
435 /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
436
437 if (start2 + ml2 < mflimit)
438 ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts);
439 else ml3 = ml2;
440
441 if (ml3 == ml2) /* No better match : 2 sequences to encode */
442 {
443 /* ip & ref are known; Now for ml */
444 if (start2 < ip+ml) ml = (int)(start2 - ip);
445 /* Now, encode 2 sequences */
446 if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
447 ip = start2;
448 if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;
449 continue;
450 }
451
452 if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */
453 {
454 if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */
455 {
456 if (start2 < ip+ml)
457 {
458 int correction = (int)(ip+ml - start2);
459 start2 += correction;
460 ref2 += correction;
461 ml2 -= correction;
462 if (ml2 < MINMATCH)
463 {
464 start2 = start3;
465 ref2 = ref3;
466 ml2 = ml3;
467 }
468 }
469
470 if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
471 ip = start3;
472 ref = ref3;
473 ml = ml3;
474
475 start0 = start2;
476 ref0 = ref2;
477 ml0 = ml2;
478 goto _Search2;
479 }
480
481 start2 = start3;
482 ref2 = ref3;
483 ml2 = ml3;
484 goto _Search3;
485 }
486
487 /*
488 * OK, now we have 3 ascending matches; let's write at least the first one
489 * ip & ref are known; Now for ml
490 */
491 if (start2 < ip+ml)
492 {
493 if ((start2 - ip) < (int)ML_MASK)
494 {
495 int correction;
496 if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
497 if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;
498 correction = ml - (int)(start2 - ip);
499 if (correction > 0)
500 {
501 start2 += correction;
502 ref2 += correction;
503 ml2 -= correction;
504 }
505 }
506 else
507 {
508 ml = (int)(start2 - ip);
509 }
510 }
511 if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
512
513 ip = start2;
514 ref = ref2;
515 ml = ml2;
516
517 start2 = start3;
518 ref2 = ref3;
519 ml2 = ml3;
520
521 goto _Search3;
522 }
523
524 /* Encode Last Literals */
525 {
526 int lastRun = (int)(iend - anchor);
527 if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
528 if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
529 else *op++ = (BYTE)(lastRun<<ML_BITS);
530 memcpy(op, anchor, iend - anchor);
531 op += iend-anchor;
532 }
533
534 /* End */
535 return (int) (((char*)op)-dest);
536 }
537
538
LZ4_compressHC2(const char * source,char * dest,int inputSize,int compressionLevel)539 int LZ4_compressHC2(const char* source, char* dest, int inputSize, int compressionLevel)
540 {
541 LZ4HC_Data_Structure ctx;
542 LZ4HC_init(&ctx, (const BYTE*)source);
543 return LZ4HC_compress_generic (&ctx, source, dest, inputSize, 0, compressionLevel, noLimit);
544 }
545
LZ4_compressHC(const char * source,char * dest,int inputSize)546 int LZ4_compressHC(const char* source, char* dest, int inputSize) { return LZ4_compressHC2(source, dest, inputSize, 0); }
547
LZ4_compressHC2_limitedOutput(const char * source,char * dest,int inputSize,int maxOutputSize,int compressionLevel)548 int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
549 {
550 LZ4HC_Data_Structure ctx;
551 LZ4HC_init(&ctx, (const BYTE*)source);
552 return LZ4HC_compress_generic (&ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
553 }
554
LZ4_compressHC_limitedOutput(const char * source,char * dest,int inputSize,int maxOutputSize)555 int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
556 {
557 return LZ4_compressHC2_limitedOutput(source, dest, inputSize, maxOutputSize, 0);
558 }
559
560
561 /*****************************
562 * Using external allocation
563 * ***************************/
LZ4_sizeofStateHC(void)564 int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); }
565
566
LZ4_compressHC2_withStateHC(void * state,const char * source,char * dest,int inputSize,int compressionLevel)567 int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel)
568 {
569 if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
570 LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
571 return LZ4HC_compress_generic (state, source, dest, inputSize, 0, compressionLevel, noLimit);
572 }
573
LZ4_compressHC_withStateHC(void * state,const char * source,char * dest,int inputSize)574 int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize)
575 { return LZ4_compressHC2_withStateHC (state, source, dest, inputSize, 0); }
576
577
LZ4_compressHC2_limitedOutput_withStateHC(void * state,const char * source,char * dest,int inputSize,int maxOutputSize,int compressionLevel)578 int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
579 {
580 if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
581 LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
582 return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
583 }
584
LZ4_compressHC_limitedOutput_withStateHC(void * state,const char * source,char * dest,int inputSize,int maxOutputSize)585 int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
586 { return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); }
587
588
589
590 /**************************************
591 * Streaming Functions
592 * ************************************/
593 /* allocation */
LZ4_createStreamHC(void)594 LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
LZ4_freeStreamHC(LZ4_streamHC_t * LZ4_streamHCPtr)595 int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }
596
597
598 /* initialization */
LZ4_resetStreamHC(LZ4_streamHC_t * LZ4_streamHCPtr,int compressionLevel)599 void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
600 {
601 LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= sizeof(LZ4_streamHC_t)); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
602 ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base = NULL;
603 ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel;
604 }
605
LZ4_loadDictHC(LZ4_streamHC_t * LZ4_streamHCPtr,const char * dictionary,int dictSize)606 int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
607 {
608 LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr;
609 if (dictSize > 64 KB)
610 {
611 dictionary += dictSize - 64 KB;
612 dictSize = 64 KB;
613 }
614 LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
615 if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3));
616 ctxPtr->end = (const BYTE*)dictionary + dictSize;
617 return dictSize;
618 }
619
620
621 /* compression */
622
LZ4HC_setExternalDict(LZ4HC_Data_Structure * ctxPtr,const BYTE * newBlock)623 static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock)
624 {
625 if (ctxPtr->end >= ctxPtr->base + 4)
626 LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
627 /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
628 ctxPtr->lowLimit = ctxPtr->dictLimit;
629 ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
630 ctxPtr->dictBase = ctxPtr->base;
631 ctxPtr->base = newBlock - ctxPtr->dictLimit;
632 ctxPtr->end = newBlock;
633 ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
634 }
635
LZ4_compressHC_continue_generic(LZ4HC_Data_Structure * ctxPtr,const char * source,char * dest,int inputSize,int maxOutputSize,limitedOutput_directive limit)636 static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr,
637 const char* source, char* dest,
638 int inputSize, int maxOutputSize, limitedOutput_directive limit)
639 {
640 /* auto-init if forgotten */
641 if (ctxPtr->base == NULL)
642 LZ4HC_init (ctxPtr, (const BYTE*) source);
643
644 /* Check overflow */
645 if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB)
646 {
647 size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
648 if (dictSize > 64 KB) dictSize = 64 KB;
649
650 LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);
651 }
652
653 /* Check if blocks follow each other */
654 if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
655
656 /* Check overlapping input/dictionary space */
657 {
658 const BYTE* sourceEnd = (const BYTE*) source + inputSize;
659 const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
660 const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
661 if ((sourceEnd > dictBegin) && ((BYTE*)source < dictEnd))
662 {
663 if (sourceEnd > dictEnd) sourceEnd = dictEnd;
664 ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
665 if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
666 }
667 }
668
669 return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit);
670 }
671
LZ4_compressHC_continue(LZ4_streamHC_t * LZ4_streamHCPtr,const char * source,char * dest,int inputSize)672 int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize)
673 {
674 return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, 0, noLimit);
675 }
676
LZ4_compressHC_limitedOutput_continue(LZ4_streamHC_t * LZ4_streamHCPtr,const char * source,char * dest,int inputSize,int maxOutputSize)677 int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
678 {
679 return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
680 }
681
682
683 /* dictionary saving */
684
LZ4_saveDictHC(LZ4_streamHC_t * LZ4_streamHCPtr,char * safeBuffer,int dictSize)685 int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
686 {
687 LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr;
688 int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
689 if (dictSize > 64 KB) dictSize = 64 KB;
690 if (dictSize < 4) dictSize = 0;
691 if (dictSize > prefixSize) dictSize = prefixSize;
692 memcpy(safeBuffer, streamPtr->end - dictSize, dictSize);
693 {
694 U32 endIndex = (U32)(streamPtr->end - streamPtr->base);
695 streamPtr->end = (const BYTE*)safeBuffer + dictSize;
696 streamPtr->base = streamPtr->end - endIndex;
697 streamPtr->dictLimit = endIndex - dictSize;
698 streamPtr->lowLimit = endIndex - dictSize;
699 if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit;
700 }
701 return dictSize;
702 }
703
704
705 /***********************************
706 * Deprecated Functions
707 ***********************************/
LZ4_sizeofStreamStateHC(void)708 int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }
709
LZ4_resetStreamStateHC(void * state,const char * inputBuffer)710 int LZ4_resetStreamStateHC(void* state, const char* inputBuffer)
711 {
712 if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
713 LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
714 return 0;
715 }
716
LZ4_createHC(const char * inputBuffer)717 void* LZ4_createHC (const char* inputBuffer)
718 {
719 void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure));
720 LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
721 return hc4;
722 }
723
LZ4_freeHC(void * LZ4HC_Data)724 int LZ4_freeHC (void* LZ4HC_Data)
725 {
726 FREEMEM(LZ4HC_Data);
727 return (0);
728 }
729
730 /*
731 int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize)
732 {
733 return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit);
734 }
735 int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
736 {
737 return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput);
738 }
739 */
740
LZ4_compressHC2_continue(void * LZ4HC_Data,const char * source,char * dest,int inputSize,int compressionLevel)741 int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
742 {
743 return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit);
744 }
745
LZ4_compressHC2_limitedOutput_continue(void * LZ4HC_Data,const char * source,char * dest,int inputSize,int maxOutputSize,int compressionLevel)746 int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
747 {
748 return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
749 }
750
LZ4_slideInputBufferHC(void * LZ4HC_Data)751 char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
752 {
753 LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
754 int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
755 return (char*)(hc4->inputBuffer + dictSize);
756 }
757