1 /*
2 fuzzer.c - Fuzzer test tool for LZ4
3 Copyright (C) Yann Collet 2012-2017
4
5 GPL v2 License
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License along
18 with this program; if not, write to the Free Software Foundation, Inc.,
19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21 You can contact the author at :
22 - LZ4 homepage : http://www.lz4.org
23 - LZ4 source repo : https://github.com/lz4/lz4
24 */
25
26 /*-************************************
27 * Compiler options
28 **************************************/
29 #ifdef _MSC_VER /* Visual Studio */
30 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
31 # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
32 # pragma warning(disable : 4310) /* disable: C4310: constant char value > 127 */
33 #endif
34
35 #define LZ4_DISABLE_DEPRECATE_WARNINGS
36
37 /*-************************************
38 * Dependencies
39 **************************************/
40 #include "platform.h" /* _CRT_SECURE_NO_WARNINGS */
41 #include "util.h" /* U32 */
42 #include <stdlib.h>
43 #include <stdio.h> /* fgets, sscanf */
44 #include <string.h> /* strcmp */
45 #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
46 #define LZ4_HC_STATIC_LINKING_ONLY
47 #include "lz4hc.h"
48 #define XXH_STATIC_LINKING_ONLY
49 #include "xxhash.h"
50
51
52 /*-************************************
53 * Basic Types
54 **************************************/
55 #if !defined(__cplusplus) && !(defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
56 typedef size_t uintptr_t; /* true on most systems, except OpenVMS-64 (which doesn't need address overflow test) */
57 #endif
58
59
60 /*-************************************
61 * Constants
62 **************************************/
63 #define NB_ATTEMPTS (1<<16)
64 #define COMPRESSIBLE_NOISE_LENGTH (1 << 21)
65 #define FUZ_MAX_BLOCK_SIZE (1 << 17)
66 #define FUZ_MAX_DICT_SIZE (1 << 15)
67 #define FUZ_COMPRESSIBILITY_DEFAULT 60
68 #define PRIME1 2654435761U
69 #define PRIME2 2246822519U
70 #define PRIME3 3266489917U
71
72 #define KB *(1U<<10)
73 #define MB *(1U<<20)
74 #define GB *(1U<<30)
75
76
77 /*-***************************************
78 * Macros
79 *****************************************/
80 #define DISPLAY(...) fprintf(stdout, __VA_ARGS__)
81 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
82 static int g_displayLevel = 2;
83
84 #define MIN(a,b) ( (a) < (b) ? (a) : (b) )
85
86
87 /*-*******************************************************
88 * Fuzzer functions
89 *********************************************************/
FUZ_GetClockSpan(clock_t clockStart)90 static clock_t FUZ_GetClockSpan(clock_t clockStart)
91 {
92 return clock() - clockStart; /* works even if overflow; max span ~ 30mn */
93 }
94
FUZ_displayUpdate(unsigned testNb)95 static void FUZ_displayUpdate(unsigned testNb)
96 {
97 static clock_t g_time = 0;
98 static const clock_t g_refreshRate = CLOCKS_PER_SEC / 5;
99 if ((FUZ_GetClockSpan(g_time) > g_refreshRate) || (g_displayLevel>=4)) {
100 g_time = clock();
101 DISPLAY("\r%5u ", testNb);
102 fflush(stdout);
103 }
104 }
105
FUZ_rotl32(U32 u32,U32 nbBits)106 static U32 FUZ_rotl32(U32 u32, U32 nbBits)
107 {
108 return ((u32 << nbBits) | (u32 >> (32 - nbBits)));
109 }
110
FUZ_rand(U32 * src)111 static U32 FUZ_rand(U32* src)
112 {
113 U32 rand32 = *src;
114 rand32 *= PRIME1;
115 rand32 ^= PRIME2;
116 rand32 = FUZ_rotl32(rand32, 13);
117 *src = rand32;
118 return rand32;
119 }
120
121
122 #define FUZ_RAND15BITS ((FUZ_rand(seed) >> 3) & 32767)
123 #define FUZ_RANDLENGTH ( ((FUZ_rand(seed) >> 7) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
FUZ_fillCompressibleNoiseBuffer(void * buffer,size_t bufferSize,double proba,U32 * seed)124 static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed)
125 {
126 BYTE* const BBuffer = (BYTE*)buffer;
127 size_t pos = 0;
128 U32 const P32 = (U32)(32768 * proba);
129
130 /* First Bytes */
131 while (pos < 20)
132 BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
133
134 while (pos < bufferSize) {
135 /* Select : Literal (noise) or copy (within 64K) */
136 if (FUZ_RAND15BITS < P32) {
137 /* Copy (within 64K) */
138 size_t const length = FUZ_RANDLENGTH + 4;
139 size_t const d = MIN(pos+length, bufferSize);
140 size_t match;
141 size_t offset = FUZ_RAND15BITS + 1;
142 while (offset > pos) offset >>= 1;
143 match = pos - offset;
144 while (pos < d) BBuffer[pos++] = BBuffer[match++];
145 } else {
146 /* Literal (noise) */
147 size_t const length = FUZ_RANDLENGTH;
148 size_t const d = MIN(pos+length, bufferSize);
149 while (pos < d) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
150 }
151 }
152 }
153
154
155 #define MAX_NB_BUFF_I134 150
156 #define BLOCKSIZE_I134 (32 MB)
157 /*! FUZ_AddressOverflow() :
158 * Aggressively pushes memory allocation limits,
159 * and generates patterns which create address space overflow.
160 * only possible in 32-bits mode */
FUZ_AddressOverflow(void)161 static int FUZ_AddressOverflow(void)
162 {
163 char* buffers[MAX_NB_BUFF_I134+1];
164 int nbBuff=0;
165 int highAddress = 0;
166
167 DISPLAY("Overflow tests : ");
168
169 /* Only possible in 32-bits */
170 if (sizeof(void*)==8) {
171 DISPLAY("64 bits mode : no overflow \n");
172 fflush(stdout);
173 return 0;
174 }
175
176 buffers[0] = (char*)malloc(BLOCKSIZE_I134);
177 buffers[1] = (char*)malloc(BLOCKSIZE_I134);
178 if ((!buffers[0]) || (!buffers[1])) {
179 free(buffers[0]); free(buffers[1]);
180 DISPLAY("not enough memory for tests \n");
181 return 0;
182 }
183
184 for (nbBuff=2; nbBuff < MAX_NB_BUFF_I134; nbBuff++) {
185 DISPLAY("%3i \b\b\b\b", nbBuff); fflush(stdout);
186 buffers[nbBuff] = (char*)malloc(BLOCKSIZE_I134);
187 if (buffers[nbBuff]==NULL) goto _endOfTests;
188
189 if (((uintptr_t)buffers[nbBuff] > (uintptr_t)0x80000000) && (!highAddress)) {
190 DISPLAY("high address detected : ");
191 fflush(stdout);
192 highAddress=1;
193 }
194
195 { size_t const sizeToGenerateOverflow = (size_t)(- ((uintptr_t)buffers[nbBuff-1]) + 512);
196 unsigned const nbOf255 = (unsigned)((sizeToGenerateOverflow / 255) + 1);
197 char* const input = buffers[nbBuff-1];
198 char* output = buffers[nbBuff];
199 int r;
200 input[0] = (char)0xF0; /* Literal length overflow */
201 input[1] = (char)0xFF;
202 input[2] = (char)0xFF;
203 input[3] = (char)0xFF;
204 { unsigned u; for(u = 4; u <= nbOf255+4; u++) input[u] = (char)0xff; }
205 r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
206 if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; }
207 input[0] = (char)0x1F; /* Match length overflow */
208 input[1] = (char)0x01;
209 input[2] = (char)0x01;
210 input[3] = (char)0x00;
211 r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
212 if (r>0) { DISPLAY("LZ4_decompress_safe = %i \n", r); goto _overflowError; }
213
214 output = buffers[nbBuff-2]; /* Reverse in/out pointer order */
215 input[0] = (char)0xF0; /* Literal length overflow */
216 input[1] = (char)0xFF;
217 input[2] = (char)0xFF;
218 input[3] = (char)0xFF;
219 r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
220 if (r>0) goto _overflowError;
221 input[0] = (char)0x1F; /* Match length overflow */
222 input[1] = (char)0x01;
223 input[2] = (char)0x01;
224 input[3] = (char)0x00;
225 r = LZ4_decompress_safe(input, output, nbOf255+64, BLOCKSIZE_I134);
226 if (r>0) goto _overflowError;
227 }
228 }
229
230 nbBuff++;
231 _endOfTests:
232 { int i; for (i=0 ; i<nbBuff; i++) free(buffers[i]); }
233 if (!highAddress) DISPLAY("high address not possible \n");
234 else DISPLAY("all overflows correctly detected \n");
235 return 0;
236
237 _overflowError:
238 DISPLAY("Address space overflow error !! \n");
239 exit(1);
240 }
241
242
243 /*! FUZ_findDiff() :
244 * find the first different byte between buff1 and buff2.
245 * presumes buff1 != buff2.
246 * presumes a difference exists before end of either buffer.
247 * Typically invoked after a checksum mismatch.
248 */
FUZ_findDiff(const void * buff1,const void * buff2)249 static void FUZ_findDiff(const void* buff1, const void* buff2)
250 {
251 const BYTE* const b1 = (const BYTE*)buff1;
252 const BYTE* const b2 = (const BYTE*)buff2;
253 size_t u = 0;
254 while (b1[u]==b2[u]) u++;
255 DISPLAY("Wrong Byte at position %u \n", (unsigned)u);
256 }
257
258
FUZ_test(U32 seed,U32 nbCycles,const U32 startCycle,const double compressibility,U32 duration_s)259 static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double compressibility, U32 duration_s)
260 {
261 unsigned long long bytes = 0;
262 unsigned long long cbytes = 0;
263 unsigned long long hcbytes = 0;
264 unsigned long long ccbytes = 0;
265 void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
266 size_t const compressedBufferSize = LZ4_compressBound(FUZ_MAX_BLOCK_SIZE);
267 char* const compressedBuffer = (char*)malloc(compressedBufferSize);
268 char* const decodedBuffer = (char*)malloc(FUZ_MAX_DICT_SIZE + FUZ_MAX_BLOCK_SIZE);
269 void* const stateLZ4 = malloc(LZ4_sizeofState());
270 void* const stateLZ4HC = malloc(LZ4_sizeofStateHC());
271 LZ4_stream_t LZ4dict;
272 LZ4_streamHC_t LZ4dictHC;
273 U32 coreRandState = seed;
274 clock_t const clockStart = clock();
275 clock_t const clockDuration = (clock_t)duration_s * CLOCKS_PER_SEC;
276 int result = 0;
277 unsigned cycleNb;
278
279 # define FUZ_CHECKTEST(cond, ...) if (cond) { printf("Test %u : ", testNb); printf(__VA_ARGS__); \
280 printf(" (seed %u, cycle %u) \n", seed, cycleNb); goto _output_error; }
281 # define FUZ_DISPLAYTEST { testNb++; g_displayLevel>=4 ? printf("%2u\b\b", testNb), fflush(stdout) : 0; }
282
283
284 /* init */
285 if(!CNBuffer || !compressedBuffer || !decodedBuffer) {
286 DISPLAY("Not enough memory to start fuzzer tests");
287 goto _output_error;
288 }
289 memset(&LZ4dict, 0, sizeof(LZ4dict));
290 { U32 randState = coreRandState ^ PRIME3;
291 FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
292 }
293
294 /* move to startCycle */
295 for (cycleNb = 0; cycleNb < startCycle; cycleNb++)
296 (void) FUZ_rand(&coreRandState); /* sync coreRandState */
297
298 /* Main test loop */
299 for (cycleNb = startCycle;
300 (cycleNb < nbCycles) || (FUZ_GetClockSpan(clockStart) < clockDuration);
301 cycleNb++) {
302 U32 testNb = 0;
303 U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
304 int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
305 int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
306 int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
307 int const dictSize = MIN(dictSizeRand, blockStart);
308 int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
309 char* const block = ((char*)CNBuffer) + blockStart;
310 const char* dict = block - dictSize;
311 int compressedSize, HCcompressedSize;
312 int blockContinueCompressedSize;
313 U32 const crcOrig = XXH32(block, blockSize, 0);
314 U32 crcCheck;
315 int ret;
316
317 FUZ_displayUpdate(cycleNb);
318
319 /* Compression tests */
320
321 /* Test compression destSize */
322 FUZ_DISPLAYTEST;
323 { int srcSize = blockSize;
324 int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
325 char endCheck = FUZ_rand(&randState) & 255;
326 compressedBuffer[targetSize] = endCheck;
327 ret = LZ4_compress_destSize(block, compressedBuffer, &srcSize, targetSize);
328 FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_destSize() result larger than dst buffer !");
329 FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_destSize() overwrite dst buffer !");
330 FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_destSize() fed more than src buffer !");
331 DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
332 if (targetSize>0) {
333 /* check correctness */
334 U32 const crcBase = XXH32(block, srcSize, 0);
335 char const canary = FUZ_rand(&randState) & 255;
336 FUZ_CHECKTEST((ret==0), "LZ4_compress_destSize() compression failed");
337 FUZ_DISPLAYTEST;
338 compressedSize = ret;
339 decodedBuffer[srcSize] = canary;
340 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
341 FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compress_destSize");
342 FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
343 FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !");
344 { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0);
345 FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); }
346
347 DISPLAYLEVEL(5, " OK \n");
348 }
349 else
350 DISPLAYLEVEL(5, " \n");
351 }
352
353 /* Test compression HC destSize */
354 FUZ_DISPLAYTEST;
355 { int srcSize = blockSize;
356 int const targetSize = srcSize * ((FUZ_rand(&randState) & 127)+1) >> 7;
357 char const endCheck = FUZ_rand(&randState) & 255;
358 void* ctx = LZ4_createHC(block);
359 FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed");
360 compressedBuffer[targetSize] = endCheck;
361 ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
362 LZ4_freeHC(ctx);
363 FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
364 FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
365 FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
366 DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
367 if (targetSize>0) {
368 /* check correctness */
369 U32 const crcBase = XXH32(block, srcSize, 0);
370 char const canary = FUZ_rand(&randState) & 255;
371 FUZ_CHECKTEST((ret==0), "LZ4_compress_HC_destSize() compression failed");
372 FUZ_DISPLAYTEST;
373 compressedSize = ret;
374 decodedBuffer[srcSize] = canary;
375 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, srcSize);
376 FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe() failed on data compressed by LZ4_compressHC_destSize");
377 FUZ_CHECKTEST(ret!=srcSize, "LZ4_decompress_safe() failed : did not fully decompressed data");
378 FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe() overwrite dst buffer !");
379 { U32 const crcDec = XXH32(decodedBuffer, srcSize, 0);
380 FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); }
381
382 DISPLAYLEVEL(5, " OK \n");
383 }
384 else
385 DISPLAYLEVEL(5, " \n");
386 }
387
388 /* Test compression HC */
389 FUZ_DISPLAYTEST;
390 ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
391 FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed");
392 HCcompressedSize = ret;
393
394 /* Test compression HC using external state */
395 FUZ_DISPLAYTEST;
396 ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
397 FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed");
398
399 /* Test compression using external state */
400 FUZ_DISPLAYTEST;
401 ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
402 FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed");
403
404 /* Test compression */
405 FUZ_DISPLAYTEST;
406 ret = LZ4_compress_default(block, compressedBuffer, blockSize, (int)compressedBufferSize);
407 FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed");
408 compressedSize = ret;
409
410 /* Decompression tests */
411
412 /* Test decoding with output size being exactly what's necessary => must work */
413 FUZ_DISPLAYTEST;
414 ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize);
415 FUZ_CHECKTEST(ret<0, "LZ4_decompress_fast failed despite correct space");
416 FUZ_CHECKTEST(ret!=compressedSize, "LZ4_decompress_fast failed : did not fully read compressed data");
417 crcCheck = XXH32(decodedBuffer, blockSize, 0);
418 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast corrupted decoded data");
419
420 /* Test decoding with one byte missing => must fail */
421 FUZ_DISPLAYTEST;
422 decodedBuffer[blockSize-1] = 0;
423 ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize-1);
424 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
425 FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast overrun specified output buffer");
426
427 /* Test decoding with one byte too much => must fail */
428 FUZ_DISPLAYTEST;
429 ret = LZ4_decompress_fast(compressedBuffer, decodedBuffer, blockSize+1);
430 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast should have failed, due to Output Size being too large");
431
432 /* Test decoding with output size exactly what's necessary => must work */
433 FUZ_DISPLAYTEST;
434 decodedBuffer[blockSize] = 0;
435 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize);
436 FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite sufficient space");
437 FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
438 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
439 crcCheck = XXH32(decodedBuffer, blockSize, 0);
440 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
441
442 // Test decoding with more than enough output size => must work
443 FUZ_DISPLAYTEST;
444 decodedBuffer[blockSize] = 0;
445 decodedBuffer[blockSize+1] = 0;
446 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize+1);
447 FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe failed despite amply sufficient space");
448 FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe did not regenerate original data");
449 //FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe wrote more than (unknown) target size"); // well, is that an issue ?
450 FUZ_CHECKTEST(decodedBuffer[blockSize+1], "LZ4_decompress_safe overrun specified output buffer size");
451 crcCheck = XXH32(decodedBuffer, blockSize, 0);
452 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe corrupted decoded data");
453
454 // Test decoding with output size being one byte too short => must fail
455 FUZ_DISPLAYTEST;
456 decodedBuffer[blockSize-1] = 0;
457 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-1);
458 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
459 FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
460
461 // Test decoding with output size being 10 bytes too short => must fail
462 FUZ_DISPLAYTEST;
463 if (blockSize>10)
464 {
465 decodedBuffer[blockSize-10] = 0;
466 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize, blockSize-10);
467 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to Output Size being 10 bytes too short");
468 FUZ_CHECKTEST(decodedBuffer[blockSize-10], "LZ4_decompress_safe overrun specified output buffer size");
469 }
470
471 // Test decoding with input size being one byte too short => must fail
472 FUZ_DISPLAYTEST;
473 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize-1, blockSize);
474 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being one byte too short (blockSize=%i, ret=%i, compressedSize=%i)", blockSize, ret, compressedSize);
475
476 // Test decoding with input size being one byte too large => must fail
477 FUZ_DISPLAYTEST;
478 decodedBuffer[blockSize] = 0;
479 ret = LZ4_decompress_safe(compressedBuffer, decodedBuffer, compressedSize+1, blockSize);
480 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe should have failed, due to input size being too large");
481 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
482
483 // Test partial decoding with target output size being max/2 => must work
484 FUZ_DISPLAYTEST;
485 ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize/2, blockSize);
486 FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
487
488 // Test partial decoding with target output size being just below max => must work
489 FUZ_DISPLAYTEST;
490 ret = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, compressedSize, blockSize-3, blockSize);
491 FUZ_CHECKTEST(ret<0, "LZ4_decompress_safe_partial failed despite sufficient space");
492
493 /* Test Compression with limited output size */
494
495 /* Test compression with output size being exactly what's necessary (should work) */
496 FUZ_DISPLAYTEST;
497 ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize);
498 FUZ_CHECKTEST(ret==0, "LZ4_compress_default() failed despite sufficient space");
499
500 /* Test compression with output size being exactly what's necessary and external state (should work) */
501 FUZ_DISPLAYTEST;
502 ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, compressedSize, 1);
503 FUZ_CHECKTEST(ret==0, "LZ4_compress_fast_extState() failed despite sufficient space");
504
505 /* Test HC compression with output size being exactly what's necessary (should work) */
506 FUZ_DISPLAYTEST;
507 ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
508 FUZ_CHECKTEST(ret==0, "LZ4_compress_HC() failed despite sufficient space");
509
510 /* Test HC compression with output size being exactly what's necessary (should work) */
511 FUZ_DISPLAYTEST;
512 ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
513 FUZ_CHECKTEST(ret==0, "LZ4_compress_HC_extStateHC() failed despite sufficient space");
514
515 /* Test compression with missing bytes into output buffer => must fail */
516 FUZ_DISPLAYTEST;
517 { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1;
518 if (missingBytes >= compressedSize) missingBytes = compressedSize-1;
519 missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
520 compressedBuffer[compressedSize-missingBytes] = 0;
521 ret = LZ4_compress_default(block, compressedBuffer, blockSize, compressedSize-missingBytes);
522 FUZ_CHECKTEST(ret, "LZ4_compress_default should have failed (output buffer too small by %i byte)", missingBytes);
523 FUZ_CHECKTEST(compressedBuffer[compressedSize-missingBytes], "LZ4_compress_default overran output buffer ! (%i missingBytes)", missingBytes)
524 }
525
526 /* Test HC compression with missing bytes into output buffer => must fail */
527 FUZ_DISPLAYTEST;
528 { int missingBytes = (FUZ_rand(&randState) % 0x3F) + 1;
529 if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1;
530 missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
531 compressedBuffer[HCcompressedSize-missingBytes] = 0;
532 ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel);
533 FUZ_CHECKTEST(ret, "LZ4_compress_HC should have failed (output buffer too small by %i byte)", missingBytes);
534 FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compress_HC overran output buffer ! (%i missingBytes)", missingBytes)
535 }
536
537
538 /*-******************/
539 /* Dictionary tests */
540 /*-******************/
541
542 /* Compress using dictionary */
543 FUZ_DISPLAYTEST;
544 { LZ4_stream_t LZ4_stream;
545 LZ4_resetStream(&LZ4_stream);
546 LZ4_compress_fast_continue (&LZ4_stream, dict, compressedBuffer, dictSize, (int)compressedBufferSize, 1); /* Just to fill hash tables */
547 blockContinueCompressedSize = LZ4_compress_fast_continue (&LZ4_stream, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
548 FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
549 }
550
551 /* Decompress with dictionary as prefix */
552 FUZ_DISPLAYTEST;
553 memcpy(decodedBuffer, dict, dictSize);
554 ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
555 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
556 crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
557 if (crcCheck!=crcOrig) {
558 int i=0;
559 while (block[i]==decodedBuffer[i]) i++;
560 printf("Wrong Byte at position %i/%i\n", i, blockSize);
561
562 }
563 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
564
565 FUZ_DISPLAYTEST;
566 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer+dictSize, blockContinueCompressedSize, blockSize, decodedBuffer, dictSize);
567 FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
568 crcCheck = XXH32(decodedBuffer+dictSize, blockSize, 0);
569 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
570
571 /* Compress using External dictionary */
572 FUZ_DISPLAYTEST;
573 dict -= (FUZ_rand(&randState) & 0xF) + 1; /* Separation, so it is an ExtDict */
574 if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
575 LZ4_loadDict(&LZ4dict, dict, dictSize);
576 blockContinueCompressedSize = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, (int)compressedBufferSize, 1);
577 FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_fast_continue failed");
578
579 FUZ_DISPLAYTEST;
580 LZ4_loadDict(&LZ4dict, dict, dictSize);
581 ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
582 FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
583
584 FUZ_DISPLAYTEST;
585 LZ4_loadDict(&LZ4dict, dict, dictSize);
586 ret = LZ4_compress_fast_continue(&LZ4dict, block, compressedBuffer, blockSize, blockContinueCompressedSize, 1);
587 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_limitedOutput_compressed size is different (%i != %i)", ret, blockContinueCompressedSize);
588 FUZ_CHECKTEST(ret<=0, "LZ4_compress_fast_continue should work : enough size available within output buffer");
589
590 /* Decompress with dictionary as external */
591 FUZ_DISPLAYTEST;
592 decodedBuffer[blockSize] = 0;
593 ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize, dict, dictSize);
594 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
595 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
596 crcCheck = XXH32(decodedBuffer, blockSize, 0);
597 if (crcCheck!=crcOrig) FUZ_findDiff(block, decodedBuffer);
598 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_fast_usingDict corrupted decoded data (dict %i)", dictSize);
599
600 FUZ_DISPLAYTEST;
601 decodedBuffer[blockSize] = 0;
602 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
603 FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
604 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
605 crcCheck = XXH32(decodedBuffer, blockSize, 0);
606 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
607
608 FUZ_DISPLAYTEST;
609 decodedBuffer[blockSize-1] = 0;
610 ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
611 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
612 FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
613
614 FUZ_DISPLAYTEST;
615 decodedBuffer[blockSize-1] = 0;
616 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
617 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
618 FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
619
620 FUZ_DISPLAYTEST;
621 { U32 const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
622 if ((U32)blockSize > missingBytes) {
623 decodedBuffer[blockSize-missingBytes] = 0;
624 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-missingBytes, dict, dictSize);
625 FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : output buffer too small (-%u byte)", missingBytes);
626 FUZ_CHECKTEST(decodedBuffer[blockSize-missingBytes], "LZ4_decompress_safe_usingDict overrun specified output buffer size (-%u byte) (blockSize=%i)", missingBytes, blockSize);
627 } }
628
629 /* Compress HC using External dictionary */
630 FUZ_DISPLAYTEST;
631 dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
632 if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
633 LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
634 LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
635 blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
636 FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue failed");
637
638 FUZ_DISPLAYTEST;
639 LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
640 ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
641 FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
642
643 FUZ_DISPLAYTEST;
644 LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
645 ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize);
646 FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_compress_HC_continue size is different (%i != %i)", ret, blockContinueCompressedSize);
647 FUZ_CHECKTEST(ret<=0, "LZ4_compress_HC_continue should work : enough size available within output buffer");
648
649 FUZ_DISPLAYTEST;
650 decodedBuffer[blockSize] = 0;
651 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize, dict, dictSize);
652 FUZ_CHECKTEST(ret!=blockSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
653 FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
654 crcCheck = XXH32(decodedBuffer, blockSize, 0);
655 if (crcCheck!=crcOrig)
656 FUZ_findDiff(block, decodedBuffer);
657 FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
658
659 /* Compress HC continue destSize */
660 FUZ_DISPLAYTEST;
661 { int const availableSpace = (FUZ_rand(&randState) % blockSize) + 5;
662 int consumedSize = blockSize;
663 FUZ_DISPLAYTEST;
664 LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
665 LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
666 blockContinueCompressedSize = LZ4_compress_HC_continue_destSize(&LZ4dictHC, block, compressedBuffer, &consumedSize, availableSpace);
667 DISPLAYLEVEL(5, " LZ4_compress_HC_continue_destSize : compressed %6i/%6i into %6i/%6i at cLevel=%i\n", consumedSize, blockSize, blockContinueCompressedSize, availableSpace, compressionLevel);
668 FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compress_HC_continue_destSize failed");
669 FUZ_CHECKTEST(blockContinueCompressedSize > availableSpace, "LZ4_compress_HC_continue_destSize write overflow");
670 FUZ_CHECKTEST(consumedSize > blockSize, "LZ4_compress_HC_continue_destSize read overflow");
671
672 FUZ_DISPLAYTEST;
673 decodedBuffer[consumedSize] = 0;
674 ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, consumedSize, dict, dictSize);
675 FUZ_CHECKTEST(ret!=consumedSize, "LZ4_decompress_safe_usingDict did not regenerate original data");
676 FUZ_CHECKTEST(decodedBuffer[consumedSize], "LZ4_decompress_safe_usingDict overrun specified output buffer size")
677 { U32 const crcSrc = XXH32(block, consumedSize, 0);
678 U32 const crcDst = XXH32(decodedBuffer, consumedSize, 0);
679 if (crcSrc!=crcDst)
680 FUZ_findDiff(block, decodedBuffer);
681 FUZ_CHECKTEST(crcSrc!=crcDst, "LZ4_decompress_safe_usingDict corrupted decoded data");
682 }
683 }
684
685 /* ***** End of tests *** */
686 /* Fill stats */
687 bytes += blockSize;
688 cbytes += compressedSize;
689 hcbytes += HCcompressedSize;
690 ccbytes += blockContinueCompressedSize;
691 }
692
693 if (nbCycles<=1) nbCycles = cycleNb; /* end by time */
694 bytes += !bytes; /* avoid division by 0 */
695 printf("\r%7u /%7u - ", cycleNb, nbCycles);
696 printf("all tests completed successfully \n");
697 printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100);
698 printf("HC compression ratio: %0.3f%%\n", (double)hcbytes/bytes*100);
699 printf("ratio with dict: %0.3f%%\n", (double)ccbytes/bytes*100);
700
701 /* release memory */
702 {
703 _exit:
704 free(CNBuffer);
705 free(compressedBuffer);
706 free(decodedBuffer);
707 free(stateLZ4);
708 free(stateLZ4HC);
709 return result;
710
711 _output_error:
712 result = 1;
713 goto _exit;
714 }
715 }
716
717
718 #define testInputSize (192 KB)
719 #define testCompressedSize (128 KB)
720 #define ringBufferSize (8 KB)
721
FUZ_unitTests(int compressionLevel)722 static void FUZ_unitTests(int compressionLevel)
723 {
724 const unsigned testNb = 0;
725 const unsigned seed = 0;
726 const unsigned cycleNb= 0;
727 char testInput[testInputSize];
728 char testCompressed[testCompressedSize];
729 char testVerify[testInputSize];
730 char ringBuffer[ringBufferSize];
731 U32 randState = 1;
732
733 /* Init */
734 FUZ_fillCompressibleNoiseBuffer(testInput, testInputSize, 0.50, &randState);
735
736 /* 32-bits address space overflow test */
737 FUZ_AddressOverflow();
738
739 /* LZ4 streaming tests */
740 { LZ4_stream_t* statePtr;
741 LZ4_stream_t streamingState;
742 U64 crcOrig;
743 int result;
744
745 /* Allocation test */
746 statePtr = LZ4_createStream();
747 FUZ_CHECKTEST(statePtr==NULL, "LZ4_createStream() allocation failed");
748 LZ4_freeStream(statePtr);
749
750 /* simple compression test */
751 crcOrig = XXH64(testInput, testCompressedSize, 0);
752 LZ4_resetStream(&streamingState);
753 result = LZ4_compress_fast_continue(&streamingState, testInput, testCompressed, testCompressedSize, testCompressedSize-1, 1);
754 FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed!");
755
756 result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize);
757 FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
758 { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
759 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
760
761 /* ring buffer test */
762 { XXH64_state_t xxhOrig;
763 XXH64_state_t xxhNew;
764 LZ4_streamDecode_t decodeState;
765 const U32 maxMessageSizeLog = 10;
766 const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
767 U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
768 U32 iNext = 0;
769 U32 rNext = 0;
770 U32 dNext = 0;
771 const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
772
773 XXH64_reset(&xxhOrig, 0);
774 XXH64_reset(&xxhNew, 0);
775 LZ4_resetStream(&streamingState);
776 LZ4_setStreamDecode(&decodeState, NULL, 0);
777
778 while (iNext + messageSize < testCompressedSize) {
779 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
780 crcOrig = XXH64_digest(&xxhOrig);
781
782 memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
783 result = LZ4_compress_fast_continue(&streamingState, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize, 1);
784 FUZ_CHECKTEST(result==0, "LZ4_compress_fast_continue() compression failed");
785
786 result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
787 FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
788
789 XXH64_update(&xxhNew, testVerify + dNext, messageSize);
790 { U64 const crcNew = XXH64_digest(&xxhNew);
791 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
792
793 /* prepare next message */
794 iNext += messageSize;
795 rNext += messageSize;
796 dNext += messageSize;
797 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
798 if (rNext + messageSize > ringBufferSize) rNext = 0;
799 if (dNext + messageSize > dBufferSize) dNext = 0;
800 }
801 }
802 }
803
804 /* LZ4 HC streaming tests */
805 { LZ4_streamHC_t* sp;
806 LZ4_streamHC_t sHC;
807 U64 crcOrig;
808 int result;
809
810 /* Allocation test */
811 sp = LZ4_createStreamHC();
812 FUZ_CHECKTEST(sp==NULL, "LZ4_createStreamHC() allocation failed");
813 LZ4_freeStreamHC(sp);
814
815 /* simple HC compression test */
816 crcOrig = XXH64(testInput, testCompressedSize, 0);
817 LZ4_resetStreamHC(&sHC, compressionLevel);
818 result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
819 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
820
821 result = LZ4_decompress_safe(testCompressed, testVerify, result, testCompressedSize);
822 FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() decompression failed");
823 { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
824 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
825
826 /* simple dictionary HC compression test */
827 crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
828 LZ4_resetStreamHC(&sHC, compressionLevel);
829 LZ4_loadDictHC(&sHC, testInput, 64 KB);
830 result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
831 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
832
833 result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 64 KB);
834 FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe() simple dictionary decompression test failed");
835 { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
836 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() simple dictionary decompression test : corruption"); }
837
838 /* multiple HC compression test with dictionary */
839 { int result1, result2;
840 int segSize = testCompressedSize / 2;
841 crcOrig = XXH64(testInput + segSize, testCompressedSize, 0);
842 LZ4_resetStreamHC(&sHC, compressionLevel);
843 LZ4_loadDictHC(&sHC, testInput, segSize);
844 result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1);
845 FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1);
846 result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*segSize, testCompressed+result1, segSize, segSize-1);
847 FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2);
848
849 result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize);
850 FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 1 failed");
851 result = LZ4_decompress_safe_usingDict(testCompressed+result1, testVerify+segSize, result2, segSize, testInput, 2*segSize);
852 FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe() dictionary decompression part 2 failed");
853 { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
854 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() dictionary decompression corruption"); }
855 }
856
857 /* remote dictionary HC compression test */
858 crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
859 LZ4_resetStreamHC(&sHC, compressionLevel);
860 LZ4_loadDictHC(&sHC, testInput, 32 KB);
861 result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
862 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result);
863
864 result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB);
865 FUZ_CHECKTEST(result!=(int)testCompressedSize, "LZ4_decompress_safe_usingDict() decompression failed following remote dictionary HC compression test");
866 { U64 const crcNew = XXH64(testVerify, testCompressedSize, 0);
867 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() decompression corruption"); }
868
869 /* multiple HC compression with ext. dictionary */
870 { XXH64_state_t crcOrigState;
871 XXH64_state_t crcNewState;
872 const char* dict = testInput + 3;
873 int dictSize = (FUZ_rand(&randState) & 8191);
874 char* dst = testVerify;
875
876 size_t segStart = dictSize + 7;
877 int segSize = (FUZ_rand(&randState) & 8191);
878 int segNb = 1;
879
880 LZ4_resetStreamHC(&sHC, compressionLevel);
881 LZ4_loadDictHC(&sHC, dict, dictSize);
882
883 XXH64_reset(&crcOrigState, 0);
884 XXH64_reset(&crcNewState, 0);
885
886 while (segStart + segSize < testInputSize) {
887 XXH64_update(&crcOrigState, testInput + segStart, segSize);
888 crcOrig = XXH64_digest(&crcOrigState);
889 result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, segSize, LZ4_compressBound(segSize));
890 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
891
892 result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, segSize, dict, dictSize);
893 FUZ_CHECKTEST(result!=segSize, "LZ4_decompress_safe_usingDict() dictionary decompression part %i failed", segNb);
894 XXH64_update(&crcNewState, dst, segSize);
895 { U64 const crcNew = XXH64_digest(&crcNewState);
896 if (crcOrig != crcNew) FUZ_findDiff(dst, testInput+segStart);
897 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe_usingDict() part %i corruption", segNb);
898 }
899
900 dict = dst;
901 //dict = testInput + segStart;
902 dictSize = segSize;
903
904 dst += segSize + 1;
905 segNb ++;
906
907 segStart += segSize + (FUZ_rand(&randState) & 0xF) + 1;
908 segSize = (FUZ_rand(&randState) & 8191);
909 }
910 }
911
912 /* ring buffer test */
913 { XXH64_state_t xxhOrig;
914 XXH64_state_t xxhNew;
915 LZ4_streamDecode_t decodeState;
916 const U32 maxMessageSizeLog = 10;
917 const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
918 U32 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
919 U32 iNext = 0;
920 U32 rNext = 0;
921 U32 dNext = 0;
922 const U32 dBufferSize = ringBufferSize + maxMessageSizeMask;
923
924 XXH64_reset(&xxhOrig, 0);
925 XXH64_reset(&xxhNew, 0);
926 LZ4_resetStreamHC(&sHC, compressionLevel);
927 LZ4_setStreamDecode(&decodeState, NULL, 0);
928
929 while (iNext + messageSize < testCompressedSize) {
930 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
931 crcOrig = XXH64_digest(&xxhOrig);
932
933 memcpy (ringBuffer + rNext, testInput + iNext, messageSize);
934 result = LZ4_compress_HC_continue(&sHC, ringBuffer + rNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
935 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
936
937 result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
938 FUZ_CHECKTEST(result!=(int)messageSize, "ringBuffer : LZ4_decompress_safe() test failed");
939
940 XXH64_update(&xxhNew, testVerify + dNext, messageSize);
941 { U64 const crcNew = XXH64_digest(&xxhNew);
942 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
943
944 /* prepare next message */
945 iNext += messageSize;
946 rNext += messageSize;
947 dNext += messageSize;
948 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
949 if (rNext + messageSize > ringBufferSize) rNext = 0;
950 if (dNext + messageSize > dBufferSize) dNext = 0;
951 }
952 }
953
954 /* small decoder-side ring buffer test */
955 { XXH64_state_t xxhOrig;
956 XXH64_state_t xxhNew;
957 LZ4_streamDecode_t decodeState;
958 const U32 maxMessageSizeLog = 12;
959 const U32 maxMessageSizeMask = (1<<maxMessageSizeLog) - 1;
960 U32 messageSize;
961 U32 totalMessageSize = 0;
962 U32 iNext = 0;
963 U32 dNext = 0;
964 const U32 dBufferSize = 64 KB;
965
966 XXH64_reset(&xxhOrig, 0);
967 XXH64_reset(&xxhNew, 0);
968 LZ4_resetStreamHC(&sHC, compressionLevel);
969 LZ4_setStreamDecode(&decodeState, NULL, 0);
970
971 #define BSIZE1 65537
972 #define BSIZE2 16435
973
974 /* first block */
975
976 messageSize = BSIZE1;
977 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
978 crcOrig = XXH64_digest(&xxhOrig);
979
980 result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
981 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
982
983 result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
984 FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
985
986 XXH64_update(&xxhNew, testVerify + dNext, messageSize);
987 { U64 const crcNew = XXH64_digest(&xxhNew);
988 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
989
990 /* prepare next message */
991 dNext += messageSize;
992 totalMessageSize += messageSize;
993 messageSize = BSIZE2;
994 iNext = 132000;
995 memcpy(testInput + iNext, testInput + 8, messageSize);
996 if (dNext > dBufferSize) dNext = 0;
997
998 while (totalMessageSize < 9 MB) {
999 XXH64_update(&xxhOrig, testInput + iNext, messageSize);
1000 crcOrig = XXH64_digest(&xxhOrig);
1001
1002 result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
1003 FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
1004
1005 result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
1006 FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
1007
1008 XXH64_update(&xxhNew, testVerify + dNext, messageSize);
1009 { U64 const crcNew = XXH64_digest(&xxhNew);
1010 if (crcOrig != crcNew) FUZ_findDiff(testInput + iNext, testVerify + dNext);
1011 FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption during small decoder-side ring buffer test");
1012 }
1013 /* prepare next message */
1014 dNext += messageSize;
1015 totalMessageSize += messageSize;
1016 messageSize = (FUZ_rand(&randState) & maxMessageSizeMask) + 1;
1017 iNext = (FUZ_rand(&randState) & 65535);
1018 if (dNext > dBufferSize) dNext = 0;
1019 }
1020 }
1021 }
1022
1023 printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel);
1024 return;
1025 _output_error:
1026 exit(1);
1027 }
1028
1029
FUZ_usage(const char * programName)1030 static int FUZ_usage(const char* programName)
1031 {
1032 DISPLAY( "Usage :\n");
1033 DISPLAY( " %s [args]\n", programName);
1034 DISPLAY( "\n");
1035 DISPLAY( "Arguments :\n");
1036 DISPLAY( " -i# : Nb of tests (default:%i) \n", NB_ATTEMPTS);
1037 DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
1038 DISPLAY( " -s# : Select seed (default:prompt user)\n");
1039 DISPLAY( " -t# : Select starting test number (default:0)\n");
1040 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
1041 DISPLAY( " -v : verbose\n");
1042 DISPLAY( " -p : pause at the end\n");
1043 DISPLAY( " -h : display help and exit\n");
1044 return 0;
1045 }
1046
1047
main(int argc,const char ** argv)1048 int main(int argc, const char** argv)
1049 {
1050 U32 seed = 0;
1051 int seedset = 0;
1052 int argNb;
1053 int nbTests = NB_ATTEMPTS;
1054 int testNb = 0;
1055 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
1056 int use_pause = 0;
1057 const char* programName = argv[0];
1058 U32 duration = 0;
1059
1060 /* Check command line */
1061 for(argNb=1; argNb<argc; argNb++) {
1062 const char* argument = argv[argNb];
1063
1064 if(!argument) continue; // Protection if argument empty
1065
1066 // Decode command (note : aggregated commands are allowed)
1067 if (argument[0]=='-') {
1068 if (!strcmp(argument, "--no-prompt")) { use_pause=0; seedset=1; g_displayLevel=1; continue; }
1069 argument++;
1070
1071 while (*argument!=0) {
1072 switch(*argument)
1073 {
1074 case 'h': /* display help */
1075 return FUZ_usage(programName);
1076
1077 case 'v': /* verbose mode */
1078 argument++;
1079 g_displayLevel++;
1080 break;
1081
1082 case 'p': /* pause at the end */
1083 argument++;
1084 use_pause=1;
1085 break;
1086
1087 case 'i':
1088 argument++;
1089 nbTests = 0; duration = 0;
1090 while ((*argument>='0') && (*argument<='9')) {
1091 nbTests *= 10;
1092 nbTests += *argument - '0';
1093 argument++;
1094 }
1095 break;
1096
1097 case 'T':
1098 argument++;
1099 nbTests = 0; duration = 0;
1100 for (;;) {
1101 switch(*argument)
1102 {
1103 case 'm': duration *= 60; argument++; continue;
1104 case 's':
1105 case 'n': argument++; continue;
1106 case '0':
1107 case '1':
1108 case '2':
1109 case '3':
1110 case '4':
1111 case '5':
1112 case '6':
1113 case '7':
1114 case '8':
1115 case '9': duration *= 10; duration += *argument++ - '0'; continue;
1116 }
1117 break;
1118 }
1119 break;
1120
1121 case 's':
1122 argument++;
1123 seed=0; seedset=1;
1124 while ((*argument>='0') && (*argument<='9')) {
1125 seed *= 10;
1126 seed += *argument - '0';
1127 argument++;
1128 }
1129 break;
1130
1131 case 't': /* select starting test nb */
1132 argument++;
1133 testNb=0;
1134 while ((*argument>='0') && (*argument<='9')) {
1135 testNb *= 10;
1136 testNb += *argument - '0';
1137 argument++;
1138 }
1139 break;
1140
1141 case 'P': /* change probability */
1142 argument++;
1143 proba=0;
1144 while ((*argument>='0') && (*argument<='9')) {
1145 proba *= 10;
1146 proba += *argument - '0';
1147 argument++;
1148 }
1149 if (proba<0) proba=0;
1150 if (proba>100) proba=100;
1151 break;
1152 default: ;
1153 }
1154 }
1155 }
1156 }
1157
1158 printf("Starting LZ4 fuzzer (%i-bits, v%s)\n", (int)(sizeof(size_t)*8), LZ4_versionString());
1159
1160 if (!seedset) {
1161 time_t const t = time(NULL);
1162 U32 const h = XXH32(&t, sizeof(t), 1);
1163 seed = h % 10000;
1164 }
1165 printf("Seed = %u\n", seed);
1166
1167 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);
1168
1169 if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); }
1170
1171 if (nbTests<=0) nbTests=1;
1172
1173 { int const result = FUZ_test(seed, nbTests, testNb, ((double)proba) / 100, duration);
1174 if (use_pause) {
1175 DISPLAY("press enter ... \n");
1176 (void)getchar();
1177 }
1178 return result;
1179 }
1180 }
1181