1 /*
2 frameTest - test tool for lz4frame
3 Copyright (C) Yann Collet 2014-2016
4
5 GPL v2 License
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License along
18 with this program; if not, write to the Free Software Foundation, Inc.,
19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21 You can contact the author at :
22 - LZ4 homepage : http://www.lz4.org
23 - LZ4 source repository : https://github.com/lz4/lz4
24 */
25
26 /*-************************************
27 * Compiler specific
28 **************************************/
29 #ifdef _MSC_VER /* Visual Studio */
30 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
31 # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
32 #endif
33
34
35 /*-************************************
36 * Includes
37 **************************************/
38 #include "util.h" /* U32 */
39 #include <stdlib.h> /* malloc, free */
40 #include <stdio.h> /* fprintf */
41 #include <string.h> /* strcmp */
42 #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
43 #include <assert.h>
44 #include "lz4frame.h" /* included multiple times to test correctness/safety */
45 #include "lz4frame.h"
46 #define LZ4F_STATIC_LINKING_ONLY
47 #include "lz4frame.h"
48 #include "lz4frame.h"
49 #define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */
50 #include "lz4.h" /* LZ4_VERSION_STRING */
51 #define XXH_STATIC_LINKING_ONLY
52 #include "xxhash.h" /* XXH64 */
53
54
55 /* unoptimized version; solves endianess & alignment issues */
FUZ_writeLE32(void * dstVoidPtr,U32 value32)56 static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
57 {
58 BYTE* dstPtr = (BYTE*)dstVoidPtr;
59 dstPtr[0] = (BYTE) value32;
60 dstPtr[1] = (BYTE)(value32 >> 8);
61 dstPtr[2] = (BYTE)(value32 >> 16);
62 dstPtr[3] = (BYTE)(value32 >> 24);
63 }
64
65
66 /*-************************************
67 * Constants
68 **************************************/
69 #define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
70
71 #define KB *(1U<<10)
72 #define MB *(1U<<20)
73 #define GB *(1U<<30)
74
75 static const U32 nbTestsDefault = 256 KB;
76 #define FUZ_COMPRESSIBILITY_DEFAULT 50
77 static const U32 prime1 = 2654435761U;
78 static const U32 prime2 = 2246822519U;
79
80
81 /*-************************************
82 * Macros
83 **************************************/
84 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
85 #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
86 #define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
87 if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \
88 { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \
89 if (displayLevel>=4) fflush(stdout); } }
90 static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
91 static clock_t g_clockTime = 0;
92
93
94 /*-***************************************
95 * Local Parameters
96 *****************************************/
97 static U32 no_prompt = 0;
98 static U32 displayLevel = 2;
99 static U32 use_pause = 0;
100
101
102 /*-*******************************************************
103 * Fuzzer functions
104 *********************************************************/
105 #define MIN(a,b) ( (a) < (b) ? (a) : (b) )
106 #define MAX(a,b) ( (a) > (b) ? (a) : (b) )
107
FUZ_GetClockSpan(clock_t clockStart)108 static clock_t FUZ_GetClockSpan(clock_t clockStart)
109 {
110 return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */
111 }
112
113
114 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(unsigned int * src)115 unsigned int FUZ_rand(unsigned int* src)
116 {
117 U32 rand32 = *src;
118 rand32 *= prime1;
119 rand32 += prime2;
120 rand32 = FUZ_rotl32(rand32, 13);
121 *src = rand32;
122 return rand32 >> 5;
123 }
124
125
126 #define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF)
127 #define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
FUZ_fillCompressibleNoiseBuffer(void * buffer,size_t bufferSize,double proba,U32 * seed)128 static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed)
129 {
130 BYTE* BBuffer = (BYTE*)buffer;
131 size_t pos = 0;
132 U32 P32 = (U32)(32768 * proba);
133
134 /* First Byte */
135 BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
136
137 while (pos < bufferSize) {
138 /* Select : Literal (noise) or copy (within 64K) */
139 if (FUZ_RAND15BITS < P32) {
140 /* Copy (within 64K) */
141 size_t const lengthRand = FUZ_RANDLENGTH + 4;
142 size_t const length = MIN(lengthRand, bufferSize - pos);
143 size_t const end = pos + length;
144 size_t const offsetRand = FUZ_RAND15BITS + 1;
145 size_t const offset = MIN(offsetRand, pos);
146 size_t match = pos - offset;
147 while (pos < end) BBuffer[pos++] = BBuffer[match++];
148 } else {
149 /* Literal (noise) */
150 size_t const lengthRand = FUZ_RANDLENGTH + 4;
151 size_t const length = MIN(lengthRand, bufferSize - pos);
152 size_t const end = pos + length;
153 while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
154 } }
155 }
156
157
FUZ_highbit(U32 v32)158 static unsigned FUZ_highbit(U32 v32)
159 {
160 unsigned nbBits = 0;
161 if (v32==0) return 0;
162 while (v32) {v32 >>= 1; nbBits ++;}
163 return nbBits;
164 }
165
166
167 /*-*******************************************************
168 * Tests
169 *********************************************************/
170 #define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) { fprintf(stderr, "%s \n", LZ4F_getErrorName(v)); goto _output_error; }
171 #define CHECK(f) { LZ4F_errorCode_t const CHECK_V(err_ , f); }
172
basicTests(U32 seed,double compressibility)173 int basicTests(U32 seed, double compressibility)
174 {
175 #define COMPRESSIBLE_NOISE_LENGTH (2 MB)
176 void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
177 size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL);
178 void* const compressedBuffer = malloc(cBuffSize);
179 void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
180 U32 randState = seed;
181 size_t cSize, testSize;
182 LZ4F_decompressionContext_t dCtx = NULL;
183 LZ4F_compressionContext_t cctx = NULL;
184 U64 crcOrig;
185 int basicTests_error = 0;
186 LZ4F_preferences_t prefs;
187 memset(&prefs, 0, sizeof(prefs));
188
189 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
190 DISPLAY("allocation error, not enough memory to start fuzzer tests \n");
191 goto _output_error;
192 }
193 FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
194 crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
195
196 /* LZ4F_compressBound() : special case : srcSize == 0 */
197 DISPLAYLEVEL(3, "LZ4F_compressBound(0) = ");
198 { size_t const cBound = LZ4F_compressBound(0, NULL);
199 if (cBound < 64 KB) goto _output_error;
200 DISPLAYLEVEL(3, " %u \n", (U32)cBound);
201 }
202
203 /* Special case : null-content frame */
204 testSize = 0;
205 DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : ");
206 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
207 DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize);
208
209 DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n");
210 CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
211
212 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n");
213 assert(cSize >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH);
214 { LZ4F_frameInfo_t frame_info;
215 size_t const fhs = LZ4F_headerSize(compressedBuffer, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH);
216 size_t avail_in = fhs;
217 CHECK( fhs );
218 CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) );
219 if (avail_in != fhs) goto _output_error; /* must consume all, since header size is supposed to be exact */
220 }
221
222 DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n");
223 CHECK( LZ4F_freeDecompressionContext(dCtx) );
224 dCtx = NULL;
225
226 /* test one-pass frame compression */
227 testSize = COMPRESSIBLE_NOISE_LENGTH;
228
229 DISPLAYLEVEL(3, "LZ4F_compressFrame, using fast level -3 : ");
230 { LZ4F_preferences_t fastCompressPrefs;
231 memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs));
232 fastCompressPrefs.compressionLevel = -3;
233 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs));
234 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
235 }
236
237 DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : ");
238 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
239 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
240
241 DISPLAYLEVEL(3, "Decompression test : \n");
242 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
243 size_t compressedBufferSize = cSize;
244
245 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
246
247 DISPLAYLEVEL(3, "Single Pass decompression : ");
248 CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) );
249 { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
250 if (crcDest != crcOrig) goto _output_error; }
251 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize);
252
253 DISPLAYLEVEL(3, "Reusing decompression context \n");
254 { size_t const missingBytes = 4;
255 size_t iSize = compressedBufferSize - missingBytes;
256 const BYTE* cBuff = (const BYTE*) compressedBuffer;
257 BYTE* const ostart = (BYTE*)decodedBuffer;
258 BYTE* op = ostart;
259 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
260 size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH;
261 DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes);
262 CHECK_V(decResult, LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL));
263 if (decResult != missingBytes) {
264 DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult);
265 goto _output_error;
266 }
267 DISPLAYLEVEL(3, "indeed, requests %u bytes \n", (unsigned)decResult);
268 cBuff += iSize;
269 iSize = decResult;
270 op += oSize;
271 oSize = (size_t)(oend-op);
272 decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL);
273 if (decResult != 0) goto _output_error; /* should finish now */
274 op += oSize;
275 if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; }
276 { U64 const crcDest = XXH64(decodedBuffer, (size_t)(op-ostart), 1);
277 if (crcDest != crcOrig) goto _output_error;
278 } }
279
280 { size_t oSize = 0;
281 size_t iSize = 0;
282 LZ4F_frameInfo_t fi;
283 const BYTE* ip = (BYTE*)compressedBuffer;
284
285 DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
286 CHECK( LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL) );
287 //DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
288 DISPLAYLEVEL(3, " OK \n");
289
290 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : ");
291 { size_t nullSize = 0;
292 size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize);
293 if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
294 DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n",
295 LZ4F_getErrorName(fiError));
296 goto _output_error;
297 }
298 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
299 }
300
301 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on not enough input : ");
302 { size_t inputSize = 6;
303 size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &inputSize);
304 if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
305 DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError));
306 goto _output_error;
307 }
308 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
309 }
310
311 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : ");
312 iSize = LZ4F_headerSize(ip, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH);
313 CHECK( iSize );
314 CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) );
315 DISPLAYLEVEL(3, " correctly decoded \n");
316 }
317
318 DISPLAYLEVEL(3, "Decode a buggy input : ");
319 assert(COMPRESSIBLE_NOISE_LENGTH > 64);
320 assert(cSize > 48);
321 memcpy(decodedBuffer, (char*)compressedBuffer+16, 32); /* save correct data */
322 memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32); /* insert noise */
323 { size_t dbSize = COMPRESSIBLE_NOISE_LENGTH;
324 size_t cbSize = cSize;
325 size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize,
326 compressedBuffer, &cbSize,
327 NULL);
328 if (!LZ4F_isError(decompressError)) goto _output_error;
329 DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError));
330 }
331 memcpy((char*)compressedBuffer+16, decodedBuffer, 32); /* restore correct data */
332
333 DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n");
334 LZ4F_resetDecompressionContext(dCtx); /* always successful */
335
336 DISPLAYLEVEL(3, "Byte after byte : ");
337 { BYTE* const ostart = (BYTE*)decodedBuffer;
338 BYTE* op = ostart;
339 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
340 const BYTE* ip = (const BYTE*) compressedBuffer;
341 const BYTE* const iend = ip + cSize;
342 while (ip < iend) {
343 size_t oSize = (size_t)(oend-op);
344 size_t iSize = 1;
345 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
346 op += oSize;
347 ip += iSize;
348 }
349 { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
350 if (crcDest != crcOrig) goto _output_error;
351 }
352 DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), (unsigned)COMPRESSIBLE_NOISE_LENGTH);
353 }
354 }
355
356 DISPLAYLEVEL(3, "Using 64 KB block : ");
357 prefs.frameInfo.blockSizeID = LZ4F_max64KB;
358 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
359 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
360 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
361
362 DISPLAYLEVEL(3, "without checksum : ");
363 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
364 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
365 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
366
367 DISPLAYLEVEL(3, "Using 256 KB block : ");
368 prefs.frameInfo.blockSizeID = LZ4F_max256KB;
369 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
370 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
371 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
372
373 DISPLAYLEVEL(3, "Decompression test : \n");
374 { size_t const decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
375 unsigned const maxBits = FUZ_highbit((U32)decodedBufferSize);
376 BYTE* const ostart = (BYTE*)decodedBuffer;
377 BYTE* op = ostart;
378 BYTE* const oend = ostart + COMPRESSIBLE_NOISE_LENGTH;
379 const BYTE* ip = (const BYTE*)compressedBuffer;
380 const BYTE* const iend = (const BYTE*)compressedBuffer + cSize;
381
382 DISPLAYLEVEL(3, "random segment sizes : ");
383 while (ip < iend) {
384 unsigned const nbBits = FUZ_rand(&randState) % maxBits;
385 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
386 size_t oSize = (size_t)(oend-op);
387 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
388 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
389 op += oSize;
390 ip += iSize;
391 }
392 { size_t const decodedSize = (size_t)(op - ostart);
393 U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
394 if (crcDest != crcOrig) goto _output_error;
395 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
396 }
397
398 CHECK( LZ4F_freeDecompressionContext(dCtx) );
399 dCtx = NULL;
400 }
401
402 DISPLAYLEVEL(3, "without checksum : ");
403 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
404 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
405 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
406
407 DISPLAYLEVEL(3, "Using 1 MB block : ");
408 prefs.frameInfo.blockSizeID = LZ4F_max1MB;
409 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
410 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
411 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
412
413 DISPLAYLEVEL(3, "without frame checksum : ");
414 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
415 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
416 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
417
418 DISPLAYLEVEL(3, "Using 4 MB block : ");
419 prefs.frameInfo.blockSizeID = LZ4F_max4MB;
420 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
421 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
422 DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
423 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
424 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
425 }
426
427 DISPLAYLEVEL(3, "without frame checksum : ");
428 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
429 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
430 DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
431 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
432 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
433 }
434
435 DISPLAYLEVEL(3, "LZ4F_compressFrame with block checksum : ");
436 memset(&prefs, 0, sizeof(prefs));
437 prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled;
438 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
439 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
440
441 DISPLAYLEVEL(3, "Decompress with block checksum : ");
442 { size_t iSize = cSize;
443 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
444 LZ4F_decompressionContext_t dctx;
445 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
446 CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) );
447 if (decodedSize != testSize) goto _output_error;
448 if (iSize != cSize) goto _output_error;
449 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
450 U64 const crcSrc = XXH64(CNBuffer, testSize, 1);
451 if (crcDest != crcSrc) goto _output_error;
452 }
453 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
454
455 CHECK( LZ4F_freeDecompressionContext(dctx) );
456 }
457
458 /* frame content size tests */
459 { size_t cErr;
460 BYTE* const ostart = (BYTE*)compressedBuffer;
461 BYTE* op = ostart;
462 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
463
464 DISPLAYLEVEL(3, "compress without frameSize : ");
465 memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
466 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
467 op += cErr;
468 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
469 op += cErr;
470 CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
471 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
472
473 DISPLAYLEVEL(3, "compress with frameSize : ");
474 prefs.frameInfo.contentSize = testSize;
475 op = ostart;
476 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
477 op += cErr;
478 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
479 op += cErr;
480 CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
481 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
482
483 DISPLAYLEVEL(3, "compress with wrong frameSize : ");
484 prefs.frameInfo.contentSize = testSize+1;
485 op = ostart;
486 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
487 op += cErr;
488 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
489 op += cErr;
490 cErr = LZ4F_compressEnd(cctx, op, testSize, NULL);
491 if (!LZ4F_isError(cErr)) goto _output_error;
492 DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(cErr));
493
494 CHECK( LZ4F_freeCompressionContext(cctx) );
495 cctx = NULL;
496 }
497
498 /* dictID tests */
499 { size_t cErr;
500 U32 const dictID = 0x99;
501 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
502
503 DISPLAYLEVEL(3, "insert a dictID : ");
504 memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
505 prefs.frameInfo.dictID = dictID;
506 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
507 DISPLAYLEVEL(3, "created frame header of size %i bytes \n", (int)cErr);
508
509 DISPLAYLEVEL(3, "read a dictID : ");
510 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
511 memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
512 CHECK( LZ4F_getFrameInfo(dCtx, &prefs.frameInfo, compressedBuffer, &cErr) );
513 if (prefs.frameInfo.dictID != dictID) goto _output_error;
514 DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID);
515
516 CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL;
517 CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
518 }
519
520 /* Dictionary compression test */
521 { size_t const dictSize = 63 KB;
522 size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
523 size_t cSizeNoDict, cSizeWithDict;
524 LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
525 if (cdict == NULL) goto _output_error;
526 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
527
528 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
529 CHECK_V(cSizeNoDict,
530 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
531 CNBuffer, dictSize,
532 NULL, NULL) );
533 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
534
535 CHECK( LZ4F_freeCompressionContext(cctx) );
536 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
537 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
538 CHECK_V(cSizeWithDict,
539 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
540 CNBuffer, dictSize,
541 cdict, NULL) );
542 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
543 (unsigned)dictSize, (unsigned)cSizeWithDict);
544 if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) goto _output_error; /* must be more efficient */
545 crcOrig = XXH64(CNBuffer, dictSize, 0);
546
547 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
548 { LZ4F_dctx* dctx;
549 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
550 size_t compressedSize = cSizeWithDict;
551 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
552 CHECK( LZ4F_decompress_usingDict(dctx,
553 decodedBuffer, &decodedSize,
554 compressedBuffer, &compressedSize,
555 CNBuffer, dictSize,
556 NULL) );
557 if (compressedSize != cSizeWithDict) goto _output_error;
558 if (decodedSize != dictSize) goto _output_error;
559 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
560 if (crcDest != crcOrig) goto _output_error; }
561 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
562 CHECK( LZ4F_freeDecompressionContext(dctx) );
563 }
564
565 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : ");
566 { size_t cSizeLevelMax;
567 LZ4F_preferences_t cParams;
568 memset(&cParams, 0, sizeof(cParams));
569 cParams.compressionLevel = -3;
570 CHECK_V(cSizeLevelMax,
571 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
572 CNBuffer, dictSize,
573 cdict, &cParams) );
574 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
575 }
576
577 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, level max : ");
578 { size_t cSizeLevelMax;
579 LZ4F_preferences_t cParams;
580 memset(&cParams, 0, sizeof(cParams));
581 cParams.compressionLevel = LZ4F_compressionLevel_max();
582 CHECK_V(cSizeLevelMax,
583 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
584 CNBuffer, dictSize,
585 cdict, &cParams) );
586 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
587 }
588
589 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple linked blocks : ");
590 { size_t cSizeContiguous;
591 size_t const inSize = dictSize * 3;
592 size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
593 LZ4F_preferences_t cParams;
594 memset(&cParams, 0, sizeof(cParams));
595 cParams.frameInfo.blockMode = LZ4F_blockLinked;
596 cParams.frameInfo.blockSizeID = LZ4F_max64KB;
597 CHECK_V(cSizeContiguous,
598 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
599 CNBuffer, inSize,
600 cdict, &cParams) );
601 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
602 (unsigned)inSize, (unsigned)cSizeContiguous);
603
604 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple linked blocks : ");
605 { LZ4F_dctx* dctx;
606 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
607 size_t compressedSize = cSizeContiguous;
608 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
609 CHECK( LZ4F_decompress_usingDict(dctx,
610 decodedBuffer, &decodedSize,
611 compressedBuffer, &compressedSize,
612 CNBuffer, dictSize,
613 NULL) );
614 if (compressedSize != cSizeContiguous) goto _output_error;
615 if (decodedSize != inSize) goto _output_error;
616 crcOrig = XXH64(CNBuffer, inSize, 0);
617 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
618 if (crcDest != crcOrig) goto _output_error; }
619 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
620 CHECK( LZ4F_freeDecompressionContext(dctx) );
621 }
622 }
623
624
625 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : ");
626 { size_t cSizeIndep;
627 size_t const inSize = dictSize * 3;
628 size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
629 LZ4F_preferences_t cParams;
630 memset(&cParams, 0, sizeof(cParams));
631 cParams.frameInfo.blockMode = LZ4F_blockIndependent;
632 cParams.frameInfo.blockSizeID = LZ4F_max64KB;
633 CHECK_V(cSizeIndep,
634 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
635 CNBuffer, inSize,
636 cdict, &cParams) );
637 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
638 (unsigned)inSize, (unsigned)cSizeIndep);
639
640 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple independent blocks : ");
641 { LZ4F_dctx* dctx;
642 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
643 size_t compressedSize = cSizeIndep;
644 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
645 CHECK( LZ4F_decompress_usingDict(dctx,
646 decodedBuffer, &decodedSize,
647 compressedBuffer, &compressedSize,
648 CNBuffer, dictSize,
649 NULL) );
650 if (compressedSize != cSizeIndep) goto _output_error;
651 if (decodedSize != inSize) goto _output_error;
652 crcOrig = XXH64(CNBuffer, inSize, 0);
653 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
654 if (crcDest != crcOrig) goto _output_error; }
655 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
656 CHECK( LZ4F_freeDecompressionContext(dctx) );
657 }
658 }
659
660 LZ4F_freeCDict(cdict);
661 CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
662 }
663
664 DISPLAYLEVEL(3, "getBlockSize test: \n");
665 { size_t result;
666 unsigned blockSizeID;
667 for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) {
668 result = LZ4F_getBlockSize(blockSizeID);
669 CHECK(result);
670 DISPLAYLEVEL(3, "Returned block size of %u bytes for blockID %u \n",
671 (unsigned)result, blockSizeID);
672 }
673
674 /* Test an invalid input that's too large */
675 result = LZ4F_getBlockSize(8);
676 if(!LZ4F_isError(result) ||
677 LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid)
678 goto _output_error;
679
680 /* Test an invalid input that's too small */
681 result = LZ4F_getBlockSize(3);
682 if(!LZ4F_isError(result) ||
683 LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid)
684 goto _output_error;
685 }
686
687
688 DISPLAYLEVEL(3, "Skippable frame test : \n");
689 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
690 unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
691 BYTE* op = (BYTE*)decodedBuffer;
692 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
693 BYTE* ip = (BYTE*)compressedBuffer;
694 BYTE* iend = (BYTE*)compressedBuffer + cSize + 8;
695
696 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
697
698 /* generate skippable frame */
699 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START);
700 FUZ_writeLE32(ip+4, (U32)cSize);
701
702 DISPLAYLEVEL(3, "random segment sizes : \n");
703 while (ip < iend) {
704 unsigned nbBits = FUZ_rand(&randState) % maxBits;
705 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
706 size_t oSize = (size_t)(oend-op);
707 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
708 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
709 op += oSize;
710 ip += iSize;
711 }
712 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize);
713
714 /* generate zero-size skippable frame */
715 DISPLAYLEVEL(3, "zero-size skippable frame\n");
716 ip = (BYTE*)compressedBuffer;
717 op = (BYTE*)decodedBuffer;
718 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1);
719 FUZ_writeLE32(ip+4, 0);
720 iend = ip+8;
721
722 while (ip < iend) {
723 unsigned const nbBits = FUZ_rand(&randState) % maxBits;
724 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
725 size_t oSize = (size_t)(oend-op);
726 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
727 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
728 op += oSize;
729 ip += iSize;
730 }
731 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
732
733 DISPLAYLEVEL(3, "Skippable frame header complete in first call \n");
734 ip = (BYTE*)compressedBuffer;
735 op = (BYTE*)decodedBuffer;
736 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2);
737 FUZ_writeLE32(ip+4, 10);
738 iend = ip+18;
739 while (ip < iend) {
740 size_t iSize = 10;
741 size_t oSize = 10;
742 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
743 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
744 op += oSize;
745 ip += iSize;
746 }
747 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
748 }
749
750 DISPLAY("Basic tests completed \n");
751 _end:
752 free(CNBuffer);
753 free(compressedBuffer);
754 free(decodedBuffer);
755 LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
756 LZ4F_freeCompressionContext(cctx); cctx = NULL;
757 return basicTests_error;
758
759 _output_error:
760 basicTests_error = 1;
761 DISPLAY("Error detected ! \n");
762 goto _end;
763 }
764
765
766 typedef enum { o_contiguous, o_noncontiguous, o_overwrite } o_scenario_e;
767
locateBuffDiff(const void * buff1,const void * buff2,size_t size,o_scenario_e o_scenario)768 static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_scenario_e o_scenario)
769 {
770 if (displayLevel >= 5) {
771 size_t p=0;
772 const BYTE* b1=(const BYTE*)buff1;
773 const BYTE* b2=(const BYTE*)buff2;
774 DISPLAY("locateBuffDiff: looking for error position \n");
775 if (o_scenario != o_contiguous) {
776 DISPLAY("mode %i: non-contiguous output (%u bytes), cannot search \n",
777 (int)o_scenario, (unsigned)size);
778 return;
779 }
780 while (p < size && b1[p]==b2[p]) p++;
781 if (p != size) {
782 DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]);
783 }
784 }
785 }
786
787 # define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
788 DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); exit(1); }
789 # undef CHECK
790 # define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } }
791
792
test_lz4f_decompression_wBuffers(const void * cSrc,size_t cSize,void * dst,size_t dstCapacity,o_scenario_e o_scenario,const void * srcRef,size_t decompressedSize,U64 crcOrig,U32 * const randState,LZ4F_dctx * const dCtx,U32 seed,U32 testNb)793 size_t test_lz4f_decompression_wBuffers(
794 const void* cSrc, size_t cSize,
795 void* dst, size_t dstCapacity, o_scenario_e o_scenario,
796 const void* srcRef, size_t decompressedSize,
797 U64 crcOrig,
798 U32* const randState,
799 LZ4F_dctx* const dCtx,
800 U32 seed, U32 testNb)
801 {
802 const BYTE* ip = (const BYTE*)cSrc;
803 const BYTE* const iend = ip + cSize;
804
805 BYTE* op = (BYTE*)dst;
806 BYTE* const oend = op + dstCapacity;
807
808 unsigned const suggestedBits = FUZ_highbit((U32)cSize);
809 unsigned const maxBits = MAX(3, suggestedBits);
810 size_t totalOut = 0;
811 size_t moreToFlush = 0;
812 XXH64_state_t xxh64;
813 XXH64_reset(&xxh64, 1);
814 assert(ip < iend);
815 while (ip < iend) {
816 unsigned const nbBitsI = (FUZ_rand(randState) % (maxBits-1)) + 1;
817 unsigned const nbBitsO = (FUZ_rand(randState) % (maxBits)) + 1;
818 size_t const iSizeCand = (FUZ_rand(randState) & ((1<<nbBitsI)-1)) + 1;
819 size_t const iSizeMax = MIN(iSizeCand, (size_t)(iend-ip));
820 size_t iSize = iSizeMax;
821 size_t const oSizeCand = (FUZ_rand(randState) & ((1<<nbBitsO)-1)) + 2;
822 size_t const oSizeMax = MIN(oSizeCand, (size_t)(oend-op));
823 size_t oSize = oSizeMax;
824 BYTE const mark = (BYTE)(FUZ_rand(randState) & 255);
825 LZ4F_decompressOptions_t dOptions;
826 memset(&dOptions, 0, sizeof(dOptions));
827 dOptions.stableDst = FUZ_rand(randState) & 1;
828 if (o_scenario == o_overwrite) dOptions.stableDst = 0; /* overwrite mode */
829 if (op + oSizeMax < oend) op[oSizeMax] = mark;
830
831 DISPLAYLEVEL(7, "dstCapacity=%u, presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize);
832
833 /* read data from byte-exact buffer to catch out-of-bound reads */
834 { void* const iBuffer = malloc(iSizeMax);
835 assert(iBuffer != NULL);
836 memcpy(iBuffer, ip, iSizeMax);
837 moreToFlush = LZ4F_decompress(dCtx, op, &oSize, iBuffer, &iSize, &dOptions);
838 free(iBuffer);
839 }
840 DISPLAYLEVEL(7, "oSize=%u, readSize=%u \n", (unsigned)oSize, (unsigned)iSize);
841
842 if (op + oSizeMax < oend) {
843 CHECK(op[oSizeMax] != mark, "op[oSizeMax] = %02X != %02X : "
844 "Decompression overwrites beyond assigned dst size",
845 op[oSizeMax], mark);
846 }
847 if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid)
848 locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
849 if (LZ4F_isError(moreToFlush)) return moreToFlush;
850
851 XXH64_update(&xxh64, op, oSize);
852 totalOut += oSize;
853 op += oSize;
854 ip += iSize;
855 if (o_scenario == o_noncontiguous) {
856 if (op == oend) return LZ4F_ERROR_GENERIC; /* can theoretically happen with bogus data */
857 op++; /* create a gap between consecutive output */
858 }
859 if (o_scenario==o_overwrite) op = (BYTE*)dst; /* overwrite destination */
860 if ( (op == oend) /* no more room for output; can happen with bogus input */
861 && (iSize == 0)) /* no input consumed */
862 break;
863 }
864 if (moreToFlush != 0) return LZ4F_ERROR_decompressionFailed;
865 if (totalOut) { /* otherwise, it's a skippable frame */
866 U64 const crcDecoded = XXH64_digest(&xxh64);
867 if (crcDecoded != crcOrig) {
868 locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
869 return LZ4F_ERROR_contentChecksum_invalid;
870 } }
871 return 0;
872 }
873
874
test_lz4f_decompression(const void * cSrc,size_t cSize,const void * srcRef,size_t decompressedSize,U64 crcOrig,U32 * const randState,LZ4F_dctx * const dCtx,U32 seed,U32 testNb)875 size_t test_lz4f_decompression(const void* cSrc, size_t cSize,
876 const void* srcRef, size_t decompressedSize,
877 U64 crcOrig,
878 U32* const randState,
879 LZ4F_dctx* const dCtx,
880 U32 seed, U32 testNb)
881 {
882 o_scenario_e const o_scenario = (o_scenario_e)(FUZ_rand(randState) % 3); /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */
883 /* tighten dst buffer conditions */
884 size_t const dstCapacity = (o_scenario == o_noncontiguous) ?
885 (decompressedSize * 2) + 128 :
886 decompressedSize;
887 size_t result;
888 void* const dstBuffer = malloc(dstCapacity);
889 assert(dstBuffer != NULL);
890
891 result = test_lz4f_decompression_wBuffers(cSrc, cSize,
892 dstBuffer, dstCapacity, o_scenario,
893 srcRef, decompressedSize,
894 crcOrig,
895 randState,
896 dCtx,
897 seed, testNb);
898
899 free(dstBuffer);
900 return result;
901 }
902
903
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,double compressibility,U32 duration_s)904 int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s)
905 {
906 unsigned testNb = 0;
907 size_t const CNBufferLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
908 void* CNBuffer = NULL;
909 size_t const compressedBufferSize = LZ4F_compressFrameBound(CNBufferLength, NULL) + 4 MB; /* needs some margin */
910 void* compressedBuffer = NULL;
911 void* decodedBuffer = NULL;
912 U32 coreRand = seed;
913 LZ4F_decompressionContext_t dCtx = NULL;
914 LZ4F_decompressionContext_t dCtxNoise = NULL;
915 LZ4F_compressionContext_t cCtx = NULL;
916 clock_t const startClock = clock();
917 clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
918
919 /* Create buffers */
920 { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
921 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
922 { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION);
923 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
924 { size_t const creationStatus = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
925 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
926 CNBuffer = malloc(CNBufferLength);
927 CHECK(CNBuffer==NULL, "CNBuffer Allocation failed");
928 compressedBuffer = malloc(compressedBufferSize);
929 CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
930 decodedBuffer = calloc(1, CNBufferLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
931 CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
932 FUZ_fillCompressibleNoiseBuffer(CNBuffer, CNBufferLength, compressibility, &coreRand);
933
934 /* jump to requested testNb */
935 for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */
936
937 /* main fuzzer test loop */
938 for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) {
939 U32 randState = coreRand ^ prime1;
940 unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(CNBufferLength-1)) - 1)) + 1;
941 size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1;
942 size_t const srcStartId = FUZ_rand(&randState) % (CNBufferLength - srcSize);
943 const BYTE* const srcStart = (const BYTE*)CNBuffer + srcStartId;
944 unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1;
945 U64 const crcOrig = XXH64(srcStart, srcSize, 1);
946 LZ4F_preferences_t prefs;
947 const LZ4F_preferences_t* prefsPtr = &prefs;
948 size_t cSize;
949
950 (void)FUZ_rand(&coreRand); /* update seed */
951 memset(&prefs, 0, sizeof(prefs));
952 prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1);
953 prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3));
954 prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)(FUZ_rand(&randState) & 1);
955 prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1);
956 prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
957 prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2;
958 prefs.compressionLevel = -5 + (int)(FUZ_rand(&randState) % 11);
959 if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
960
961 DISPLAYUPDATE(2, "\r%5u ", testNb);
962
963 if ((FUZ_rand(&randState) & 0xFFF) == 0) {
964 /* create a skippable frame (rare case) */
965 BYTE* op = (BYTE*)compressedBuffer;
966 FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15));
967 FUZ_writeLE32(op+4, (U32)srcSize);
968 cSize = srcSize+8;
969
970 } else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */
971 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr);
972 CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
973
974 } else { /* multi-segments compression */
975 const BYTE* ip = srcStart;
976 const BYTE* const iend = srcStart + srcSize;
977 BYTE* op = (BYTE*)compressedBuffer;
978 BYTE* const oend = op + (neverFlush ? LZ4F_compressFrameBound(srcSize, prefsPtr) : compressedBufferSize); /* when flushes are possible, can't guarantee a max compressed size */
979 unsigned const maxBits = FUZ_highbit((U32)srcSize);
980 size_t cSegmentSize;
981 LZ4F_compressOptions_t cOptions;
982 memset(&cOptions, 0, sizeof(cOptions));
983 cSegmentSize = LZ4F_compressBegin(cCtx, op, (size_t)(oend-op), prefsPtr);
984 CHECK(LZ4F_isError(cSegmentSize), "Compression header failed (error %i)",
985 (int)cSegmentSize);
986 op += cSegmentSize;
987 while (ip < iend) {
988 unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits;
989 size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
990 size_t const iSize = MIN(sampleMax, (size_t)(iend-ip));
991 size_t const oSize = LZ4F_compressBound(iSize, prefsPtr);
992 size_t flushedSize;
993 cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
994 DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n",
995 (unsigned)iSize, cOptions.stableSrc);
996
997 flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
998 CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)",
999 (int)flushedSize, LZ4F_getErrorName(flushedSize));
1000 op += flushedSize;
1001 ip += iSize;
1002
1003 { unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1);
1004 if (forceFlush) {
1005 size_t const flushSize = LZ4F_flush(cCtx, op, (size_t)(oend-op), &cOptions);
1006 DISPLAYLEVEL(6,"flushing %u bytes \n", (unsigned)flushSize);
1007 CHECK(LZ4F_isError(flushSize), "Compression failed (error %i)", (int)flushSize);
1008 op += flushSize;
1009 } }
1010 }
1011 CHECK(op>=oend, "LZ4F_compressFrameBound overflow");
1012 { size_t const dstEndSafeSize = LZ4F_compressBound(0, prefsPtr);
1013 int const tooSmallDstEnd = ((FUZ_rand(&randState) & 31) == 3);
1014 size_t const dstEndTooSmallSize = (FUZ_rand(&randState) % dstEndSafeSize) + 1;
1015 size_t const dstEndSize = tooSmallDstEnd ? dstEndTooSmallSize : dstEndSafeSize;
1016 BYTE const canaryByte = (BYTE)(FUZ_rand(&randState) & 255);
1017 size_t flushedSize;
1018 DISPLAYLEVEL(7,"canaryByte at pos %u / %u \n",
1019 (unsigned)((size_t)(op - (BYTE*)compressedBuffer) + dstEndSize),
1020 (unsigned)compressedBufferSize);
1021 assert(op + dstEndSize < (BYTE*)compressedBuffer + compressedBufferSize);
1022 op[dstEndSize] = canaryByte;
1023 flushedSize = LZ4F_compressEnd(cCtx, op, dstEndSize, &cOptions);
1024 CHECK(op[dstEndSize] != canaryByte, "LZ4F_compressEnd writes beyond dstCapacity !");
1025 if (LZ4F_isError(flushedSize)) {
1026 if (tooSmallDstEnd) /* failure is allowed */ continue;
1027 CHECK(1, "Compression completion failed (error %i : %s)",
1028 (int)flushedSize, LZ4F_getErrorName(flushedSize));
1029 }
1030 op += flushedSize;
1031 }
1032 cSize = (size_t)(op - (BYTE*)compressedBuffer);
1033 DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize);
1034 }
1035
1036
1037 /* multi-segments decompression */
1038 DISPLAYLEVEL(6, "normal decompression \n");
1039 { size_t result = test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtx, seed, testNb);
1040 CHECK (LZ4F_isError(result), "multi-segment decompression failed (error %i => %s)",
1041 (int)result, LZ4F_getErrorName(result));
1042 }
1043
1044 #if 1
1045 /* insert noise into src */
1046 { U32 const maxNbBits = FUZ_highbit((U32)cSize);
1047 size_t pos = 0;
1048 for (;;) {
1049 /* keep some original src */
1050 { U32 const nbBits = FUZ_rand(&randState) % maxNbBits;
1051 size_t const mask = (1<<nbBits) - 1;
1052 size_t const skipLength = FUZ_rand(&randState) & mask;
1053 pos += skipLength;
1054 }
1055 if (pos >= cSize) break;
1056 /* add noise */
1057 { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits;
1058 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
1059 size_t const mask = (1<<nbBits) - 1;
1060 size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1;
1061 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
1062 size_t const noiseStart = FUZ_rand(&randState) % (CNBufferLength - noiseLength);
1063 memcpy((BYTE*)compressedBuffer + pos, (const char*)CNBuffer + noiseStart, noiseLength);
1064 pos += noiseLength;
1065 } } }
1066
1067 /* test decompression on noisy src */
1068 DISPLAYLEVEL(6, "noisy decompression \n");
1069 test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtxNoise, seed, testNb);
1070 /* note : we don't analyze result here : it probably failed, which is expected.
1071 * We just check for potential out-of-bound reads and writes. */
1072 LZ4F_resetDecompressionContext(dCtxNoise); /* context must be reset after an error */
1073 #endif
1074
1075 } /* for ( ; (testNb < nbTests) ; ) */
1076
1077 DISPLAYLEVEL(2, "\rAll tests completed \n");
1078
1079 LZ4F_freeDecompressionContext(dCtx);
1080 LZ4F_freeDecompressionContext(dCtxNoise);
1081 LZ4F_freeCompressionContext(cCtx);
1082 free(CNBuffer);
1083 free(compressedBuffer);
1084 free(decodedBuffer);
1085
1086 if (use_pause) {
1087 DISPLAY("press enter to finish \n");
1088 (void)getchar();
1089 }
1090 return 0;
1091 }
1092
1093
FUZ_usage(const char * programName)1094 int FUZ_usage(const char* programName)
1095 {
1096 DISPLAY( "Usage :\n");
1097 DISPLAY( " %s [args]\n", programName);
1098 DISPLAY( "\n");
1099 DISPLAY( "Arguments :\n");
1100 DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
1101 DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
1102 DISPLAY( " -s# : Select seed (default:prompt user)\n");
1103 DISPLAY( " -t# : Select starting test number (default:0)\n");
1104 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
1105 DISPLAY( " -v : verbose\n");
1106 DISPLAY( " -h : display help and exit\n");
1107 return 0;
1108 }
1109
1110
main(int argc,const char ** argv)1111 int main(int argc, const char** argv)
1112 {
1113 U32 seed=0;
1114 int seedset=0;
1115 int argNb;
1116 unsigned nbTests = nbTestsDefault;
1117 unsigned testNb = 0;
1118 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
1119 int result=0;
1120 U32 duration=0;
1121 const char* const programName = argv[0];
1122
1123 /* Check command line */
1124 for (argNb=1; argNb<argc; argNb++) {
1125 const char* argument = argv[argNb];
1126
1127 if(!argument) continue; /* Protection if argument empty */
1128
1129 /* Decode command (note : aggregated short commands are allowed) */
1130 if (argument[0]=='-') {
1131 if (!strcmp(argument, "--no-prompt")) {
1132 no_prompt=1;
1133 seedset=1;
1134 displayLevel=1;
1135 continue;
1136 }
1137 argument++;
1138
1139 while (*argument!=0) {
1140 switch(*argument)
1141 {
1142 case 'h':
1143 return FUZ_usage(programName);
1144 case 'v':
1145 argument++;
1146 displayLevel++;
1147 break;
1148 case 'q':
1149 argument++;
1150 displayLevel--;
1151 break;
1152 case 'p': /* pause at the end */
1153 argument++;
1154 use_pause = 1;
1155 break;
1156
1157 case 'i':
1158 argument++;
1159 nbTests=0; duration=0;
1160 while ((*argument>='0') && (*argument<='9')) {
1161 nbTests *= 10;
1162 nbTests += (unsigned)(*argument - '0');
1163 argument++;
1164 }
1165 break;
1166
1167 case 'T':
1168 argument++;
1169 nbTests = 0; duration = 0;
1170 for (;;) {
1171 switch(*argument)
1172 {
1173 case 'm': duration *= 60; argument++; continue;
1174 case 's':
1175 case 'n': argument++; continue;
1176 case '0':
1177 case '1':
1178 case '2':
1179 case '3':
1180 case '4':
1181 case '5':
1182 case '6':
1183 case '7':
1184 case '8':
1185 case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue;
1186 }
1187 break;
1188 }
1189 break;
1190
1191 case 's':
1192 argument++;
1193 seed=0;
1194 seedset=1;
1195 while ((*argument>='0') && (*argument<='9')) {
1196 seed *= 10;
1197 seed += (U32)(*argument - '0');
1198 argument++;
1199 }
1200 break;
1201 case 't':
1202 argument++;
1203 testNb=0;
1204 while ((*argument>='0') && (*argument<='9')) {
1205 testNb *= 10;
1206 testNb += (unsigned)(*argument - '0');
1207 argument++;
1208 }
1209 break;
1210 case 'P': /* compressibility % */
1211 argument++;
1212 proba=0;
1213 while ((*argument>='0') && (*argument<='9')) {
1214 proba *= 10;
1215 proba += *argument - '0';
1216 argument++;
1217 }
1218 if (proba<0) proba=0;
1219 if (proba>100) proba=100;
1220 break;
1221 default:
1222 ;
1223 return FUZ_usage(programName);
1224 }
1225 }
1226 }
1227 }
1228
1229 /* Get Seed */
1230 DISPLAY("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING);
1231
1232 if (!seedset) {
1233 time_t const t = time(NULL);
1234 U32 const h = XXH32(&t, sizeof(t), 1);
1235 seed = h % 10000;
1236 }
1237 DISPLAY("Seed = %u\n", seed);
1238 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
1239
1240 nbTests += (nbTests==0); /* avoid zero */
1241
1242 if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
1243 if (result) return 1;
1244 return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration);
1245 }
1246