1 /*
2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12 /*-************************************
13 * Compiler specific
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
19 #endif
20
21
22 /*-************************************
23 * Includes
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #include <assert.h> /* assert */
29 #include "timefn.h" /* UTIL_time_t, UTIL_getTime */
30 #include "mem.h"
31 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
32 #include "zstd.h" /* ZSTD_compressBound */
33 #include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */
34 #include "zstdmt_compress.h"
35 #include "zdict.h" /* ZDICT_trainFromBuffer */
36 #include "datagen.h" /* RDG_genBuffer */
37 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
38 #include "xxhash.h" /* XXH64_* */
39 #include "seqgen.h"
40 #include "util.h"
41 #include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
42
43
44 /*-************************************
45 * Constants
46 **************************************/
47 #define KB *(1U<<10)
48 #define MB *(1U<<20)
49 #define GB *(1U<<30)
50
51 static const int nbTestsDefault = 10000;
52 static const U32 g_cLevelMax_smallTests = 10;
53 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
54 #define FUZ_COMPRESSIBILITY_DEFAULT 50
55 static const U32 prime32 = 2654435761U;
56
57
58 /*-************************************
59 * Display Macros
60 **************************************/
61 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
62 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \
63 DISPLAY(__VA_ARGS__); \
64 if (g_displayLevel>=4) fflush(stderr); }
65 static U32 g_displayLevel = 2;
66
67 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
68 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
69
70 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
71 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
72 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
73 if (g_displayLevel>=4) fflush(stderr); } }
74
75 static U64 g_clockTime = 0;
76
77
78 /*-*******************************************************
79 * Check macros
80 *********************************************************/
81 #undef MIN
82 #undef MAX
83 #define MIN(a,b) ((a)<(b)?(a):(b))
84 #define MAX(a,b) ((a)>(b)?(a):(b))
85 /*! FUZ_rand() :
86 @return : a 27 bits random value, from a 32-bits `seed`.
87 `seed` is also modified */
88 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(U32 * seedPtr)89 static U32 FUZ_rand(U32* seedPtr)
90 {
91 static const U32 prime2 = 2246822519U;
92 U32 rand32 = *seedPtr;
93 rand32 *= prime32;
94 rand32 += prime2;
95 rand32 = FUZ_rotl32(rand32, 13);
96 *seedPtr = rand32;
97 return rand32 >> 5;
98 }
99
100 #define CHECK(cond, ...) { \
101 if (cond) { \
102 DISPLAY("Error => "); \
103 DISPLAY(__VA_ARGS__); \
104 DISPLAY(" (seed %u, test nb %u, line %u) \n", \
105 (unsigned)seed, testNb, __LINE__); \
106 goto _output_error; \
107 } }
108
109 #define CHECK_Z(f) { \
110 size_t const err = f; \
111 CHECK(ZSTD_isError(err), "%s : %s ", \
112 #f, ZSTD_getErrorName(err)); \
113 }
114
115 #define CHECK_RET(ret, cond, ...) { \
116 if (cond) { \
117 DISPLAY("Error %llu => ", (unsigned long long)ret); \
118 DISPLAY(__VA_ARGS__); \
119 DISPLAY(" (line %u)\n", __LINE__); \
120 return ret; \
121 } }
122
123 #define CHECK_RET_Z(f) { \
124 size_t const err = f; \
125 CHECK_RET(err, ZSTD_isError(err), "%s : %s ", \
126 #f, ZSTD_getErrorName(err)); \
127 }
128
129
130 /*======================================================
131 * Basic Unit tests
132 *======================================================*/
133
134 typedef struct {
135 void* start;
136 size_t size;
137 size_t filled;
138 } buffer_t;
139
140 static const buffer_t kBuffNull = { NULL, 0 , 0 };
141
FUZ_freeDictionary(buffer_t dict)142 static void FUZ_freeDictionary(buffer_t dict)
143 {
144 free(dict.start);
145 }
146
FUZ_createDictionary(const void * src,size_t srcSize,size_t blockSize,size_t requestedDictSize)147 static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
148 {
149 buffer_t dict = kBuffNull;
150 size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
151 size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t));
152 if (!blockSizes) return kBuffNull;
153 dict.start = malloc(requestedDictSize);
154 if (!dict.start) { free(blockSizes); return kBuffNull; }
155 { size_t nb;
156 for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
157 blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
158 }
159 { size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
160 free(blockSizes);
161 if (ZDICT_isError(dictSize)) { FUZ_freeDictionary(dict); return kBuffNull; }
162 dict.size = requestedDictSize;
163 dict.filled = dictSize;
164 return dict;
165 }
166 }
167
168 /* Round trips data and updates xxh with the decompressed data produced */
SEQ_roundTrip(ZSTD_CCtx * cctx,ZSTD_DCtx * dctx,XXH64_state_t * xxh,void * data,size_t size,ZSTD_EndDirective endOp)169 static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
170 XXH64_state_t* xxh, void* data, size_t size,
171 ZSTD_EndDirective endOp)
172 {
173 static BYTE compressed[1024];
174 static BYTE uncompressed[1024];
175
176 ZSTD_inBuffer cin = {data, size, 0};
177 size_t cret;
178
179 do {
180 ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
181 ZSTD_inBuffer din = { compressed, 0, 0 };
182 ZSTD_outBuffer dout = { uncompressed, 0, 0 };
183
184 cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
185 if (ZSTD_isError(cret))
186 return cret;
187
188 din.size = cout.pos;
189 while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
190 size_t dret;
191
192 dout.pos = 0;
193 dout.size = sizeof(uncompressed);
194 dret = ZSTD_decompressStream(dctx, &dout, &din);
195 if (ZSTD_isError(dret))
196 return dret;
197 XXH64_update(xxh, dout.dst, dout.pos);
198 if (dret == 0)
199 break;
200 }
201 } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
202 return 0;
203 }
204
205 /* Generates some data and round trips it */
SEQ_generateRoundTrip(ZSTD_CCtx * cctx,ZSTD_DCtx * dctx,XXH64_state_t * xxh,SEQ_stream * seq,SEQ_gen_type type,unsigned value)206 static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
207 XXH64_state_t* xxh, SEQ_stream* seq,
208 SEQ_gen_type type, unsigned value)
209 {
210 static BYTE data[1024];
211 size_t gen;
212
213 do {
214 SEQ_outBuffer sout = {data, sizeof(data), 0};
215 size_t ret;
216 gen = SEQ_gen(seq, type, value, &sout);
217
218 ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
219 if (ZSTD_isError(ret))
220 return ret;
221 } while (gen != 0);
222
223 return 0;
224 }
225
getCCtxParams(ZSTD_CCtx * zc,ZSTD_parameters * savedParams)226 static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
227 {
228 int value;
229 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog));
230 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog));
231 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog));
232 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
233 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
234 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
235 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value));
236 savedParams->cParams.strategy = value;
237
238 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
239 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
240 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value));
241 savedParams->fParams.noDictIDFlag = !value;
242 return 0;
243 }
244
badParameters(ZSTD_CCtx * zc,ZSTD_parameters const savedParams)245 static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
246 {
247 ZSTD_parameters params;
248 if (ZSTD_isError(getCCtxParams(zc, ¶ms))) return 10;
249 CHECK_RET(1, params.cParams.windowLog != savedParams.cParams.windowLog, "windowLog");
250 CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog");
251 CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog");
252 CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog");
253 CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch");
254 CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength");
255
256 CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag");
257 CHECK_RET(8, params.fParams.contentSizeFlag != savedParams.fParams.contentSizeFlag, "contentSizeFlag");
258 CHECK_RET(9, params.fParams.noDictIDFlag != savedParams.fParams.noDictIDFlag, "noDictIDFlag");
259 return 0;
260 }
261
basicUnitTests(U32 seed,double compressibility)262 static int basicUnitTests(U32 seed, double compressibility)
263 {
264 size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
265 void* CNBuffer = malloc(CNBufferSize);
266 size_t const skippableFrameSize = 200 KB;
267 size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
268 void* compressedBuffer = malloc(compressedBufferSize);
269 size_t const decodedBufferSize = CNBufferSize;
270 void* decodedBuffer = malloc(decodedBufferSize);
271 size_t cSize;
272 int testResult = 0;
273 int testNb = 1;
274 U32 coreSeed = 0; /* this name to conform with CHECK_Z macro display */
275 ZSTD_CStream* zc = ZSTD_createCStream();
276 ZSTD_DStream* zd = ZSTD_createDStream();
277 ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2);
278
279 ZSTD_inBuffer inBuff, inBuff2;
280 ZSTD_outBuffer outBuff;
281 buffer_t dictionary = kBuffNull;
282 size_t const dictSize = 128 KB;
283 unsigned dictID = 0;
284
285 /* Create compressible test buffer */
286 if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
287 DISPLAY("Not enough memory, aborting \n");
288 goto _output_error;
289 }
290 RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
291
292 /* Create dictionary */
293 DISPLAYLEVEL(3, "creating dictionary for unit tests \n");
294 dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB);
295 if (!dictionary.start) {
296 DISPLAY("Error creating dictionary, aborting \n");
297 goto _output_error;
298 }
299 dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
300
301 /* Basic compression test */
302 DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
303 CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
304 outBuff.dst = (char*)(compressedBuffer);
305 outBuff.size = compressedBufferSize;
306 outBuff.pos = 0;
307 inBuff.src = CNBuffer;
308 inBuff.size = CNBufferSize;
309 inBuff.pos = 0;
310 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
311 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
312 { size_t const r = ZSTD_endStream(zc, &outBuff);
313 if (r != 0) goto _output_error; } /* error, or some data not flushed */
314 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)outBuff.pos);
315
316 /* generate skippable frame */
317 MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
318 MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
319 cSize = skippableFrameSize + 8;
320
321 /* Basic compression test using dict */
322 DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
323 CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
324 outBuff.dst = (char*)(compressedBuffer)+cSize;
325 assert(compressedBufferSize > cSize);
326 outBuff.size = compressedBufferSize - cSize;
327 outBuff.pos = 0;
328 inBuff.src = CNBuffer;
329 inBuff.size = CNBufferSize;
330 inBuff.pos = 0;
331 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
332 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
333 { size_t const r = ZSTD_endStream(zc, &outBuff);
334 if (r != 0) goto _output_error; } /* error, or some data not flushed */
335 cSize += outBuff.pos;
336 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
337 (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
338
339 /* context size functions */
340 DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
341 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
342 size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams);
343 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */
344 if (ZSTD_isError(cstreamSize)) goto _output_error;
345 if (ZSTD_isError(cdictSize)) goto _output_error;
346 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize));
347 }
348
349 /* context size functions */
350 DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++);
351 { ZSTD_CCtx_params* const params = ZSTD_createCCtxParams();
352 size_t cstreamSize, cctxSize;
353 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) );
354 cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
355 CHECK_Z(cstreamSize);
356 cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
357 CHECK_Z(cctxSize);
358 if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error;
359 ZSTD_freeCCtxParams(params);
360 DISPLAYLEVEL(3, "OK \n");
361 }
362
363 DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
364 { size_t const s = ZSTD_sizeof_CStream(zc);
365 if (ZSTD_isError(s)) goto _output_error;
366 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
367 }
368
369 /* Attempt bad compression parameters */
370 DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++);
371 { size_t r;
372 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
373 params.cParams.minMatch = 2;
374 r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
375 if (!ZSTD_isError(r)) goto _output_error;
376 DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
377 }
378
379 /* skippable frame test */
380 DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
381 CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) );
382 inBuff.src = compressedBuffer;
383 inBuff.size = cSize;
384 inBuff.pos = 0;
385 outBuff.dst = decodedBuffer;
386 outBuff.size = CNBufferSize;
387 outBuff.pos = 0;
388 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
389 DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (unsigned)r);
390 if (r != 0) goto _output_error;
391 }
392 if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */
393 DISPLAYLEVEL(3, "OK \n");
394
395 /* Basic decompression test */
396 inBuff2 = inBuff;
397 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
398 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
399 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */
400 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
401 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
402 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
403 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
404 DISPLAYLEVEL(3, "OK \n");
405
406 /* Re-use without init */
407 DISPLAYLEVEL(3, "test%3i : decompress again without init (re-use previous settings): ", testNb++);
408 outBuff.pos = 0;
409 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2);
410 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
411 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
412 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
413 DISPLAYLEVEL(3, "OK \n");
414
415 /* check regenerated data is byte exact */
416 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
417 { size_t i;
418 for (i=0; i<CNBufferSize; i++) {
419 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
420 } }
421 DISPLAYLEVEL(3, "OK \n");
422
423 /* context size functions */
424 DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
425 { ZSTD_frameHeader fhi;
426 const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
427 size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
428 if (gfhError!=0) goto _output_error;
429 DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize);
430 { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
431 /* uses ZSTD_initDStream_usingDict() */
432 + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
433 if (ZSTD_isError(s)) goto _output_error;
434 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
435 } }
436
437 DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
438 { size_t const s = ZSTD_sizeof_DStream(zd);
439 if (ZSTD_isError(s)) goto _output_error;
440 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
441 }
442
443 /* Decompression by small increment */
444 DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
445 { /* skippable frame */
446 size_t r = 1;
447 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
448 inBuff.src = compressedBuffer;
449 outBuff.dst = decodedBuffer;
450 inBuff.pos = 0;
451 outBuff.pos = 0;
452 while (r) { /* skippable frame */
453 size_t const inSize = (FUZ_rand(&coreSeed) & 15) + 1;
454 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + 1;
455 inBuff.size = inBuff.pos + inSize;
456 outBuff.size = outBuff.pos + outSize;
457 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
458 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream on skippable frame error : %s \n", ZSTD_getErrorName(r));
459 if (ZSTD_isError(r)) goto _output_error;
460 }
461 /* normal frame */
462 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
463 r=1;
464 while (r) {
465 size_t const inSize = FUZ_rand(&coreSeed) & 15;
466 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + (!inSize); /* avoid having both sizes at 0 => would trigger a no_forward_progress error */
467 inBuff.size = inBuff.pos + inSize;
468 outBuff.size = outBuff.pos + outSize;
469 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
470 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(r));
471 if (ZSTD_isError(r)) goto _output_error;
472 }
473 }
474 if (outBuff.pos != CNBufferSize) DISPLAYLEVEL(4, "outBuff.pos != CNBufferSize : should have regenerated same amount ! \n");
475 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
476 if (inBuff.pos != cSize) DISPLAYLEVEL(4, "inBuff.pos != cSize : should have real all input ! \n");
477 if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
478 DISPLAYLEVEL(3, "OK \n");
479
480 /* check regenerated data is byte exact */
481 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
482 { size_t i;
483 for (i=0; i<CNBufferSize; i++) {
484 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
485 } }
486 DISPLAYLEVEL(3, "OK \n");
487
488 /* Decompression forward progress */
489 DISPLAYLEVEL(3, "test%3i : generate error when ZSTD_decompressStream() doesn't progress : ", testNb++);
490 { /* skippable frame */
491 size_t r = 0;
492 int decNb = 0;
493 int const maxDec = 100;
494 inBuff.src = compressedBuffer;
495 inBuff.size = cSize;
496 inBuff.pos = 0;
497
498 outBuff.dst = decodedBuffer;
499 outBuff.pos = 0;
500 outBuff.size = CNBufferSize-1; /* 1 byte missing */
501
502 for (decNb=0; decNb<maxDec; decNb++) {
503 if (r==0) ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
504 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
505 if (ZSTD_isError(r)) break;
506 }
507 if (!ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream should have triggered a no_forward_progress error \n");
508 if (!ZSTD_isError(r)) goto _output_error; /* should have triggered no_forward_progress error */
509 }
510 DISPLAYLEVEL(3, "OK \n");
511
512 /* _srcSize compression test */
513 DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
514 CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) );
515 outBuff.dst = (char*)(compressedBuffer);
516 outBuff.size = compressedBufferSize;
517 outBuff.pos = 0;
518 inBuff.src = CNBuffer;
519 inBuff.size = CNBufferSize;
520 inBuff.pos = 0;
521 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
522 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
523 { size_t const r = ZSTD_endStream(zc, &outBuff);
524 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
525 }
526 { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
527 CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
528 CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
529 }
530 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
531
532 /* wrong _srcSize compression test */
533 DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
534 ZSTD_initCStream_srcSize(zc, 1, CNBufferSize+1);
535 outBuff.dst = (char*)(compressedBuffer);
536 outBuff.size = compressedBufferSize;
537 outBuff.pos = 0;
538 inBuff.src = CNBuffer;
539 inBuff.size = CNBufferSize;
540 inBuff.pos = 0;
541 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
542 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
543 { size_t const r = ZSTD_endStream(zc, &outBuff);
544 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
545 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
546
547 /* wrong _srcSize compression test */
548 DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
549 ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1);
550 outBuff.dst = (char*)(compressedBuffer);
551 outBuff.size = compressedBufferSize;
552 outBuff.pos = 0;
553 inBuff.src = CNBuffer;
554 inBuff.size = CNBufferSize;
555 inBuff.pos = 0;
556 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
557 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
558 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
559 }
560
561 DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
562 { ZSTD_parameters params = ZSTD_getParams(1, CNBufferSize, 0);
563 params.fParams.contentSizeFlag = 0;
564 CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, CNBufferSize - MIN(CNBufferSize, 200 KB)));
565 outBuff.dst = (char*)compressedBuffer;
566 outBuff.size = compressedBufferSize;
567 outBuff.pos = 0;
568 inBuff.src = CNBuffer;
569 inBuff.size = CNBufferSize;
570 inBuff.pos = 0;
571 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
572 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
573 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
574 } }
575
576 /* Complex context re-use scenario */
577 DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
578 ZSTD_freeCStream(zc);
579 zc = ZSTD_createCStream();
580 if (zc==NULL) goto _output_error; /* memory allocation issue */
581 /* use 1 */
582 { size_t const inSize = 513;
583 DISPLAYLEVEL(5, "use1 ");
584 ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */
585 inBuff.src = CNBuffer;
586 inBuff.size = inSize;
587 inBuff.pos = 0;
588 outBuff.dst = (char*)(compressedBuffer)+cSize;
589 outBuff.size = ZSTD_compressBound(inSize);
590 outBuff.pos = 0;
591 DISPLAYLEVEL(5, "compress1 ");
592 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
593 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
594 DISPLAYLEVEL(5, "end1 ");
595 { size_t const r = ZSTD_endStream(zc, &outBuff);
596 if (r != 0) goto _output_error; } /* error, or some data not flushed */
597 }
598 /* use 2 */
599 { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
600 DISPLAYLEVEL(5, "use2 ");
601 ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */
602 inBuff.src = CNBuffer;
603 inBuff.size = inSize;
604 inBuff.pos = 0;
605 outBuff.dst = (char*)(compressedBuffer)+cSize;
606 outBuff.size = ZSTD_compressBound(inSize);
607 outBuff.pos = 0;
608 DISPLAYLEVEL(5, "compress2 ");
609 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
610 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
611 DISPLAYLEVEL(5, "end2 ");
612 { size_t const r = ZSTD_endStream(zc, &outBuff);
613 if (r != 0) goto _output_error; } /* error, or some data not flushed */
614 }
615 DISPLAYLEVEL(3, "OK \n");
616
617 /* CDict scenario */
618 DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
619 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
620 size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
621 DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (unsigned)initError);
622 if (ZSTD_isError(initError)) goto _output_error;
623 outBuff.dst = compressedBuffer;
624 outBuff.size = compressedBufferSize;
625 outBuff.pos = 0;
626 inBuff.src = CNBuffer;
627 inBuff.size = CNBufferSize;
628 inBuff.pos = 0;
629 DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
630 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
631 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
632 { size_t const r = ZSTD_endStream(zc, &outBuff);
633 DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (unsigned)r);
634 if (r != 0) goto _output_error; /* error, or some data not flushed */
635 }
636 cSize = outBuff.pos;
637 ZSTD_freeCDict(cdict);
638 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
639 }
640
641 DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
642 { size_t const s = ZSTD_sizeof_CStream(zc);
643 if (ZSTD_isError(s)) goto _output_error;
644 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
645 }
646
647 DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++);
648 { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
649 if (dID != dictID) goto _output_error;
650 DISPLAYLEVEL(4, "OK (%u) \n", dID);
651 }
652
653 /* DDict scenario */
654 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (unsigned)CNBufferSize);
655 { ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
656 size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
657 if (ZSTD_isError(initError)) goto _output_error;
658 outBuff.dst = decodedBuffer;
659 outBuff.size = CNBufferSize;
660 outBuff.pos = 0;
661 inBuff.src = compressedBuffer;
662 inBuff.size = cSize;
663 inBuff.pos = 0;
664 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
665 if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
666 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
667 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
668 ZSTD_freeDDict(ddict);
669 DISPLAYLEVEL(3, "OK \n");
670 }
671
672 /* Memory restriction */
673 DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
674 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
675 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */
676 outBuff.dst = decodedBuffer;
677 outBuff.size = CNBufferSize;
678 outBuff.pos = 0;
679 inBuff.src = compressedBuffer;
680 inBuff.size = cSize;
681 inBuff.pos = 0;
682 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
683 if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
684 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
685 ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */
686
687 DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
688 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
689 int const maxLevel = 16; /* first level with zstd_opt */
690 int level;
691 assert(maxLevel < ZSTD_maxCLevel());
692 CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) );
693 for (level = 1; level <= maxLevel; ++level) {
694 ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level);
695 size_t const maxSize = MIN(1 MB, CNBufferSize);
696 size_t size;
697 for (size = 512; size <= maxSize; size <<= 1) {
698 U64 const crcOrig = XXH64(CNBuffer, size, 0);
699 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
700 ZSTD_parameters savedParams;
701 getCCtxParams(cctx, &savedParams);
702 outBuff.dst = compressedBuffer;
703 outBuff.size = compressedBufferSize;
704 outBuff.pos = 0;
705 inBuff.src = CNBuffer;
706 inBuff.size = size;
707 inBuff.pos = 0;
708 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
709 CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
710 CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
711 if (inBuff.pos != inBuff.size) goto _output_error;
712 { ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
713 ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
714 CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) );
715 if (decIn.pos != decIn.size) goto _output_error;
716 if (decOut.pos != size) goto _output_error;
717 { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
718 if (crcDec != crcOrig) goto _output_error;
719 } }
720 ZSTD_freeCCtx(cctx);
721 }
722 ZSTD_freeCDict(cdict);
723 }
724 ZSTD_freeDCtx(dctx);
725 }
726 DISPLAYLEVEL(3, "OK\n");
727
728 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
729 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) );
730 cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB));
731 CHECK_Z(cSize);
732 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++);
733 {
734 ZSTD_DCtx* dctx = ZSTD_createDCtx();
735 /* We should fail to decompress without a dictionary. */
736 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
737 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
738 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
739 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
740 if (!ZSTD_isError(ret)) goto _output_error;
741 }
742 /* We should succeed to decompress with the dictionary. */
743 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
744 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
745 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
746 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
747 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
748 if (in.pos != in.size) goto _output_error;
749 }
750 /* The dictionary should presist across calls. */
751 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
752 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
753 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
754 if (in.pos != in.size) goto _output_error;
755 }
756 /* The dictionary should not be cleared by ZSTD_reset_session_only. */
757 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
758 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
759 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
760 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
761 if (in.pos != in.size) goto _output_error;
762 }
763 /* When we reset the context the dictionary is cleared. */
764 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
765 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
766 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
767 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
768 if (!ZSTD_isError(ret)) goto _output_error;
769 }
770 ZSTD_freeDCtx(dctx);
771 }
772 DISPLAYLEVEL(3, "OK \n");
773
774 DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++);
775 {
776 ZSTD_DCtx* dctx = ZSTD_createDCtx();
777 /* We should succeed to decompress with the dictionary. */
778 ZSTD_resetDStream(dctx);
779 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
780 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
781 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
782 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
783 if (in.pos != in.size) goto _output_error;
784 }
785 /* The dictionary should not be cleared by ZSTD_resetDStream(). */
786 ZSTD_resetDStream(dctx);
787 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
788 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
789 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
790 if (in.pos != in.size) goto _output_error;
791 }
792 /* The dictionary should be cleared by ZSTD_initDStream(). */
793 CHECK_Z( ZSTD_initDStream(dctx) );
794 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
795 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
796 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
797 if (!ZSTD_isError(ret)) goto _output_error;
798 }
799 ZSTD_freeDCtx(dctx);
800 }
801 DISPLAYLEVEL(3, "OK \n");
802
803 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++);
804 {
805 ZSTD_DCtx* dctx = ZSTD_createDCtx();
806 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
807 /* We should succeed to decompress with the ddict. */
808 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
809 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
810 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
811 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
812 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
813 if (in.pos != in.size) goto _output_error;
814 }
815 /* The ddict should presist across calls. */
816 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
817 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
818 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
819 if (in.pos != in.size) goto _output_error;
820 }
821 /* When we reset the context the ddict is cleared. */
822 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
823 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
824 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
825 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
826 if (!ZSTD_isError(ret)) goto _output_error;
827 }
828 ZSTD_freeDCtx(dctx);
829 ZSTD_freeDDict(ddict);
830 }
831 DISPLAYLEVEL(3, "OK \n");
832
833 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
834 {
835 ZSTD_DCtx* dctx = ZSTD_createDCtx();
836 /* We should succeed to decompress with the prefix. */
837 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
838 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) );
839 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
840 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
841 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
842 if (in.pos != in.size) goto _output_error;
843 }
844 /* The prefix should be cleared after the first compression. */
845 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
846 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
847 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
848 if (!ZSTD_isError(ret)) goto _output_error;
849 }
850 ZSTD_freeDCtx(dctx);
851 }
852 DISPLAYLEVEL(3, "OK \n");
853
854 DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++);
855 {
856 ZSTD_DCtx* dctx = ZSTD_createDCtx();
857 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
858 size_t ret;
859 /* We should succeed to decompress with the dictionary. */
860 CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) );
861 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
862 /* The dictionary should presist across calls. */
863 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
864 /* We should succeed to decompress with the ddict. */
865 CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) );
866 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
867 /* The ddict should presist across calls. */
868 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
869 /* When we reset the context the ddict is cleared. */
870 CHECK_Z( ZSTD_initDStream(dctx) );
871 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize);
872 if (!ZSTD_isError(ret)) goto _output_error;
873 ZSTD_freeDCtx(dctx);
874 ZSTD_freeDDict(ddict);
875 }
876 DISPLAYLEVEL(3, "OK \n");
877
878 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
879 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
880 ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
881 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
882 size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
883 if (ZSTD_isError(initError)) goto _output_error;
884 outBuff.dst = compressedBuffer;
885 outBuff.size = compressedBufferSize;
886 outBuff.pos = 0;
887 inBuff.src = CNBuffer;
888 inBuff.size = CNBufferSize;
889 inBuff.pos = 0;
890 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
891 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
892 { size_t const r = ZSTD_endStream(zc, &outBuff);
893 if (r != 0) goto _output_error; } /* error, or some data not flushed */
894 cSize = outBuff.pos;
895 ZSTD_freeCDict(cdict);
896 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
897 }
898
899 DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
900 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
901 if (did != 0) goto _output_error;
902 }
903 DISPLAYLEVEL(3, "OK (not detected) \n");
904
905 DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
906 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
907 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
908 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
909 }
910
911 DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
912 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) );
913 outBuff.dst = compressedBuffer;
914 outBuff.size = compressedBufferSize;
915 outBuff.pos = 0;
916 inBuff.src = CNBuffer;
917 inBuff.size = CNBufferSize;
918 inBuff.pos = 0;
919 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
920 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
921 cSize = outBuff.pos;
922 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
923
924 DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++);
925 CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) );
926 outBuff.dst = decodedBuffer;
927 outBuff.size = CNBufferSize;
928 outBuff.pos = 0;
929 inBuff.src = compressedBuffer;
930 inBuff.size = cSize;
931 inBuff.pos = 0;
932 CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) );
933 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
934 if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */
935 DISPLAYLEVEL(3, "OK \n");
936
937 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
938 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
939 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
940 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
941 }
942
943 DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
944 outBuff.dst = compressedBuffer;
945 outBuff.size = compressedBufferSize;
946 outBuff.pos = 0;
947 inBuff.src = CNBuffer;
948 inBuff.size = CNBufferSize;
949 inBuff.pos = 0;
950 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
951 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
952 cSize = outBuff.pos;
953 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
954
955 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
956 CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) );
957 DISPLAYLEVEL(3, "OK \n");
958
959 /* Empty srcSize */
960 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
961 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
962 params.fParams.contentSizeFlag = 1;
963 CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
964 } /* cstream advanced shall write content size = 0 */
965 outBuff.dst = compressedBuffer;
966 outBuff.size = compressedBufferSize;
967 outBuff.pos = 0;
968 inBuff.src = CNBuffer;
969 inBuff.size = 0;
970 inBuff.pos = 0;
971 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
972 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
973 cSize = outBuff.pos;
974 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
975 DISPLAYLEVEL(3, "OK \n");
976
977 DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
978 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
979 params.fParams.contentSizeFlag = 1;
980 CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) );
981 } /* cstream advanced shall write content size = 0 */
982 inBuff.src = CNBuffer;
983 inBuff.size = 0;
984 inBuff.pos = 0;
985 outBuff.dst = compressedBuffer;
986 outBuff.size = compressedBufferSize;
987 outBuff.pos = 0;
988 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
989 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
990 cSize = outBuff.pos;
991 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
992
993 ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */
994 outBuff.dst = compressedBuffer;
995 outBuff.size = compressedBufferSize;
996 outBuff.pos = 0;
997 inBuff.src = CNBuffer;
998 inBuff.size = 0;
999 inBuff.pos = 0;
1000 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1001 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1002 cSize = outBuff.pos;
1003 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
1004 DISPLAYLEVEL(3, "OK \n");
1005
1006 /* Basic multithreading compression test */
1007 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
1008 { ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
1009 int jobSize;
1010 CHECK_Z( ZSTDMT_getMTCtxParameter(mtctx, ZSTDMT_p_jobSize, &jobSize));
1011 CHECK(jobSize != 0, "job size non-zero");
1012 CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) );
1013 CHECK_Z( ZSTDMT_getMTCtxParameter(mtctx, ZSTDMT_p_jobSize, &jobSize));
1014 CHECK(jobSize != 0, "job size non-zero");
1015 }
1016 outBuff.dst = compressedBuffer;
1017 outBuff.size = compressedBufferSize;
1018 outBuff.pos = 0;
1019 inBuff.src = CNBuffer;
1020 inBuff.size = CNBufferSize;
1021 inBuff.pos = 0;
1022 { size_t const compressResult = ZSTDMT_compressStream_generic(mtctx, &outBuff, &inBuff, ZSTD_e_end);
1023 if (compressResult != 0) goto _output_error; /* compression must be completed in a single round */
1024 }
1025 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1026 { size_t const compressedSize = ZSTD_findFrameCompressedSize(compressedBuffer, outBuff.pos);
1027 if (compressedSize != outBuff.pos) goto _output_error; /* must be a full valid frame */
1028 }
1029 DISPLAYLEVEL(3, "OK \n");
1030
1031 /* Complex multithreading + dictionary test */
1032 { U32 const nbWorkers = 2;
1033 size_t const jobSize = 4 * 1 MB;
1034 size_t const srcSize = jobSize * nbWorkers; /* we want each job to have predictable size */
1035 size_t const segLength = 2 KB;
1036 size_t const offset = 600 KB; /* must be larger than window defined in cdict */
1037 size_t const start = jobSize + (offset-1);
1038 const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
1039 BYTE* const dst = (BYTE*)CNBuffer + start - offset;
1040 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize);
1041 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
1042 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
1043 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) );
1044 assert(start > offset);
1045 assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
1046 memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
1047 outBuff.dst = compressedBuffer;
1048 outBuff.size = compressedBufferSize;
1049 outBuff.pos = 0;
1050 inBuff.src = CNBuffer;
1051 inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
1052 inBuff.pos = 0;
1053 }
1054 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */
1055 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1056 DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
1057 CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
1058 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1059 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
1060 ZSTD_freeCDict(cdict);
1061 }
1062 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1063 cSize = outBuff.pos;
1064 DISPLAYLEVEL(3, "OK \n");
1065
1066 DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
1067 { ZSTD_DStream* const dstream = ZSTD_createDCtx();
1068 ZSTD_frameHeader zfh;
1069 ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
1070 DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize);
1071 outBuff.dst = decodedBuffer;
1072 outBuff.size = CNBufferSize;
1073 outBuff.pos = 0;
1074 inBuff.src = compressedBuffer;
1075 inBuff.pos = 0;
1076 CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
1077 inBuff.size = 1; /* avoid shortcut to single-pass mode */
1078 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1079 inBuff.size = cSize;
1080 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1081 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1082 ZSTD_freeDStream(dstream);
1083 }
1084 DISPLAYLEVEL(3, "OK \n");
1085
1086 DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
1087 { unsigned const kMaxWindowLog = 24;
1088 unsigned value;
1089 ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
1090 ZSTD_CDict* cdict;
1091 ZSTD_DDict* ddict;
1092 SEQ_stream seq = SEQ_initStream(0x87654321);
1093 SEQ_gen_type type;
1094 XXH64_state_t xxh;
1095
1096 XXH64_reset(&xxh, 0);
1097 cParams.windowLog = kMaxWindowLog;
1098 cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1099 ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1100
1101 if (!cdict || !ddict) goto _output_error;
1102
1103 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1104 ZSTD_resetDStream(zd);
1105 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1106 CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
1107 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog));
1108 /* Test all values < 300 */
1109 for (value = 0; value < 300; ++value) {
1110 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1111 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1112 }
1113 }
1114 /* Test values 2^8 to 2^17 */
1115 for (value = (1 << 8); value < (1 << 17); value <<= 1) {
1116 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1117 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1118 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
1119 }
1120 }
1121 /* Test offset values up to the max window log */
1122 for (value = 8; value <= kMaxWindowLog; ++value) {
1123 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
1124 }
1125
1126 CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
1127 CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
1128
1129 ZSTD_freeCDict(cdict);
1130 ZSTD_freeDDict(ddict);
1131 }
1132 DISPLAYLEVEL(3, "OK \n");
1133
1134 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++);
1135 { int level;
1136 CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
1137 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1138 CHECK(level != 11, "Compression level does not match");
1139 ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN);
1140 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1141 CHECK(level != 11, "Compression level does not match");
1142 }
1143 DISPLAYLEVEL(3, "OK \n");
1144
1145 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced sets requestedParams : ", testNb++);
1146 { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0);
1147 CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN));
1148 CHECK(badParameters(zc, params), "Compression parameters do not match");
1149 ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN);
1150 CHECK(badParameters(zc, params), "Compression parameters do not match");
1151 }
1152 DISPLAYLEVEL(3, "OK \n");
1153
1154 DISPLAYLEVEL(3, "test%3i : ZSTD_c_srcSizeHint bounds : ", testNb++);
1155 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1156 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, INT_MAX));
1157 { int srcSizeHint;
1158 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_srcSizeHint, &srcSizeHint));
1159 CHECK(!(srcSizeHint == INT_MAX), "srcSizeHint doesn't match");
1160 }
1161 CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
1162 DISPLAYLEVEL(3, "OK \n");
1163
1164 /* Overlen overwriting window data bug */
1165 DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
1166 { /* This test has a window size of 1024 bytes and consists of 3 blocks:
1167 1. 'a' repeated 517 times
1168 2. 'b' repeated 516 times
1169 3. a compressed block with no literals and 3 sequence commands:
1170 litlength = 0, offset = 24, match length = 24
1171 litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3)
1172 litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */
1173
1174 const char* testCase =
1175 "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
1176 "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
1177 "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
1178 ZSTD_DStream* const zds = ZSTD_createDStream();
1179 if (zds==NULL) goto _output_error;
1180
1181 CHECK_Z( ZSTD_initDStream(zds) );
1182 inBuff.src = testCase;
1183 inBuff.size = 47;
1184 inBuff.pos = 0;
1185 outBuff.dst = decodedBuffer;
1186 outBuff.size = CNBufferSize;
1187 outBuff.pos = 0;
1188
1189 while (inBuff.pos < inBuff.size) {
1190 CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) );
1191 }
1192
1193 ZSTD_freeDStream(zds);
1194 }
1195 DISPLAYLEVEL(3, "OK \n");
1196
1197 /* Small Sequence Section bug */
1198 DISPLAYLEVEL(3, "test%3i : decompress blocks with small sequences section : ", testNb++);
1199 { /* This test consists of 3 blocks. Each block has one sequence.
1200 The sequence has literal length of 10, match length of 10 and offset of 10.
1201 The sequence value and compression mode for the blocks are following:
1202 The order of values are ll, ml, of.
1203 - First block : (10, 7, 13) (rle, rle, rle)
1204 - size of sequences section: 6 bytes (1 byte for nbSeq, 1 byte for encoding mode, 3 bytes for rle, 1 byte bitstream)
1205 - Second block : (10, 7, 1) (repeat, repeat, rle)
1206 - size of sequences section: 4 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 bytes for rle, 1 byte bitstream)
1207 - Third block : (10, 7, 1) (repeat, repeat, repeat)
1208 - size of sequences section: 3 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 byte bitstream) */
1209
1210 unsigned char compressed[] = {
1211 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x3c, 0x35, 0x01, 0x00, 0xf0, 0x85, 0x08,
1212 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1213 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac,
1214 0x69, 0x94, 0x89, 0x1c, 0x03, 0x44, 0x0a, 0x07, 0x00, 0xb4, 0x04, 0x80,
1215 0x40, 0x0a, 0xa4
1216 };
1217 unsigned int compressedSize = 51;
1218 unsigned char decompressed[] = {
1219 0x85, 0x08, 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x85, 0x08,
1220 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1221 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0x4c, 0x6b, 0xa9, 0x8b, 0xbc, 0xc5,
1222 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94,
1223 0x89, 0x1c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94, 0x89, 0x1c
1224 };
1225 unsigned int decompressedSize = 60;
1226
1227 ZSTD_DStream* const zds = ZSTD_createDStream();
1228 if (zds==NULL) goto _output_error;
1229
1230 CHECK_Z( ZSTD_initDStream(zds) );
1231 inBuff.src = compressed;
1232 inBuff.size = compressedSize;
1233 inBuff.pos = 0;
1234 outBuff.dst = decodedBuffer;
1235 outBuff.size = CNBufferSize;
1236 outBuff.pos = 0;
1237
1238 CHECK(ZSTD_decompressStream(zds, &outBuff, &inBuff) != 0,
1239 "Decompress did not reach the end of frame");
1240 CHECK(inBuff.pos != inBuff.size, "Decompress did not fully consume input");
1241 CHECK(outBuff.pos != decompressedSize, "Decompressed size does not match");
1242 CHECK(memcmp(outBuff.dst, decompressed, decompressedSize) != 0,
1243 "Decompressed data does not match");
1244
1245 ZSTD_freeDStream(zds);
1246 }
1247 DISPLAYLEVEL(3, "OK \n");
1248
1249 DISPLAYLEVEL(3, "test%3i : dictionary + uncompressible block + reusing tables checks offset table validity: ", testNb++);
1250 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1251 dictionary.start, dictionary.filled,
1252 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1253 ZSTD_getCParams(3, 0, dictionary.filled),
1254 ZSTD_defaultCMem);
1255 const size_t inbufsize = 2 * 128 * 1024; /* 2 blocks */
1256 const size_t outbufsize = ZSTD_compressBound(inbufsize);
1257 size_t inbufpos = 0;
1258 size_t cursegmentlen;
1259 BYTE *inbuf = (BYTE *)malloc(inbufsize);
1260 BYTE *outbuf = (BYTE *)malloc(outbufsize);
1261 BYTE *checkbuf = (BYTE *)malloc(inbufsize);
1262 size_t ret;
1263
1264 CHECK(cdict == NULL, "failed to alloc cdict");
1265 CHECK(inbuf == NULL, "failed to alloc input buffer");
1266
1267 /* first block is uncompressible */
1268 cursegmentlen = 128 * 1024;
1269 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0., 0., seed);
1270 inbufpos += cursegmentlen;
1271
1272 /* second block is compressible */
1273 cursegmentlen = 128 * 1024 - 256;
1274 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0.05, 0., seed);
1275 inbufpos += cursegmentlen;
1276
1277 /* and includes a very long backref */
1278 cursegmentlen = 128;
1279 memcpy(inbuf + inbufpos, dictionary.start + 256, cursegmentlen);
1280 inbufpos += cursegmentlen;
1281
1282 /* and includes a very long backref */
1283 cursegmentlen = 128;
1284 memcpy(inbuf + inbufpos, dictionary.start + 128, cursegmentlen);
1285 inbufpos += cursegmentlen;
1286
1287 ret = ZSTD_compress_usingCDict(zc, outbuf, outbufsize, inbuf, inbufpos, cdict);
1288 CHECK_Z(ret);
1289
1290 ret = ZSTD_decompress_usingDict(zd, checkbuf, inbufsize, outbuf, ret, dictionary.start, dictionary.filled);
1291 CHECK_Z(ret);
1292
1293 CHECK(memcmp(inbuf, checkbuf, inbufpos), "start and finish buffers don't match");
1294
1295 ZSTD_freeCDict(cdict);
1296 free(inbuf);
1297 free(outbuf);
1298 free(checkbuf);
1299 }
1300 DISPLAYLEVEL(3, "OK \n");
1301
1302 DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++);
1303 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1304 dictionary.start, dictionary.filled,
1305 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1306 ZSTD_getCParams(3, 0, dictionary.filled),
1307 ZSTD_defaultCMem);
1308 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1309 int remainingInput = 256 * 1024;
1310 int offset;
1311
1312 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1313 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1314 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1315 /* Write a bunch of 6 byte blocks */
1316 while (remainingInput > 0) {
1317 char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
1318 const size_t kSmallBlockSize = sizeof(testBuffer);
1319 ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
1320
1321 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
1322 CHECK(in.pos != in.size, "input not fully consumed");
1323 remainingInput -= kSmallBlockSize;
1324 }
1325 /* Write several very long offset matches into the dictionary */
1326 for (offset = 1024; offset >= 0; offset -= 128) {
1327 ZSTD_inBuffer in = {dictionary.start + offset, 128, 0};
1328 ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
1329 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
1330 CHECK(in.pos != in.size, "input not fully consumed");
1331 }
1332 /* Ensure decompression works */
1333 CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled));
1334
1335 ZSTD_freeCDict(cdict);
1336 }
1337 DISPLAYLEVEL(3, "OK \n");
1338
1339 _end:
1340 FUZ_freeDictionary(dictionary);
1341 ZSTD_freeCStream(zc);
1342 ZSTD_freeDStream(zd);
1343 ZSTDMT_freeCCtx(mtctx);
1344 free(CNBuffer);
1345 free(compressedBuffer);
1346 free(decodedBuffer);
1347 return testResult;
1348
1349 _output_error:
1350 testResult = 1;
1351 DISPLAY("Error detected in Unit tests ! \n");
1352 goto _end;
1353 }
1354
1355
1356 /* ====== Fuzzer tests ====== */
1357
findDiff(const void * buf1,const void * buf2,size_t max)1358 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
1359 {
1360 const BYTE* b1 = (const BYTE*)buf1;
1361 const BYTE* b2 = (const BYTE*)buf2;
1362 size_t u;
1363 for (u=0; u<max; u++) {
1364 if (b1[u] != b2[u]) break;
1365 }
1366 if (u==max) {
1367 DISPLAY("=> No difference detected within %u bytes \n", (unsigned)max);
1368 return u;
1369 }
1370 DISPLAY("Error at position %u / %u \n", (unsigned)u, (unsigned)max);
1371 if (u>=3)
1372 DISPLAY(" %02X %02X %02X ",
1373 b1[u-3], b1[u-2], b1[u-1]);
1374 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
1375 b1[u], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]);
1376 if (u>=3)
1377 DISPLAY(" %02X %02X %02X ",
1378 b2[u-3], b2[u-2], b2[u-1]);
1379 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
1380 b2[u], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]);
1381 return u;
1382 }
1383
FUZ_rLogLength(U32 * seed,U32 logLength)1384 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
1385 {
1386 size_t const lengthMask = ((size_t)1 << logLength) - 1;
1387 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
1388 }
1389
FUZ_randomLength(U32 * seed,U32 maxLog)1390 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
1391 {
1392 U32 const logLength = FUZ_rand(seed) % maxLog;
1393 return FUZ_rLogLength(seed, logLength);
1394 }
1395
1396 /* Return value in range minVal <= v <= maxVal */
FUZ_randomClampedLength(U32 * seed,U32 minVal,U32 maxVal)1397 static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
1398 {
1399 U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal;
1400 return (U32)((FUZ_rand(seed) % mod) + minVal);
1401 }
1402
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,double compressibility,int bigTests)1403 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, int bigTests)
1404 {
1405 U32 const maxSrcLog = bigTests ? 24 : 22;
1406 static const U32 maxSampleLog = 19;
1407 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
1408 BYTE* cNoiseBuffer[5];
1409 size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog);
1410 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
1411 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
1412 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
1413 size_t const dstBufferSize = srcBufferSize;
1414 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
1415 U32 result = 0;
1416 unsigned testNb = 0;
1417 U32 coreSeed = seed;
1418 ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
1419 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */
1420 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
1421 UTIL_time_t const startClock = UTIL_getTime();
1422 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
1423 size_t dictSize = 0;
1424 U32 oldTestLog = 0;
1425 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
1426
1427 /* allocations */
1428 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
1429 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
1430 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
1431 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
1432 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
1433 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
1434 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
1435 "Not enough memory, fuzzer tests cancelled");
1436
1437 /* Create initial samples */
1438 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
1439 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
1440 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
1441 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
1442 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
1443 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
1444 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
1445
1446 /* catch up testNb */
1447 for (testNb=1; testNb < startTest; testNb++)
1448 FUZ_rand(&coreSeed);
1449
1450 /* test loop */
1451 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
1452 U32 lseed;
1453 const BYTE* srcBuffer;
1454 size_t totalTestSize, totalGenSize, cSize;
1455 XXH64_state_t xxhState;
1456 U64 crcOrig;
1457 U32 resetAllowed = 1;
1458 size_t maxTestSize;
1459
1460 /* init */
1461 FUZ_rand(&coreSeed);
1462 lseed = coreSeed ^ prime32;
1463 if (nbTests >= testNb) {
1464 DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
1465 } else {
1466 DISPLAYUPDATE(2, "\r%6u ", testNb);
1467 }
1468
1469 /* states full reset (deliberately not synchronized) */
1470 /* some issues can only happen when reusing states */
1471 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
1472 ZSTD_freeCStream(zc);
1473 zc = ZSTD_createCStream();
1474 CHECK(zc==NULL, "ZSTD_createCStream : allocation error");
1475 resetAllowed=0;
1476 }
1477 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
1478 ZSTD_freeDStream(zd);
1479 zd = ZSTD_createDStream();
1480 CHECK(zd==NULL, "ZSTD_createDStream : allocation error");
1481 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
1482 }
1483
1484 /* srcBuffer selection [0-4] */
1485 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
1486 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
1487 else {
1488 buffNb >>= 3;
1489 if (buffNb & 7) {
1490 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
1491 buffNb = tnb[buffNb >> 3];
1492 } else {
1493 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
1494 buffNb = tnb[buffNb >> 3];
1495 } }
1496 srcBuffer = cNoiseBuffer[buffNb];
1497 }
1498
1499 /* compression init */
1500 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
1501 && oldTestLog /* at least one test happened */ && resetAllowed) {
1502 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
1503 maxTestSize = MIN(maxTestSize, srcBufferSize-16);
1504 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
1505 CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) );
1506 }
1507 } else {
1508 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
1509 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
1510 U32 const cLevelCandidate = ( FUZ_rand(&lseed) %
1511 (ZSTD_maxCLevel() -
1512 (MAX(testLog, dictLog) / 3)))
1513 + 1;
1514 U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
1515 maxTestSize = FUZ_rLogLength(&lseed, testLog);
1516 oldTestLog = testLog;
1517 /* random dictionary selection */
1518 dictSize = ((FUZ_rand(&lseed)&7)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
1519 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
1520 dict = srcBuffer + dictStart;
1521 }
1522 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
1523 ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
1524 params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
1525 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
1526 params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
1527 CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
1528 } }
1529
1530 /* multi-segments compression test */
1531 XXH64_reset(&xxhState, 0);
1532 { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
1533 U32 n;
1534 for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
1535 /* compress random chunks into randomly sized dst buffers */
1536 { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1537 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
1538 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
1539 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1540 size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
1541 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
1542 outBuff.size = outBuff.pos + dstBuffSize;
1543
1544 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1545
1546 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
1547 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
1548 totalTestSize += inBuff.pos;
1549 }
1550
1551 /* random flush operation, to mess around */
1552 if ((FUZ_rand(&lseed) & 15) == 0) {
1553 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1554 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
1555 outBuff.size = outBuff.pos + adjustedDstSize;
1556 CHECK_Z( ZSTD_flushStream(zc, &outBuff) );
1557 } }
1558
1559 /* final frame epilogue */
1560 { size_t remainingToFlush = (size_t)(-1);
1561 while (remainingToFlush) {
1562 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1563 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
1564 outBuff.size = outBuff.pos + adjustedDstSize;
1565 remainingToFlush = ZSTD_endStream(zc, &outBuff);
1566 CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush));
1567 } }
1568 crcOrig = XXH64_digest(&xxhState);
1569 cSize = outBuff.pos;
1570 }
1571
1572 /* multi - fragments decompression test */
1573 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
1574 CHECK_Z ( ZSTD_resetDStream(zd) );
1575 } else {
1576 CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
1577 }
1578 { size_t decompressionResult = 1;
1579 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
1580 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
1581 for (totalGenSize = 0 ; decompressionResult ; ) {
1582 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1583 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1584 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
1585 inBuff.size = inBuff.pos + readCSrcSize;
1586 outBuff.size = outBuff.pos + dstBuffSize;
1587 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1588 if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) {
1589 DISPLAY("checksum error : \n");
1590 findDiff(copyBuffer, dstBuffer, totalTestSize);
1591 }
1592 CHECK( ZSTD_isError(decompressionResult), "decompression error : %s",
1593 ZSTD_getErrorName(decompressionResult) );
1594 }
1595 CHECK (decompressionResult != 0, "frame not fully decoded");
1596 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)",
1597 (unsigned)outBuff.pos, (unsigned)totalTestSize);
1598 CHECK (inBuff.pos != cSize, "compressed data should be fully read")
1599 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
1600 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
1601 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
1602 } }
1603
1604 /*===== noisy/erroneous src decompression test =====*/
1605
1606 /* add some noise */
1607 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
1608 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
1609 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
1610 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
1611 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
1612 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
1613 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
1614 } }
1615
1616 /* try decompression on noisy data */
1617 CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
1618 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
1619 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
1620 while (outBuff.pos < dstBufferSize) {
1621 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1622 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1623 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
1624 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
1625 outBuff.size = outBuff.pos + adjustedDstSize;
1626 inBuff.size = inBuff.pos + adjustedCSrcSize;
1627 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1628 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
1629 /* No forward progress possible */
1630 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
1631 } } } }
1632 DISPLAY("\r%u fuzzer tests completed \n", testNb);
1633
1634 _cleanup:
1635 ZSTD_freeCStream(zc);
1636 ZSTD_freeDStream(zd);
1637 ZSTD_freeDStream(zd_noise);
1638 free(cNoiseBuffer[0]);
1639 free(cNoiseBuffer[1]);
1640 free(cNoiseBuffer[2]);
1641 free(cNoiseBuffer[3]);
1642 free(cNoiseBuffer[4]);
1643 free(copyBuffer);
1644 free(cBuffer);
1645 free(dstBuffer);
1646 return result;
1647
1648 _output_error:
1649 result = 1;
1650 goto _cleanup;
1651 }
1652
1653
1654 /* fuzzing ZSTDMT_* interface */
fuzzerTests_MT(U32 seed,int nbTests,int startTest,double compressibility,int bigTests)1655 static int fuzzerTests_MT(U32 seed, int nbTests, int startTest,
1656 double compressibility, int bigTests)
1657 {
1658 const U32 maxSrcLog = bigTests ? 24 : 22;
1659 static const U32 maxSampleLog = 19;
1660 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
1661 BYTE* cNoiseBuffer[5];
1662 size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
1663 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
1664 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
1665 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
1666 size_t const dstBufferSize = srcBufferSize;
1667 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
1668 U32 result = 0;
1669 int testNb = 0;
1670 U32 coreSeed = seed;
1671 int nbThreads = 2;
1672 ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads); /* will be reset sometimes */
1673 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
1674 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
1675 UTIL_time_t const startClock = UTIL_getTime();
1676 const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */
1677 size_t dictSize = 0;
1678 int const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
1679 U32 const nbThreadsMax = bigTests ? 4 : 2;
1680
1681 /* allocations */
1682 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
1683 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
1684 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
1685 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
1686 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
1687 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
1688 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
1689 "Not enough memory, fuzzer tests cancelled");
1690
1691 /* Create initial samples */
1692 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
1693 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
1694 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
1695 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
1696 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
1697 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
1698 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
1699 DISPLAYLEVEL(6, "Creating initial context with %i threads \n", nbThreads);
1700
1701 /* catch up testNb */
1702 for (testNb=1; testNb < startTest; testNb++)
1703 FUZ_rand(&coreSeed);
1704
1705 /* test loop */
1706 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
1707 U32 lseed;
1708 const BYTE* srcBuffer;
1709 size_t totalTestSize, totalGenSize, cSize;
1710 XXH64_state_t xxhState;
1711 U64 crcOrig;
1712 size_t maxTestSize;
1713
1714 FUZ_rand(&coreSeed);
1715 if (nbTests >= testNb) {
1716 DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
1717 } else {
1718 DISPLAYUPDATE(2, "\r%6u ", testNb);
1719 }
1720 lseed = coreSeed ^ prime32;
1721
1722 /* states full reset (deliberately not synchronized) */
1723 /* some issues can only happen when reusing states */
1724 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
1725 nbThreads = (FUZ_rand(&lseed) % nbThreadsMax) + 1;
1726 DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads);
1727 ZSTDMT_freeCCtx(zc);
1728 zc = ZSTDMT_createCCtx(nbThreads);
1729 CHECK(zc==NULL, "ZSTDMT_createCCtx allocation error")
1730 }
1731 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
1732 ZSTD_freeDStream(zd);
1733 zd = ZSTD_createDStream();
1734 CHECK(zd==NULL, "ZSTDMT_createCCtx allocation error")
1735 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
1736 }
1737
1738 /* srcBuffer selection [0-4] */
1739 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
1740 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
1741 else {
1742 buffNb >>= 3;
1743 if (buffNb & 7) {
1744 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
1745 buffNb = tnb[buffNb >> 3];
1746 } else {
1747 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
1748 buffNb = tnb[buffNb >> 3];
1749 } }
1750 srcBuffer = cNoiseBuffer[buffNb];
1751 }
1752
1753 /* compression init */
1754 { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
1755 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
1756 int const cLevelCandidate = ( FUZ_rand(&lseed)
1757 % (ZSTD_maxCLevel() - (MAX(testLog, dictLog) / 2)) )
1758 + 1;
1759 int const cLevelThreadAdjusted = cLevelCandidate - (nbThreads * 2) + 2; /* reduce cLevel when multiple threads to reduce memory consumption */
1760 int const cLevelMin = MAX(cLevelThreadAdjusted, 1); /* no negative cLevel yet */
1761 int const cLevel = MIN(cLevelMin, cLevelMax);
1762 maxTestSize = FUZ_rLogLength(&lseed, testLog);
1763
1764 if (FUZ_rand(&lseed)&1) { /* simple init */
1765 int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
1766 DISPLAYLEVEL(5, "Init with compression level = %i \n", compressionLevel);
1767 CHECK_Z( ZSTDMT_initCStream(zc, compressionLevel) );
1768 } else { /* advanced init */
1769 /* random dictionary selection */
1770 dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
1771 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
1772 dict = srcBuffer + dictStart;
1773 }
1774 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
1775 ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
1776 DISPLAYLEVEL(5, "Init with windowLog = %u, pledgedSrcSize = %u, dictSize = %u \n",
1777 params.cParams.windowLog, (unsigned)pledgedSrcSize, (unsigned)dictSize);
1778 params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
1779 params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
1780 params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
1781 DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag);
1782 CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapLog, FUZ_rand(&lseed) % 12) );
1783 CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custom job size */
1784 CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
1785 } } }
1786
1787 /* multi-segments compression test */
1788 XXH64_reset(&xxhState, 0);
1789 { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
1790 U32 n;
1791 for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
1792 /* compress random chunks into randomly sized dst buffers */
1793 { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1794 size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize);
1795 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
1796 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1797 size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
1798 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
1799 outBuff.size = outBuff.pos + dstBuffSize;
1800
1801 DISPLAYLEVEL(6, "Sending %u bytes to compress \n", (unsigned)srcSize);
1802 CHECK_Z( ZSTDMT_compressStream(zc, &outBuff, &inBuff) );
1803 DISPLAYLEVEL(6, "%u bytes read by ZSTDMT_compressStream \n", (unsigned)inBuff.pos);
1804
1805 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
1806 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
1807 totalTestSize += inBuff.pos;
1808 }
1809
1810 /* random flush operation, to mess around */
1811 if ((FUZ_rand(&lseed) & 15) == 0) {
1812 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1813 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
1814 size_t const previousPos = outBuff.pos;
1815 outBuff.size = outBuff.pos + adjustedDstSize;
1816 DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (unsigned)adjustedDstSize);
1817 CHECK_Z( ZSTDMT_flushStream(zc, &outBuff) );
1818 assert(outBuff.pos >= previousPos);
1819 DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_flushStream \n", (unsigned)(outBuff.pos-previousPos));
1820 } }
1821
1822 /* final frame epilogue */
1823 { size_t remainingToFlush = (size_t)(-1);
1824 while (remainingToFlush) {
1825 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1826 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
1827 size_t const previousPos = outBuff.pos;
1828 outBuff.size = outBuff.pos + adjustedDstSize;
1829 DISPLAYLEVEL(5, "Ending into dst buffer of size %u \n", (unsigned)adjustedDstSize);
1830 remainingToFlush = ZSTDMT_endStream(zc, &outBuff);
1831 CHECK (ZSTD_isError(remainingToFlush), "ZSTDMT_endStream error : %s", ZSTD_getErrorName(remainingToFlush));
1832 assert(outBuff.pos >= previousPos);
1833 DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_endStream \n", (unsigned)(outBuff.pos-previousPos));
1834 DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (unsigned)remainingToFlush);
1835 } }
1836 crcOrig = XXH64_digest(&xxhState);
1837 cSize = outBuff.pos;
1838 DISPLAYLEVEL(5, "Frame completed : %u bytes compressed into %u bytes \n",
1839 (unsigned)totalTestSize, (unsigned)cSize);
1840 }
1841
1842 /* multi - fragments decompression test */
1843 assert(totalTestSize < dstBufferSize);
1844 memset(dstBuffer, 170, totalTestSize); /* init dest area */
1845 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
1846 CHECK_Z( ZSTD_resetDStream(zd) );
1847 } else {
1848 CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
1849 }
1850 { size_t decompressionResult = 1;
1851 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
1852 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
1853 for (totalGenSize = 0 ; decompressionResult ; ) {
1854 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1855 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1856 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
1857 inBuff.size = inBuff.pos + readCSrcSize;
1858 outBuff.size = outBuff.pos + dstBuffSize;
1859 DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes into outBuff %u bytes \n",
1860 (unsigned)readCSrcSize, (unsigned)dstBuffSize);
1861 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1862 if (ZSTD_isError(decompressionResult)) {
1863 DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult));
1864 findDiff(copyBuffer, dstBuffer, totalTestSize);
1865 }
1866 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
1867 DISPLAYLEVEL(6, "total ingested (inBuff.pos) = %u and produced (outBuff.pos) = %u \n",
1868 (unsigned)inBuff.pos, (unsigned)outBuff.pos);
1869 }
1870 CHECK (outBuff.pos != totalTestSize,
1871 "decompressed data : wrong size (%u != %u)",
1872 (unsigned)outBuff.pos, (unsigned)totalTestSize );
1873 CHECK (inBuff.pos != cSize,
1874 "compressed data should be fully read (%u != %u)",
1875 (unsigned)inBuff.pos, (unsigned)cSize );
1876 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
1877 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
1878 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
1879 } }
1880
1881 /*===== noisy/erroneous src decompression test =====*/
1882
1883 /* add some noise */
1884 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
1885 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
1886 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
1887 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
1888 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
1889 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
1890 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
1891 } }
1892
1893 /* try decompression on noisy data */
1894 CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
1895 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
1896 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
1897 while (outBuff.pos < dstBufferSize) {
1898 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1899 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1900 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
1901 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
1902 outBuff.size = outBuff.pos + adjustedDstSize;
1903 inBuff.size = inBuff.pos + adjustedCSrcSize;
1904 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1905 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
1906 /* No forward progress possible */
1907 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
1908 } } } }
1909 DISPLAY("\r%u fuzzer tests completed \n", testNb);
1910
1911 _cleanup:
1912 ZSTDMT_freeCCtx(zc);
1913 ZSTD_freeDStream(zd);
1914 ZSTD_freeDStream(zd_noise);
1915 free(cNoiseBuffer[0]);
1916 free(cNoiseBuffer[1]);
1917 free(cNoiseBuffer[2]);
1918 free(cNoiseBuffer[3]);
1919 free(cNoiseBuffer[4]);
1920 free(copyBuffer);
1921 free(cBuffer);
1922 free(dstBuffer);
1923 return result;
1924
1925 _output_error:
1926 result = 1;
1927 goto _cleanup;
1928 }
1929
1930 /** If useOpaqueAPI, sets param in cctxParams.
1931 * Otherwise, sets the param in zc. */
setCCtxParameter(ZSTD_CCtx * zc,ZSTD_CCtx_params * cctxParams,ZSTD_cParameter param,unsigned value,int useOpaqueAPI)1932 static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
1933 ZSTD_cParameter param, unsigned value,
1934 int useOpaqueAPI)
1935 {
1936 if (useOpaqueAPI) {
1937 return ZSTD_CCtxParams_setParameter(cctxParams, param, value);
1938 } else {
1939 return ZSTD_CCtx_setParameter(zc, param, value);
1940 }
1941 }
1942
1943 /* Tests for ZSTD_compress_generic() API */
fuzzerTests_newAPI(U32 seed,int nbTests,int startTest,double compressibility,int bigTests)1944 static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
1945 double compressibility, int bigTests)
1946 {
1947 U32 const maxSrcLog = bigTests ? 24 : 22;
1948 static const U32 maxSampleLog = 19;
1949 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
1950 BYTE* cNoiseBuffer[5];
1951 size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
1952 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
1953 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
1954 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
1955 size_t const dstBufferSize = srcBufferSize;
1956 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
1957 U32 result = 0;
1958 int testNb = 0;
1959 U32 coreSeed = seed;
1960 ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
1961 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
1962 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
1963 UTIL_time_t const startClock = UTIL_getTime();
1964 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
1965 size_t dictSize = 0;
1966 U32 oldTestLog = 0;
1967 U32 windowLogMalus = 0; /* can survive between 2 loops */
1968 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
1969 U32 const nbThreadsMax = bigTests ? 4 : 2;
1970 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
1971
1972 /* allocations */
1973 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
1974 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
1975 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
1976 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
1977 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
1978 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
1979 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
1980 "Not enough memory, fuzzer tests cancelled");
1981
1982 /* Create initial samples */
1983 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
1984 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
1985 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
1986 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
1987 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
1988 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
1989 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
1990
1991 /* catch up testNb */
1992 for (testNb=1; testNb < startTest; testNb++)
1993 FUZ_rand(&coreSeed);
1994
1995 /* test loop */
1996 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
1997 U32 lseed;
1998 int opaqueAPI;
1999 const BYTE* srcBuffer;
2000 size_t totalTestSize, totalGenSize, cSize;
2001 XXH64_state_t xxhState;
2002 U64 crcOrig;
2003 U32 resetAllowed = 1;
2004 size_t maxTestSize;
2005 ZSTD_parameters savedParams;
2006
2007 /* init */
2008 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
2009 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
2010 FUZ_rand(&coreSeed);
2011 lseed = coreSeed ^ prime32;
2012 DISPLAYLEVEL(5, " *** Test %u *** \n", testNb);
2013 opaqueAPI = FUZ_rand(&lseed) & 1;
2014
2015 /* states full reset (deliberately not synchronized) */
2016 /* some issues can only happen when reusing states */
2017 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2018 DISPLAYLEVEL(5, "Creating new context \n");
2019 ZSTD_freeCCtx(zc);
2020 zc = ZSTD_createCCtx();
2021 CHECK(zc == NULL, "ZSTD_createCCtx allocation error");
2022 resetAllowed = 0;
2023 }
2024 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2025 ZSTD_freeDStream(zd);
2026 zd = ZSTD_createDStream();
2027 CHECK(zd == NULL, "ZSTD_createDStream allocation error");
2028 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2029 }
2030
2031 /* srcBuffer selection [0-4] */
2032 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2033 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2034 else {
2035 buffNb >>= 3;
2036 if (buffNb & 7) {
2037 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2038 buffNb = tnb[buffNb >> 3];
2039 } else {
2040 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2041 buffNb = tnb[buffNb >> 3];
2042 } }
2043 srcBuffer = cNoiseBuffer[buffNb];
2044 }
2045
2046 /* compression init */
2047 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
2048 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2049 && oldTestLog /* at least one test happened */
2050 && resetAllowed) {
2051 /* just set a compression level */
2052 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2053 if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
2054 { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
2055 DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel);
2056 CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) );
2057 }
2058 } else {
2059 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2060 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2061 U32 const cLevelCandidate = (FUZ_rand(&lseed) %
2062 (ZSTD_maxCLevel() -
2063 (MAX(testLog, dictLog) / 2))) +
2064 1;
2065 int const cLevel = MIN(cLevelCandidate, cLevelMax);
2066 DISPLAYLEVEL(5, "t%i: base cLevel : %u \n", testNb, cLevel);
2067 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2068 DISPLAYLEVEL(5, "t%i: maxTestSize : %u \n", testNb, (unsigned)maxTestSize);
2069 oldTestLog = testLog;
2070 /* random dictionary selection */
2071 dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2072 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2073 dict = srcBuffer + dictStart;
2074 if (!dictSize) dict=NULL;
2075 }
2076 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2077 ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
2078 const U32 windowLogMax = bigTests ? 24 : 20;
2079 const U32 searchLogMax = bigTests ? 15 : 13;
2080 if (dictSize)
2081 DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize);
2082
2083 /* mess with compression parameters */
2084 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
2085 cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
2086 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
2087 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
2088 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
2089 cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
2090 cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1;
2091 cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
2092 cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);
2093
2094 if (FUZ_rand(&lseed) & 1) {
2095 DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
2096 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) );
2097 assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
2098 windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
2099 }
2100 if (FUZ_rand(&lseed) & 1) {
2101 DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
2102 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) );
2103 }
2104 if (FUZ_rand(&lseed) & 1) {
2105 DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
2106 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) );
2107 }
2108 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) );
2109 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) );
2110 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) );
2111
2112 /* mess with long distance matching parameters */
2113 if (bigTests) {
2114 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) );
2115 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
2116 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
2117 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
2118 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) );
2119 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_srcSizeHint, FUZ_randomClampedLength(&lseed, ZSTD_SRCSIZEHINT_MIN, ZSTD_SRCSIZEHINT_MAX), opaqueAPI) );
2120 }
2121
2122 /* mess with frame parameters */
2123 if (FUZ_rand(&lseed) & 1) {
2124 int const checksumFlag = FUZ_rand(&lseed) & 1;
2125 DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag);
2126 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) );
2127 }
2128 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2129 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2130 if (FUZ_rand(&lseed) & 1) {
2131 DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize);
2132 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2133 }
2134
2135 /* multi-threading parameters. Only adjust occasionally for small tests. */
2136 if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
2137 U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
2138 U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
2139 int const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
2140 DISPLAYLEVEL(5, "t%i: nbThreads : %u \n", testNb, nbThreads);
2141 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) );
2142 if (nbThreads > 1) {
2143 U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
2144 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
2145 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
2146 }
2147 }
2148 /* Enable rsyncable mode 1 in 4 times. */
2149 setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, (FUZ_rand(&lseed) % 4 == 0), opaqueAPI);
2150
2151 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
2152
2153 /* Apply parameters */
2154 if (opaqueAPI) {
2155 DISPLAYLEVEL(5, "t%u: applying CCtxParams \n", testNb);
2156 CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
2157 }
2158
2159 if (FUZ_rand(&lseed) & 1) {
2160 if (FUZ_rand(&lseed) & 1) {
2161 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
2162 } else {
2163 CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) );
2164 }
2165 } else {
2166 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
2167 }
2168 } }
2169
2170 CHECK_Z(getCCtxParams(zc, &savedParams));
2171
2172 /* multi-segments compression test */
2173 XXH64_reset(&xxhState, 0);
2174 { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
2175 for (cSize=0, totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
2176 /* compress random chunks into randomly sized dst buffers */
2177 size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2178 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
2179 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
2180 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
2181 size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
2182 ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
2183 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
2184 outBuff.size = outBuff.pos + dstBuffSize;
2185
2186 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
2187 DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
2188 testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos);
2189
2190 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
2191 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
2192 totalTestSize += inBuff.pos;
2193 }
2194
2195 /* final frame epilogue */
2196 { size_t remainingToFlush = 1;
2197 while (remainingToFlush) {
2198 ZSTD_inBuffer inBuff = { NULL, 0, 0 };
2199 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
2200 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2201 outBuff.size = outBuff.pos + adjustedDstSize;
2202 DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize);
2203 remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
2204 DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos);
2205 CHECK( ZSTD_isError(remainingToFlush),
2206 "ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
2207 ZSTD_getErrorName(remainingToFlush) );
2208 } }
2209 crcOrig = XXH64_digest(&xxhState);
2210 cSize = outBuff.pos;
2211 DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize);
2212 }
2213
2214 CHECK(badParameters(zc, savedParams), "CCtx params are wrong");
2215
2216 /* multi - fragments decompression test */
2217 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
2218 DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", dict);
2219 CHECK_Z( ZSTD_resetDStream(zd) );
2220 } else {
2221 if (dictSize)
2222 DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize);
2223 CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
2224 }
2225 { size_t decompressionResult = 1;
2226 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2227 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2228 for (totalGenSize = 0 ; decompressionResult ; ) {
2229 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2230 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2231 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
2232 inBuff.size = inBuff.pos + readCSrcSize;
2233 outBuff.size = outBuff.pos + dstBuffSize;
2234 DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n",
2235 (unsigned)readCSrcSize, (unsigned)inBuff.pos, (unsigned)cSize);
2236 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2237 DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n",
2238 (unsigned)inBuff.pos, (unsigned)outBuff.pos);
2239 if (ZSTD_isError(decompressionResult)) {
2240 DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult));
2241 findDiff(copyBuffer, dstBuffer, totalTestSize);
2242 }
2243 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
2244 CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (unsigned)inBuff.pos, (unsigned)cSize);
2245 }
2246 CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (unsigned)inBuff.pos, (unsigned)cSize);
2247 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (unsigned)outBuff.pos, (unsigned)totalTestSize);
2248 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
2249 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
2250 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
2251 } }
2252
2253 /*===== noisy/erroneous src decompression test =====*/
2254
2255 /* add some noise */
2256 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
2257 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
2258 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
2259 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
2260 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
2261 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
2262 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
2263 } }
2264
2265 /* try decompression on noisy data */
2266 CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
2267 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2268 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2269 while (outBuff.pos < dstBufferSize) {
2270 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2271 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2272 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
2273 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
2274 outBuff.size = outBuff.pos + adjustedDstSize;
2275 inBuff.size = inBuff.pos + adjustedCSrcSize;
2276 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2277 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
2278 /* Good so far, but no more progress possible */
2279 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
2280 } } } }
2281 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
2282
2283 _cleanup:
2284 ZSTD_freeCCtx(zc);
2285 ZSTD_freeDStream(zd);
2286 ZSTD_freeDStream(zd_noise);
2287 ZSTD_freeCCtxParams(cctxParams);
2288 free(cNoiseBuffer[0]);
2289 free(cNoiseBuffer[1]);
2290 free(cNoiseBuffer[2]);
2291 free(cNoiseBuffer[3]);
2292 free(cNoiseBuffer[4]);
2293 free(copyBuffer);
2294 free(cBuffer);
2295 free(dstBuffer);
2296 return result;
2297
2298 _output_error:
2299 result = 1;
2300 goto _cleanup;
2301 }
2302
2303 /*-*******************************************************
2304 * Command line
2305 *********************************************************/
FUZ_usage(const char * programName)2306 static int FUZ_usage(const char* programName)
2307 {
2308 DISPLAY( "Usage :\n");
2309 DISPLAY( " %s [args]\n", programName);
2310 DISPLAY( "\n");
2311 DISPLAY( "Arguments :\n");
2312 DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
2313 DISPLAY( " -s# : Select seed (default:prompt user)\n");
2314 DISPLAY( " -t# : Select starting test number (default:0)\n");
2315 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
2316 DISPLAY( " -v : verbose\n");
2317 DISPLAY( " -p : pause at the end\n");
2318 DISPLAY( " -h : display help and exit\n");
2319 return 0;
2320 }
2321
2322 typedef enum { simple_api, mt_api, advanced_api } e_api;
2323
main(int argc,const char ** argv)2324 int main(int argc, const char** argv)
2325 {
2326 U32 seed = 0;
2327 int seedset = 0;
2328 int nbTests = nbTestsDefault;
2329 int testNb = 0;
2330 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
2331 int result = 0;
2332 int mainPause = 0;
2333 int bigTests = (sizeof(size_t) == 8);
2334 e_api selected_api = simple_api;
2335 const char* const programName = argv[0];
2336 int argNb;
2337
2338 /* Check command line */
2339 for(argNb=1; argNb<argc; argNb++) {
2340 const char* argument = argv[argNb];
2341 assert(argument != NULL);
2342
2343 /* Parsing commands. Aggregated commands are allowed */
2344 if (argument[0]=='-') {
2345
2346 if (!strcmp(argument, "--mt")) { selected_api=mt_api; testNb += !testNb; continue; }
2347 if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
2348 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
2349
2350 argument++;
2351 while (*argument!=0) {
2352 switch(*argument)
2353 {
2354 case 'h':
2355 return FUZ_usage(programName);
2356
2357 case 'v':
2358 argument++;
2359 g_displayLevel++;
2360 break;
2361
2362 case 'q':
2363 argument++;
2364 g_displayLevel--;
2365 break;
2366
2367 case 'p': /* pause at the end */
2368 argument++;
2369 mainPause = 1;
2370 break;
2371
2372 case 'i': /* limit tests by nb of iterations (default) */
2373 argument++;
2374 nbTests=0; g_clockTime=0;
2375 while ((*argument>='0') && (*argument<='9')) {
2376 nbTests *= 10;
2377 nbTests += *argument - '0';
2378 argument++;
2379 }
2380 break;
2381
2382 case 'T': /* limit tests by time */
2383 argument++;
2384 nbTests=0; g_clockTime=0;
2385 while ((*argument>='0') && (*argument<='9')) {
2386 g_clockTime *= 10;
2387 g_clockTime += *argument - '0';
2388 argument++;
2389 }
2390 if (*argument=='m') { /* -T1m == -T60 */
2391 g_clockTime *=60, argument++;
2392 if (*argument=='n') argument++; /* -T1mn == -T60 */
2393 } else if (*argument=='s') argument++; /* -T10s == -T10 */
2394 g_clockTime *= SEC_TO_MICRO;
2395 break;
2396
2397 case 's': /* manually select seed */
2398 argument++;
2399 seedset=1;
2400 seed=0;
2401 while ((*argument>='0') && (*argument<='9')) {
2402 seed *= 10;
2403 seed += *argument - '0';
2404 argument++;
2405 }
2406 break;
2407
2408 case 't': /* select starting test number */
2409 argument++;
2410 testNb=0;
2411 while ((*argument>='0') && (*argument<='9')) {
2412 testNb *= 10;
2413 testNb += *argument - '0';
2414 argument++;
2415 }
2416 break;
2417
2418 case 'P': /* compressibility % */
2419 argument++;
2420 proba=0;
2421 while ((*argument>='0') && (*argument<='9')) {
2422 proba *= 10;
2423 proba += *argument - '0';
2424 argument++;
2425 }
2426 if (proba<0) proba=0;
2427 if (proba>100) proba=100;
2428 break;
2429
2430 default:
2431 return FUZ_usage(programName);
2432 }
2433 } } } /* for(argNb=1; argNb<argc; argNb++) */
2434
2435 /* Get Seed */
2436 DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
2437
2438 if (!seedset) {
2439 time_t const t = time(NULL);
2440 U32 const h = XXH32(&t, sizeof(t), 1);
2441 seed = h % 10000;
2442 }
2443
2444 DISPLAY("Seed = %u\n", (unsigned)seed);
2445 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
2446
2447 if (nbTests<=0) nbTests=1;
2448
2449 if (testNb==0) {
2450 result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
2451 }
2452
2453 if (!result) {
2454 switch(selected_api)
2455 {
2456 case simple_api :
2457 result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
2458 break;
2459 case mt_api :
2460 result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
2461 break;
2462 case advanced_api :
2463 result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
2464 break;
2465 default :
2466 assert(0); /* impossible */
2467 }
2468 }
2469
2470 if (mainPause) {
2471 int unused;
2472 DISPLAY("Press Enter \n");
2473 unused = getchar();
2474 (void)unused;
2475 }
2476 return result;
2477 }
2478